feat(sense): add reconciliation safety controls [T-012]
This commit is contained in:
+22
-5
@@ -1,6 +1,6 @@
|
||||
# Sense M1/M2 接入骨架
|
||||
|
||||
本目录是 YoVision Sense 的 M1/M2 接入骨架。数据库保存期望态,ONVIF 和 MediaMTX 通过端口隔离;M1 默认使用 SQLite,T-009~T-011 增加 PostgreSQL 双 schema、Area 准入、本地审计 Outbox 和 Control API v1。默认关闭真实 ONVIF 与公共业务路由;T-006 的真实样机结论仅覆盖已批准的精确海康基线,不能据此宣称多品牌兼容。
|
||||
本目录是 YoVision Sense 的 M1/M2 接入骨架。数据库保存期望态,ONVIF 和 MediaMTX 通过端口隔离;M1 默认使用 SQLite,T-009~T-012 增加 PostgreSQL 双 schema、Area 准入、本地审计 Outbox、Control API v1、多实例调和 fencing 和孤儿受控处置。默认关闭真实 ONVIF 与公共业务路由;T-006 的真实样机结论仅覆盖已批准的精确海康基线,不能据此宣称多品牌兼容。
|
||||
|
||||
## 常用命令
|
||||
|
||||
@@ -14,7 +14,7 @@ go build -o bin/sense-api.exe ./cmd/sense-api
|
||||
go run ./cmd/sense-api
|
||||
```
|
||||
|
||||
Unix 将构建产物改为 `bin/sense-api`。服务默认监听 `127.0.0.1:8080`,SQLite 默认写入 `Sense/data/sense.db`,MediaMTX 控制 API 默认是 `http://127.0.0.1:9997`。默认运行只暴露 `/healthz` 与 `/readyz`;只有显式选择 PostgreSQL 并完成安全配置后才注册 7 个 `/api/v1` Control API 路由。
|
||||
Unix 将构建产物改为 `bin/sense-api`。服务默认监听 `127.0.0.1:8080`,SQLite 默认写入 `Sense/data/sense.db`,MediaMTX 控制 API 默认是 `http://127.0.0.1:9997`。默认运行暴露 `/healthz`、`/readyz` 和不含租户/设备标签的 `/metrics`;只有显式选择 PostgreSQL 并完成安全配置后才注册 7 个 `/api/v1` Control API 路由。
|
||||
|
||||
常用环境变量:
|
||||
|
||||
@@ -26,7 +26,13 @@ Unix 将构建产物改为 `bin/sense-api`。服务默认监听 `127.0.0.1:8080`
|
||||
| `SENSE_DB_DSN` | `file:data/sense.db` | 所选 driver 的私有 DSN;不得写入日志、文档或仓库,PostgreSQL 密码优先由 `PGPASSFILE`/环境密钥提供 |
|
||||
| `SENSE_MEDIAMTX_URL` | `http://127.0.0.1:9997` | MediaMTX 控制 API;不得包含 userinfo |
|
||||
| `SENSE_RECONCILE_INTERVAL` | `5s` | 对账周期 |
|
||||
| `SENSE_RECONCILE_LEASE_DURATION` | `30s` | PostgreSQL due-row 租期;最大 5 分钟 |
|
||||
| `SENSE_RECONCILE_OPERATION_TIMEOUT` | `20s` | 单项 ONVIF/MediaMTX deadline;必须严格短于租期 |
|
||||
| `SENSE_PROBE_INTERVAL` | `10s` | path 探活周期 |
|
||||
| `SENSE_INSTANCE_ID` | 随进程随机生成 | 最多 64 位低基数字符串;多实例部署建议显式注入唯一实例 ID |
|
||||
| `SENSE_METRICS_ENABLED` | `true` | 是否注册低基数 Prometheus 文本 `/metrics` |
|
||||
| `SENSE_ORPHAN_SCAN_ENABLED` | PostgreSQL 为 `true`,SQLite 为 `false` | 周期执行只读 MediaMTX Path 差异报告;从不自动删除 |
|
||||
| `SENSE_ORPHAN_SCAN_INTERVAL` | `1m` | 孤儿只读扫描周期,最短 10 秒 |
|
||||
| `SENSE_ONVIF_MODE` | `disabled` | `disabled` 或 `standard`;默认不访问真实摄像头 |
|
||||
| `SENSE_ONVIF_RTSP_REWRITE_HOST` | 空 | NAT 或故障代理场景下重写 ONVIF 返回的 RTSP 主机 |
|
||||
| `SENSE_ONVIF_RTSP_REWRITE_PORT` | `0` | 非零时重写 ONVIF 返回的 RTSP 端口 |
|
||||
@@ -50,9 +56,9 @@ SENSE_CREDENTIAL_<KEY>_RTSP_PASSWORD
|
||||
|
||||
MediaMTX `v1.19.3` 应作为独立二进制启动并只在可信网络开放 API。获取与 SHA-256 校验值见 `docs/03-tech-stack.md`。生成客户端使用固定版本工具和 vendored 官方 OpenAPI;`internal/mtx/generated/client.gen.go` 不可手改。
|
||||
|
||||
## T-009~T-011 PostgreSQL 17.10
|
||||
## T-009~T-012 PostgreSQL 17.10
|
||||
|
||||
初始化与增量 SQL 位于 `deploy/postgres/`,由高权限部署步骤按文件名前缀执行;Sense 进程不会自动创建角色、schema 或 Bell 对象。`bell_app` 拥有 Site/Area、配额、`capture_policy` 及两个版本化视图,`sense_app` 只能读取两个视图,不能读取或写入 Bell 源表。T-011 的 v4 schema 另有资源版本、24 小时幂等收据和 batch operation 表;收据只保存 scope/body 摘要与脱敏响应快照。应用登录角色和密码由部署环境创建,不进入仓库。
|
||||
初始化与增量 SQL 位于 `deploy/postgres/`,由高权限部署步骤按文件名前缀执行;Sense 进程不会自动创建角色、schema 或 Bell 对象。`bell_app` 拥有 Site/Area、配额、`capture_policy` 及两个版本化视图,`sense_app` 只能读取两个视图,不能读取或写入 Bell 源表。T-011 的 v4 schema 增加资源版本、24 小时幂等收据和 batch operation;T-012 的 v5 schema 增加数据库时钟租约、Path 历史归属及脱敏孤儿报告/处置结果。表中不保存 MediaMTX source URI。应用登录角色和密码由部署环境创建,不进入仓库。
|
||||
|
||||
PostgreSQL 新建设备必须携带匹配 tenant/Site 的 `area_id`。具有 `video_capture` 能力的设备在创建、移动 Area 和从 disabled 切到 enabled 时执行 Area 准入;`non_imaging_only` 拒绝成像设备但允许非成像设备。投影缺失、非法或版本回退只拒绝新变更,不关闭已有流。创建、配置修改和期望态受理都与对应脱敏 Outbox 事实同事务;停用后调和器只删除该设备的精确 MediaMTX path 并收敛为 offline,不枚举未知 path。Bell relay 尚未实现。
|
||||
|
||||
@@ -72,7 +78,18 @@ $env:SENSE_DB_DSN = '由部署环境私下设置'
|
||||
go run ./cmd/sense-api
|
||||
```
|
||||
|
||||
PostgreSQL 启动会检查 Sense migration 版本及当前角色对两个 Bell 投影视图的最小权限;权限过宽、视图不可读或 schema 未安装时 readiness 初始化失败。默认 SQLite 路径和 `cmd/sense-lab` 保持不变,但 SQLite 不实现生产 Area/Outbox 一致性,Control API feature flag 在 SQLite 下会直接拒绝启动。
|
||||
PostgreSQL 启动会检查 Sense v5 migration、当前角色对两个 Bell 投影视图和本地控制/对账表的最小权限;权限过宽、视图不可读或 schema 未安装时初始化失败。默认 SQLite 路径和 `cmd/sense-lab` 保持不变,但 SQLite 不实现生产 Area/Outbox、多实例租约或孤儿处置语义,Control API 与孤儿扫描在 SQLite 下不会启动。
|
||||
|
||||
### 孤儿报告与受控处置
|
||||
|
||||
`sense-api` 的 PostgreSQL 模式默认每分钟只读枚举一次 MediaMTX 配置 Path,并持久化汇总。`unowned` 表示没有 Sense 历史归属证据,永远不会由 Sense 删除;只有 `owned_stale` 可以进入处置候选。需要人工处置时从 `Sense/` 执行:
|
||||
|
||||
```powershell
|
||||
go run ./cmd/sense-orphan -mode report
|
||||
go run ./cmd/sense-orphan -mode apply -scan-id scan_... -actor operator-id -confirm "DELETE scan_..."
|
||||
```
|
||||
|
||||
`report` 输出不含 source URI 的 JSON 摘要。`apply` 会再次读取数据库和 MediaMTX,只处理仍属于原快照的 `owned_stale`,并要求快照不超过 15 分钟、候选不超过 128 且候选占当前全部配置 Path 不超过 10%;任何门禁失败均为零删除,没有 `force`。完整流程与恢复方法见 [`../docs/runbooks/sense-reconciliation.md`](../docs/runbooks/sense-reconciliation.md)。
|
||||
|
||||
### 开启 Control API
|
||||
|
||||
|
||||
+38
-10
@@ -15,8 +15,10 @@ import (
|
||||
"yovision/sense/internal/auth"
|
||||
"yovision/sense/internal/config"
|
||||
"yovision/sense/internal/controlapi"
|
||||
"yovision/sense/internal/metrics"
|
||||
"yovision/sense/internal/mtx"
|
||||
"yovision/sense/internal/onvif"
|
||||
"yovision/sense/internal/orphan"
|
||||
"yovision/sense/internal/probe"
|
||||
"yovision/sense/internal/reconcile"
|
||||
"yovision/sense/internal/store"
|
||||
@@ -37,6 +39,14 @@ func run(logger *slog.Logger) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("load configuration: %w", err)
|
||||
}
|
||||
instanceID := cfg.InstanceID
|
||||
if instanceID == "" {
|
||||
instanceID, err = metrics.GenerateInstanceID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
registry := metrics.New(instanceID, version)
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
@@ -76,22 +86,36 @@ func run(logger *slog.Logger) error {
|
||||
})
|
||||
}
|
||||
discovery := onvif.NewRouter(cameraAdapter, credentials)
|
||||
reconciler := reconcile.New(repository, discovery, mediaClient)
|
||||
reconciler := reconcile.NewWithOptions(repository, discovery, mediaClient, reconcile.Options{
|
||||
InstanceID: instanceID, LeaseDuration: cfg.ReconcileLeaseDuration,
|
||||
OperationTimeout: cfg.ReconcileOperationTimeout, Metrics: registry,
|
||||
})
|
||||
checker := probe.New(repository, mediaClient)
|
||||
var orphanScanner *orphan.Manager
|
||||
if cfg.OrphanScanEnabled {
|
||||
orphanStore, ok := repository.(store.OrphanRepository)
|
||||
if !ok {
|
||||
return errors.New("selected repository does not support orphan scanning")
|
||||
}
|
||||
orphanScanner = orphan.New(orphanStore, mediaClient, instanceID, registry)
|
||||
}
|
||||
report := func(err error) {
|
||||
// Domain and MediaMTX errors intentionally omit stream URIs and credentials.
|
||||
logger.Warn("background convergence error", "error", err)
|
||||
}
|
||||
var background sync.WaitGroup
|
||||
background.Add(2)
|
||||
go func() {
|
||||
defer background.Done()
|
||||
reconciler.Run(ctx, cfg.ReconcileInterval, report)
|
||||
}()
|
||||
go func() {
|
||||
defer background.Done()
|
||||
checker.Run(ctx, cfg.ProbeInterval, report)
|
||||
}()
|
||||
startBackground := func(run func()) {
|
||||
background.Add(1)
|
||||
go func() {
|
||||
defer background.Done()
|
||||
run()
|
||||
}()
|
||||
}
|
||||
startBackground(func() { reconciler.Run(ctx, cfg.ReconcileInterval, report) })
|
||||
startBackground(func() { checker.Run(ctx, cfg.ProbeInterval, report) })
|
||||
if orphanScanner != nil {
|
||||
startBackground(func() { orphanScanner.Run(ctx, cfg.OrphanScanInterval, report) })
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /healthz", func(writer http.ResponseWriter, _ *http.Request) {
|
||||
@@ -104,6 +128,9 @@ func run(logger *slog.Logger) error {
|
||||
writer.WriteHeader(http.StatusOK)
|
||||
_, _ = writer.Write([]byte(`{"status":"ready"}`))
|
||||
})
|
||||
if cfg.MetricsEnabled {
|
||||
mux.Handle("GET /metrics", registry.Handler())
|
||||
}
|
||||
if cfg.ControlAPIEnabled {
|
||||
mux.Handle("/api/v1/", controlHandler)
|
||||
}
|
||||
@@ -118,6 +145,7 @@ func run(logger *slog.Logger) error {
|
||||
serverErrors := make(chan error, 1)
|
||||
go func() {
|
||||
logger.Info("Sense listening", "address", cfg.HTTPAddress, "version", version,
|
||||
"instance_id", instanceID,
|
||||
"control_api_enabled", cfg.ControlAPIEnabled)
|
||||
serverErrors <- server.ListenAndServe()
|
||||
}()
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
// sense-orphan is a local, PostgreSQL-backed operational command. Report is
|
||||
// read-only; apply requires a fresh scan ID, actor and exact confirmation.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"yovision/sense/internal/config"
|
||||
"yovision/sense/internal/metrics"
|
||||
"yovision/sense/internal/mtx"
|
||||
"yovision/sense/internal/orphan"
|
||||
"yovision/sense/internal/store"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
_, _ = fmt.Fprintln(os.Stderr, "sense-orphan failed:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run() error {
|
||||
mode := flag.String("mode", "report", "report or apply")
|
||||
scanID := flag.String("scan-id", "", "fresh scan ID required by apply")
|
||||
actor := flag.String("actor", "", "operator identifier required by apply")
|
||||
confirm := flag.String("confirm", "", "exact confirmation: DELETE <scan-id>")
|
||||
flag.Parse()
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !strings.EqualFold(strings.TrimSpace(cfg.DatabaseDriver), store.DriverPostgres) {
|
||||
return errors.New("sense-orphan requires SENSE_DB_DRIVER=postgres")
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
repository, err := store.OpenPostgres(ctx, cfg.DatabaseDSN)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer repository.Close()
|
||||
media, err := mtx.NewClient(cfg.MediaMTXURL, &http.Client{Timeout: cfg.ReconcileOperationTimeout})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
instanceID := cfg.InstanceID
|
||||
if instanceID == "" {
|
||||
instanceID, err = metrics.GenerateInstanceID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
manager := orphan.New(repository, media, instanceID, metrics.New(instanceID, "operator"))
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
switch *mode {
|
||||
case "report":
|
||||
scan, err := manager.Report(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return encoder.Encode(struct {
|
||||
ScanID string `json:"scan_id"`
|
||||
Observed int `json:"observed"`
|
||||
OwnedStale int `json:"owned_stale"`
|
||||
Unowned int `json:"unowned"`
|
||||
SafetyAllowed bool `json:"safety_allowed"`
|
||||
SafetyReason string `json:"safety_reason"`
|
||||
ExpiresAt string `json:"expires_at"`
|
||||
}{
|
||||
ScanID: scan.ID, Observed: scan.ObservedCount,
|
||||
OwnedStale: scan.OwnedStaleCount, Unowned: scan.UnownedCount,
|
||||
SafetyAllowed: scan.SafetyAllowed, SafetyReason: scan.SafetyReason,
|
||||
ExpiresAt: scan.ExpiresAt.UTC().Format("2006-01-02T15:04:05Z07:00"),
|
||||
})
|
||||
case "apply":
|
||||
result, err := manager.Apply(ctx, *scanID, *actor, *confirm)
|
||||
if encodeErr := encoder.Encode(result); encodeErr != nil {
|
||||
return encodeErr
|
||||
}
|
||||
return err
|
||||
default:
|
||||
return errors.New("mode must be report or apply")
|
||||
}
|
||||
}
|
||||
+110
-40
@@ -7,39 +7,52 @@ import (
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultHTTPAddress = "127.0.0.1:8080"
|
||||
defaultDatabaseDriver = "sqlite"
|
||||
defaultDatabaseDSN = "file:data/sense.db"
|
||||
defaultMediaMTXURL = "http://127.0.0.1:9997"
|
||||
defaultReconcilePeriod = 5 * time.Second
|
||||
defaultProbePeriod = 10 * time.Second
|
||||
defaultONVIFMode = "disabled"
|
||||
defaultControlAuthMode = "static-sha256"
|
||||
defaultHTTPAddress = "127.0.0.1:8080"
|
||||
defaultDatabaseDriver = "sqlite"
|
||||
postgresDatabaseDriver = "postgres"
|
||||
defaultDatabaseDSN = "file:data/sense.db"
|
||||
defaultMediaMTXURL = "http://127.0.0.1:9997"
|
||||
defaultReconcilePeriod = 5 * time.Second
|
||||
defaultReconcileLease = 30 * time.Second
|
||||
defaultOperationTimeout = 20 * time.Second
|
||||
defaultProbePeriod = 10 * time.Second
|
||||
defaultOrphanScanPeriod = time.Minute
|
||||
defaultONVIFMode = "disabled"
|
||||
defaultControlAuthMode = "static-sha256"
|
||||
)
|
||||
|
||||
var instanceIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`)
|
||||
|
||||
type Config struct {
|
||||
HTTPAddress string
|
||||
AllowNonLoopback bool
|
||||
DatabaseDriver string
|
||||
DatabaseDSN string
|
||||
MediaMTXURL string
|
||||
ReconcileInterval time.Duration
|
||||
ProbeInterval time.Duration
|
||||
ONVIFMode string
|
||||
RTSPRewriteHost string
|
||||
RTSPRewritePort int
|
||||
RTSPStripQuery bool
|
||||
ControlAPIEnabled bool
|
||||
ControlAuthMode string
|
||||
ControlAuthFile string
|
||||
ControlCursorKeyFile string
|
||||
ControlAllowInsecureHTTP bool
|
||||
HTTPAddress string
|
||||
AllowNonLoopback bool
|
||||
DatabaseDriver string
|
||||
DatabaseDSN string
|
||||
MediaMTXURL string
|
||||
ReconcileInterval time.Duration
|
||||
ReconcileLeaseDuration time.Duration
|
||||
ReconcileOperationTimeout time.Duration
|
||||
ProbeInterval time.Duration
|
||||
InstanceID string
|
||||
MetricsEnabled bool
|
||||
OrphanScanEnabled bool
|
||||
OrphanScanInterval time.Duration
|
||||
ONVIFMode string
|
||||
RTSPRewriteHost string
|
||||
RTSPRewritePort int
|
||||
RTSPStripQuery bool
|
||||
ControlAPIEnabled bool
|
||||
ControlAuthMode string
|
||||
ControlAuthFile string
|
||||
ControlCursorKeyFile string
|
||||
ControlAllowInsecureHTTP bool
|
||||
}
|
||||
|
||||
func Load() (Config, error) {
|
||||
@@ -51,6 +64,14 @@ func Load() (Config, error) {
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
reconcileLease, err := durationEnv("SENSE_RECONCILE_LEASE_DURATION", defaultReconcileLease)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
operationTimeout, err := durationEnv("SENSE_RECONCILE_OPERATION_TIMEOUT", defaultOperationTimeout)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
probePeriod, err := durationEnv("SENSE_PROBE_INTERVAL", defaultProbePeriod)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
@@ -71,24 +92,44 @@ func Load() (Config, error) {
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
metricsEnabled, err := boolEnv("SENSE_METRICS_ENABLED", true)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
databaseDriver := stringEnv("SENSE_DB_DRIVER", defaultDatabaseDriver)
|
||||
orphanDefault := strings.EqualFold(strings.TrimSpace(databaseDriver), postgresDatabaseDriver)
|
||||
orphanEnabled, err := boolEnv("SENSE_ORPHAN_SCAN_ENABLED", orphanDefault)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
orphanPeriod, err := durationEnv("SENSE_ORPHAN_SCAN_INTERVAL", defaultOrphanScanPeriod)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
|
||||
cfg := Config{
|
||||
HTTPAddress: stringEnv("SENSE_HTTP_ADDR", defaultHTTPAddress),
|
||||
AllowNonLoopback: allow,
|
||||
DatabaseDriver: stringEnv("SENSE_DB_DRIVER", defaultDatabaseDriver),
|
||||
DatabaseDSN: stringEnv("SENSE_DB_DSN", defaultDatabaseDSN),
|
||||
MediaMTXURL: stringEnv("SENSE_MEDIAMTX_URL", defaultMediaMTXURL),
|
||||
ReconcileInterval: reconcilePeriod,
|
||||
ProbeInterval: probePeriod,
|
||||
ONVIFMode: stringEnv("SENSE_ONVIF_MODE", defaultONVIFMode),
|
||||
RTSPRewriteHost: stringEnv("SENSE_ONVIF_RTSP_REWRITE_HOST", ""),
|
||||
RTSPRewritePort: rewritePort,
|
||||
RTSPStripQuery: stripQuery,
|
||||
ControlAPIEnabled: controlEnabled,
|
||||
ControlAuthMode: stringEnv("SENSE_CONTROL_AUTH_MODE", defaultControlAuthMode),
|
||||
ControlAuthFile: stringEnv("SENSE_CONTROL_AUTH_FILE", ""),
|
||||
ControlCursorKeyFile: stringEnv("SENSE_CONTROL_CURSOR_KEY_FILE", ""),
|
||||
ControlAllowInsecureHTTP: controlAllowInsecure,
|
||||
HTTPAddress: stringEnv("SENSE_HTTP_ADDR", defaultHTTPAddress),
|
||||
AllowNonLoopback: allow,
|
||||
DatabaseDriver: databaseDriver,
|
||||
DatabaseDSN: stringEnv("SENSE_DB_DSN", defaultDatabaseDSN),
|
||||
MediaMTXURL: stringEnv("SENSE_MEDIAMTX_URL", defaultMediaMTXURL),
|
||||
ReconcileInterval: reconcilePeriod,
|
||||
ReconcileLeaseDuration: reconcileLease,
|
||||
ReconcileOperationTimeout: operationTimeout,
|
||||
ProbeInterval: probePeriod,
|
||||
InstanceID: stringEnv("SENSE_INSTANCE_ID", ""),
|
||||
MetricsEnabled: metricsEnabled,
|
||||
OrphanScanEnabled: orphanEnabled,
|
||||
OrphanScanInterval: orphanPeriod,
|
||||
ONVIFMode: stringEnv("SENSE_ONVIF_MODE", defaultONVIFMode),
|
||||
RTSPRewriteHost: stringEnv("SENSE_ONVIF_RTSP_REWRITE_HOST", ""),
|
||||
RTSPRewritePort: rewritePort,
|
||||
RTSPStripQuery: stripQuery,
|
||||
ControlAPIEnabled: controlEnabled,
|
||||
ControlAuthMode: stringEnv("SENSE_CONTROL_AUTH_MODE", defaultControlAuthMode),
|
||||
ControlAuthFile: stringEnv("SENSE_CONTROL_AUTH_FILE", ""),
|
||||
ControlCursorKeyFile: stringEnv("SENSE_CONTROL_CURSOR_KEY_FILE", ""),
|
||||
ControlAllowInsecureHTTP: controlAllowInsecure,
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return Config{}, err
|
||||
@@ -129,6 +170,35 @@ func (c Config) Validate() error {
|
||||
if c.ReconcileInterval <= 0 || c.ProbeInterval <= 0 {
|
||||
return fmt.Errorf("loop intervals must be positive")
|
||||
}
|
||||
leaseDuration := c.ReconcileLeaseDuration
|
||||
if leaseDuration == 0 {
|
||||
leaseDuration = defaultReconcileLease
|
||||
}
|
||||
operationTimeout := c.ReconcileOperationTimeout
|
||||
if operationTimeout == 0 {
|
||||
operationTimeout = defaultOperationTimeout
|
||||
}
|
||||
if leaseDuration <= 0 || leaseDuration > 5*time.Minute {
|
||||
return fmt.Errorf("SENSE_RECONCILE_LEASE_DURATION must be positive and at most 5m")
|
||||
}
|
||||
if operationTimeout <= 0 || operationTimeout >= leaseDuration {
|
||||
return fmt.Errorf("SENSE_RECONCILE_OPERATION_TIMEOUT must be positive and shorter than the lease")
|
||||
}
|
||||
if c.InstanceID != "" && !instanceIDPattern.MatchString(c.InstanceID) {
|
||||
return fmt.Errorf("invalid SENSE_INSTANCE_ID")
|
||||
}
|
||||
if c.OrphanScanEnabled {
|
||||
if databaseDriver != postgresDatabaseDriver {
|
||||
return fmt.Errorf("orphan scanning requires SENSE_DB_DRIVER=postgres")
|
||||
}
|
||||
orphanInterval := c.OrphanScanInterval
|
||||
if orphanInterval == 0 {
|
||||
orphanInterval = defaultOrphanScanPeriod
|
||||
}
|
||||
if orphanInterval < 10*time.Second {
|
||||
return fmt.Errorf("SENSE_ORPHAN_SCAN_INTERVAL must be at least 10s")
|
||||
}
|
||||
}
|
||||
if c.ONVIFMode != "" && c.ONVIFMode != "disabled" && c.ONVIFMode != "standard" {
|
||||
return fmt.Errorf("SENSE_ONVIF_MODE must be disabled or standard")
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package config
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestValidateRejectsNonLoopbackByDefault(t *testing.T) {
|
||||
@@ -23,6 +24,35 @@ func TestValidateRejectsNonLoopbackByDefault(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateReconcileLeaseAndOrphanProductionBoundary(t *testing.T) {
|
||||
base := Config{
|
||||
HTTPAddress: "127.0.0.1:8080", DatabaseDriver: "postgres",
|
||||
DatabaseDSN: "postgres://sense-runtime@127.0.0.1/yovision?sslmode=disable",
|
||||
MediaMTXURL: "http://127.0.0.1:9997", ReconcileInterval: time.Second,
|
||||
ProbeInterval: time.Second, ReconcileLeaseDuration: 30 * time.Second,
|
||||
ReconcileOperationTimeout: 20 * time.Second, OrphanScanEnabled: true,
|
||||
OrphanScanInterval: time.Minute, InstanceID: "ins_edge-01",
|
||||
}
|
||||
if err := base.Validate(); err != nil {
|
||||
t.Fatalf("valid multi-instance configuration failed: %v", err)
|
||||
}
|
||||
invalidTimeout := base
|
||||
invalidTimeout.ReconcileOperationTimeout = invalidTimeout.ReconcileLeaseDuration
|
||||
if err := invalidTimeout.Validate(); err == nil {
|
||||
t.Fatal("operation timeout equal to the lease was accepted")
|
||||
}
|
||||
invalidInstance := base
|
||||
invalidInstance.InstanceID = "tenant/site"
|
||||
if err := invalidInstance.Validate(); err == nil {
|
||||
t.Fatal("unbounded instance label was accepted")
|
||||
}
|
||||
sqlite := base
|
||||
sqlite.DatabaseDriver, sqlite.DatabaseDSN = "sqlite", "file:test.db"
|
||||
if err := sqlite.Validate(); err == nil {
|
||||
t.Fatal("orphan scanner was accepted on SQLite")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsCredentialsInMediaMTXURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := Config{
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
// Package metrics exposes a deliberately small, low-cardinality Prometheus
|
||||
// surface without adding a runtime dependency. Tenant, Site, device and Path
|
||||
// identifiers never become labels.
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
func GenerateInstanceID() (string, error) {
|
||||
value := make([]byte, 12)
|
||||
if _, err := rand.Read(value); err != nil {
|
||||
return "", errors.New("generate Sense instance identifier")
|
||||
}
|
||||
return "ins_" + hex.EncodeToString(value), nil
|
||||
}
|
||||
|
||||
type Registry struct {
|
||||
instanceID string
|
||||
version string
|
||||
|
||||
reconcileRunOK atomic.Uint64
|
||||
reconcileRunError atomic.Uint64
|
||||
reconcileItemOK atomic.Uint64
|
||||
reconcileItemError atomic.Uint64
|
||||
reconcileItemLost atomic.Uint64
|
||||
reconcileDurationNS atomic.Uint64
|
||||
reconcileTotal atomic.Int64
|
||||
reconcileUnconverged atomic.Int64
|
||||
|
||||
orphanScanOK atomic.Uint64
|
||||
orphanScanError atomic.Uint64
|
||||
orphanObserved atomic.Int64
|
||||
orphanOwnedStale atomic.Int64
|
||||
orphanUnowned atomic.Int64
|
||||
orphanBlockedRatio atomic.Uint64
|
||||
orphanBlockedAge atomic.Uint64
|
||||
orphanBlockedScope atomic.Uint64
|
||||
orphanDeleted atomic.Uint64
|
||||
orphanDeleteFailed atomic.Uint64
|
||||
}
|
||||
|
||||
func New(instanceID, version string) *Registry {
|
||||
return &Registry{instanceID: instanceID, version: version}
|
||||
}
|
||||
|
||||
func (r *Registry) ObserveReconcileRun(err error, duration time.Duration) {
|
||||
if err == nil {
|
||||
r.reconcileRunOK.Add(1)
|
||||
} else {
|
||||
r.reconcileRunError.Add(1)
|
||||
}
|
||||
if duration < 0 {
|
||||
duration = 0
|
||||
}
|
||||
r.reconcileDurationNS.Store(uint64(duration))
|
||||
}
|
||||
|
||||
func (r *Registry) ObserveReconcileItem(result string) {
|
||||
switch result {
|
||||
case "success":
|
||||
r.reconcileItemOK.Add(1)
|
||||
case "lease_lost":
|
||||
r.reconcileItemLost.Add(1)
|
||||
default:
|
||||
r.reconcileItemError.Add(1)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Registry) SetConvergence(total, unconverged int) {
|
||||
r.reconcileTotal.Store(int64(total))
|
||||
r.reconcileUnconverged.Store(int64(unconverged))
|
||||
}
|
||||
|
||||
func (r *Registry) ObserveOrphanScan(observed, ownedStale, unowned int, err error) {
|
||||
if err == nil {
|
||||
r.orphanScanOK.Add(1)
|
||||
r.orphanObserved.Store(int64(observed))
|
||||
r.orphanOwnedStale.Store(int64(ownedStale))
|
||||
r.orphanUnowned.Store(int64(unowned))
|
||||
} else {
|
||||
r.orphanScanError.Add(1)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Registry) ObserveOrphanCleanupBlocked(reason string) {
|
||||
switch reason {
|
||||
case "snapshot_expired":
|
||||
r.orphanBlockedAge.Add(1)
|
||||
case "ratio_exceeded":
|
||||
r.orphanBlockedRatio.Add(1)
|
||||
default:
|
||||
r.orphanBlockedScope.Add(1)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Registry) ObserveOrphanCleanup(deleted, failed int) {
|
||||
if deleted > 0 {
|
||||
r.orphanDeleted.Add(uint64(deleted))
|
||||
}
|
||||
if failed > 0 {
|
||||
r.orphanDeleteFailed.Add(uint64(failed))
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Registry) Handler() http.Handler {
|
||||
return http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
writer.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
||||
writer.Header().Set("Cache-Control", "no-store")
|
||||
_, _ = writer.Write([]byte(r.render()))
|
||||
})
|
||||
}
|
||||
|
||||
func (r *Registry) render() string {
|
||||
var output strings.Builder
|
||||
writeMetric(&output, "sense_build_info", "Sense process build and instance information.",
|
||||
fmt.Sprintf("{instance_id=%s,version=%s}", quoteLabel(r.instanceID), quoteLabel(r.version)), 1)
|
||||
writeMetric(&output, "sense_reconcile_runs_total", "Reconciliation runs by result.",
|
||||
`{result="success"}`, r.reconcileRunOK.Load())
|
||||
writeSample(&output, "sense_reconcile_runs_total", `{result="error"}`, r.reconcileRunError.Load())
|
||||
writeMetric(&output, "sense_reconcile_items_total", "Reconciliation items by fixed result.",
|
||||
`{result="success"}`, r.reconcileItemOK.Load())
|
||||
writeSample(&output, "sense_reconcile_items_total", `{result="error"}`, r.reconcileItemError.Load())
|
||||
writeSample(&output, "sense_reconcile_items_total", `{result="lease_lost"}`, r.reconcileItemLost.Load())
|
||||
writeMetric(&output, "sense_reconcile_last_duration_seconds", "Duration of the last reconciliation run.",
|
||||
"", float64(r.reconcileDurationNS.Load())/float64(time.Second))
|
||||
writeMetric(&output, "sense_reconcile_devices", "Enabled video devices by convergence state.",
|
||||
`{state="total"}`, r.reconcileTotal.Load())
|
||||
writeSample(&output, "sense_reconcile_devices", `{state="unconverged"}`, r.reconcileUnconverged.Load())
|
||||
writeMetric(&output, "sense_orphan_scan_runs_total", "MediaMTX orphan scans by result.",
|
||||
`{result="success"}`, r.orphanScanOK.Load())
|
||||
writeSample(&output, "sense_orphan_scan_runs_total", `{result="error"}`, r.orphanScanError.Load())
|
||||
writeMetric(&output, "sense_orphan_paths", "Path counts from the last successful orphan scan.",
|
||||
`{classification="observed"}`, r.orphanObserved.Load())
|
||||
writeSample(&output, "sense_orphan_paths", `{classification="owned_stale"}`, r.orphanOwnedStale.Load())
|
||||
writeSample(&output, "sense_orphan_paths", `{classification="unowned"}`, r.orphanUnowned.Load())
|
||||
writeMetric(&output, "sense_orphan_cleanup_blocked_total", "Orphan cleanups blocked by a fixed safety reason.",
|
||||
`{reason="ratio_exceeded"}`, r.orphanBlockedRatio.Load())
|
||||
writeSample(&output, "sense_orphan_cleanup_blocked_total", `{reason="snapshot_expired"}`, r.orphanBlockedAge.Load())
|
||||
writeSample(&output, "sense_orphan_cleanup_blocked_total", `{reason="scope_changed"}`, r.orphanBlockedScope.Load())
|
||||
writeMetric(&output, "sense_orphan_cleanup_items_total", "Owned stale path cleanup results.",
|
||||
`{result="deleted"}`, r.orphanDeleted.Load())
|
||||
writeSample(&output, "sense_orphan_cleanup_items_total", `{result="failed"}`, r.orphanDeleteFailed.Load())
|
||||
return output.String()
|
||||
}
|
||||
|
||||
func writeMetric(builder *strings.Builder, name, help, labels string, value any) {
|
||||
metricType := "gauge"
|
||||
if strings.HasSuffix(name, "_total") {
|
||||
metricType = "counter"
|
||||
}
|
||||
builder.WriteString("# HELP ")
|
||||
builder.WriteString(name)
|
||||
builder.WriteByte(' ')
|
||||
builder.WriteString(help)
|
||||
builder.WriteByte('\n')
|
||||
builder.WriteString("# TYPE ")
|
||||
builder.WriteString(name)
|
||||
builder.WriteByte(' ')
|
||||
builder.WriteString(metricType)
|
||||
builder.WriteByte('\n')
|
||||
writeSample(builder, name, labels, value)
|
||||
}
|
||||
|
||||
func writeSample(builder *strings.Builder, name, labels string, value any) {
|
||||
builder.WriteString(name)
|
||||
builder.WriteString(labels)
|
||||
builder.WriteByte(' ')
|
||||
switch typed := value.(type) {
|
||||
case float64:
|
||||
builder.WriteString(strconv.FormatFloat(typed, 'f', 6, 64))
|
||||
default:
|
||||
builder.WriteString(fmt.Sprint(typed))
|
||||
}
|
||||
builder.WriteByte('\n')
|
||||
}
|
||||
|
||||
func quoteLabel(value string) string {
|
||||
replacer := strings.NewReplacer(`\`, `\\`, "\n", `\n`, `"`, `\"`)
|
||||
return `"` + replacer.Replace(value) + `"`
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestHandlerUsesOnlyFixedLowCardinalityLabels(t *testing.T) {
|
||||
t.Parallel()
|
||||
registry := New("ins_test-1", "v-test")
|
||||
registry.ObserveReconcileRun(nil, 1500*time.Millisecond)
|
||||
registry.ObserveReconcileItem("success")
|
||||
registry.ObserveReconcileItem("lease_lost")
|
||||
registry.SetConvergence(16, 2)
|
||||
registry.ObserveOrphanScan(20, 1, 3, nil)
|
||||
registry.ObserveOrphanCleanupBlocked("ratio_exceeded")
|
||||
registry.ObserveOrphanCleanup(1, 1)
|
||||
|
||||
request := httptest.NewRequest("GET", "/metrics", nil)
|
||||
response := httptest.NewRecorder()
|
||||
registry.Handler().ServeHTTP(response, request)
|
||||
body := response.Body.String()
|
||||
for _, expected := range []string{
|
||||
`sense_build_info{instance_id="ins_test-1",version="v-test"} 1`,
|
||||
`sense_reconcile_items_total{result="lease_lost"} 1`,
|
||||
`sense_reconcile_devices{state="unconverged"} 2`,
|
||||
`sense_orphan_paths{classification="owned_stale"} 1`,
|
||||
`sense_orphan_cleanup_blocked_total{reason="ratio_exceeded"} 1`,
|
||||
} {
|
||||
if !strings.Contains(body, expected) {
|
||||
t.Fatalf("metrics output lacks %q:\n%s", expected, body)
|
||||
}
|
||||
}
|
||||
for _, forbidden := range []string{
|
||||
"tenant_id", "site_id", "device_id", "path_name", "rtsp://", "credential",
|
||||
} {
|
||||
if strings.Contains(body, forbidden) {
|
||||
t.Fatalf("high-cardinality or secret-bearing label appeared: %q", forbidden)
|
||||
}
|
||||
}
|
||||
if response.Header().Get("Cache-Control") != "no-store" {
|
||||
t.Fatal("metrics responses must not be cached")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryIsSafeForConcurrentObservers(t *testing.T) {
|
||||
t.Parallel()
|
||||
registry := New("ins_concurrent", "dev")
|
||||
var wait sync.WaitGroup
|
||||
for index := 0; index < 32; index++ {
|
||||
wait.Add(1)
|
||||
go func() {
|
||||
defer wait.Done()
|
||||
for count := 0; count < 100; count++ {
|
||||
registry.ObserveReconcileItem("success")
|
||||
registry.SetConvergence(128, count%17)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wait.Wait()
|
||||
response := httptest.NewRecorder()
|
||||
registry.Handler().ServeHTTP(response, httptest.NewRequest("GET", "/metrics", nil))
|
||||
if !strings.Contains(response.Body.String(), `sense_reconcile_items_total{result="success"} 3200`) {
|
||||
t.Fatalf("concurrent counter lost updates: %s", response.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateInstanceIDShape(t *testing.T) {
|
||||
t.Parallel()
|
||||
first, err := GenerateInstanceID()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := GenerateInstanceID()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(first) != 28 || !strings.HasPrefix(first, "ins_") || first == second {
|
||||
t.Fatalf("unexpected instance identifiers: %q %q", first, second)
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
mediamtxapi "yovision/sense/internal/mtx/generated"
|
||||
@@ -31,9 +32,63 @@ type pathAPI interface {
|
||||
ConfigPathsGetWithResponse(context.Context, string, ...mediamtxapi.RequestEditorFn) (*mediamtxapi.ConfigPathsGetResponse, error)
|
||||
ConfigPathsPatchWithResponse(context.Context, string, mediamtxapi.ConfigPathsPatchJSONRequestBody, ...mediamtxapi.RequestEditorFn) (*mediamtxapi.ConfigPathsPatchResponse, error)
|
||||
ConfigPathsDeleteWithResponse(context.Context, string, ...mediamtxapi.RequestEditorFn) (*mediamtxapi.ConfigPathsDeleteResponse, error)
|
||||
ConfigPathsListWithResponse(context.Context, *mediamtxapi.ConfigPathsListParams, ...mediamtxapi.RequestEditorFn) (*mediamtxapi.ConfigPathsListResponse, error)
|
||||
PathsGetWithResponse(context.Context, string, ...mediamtxapi.RequestEditorFn) (*mediamtxapi.PathsGetResponse, error)
|
||||
}
|
||||
|
||||
// ListPathNames enumerates only configuration names. Sources are deliberately
|
||||
// discarded so inventory and orphan reports cannot expose stream URIs.
|
||||
func (c *Client) ListPathNames(ctx context.Context) ([]string, error) {
|
||||
const (
|
||||
itemsPerPage = 100
|
||||
maxPages = 1000
|
||||
)
|
||||
result := make(map[string]struct{})
|
||||
seenPages := make(map[string]struct{})
|
||||
for page := 0; page < maxPages; page++ {
|
||||
pageValue, limitValue := page, itemsPerPage
|
||||
response, err := c.api.ConfigPathsListWithResponse(ctx, &mediamtxapi.ConfigPathsListParams{
|
||||
Page: &pageValue, ItemsPerPage: &limitValue,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("MediaMTX list paths transport: %w", err)
|
||||
}
|
||||
if response.StatusCode() != http.StatusOK || response.JSON200 == nil ||
|
||||
response.JSON200.PageCount == nil || response.JSON200.Items == nil {
|
||||
return nil, &APIError{Operation: "list paths", StatusCode: response.StatusCode()}
|
||||
}
|
||||
pageCount := *response.JSON200.PageCount
|
||||
if pageCount < 0 || pageCount > maxPages {
|
||||
return nil, &APIError{Operation: "list paths pagination", StatusCode: response.StatusCode()}
|
||||
}
|
||||
pageNames := make([]string, 0, len(*response.JSON200.Items))
|
||||
for _, item := range *response.JSON200.Items {
|
||||
if item.Name == nil || strings.TrimSpace(*item.Name) == "" {
|
||||
return nil, &APIError{Operation: "list paths response", StatusCode: response.StatusCode()}
|
||||
}
|
||||
pageNames = append(pageNames, *item.Name)
|
||||
result[*item.Name] = struct{}{}
|
||||
}
|
||||
sort.Strings(pageNames)
|
||||
signature := strings.Join(pageNames, "\x00")
|
||||
if page > 0 && signature != "" {
|
||||
if _, duplicate := seenPages[signature]; duplicate {
|
||||
return nil, &APIError{Operation: "list paths repeated page", StatusCode: response.StatusCode()}
|
||||
}
|
||||
}
|
||||
seenPages[signature] = struct{}{}
|
||||
if int64(page+1) >= pageCount {
|
||||
values := make([]string, 0, len(result))
|
||||
for name := range result {
|
||||
values = append(values, name)
|
||||
}
|
||||
sort.Strings(values)
|
||||
return values, nil
|
||||
}
|
||||
}
|
||||
return nil, &APIError{Operation: "list paths page limit", StatusCode: http.StatusOK}
|
||||
}
|
||||
|
||||
type Client struct {
|
||||
api pathAPI
|
||||
}
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -21,6 +23,38 @@ func (f *fakeMediaMTX) ServeHTTP(writer http.ResponseWriter, request *http.Reque
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
if request.URL.Path == "/v3/config/paths/list" {
|
||||
page, _ := strconv.Atoi(request.URL.Query().Get("page"))
|
||||
limit, _ := strconv.Atoi(request.URL.Query().Get("itemsPerPage"))
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
names := make([]string, 0, len(f.paths))
|
||||
for name := range f.paths {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
start := page * limit
|
||||
if start > len(names) {
|
||||
start = len(names)
|
||||
}
|
||||
end := start + limit
|
||||
if end > len(names) {
|
||||
end = len(names)
|
||||
}
|
||||
items := make([]map[string]any, 0, end-start)
|
||||
for _, name := range names[start:end] {
|
||||
items = append(items, map[string]any{"name": name})
|
||||
}
|
||||
pageCount := 0
|
||||
if len(names) > 0 {
|
||||
pageCount = (len(names) + limit - 1) / limit
|
||||
}
|
||||
_ = json.NewEncoder(writer).Encode(map[string]any{
|
||||
"itemCount": len(names), "pageCount": pageCount, "items": items,
|
||||
})
|
||||
return
|
||||
}
|
||||
prefixes := map[string]string{
|
||||
"/v3/config/paths/get/": "get",
|
||||
"/v3/config/paths/add/": "add",
|
||||
@@ -72,6 +106,54 @@ func (f *fakeMediaMTX) ServeHTTP(writer http.ResponseWriter, request *http.Reque
|
||||
http.NotFound(writer, request)
|
||||
}
|
||||
|
||||
func TestListPathNamesUsesPaginationAndDoesNotReturnSources(t *testing.T) {
|
||||
t.Parallel()
|
||||
paths := make(map[string]string)
|
||||
for index := 0; index < 205; index++ {
|
||||
paths[fmt.Sprintf("camera-%03d", index)] = fmt.Sprintf("rtsp://secret.invalid/%d", index)
|
||||
}
|
||||
fake := &fakeMediaMTX{paths: paths}
|
||||
server := httptest.NewServer(fake)
|
||||
defer server.Close()
|
||||
client, err := NewClient(server.URL, server.Client())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
values, err := client.ListPathNames(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(values) != 205 || values[0] != "camera-000" || values[204] != "camera-204" {
|
||||
t.Fatalf("unexpected path inventory: len=%d first=%q last=%q", len(values), values[0], values[len(values)-1])
|
||||
}
|
||||
for _, value := range values {
|
||||
if strings.Contains(value, "rtsp") || strings.Contains(value, "secret") {
|
||||
t.Fatalf("source leaked from path inventory: %q", value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListPathNamesRejectsRepeatedPage(t *testing.T) {
|
||||
t.Parallel()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
if request.URL.Path != "/v3/config/paths/list" {
|
||||
http.NotFound(writer, request)
|
||||
return
|
||||
}
|
||||
_, _ = writer.Write([]byte(`{"itemCount":2,"pageCount":2,"items":[{"name":"same"}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := NewClient(server.URL, server.Client())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := client.ListPathNames(context.Background()); err == nil ||
|
||||
!strings.Contains(err.Error(), "repeated page") {
|
||||
t.Fatalf("repeated pagination was accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratedClientCreateReadDeleteMapping(t *testing.T) {
|
||||
t.Parallel()
|
||||
fake := &fakeMediaMTX{paths: make(map[string]string)}
|
||||
|
||||
@@ -0,0 +1,400 @@
|
||||
// Package orphan reports MediaMTX configuration paths that do not match the
|
||||
// current Sense ledger. Only paths with durable Sense ownership evidence can
|
||||
// ever enter the controlled cleanup set.
|
||||
package orphan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"yovision/sense/internal/metrics"
|
||||
"yovision/sense/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultLeaseDuration = 30 * time.Second
|
||||
defaultOperationTimeout = 20 * time.Second
|
||||
reportTTL = 15 * time.Minute
|
||||
maxCleanupItems = 128
|
||||
crockford = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrLeaseHeld = errors.New("orphan operation lease held")
|
||||
ErrConfirmation = errors.New("orphan cleanup confirmation invalid")
|
||||
ErrSnapshotExpired = errors.New("orphan scan snapshot expired")
|
||||
ErrSafetyBlocked = errors.New("orphan cleanup safety gate blocked")
|
||||
operatorIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`)
|
||||
)
|
||||
|
||||
type Runtime interface {
|
||||
ListPathNames(context.Context) ([]string, error)
|
||||
DeletePath(context.Context, string) error
|
||||
}
|
||||
|
||||
type Manager struct {
|
||||
repository store.OrphanRepository
|
||||
runtime Runtime
|
||||
instanceID string
|
||||
metrics *metrics.Registry
|
||||
now func() time.Time
|
||||
leaseDuration time.Duration
|
||||
operationTimeout time.Duration
|
||||
}
|
||||
|
||||
type CleanupResult struct {
|
||||
ScanID string `json:"scan_id"`
|
||||
Deleted int `json:"deleted"`
|
||||
Failed int `json:"failed"`
|
||||
}
|
||||
|
||||
func New(
|
||||
repository store.OrphanRepository,
|
||||
runtime Runtime,
|
||||
instanceID string,
|
||||
registry *metrics.Registry,
|
||||
) *Manager {
|
||||
return &Manager{
|
||||
repository: repository, runtime: runtime, instanceID: instanceID, metrics: registry,
|
||||
now: time.Now, leaseDuration: defaultLeaseDuration, operationTimeout: defaultOperationTimeout,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) Report(ctx context.Context) (store.OrphanScan, error) {
|
||||
now := m.now().UTC()
|
||||
token, err := randomToken()
|
||||
if err != nil {
|
||||
return store.OrphanScan{}, err
|
||||
}
|
||||
acquired, err := m.repository.AcquireOperationalLease(
|
||||
ctx, store.OperationalLeaseOrphanScan, m.instanceID, token, now, m.leaseDuration,
|
||||
)
|
||||
if err != nil {
|
||||
m.observeScan(0, 0, 0, err)
|
||||
return store.OrphanScan{}, err
|
||||
}
|
||||
if !acquired {
|
||||
return store.OrphanScan{}, ErrLeaseHeld
|
||||
}
|
||||
saved := false
|
||||
defer func() {
|
||||
if saved {
|
||||
return
|
||||
}
|
||||
releaseCtx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
_ = m.repository.ReleaseOperationalLease(
|
||||
releaseCtx, store.OperationalLeaseOrphanScan, m.instanceID, token, m.now().UTC(),
|
||||
)
|
||||
}()
|
||||
|
||||
operationCtx, cancel := context.WithTimeout(ctx, m.operationTimeout)
|
||||
paths, err := m.runtime.ListPathNames(operationCtx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
m.observeScan(0, 0, 0, err)
|
||||
return store.OrphanScan{}, fmt.Errorf("list MediaMTX paths: %w", err)
|
||||
}
|
||||
ownership, err := m.repository.ListMediaPathOwnership(ctx)
|
||||
if err != nil {
|
||||
m.observeScan(0, 0, 0, err)
|
||||
return store.OrphanScan{}, err
|
||||
}
|
||||
findings, stale, unowned := classify(paths, ownership)
|
||||
allowed, reason := safetyGate(stale, len(paths))
|
||||
completedAt := m.now().UTC()
|
||||
acquired, err = m.repository.AcquireOperationalLease(
|
||||
ctx, store.OperationalLeaseOrphanScan, m.instanceID, token, completedAt, m.leaseDuration,
|
||||
)
|
||||
if err != nil || !acquired {
|
||||
if err == nil {
|
||||
err = store.ErrOperationalLeaseLost
|
||||
}
|
||||
m.observeScan(len(paths), stale, unowned, err)
|
||||
return store.OrphanScan{}, err
|
||||
}
|
||||
id, err := newScanID(completedAt)
|
||||
if err != nil {
|
||||
m.observeScan(0, 0, 0, err)
|
||||
return store.OrphanScan{}, err
|
||||
}
|
||||
scan := store.OrphanScan{
|
||||
ID: id, InstanceID: m.instanceID, ObservedCount: len(paths),
|
||||
OwnedStaleCount: stale, UnownedCount: unowned,
|
||||
SafetyAllowed: allowed, SafetyReason: reason,
|
||||
CompletedAt: completedAt, ExpiresAt: completedAt.Add(reportTTL), Findings: findings,
|
||||
}
|
||||
if err := m.repository.SaveOrphanScan(ctx, scan, m.instanceID, token); err != nil {
|
||||
m.observeScan(len(paths), stale, unowned, err)
|
||||
return store.OrphanScan{}, err
|
||||
}
|
||||
saved = true
|
||||
m.observeScan(len(paths), stale, unowned, nil)
|
||||
return scan, nil
|
||||
}
|
||||
|
||||
func classify(
|
||||
paths []string,
|
||||
ownership []store.MediaPathOwnership,
|
||||
) ([]store.OrphanFinding, int, int) {
|
||||
history := make(map[string]store.MediaPathOwnership, len(ownership))
|
||||
for _, value := range ownership {
|
||||
history[value.PathName] = value
|
||||
}
|
||||
unique := make(map[string]struct{}, len(paths))
|
||||
for _, path := range paths {
|
||||
unique[path] = struct{}{}
|
||||
}
|
||||
ordered := make([]string, 0, len(unique))
|
||||
for path := range unique {
|
||||
ordered = append(ordered, path)
|
||||
}
|
||||
sort.Strings(ordered)
|
||||
findings := make([]store.OrphanFinding, 0)
|
||||
stale, unowned := 0, 0
|
||||
for _, path := range ordered {
|
||||
value, known := history[path]
|
||||
switch {
|
||||
case known && value.CurrentClaim:
|
||||
continue
|
||||
case known:
|
||||
stale++
|
||||
findings = append(findings, store.OrphanFinding{
|
||||
PathName: path, Classification: store.OrphanOwnedStale, DeviceID: value.DeviceID,
|
||||
})
|
||||
default:
|
||||
unowned++
|
||||
findings = append(findings, store.OrphanFinding{
|
||||
PathName: path, Classification: store.OrphanUnowned,
|
||||
})
|
||||
}
|
||||
}
|
||||
return findings, stale, unowned
|
||||
}
|
||||
|
||||
func safetyGate(candidates, observed int) (bool, string) {
|
||||
switch {
|
||||
case candidates <= 0:
|
||||
return false, "no_candidates"
|
||||
case observed <= 0:
|
||||
return false, "empty_inventory"
|
||||
case candidates > maxCleanupItems:
|
||||
return false, "scope_exceeded"
|
||||
case candidates*100 > observed*10:
|
||||
return false, "ratio_exceeded"
|
||||
default:
|
||||
return true, "allowed"
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) Apply(
|
||||
ctx context.Context,
|
||||
scanID, actorID, confirmation string,
|
||||
) (CleanupResult, error) {
|
||||
result := CleanupResult{ScanID: scanID}
|
||||
if !operatorIDPattern.MatchString(actorID) || confirmation != "DELETE "+scanID {
|
||||
m.blocked("scope_changed")
|
||||
return result, ErrConfirmation
|
||||
}
|
||||
now := m.now().UTC()
|
||||
token, err := randomToken()
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
acquired, err := m.repository.AcquireOperationalLease(
|
||||
ctx, store.OperationalLeaseOrphanCleanup, m.instanceID, token, now, m.leaseDuration,
|
||||
)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
if !acquired {
|
||||
return result, ErrLeaseHeld
|
||||
}
|
||||
defer func() {
|
||||
releaseCtx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
_ = m.repository.ReleaseOperationalLease(
|
||||
releaseCtx, store.OperationalLeaseOrphanCleanup, m.instanceID, token, m.now().UTC(),
|
||||
)
|
||||
}()
|
||||
|
||||
scan, err := m.repository.GetOrphanScan(ctx, scanID)
|
||||
if err != nil {
|
||||
m.blocked("scope_changed")
|
||||
return result, err
|
||||
}
|
||||
if !scan.ExpiresAt.After(now) {
|
||||
m.blocked("snapshot_expired")
|
||||
return result, ErrSnapshotExpired
|
||||
}
|
||||
if !scan.SafetyAllowed {
|
||||
m.blocked(scan.SafetyReason)
|
||||
return result, fmt.Errorf("%w: %s", ErrSafetyBlocked, scan.SafetyReason)
|
||||
}
|
||||
operationCtx, cancel := context.WithTimeout(ctx, m.operationTimeout)
|
||||
paths, err := m.runtime.ListPathNames(operationCtx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("refresh MediaMTX path inventory: %w", err)
|
||||
}
|
||||
ownership, err := m.repository.ListMediaPathOwnership(ctx)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
candidates := cleanupCandidates(scan, paths, ownership)
|
||||
if len(candidates) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
allowed, reason := safetyGate(len(candidates), len(uniqueNames(paths)))
|
||||
if !allowed {
|
||||
m.blocked(reason)
|
||||
return result, fmt.Errorf("%w: %s", ErrSafetyBlocked, reason)
|
||||
}
|
||||
var failures []error
|
||||
for _, path := range candidates {
|
||||
now = m.now().UTC()
|
||||
acquired, err = m.repository.AcquireOperationalLease(
|
||||
ctx, store.OperationalLeaseOrphanCleanup, m.instanceID, token, now, m.leaseDuration,
|
||||
)
|
||||
if err != nil || !acquired {
|
||||
if err == nil {
|
||||
err = ErrLeaseHeld
|
||||
}
|
||||
failures = append(failures, err)
|
||||
break
|
||||
}
|
||||
itemCtx, itemCancel := context.WithTimeout(ctx, m.operationTimeout)
|
||||
deleteErr := m.runtime.DeletePath(itemCtx, path)
|
||||
itemCancel()
|
||||
status, code := "deleted", ""
|
||||
if deleteErr != nil {
|
||||
status, code = "failed", "media_error"
|
||||
result.Failed++
|
||||
failures = append(failures, fmt.Errorf("delete owned stale path: %w", deleteErr))
|
||||
} else {
|
||||
result.Deleted++
|
||||
}
|
||||
if err := m.repository.RecordOrphanCleanup(
|
||||
ctx, scanID, path, actorID, status, code, m.now().UTC(),
|
||||
); err != nil {
|
||||
failures = append(failures, err)
|
||||
break
|
||||
}
|
||||
}
|
||||
if m.metrics != nil {
|
||||
m.metrics.ObserveOrphanCleanup(result.Deleted, result.Failed)
|
||||
}
|
||||
return result, errors.Join(failures...)
|
||||
}
|
||||
|
||||
func cleanupCandidates(
|
||||
scan store.OrphanScan,
|
||||
paths []string,
|
||||
ownership []store.MediaPathOwnership,
|
||||
) []string {
|
||||
runtime := uniqueNames(paths)
|
||||
history := make(map[string]store.MediaPathOwnership, len(ownership))
|
||||
for _, value := range ownership {
|
||||
history[value.PathName] = value
|
||||
}
|
||||
values := make([]string, 0)
|
||||
for _, finding := range scan.Findings {
|
||||
if finding.Classification != store.OrphanOwnedStale || finding.Deleted {
|
||||
continue
|
||||
}
|
||||
_, present := runtime[finding.PathName]
|
||||
owner, known := history[finding.PathName]
|
||||
if present && known && !owner.CurrentClaim && owner.DeviceID == finding.DeviceID {
|
||||
values = append(values, finding.PathName)
|
||||
}
|
||||
}
|
||||
sort.Strings(values)
|
||||
return values
|
||||
}
|
||||
|
||||
func uniqueNames(values []string) map[string]struct{} {
|
||||
result := make(map[string]struct{}, len(values))
|
||||
for _, value := range values {
|
||||
result[value] = struct{}{}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (m *Manager) Run(ctx context.Context, interval time.Duration, report func(error)) {
|
||||
run := func() {
|
||||
_, err := m.Report(ctx)
|
||||
if err != nil && !errors.Is(err, ErrLeaseHeld) && ctx.Err() == nil && report != nil {
|
||||
report(err)
|
||||
}
|
||||
}
|
||||
run()
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
run()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) observeScan(observed, stale, unowned int, err error) {
|
||||
if m.metrics != nil {
|
||||
m.metrics.ObserveOrphanScan(observed, stale, unowned, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) blocked(reason string) {
|
||||
if m.metrics != nil {
|
||||
m.metrics.ObserveOrphanCleanupBlocked(reason)
|
||||
}
|
||||
}
|
||||
|
||||
func randomToken() (string, error) {
|
||||
value := make([]byte, 16)
|
||||
if _, err := rand.Read(value); err != nil {
|
||||
return "", errors.New("generate fencing token")
|
||||
}
|
||||
const alphabet = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||
result := make([]byte, 22)
|
||||
number := new(big.Int).SetBytes(value)
|
||||
base := big.NewInt(int64(len(alphabet)))
|
||||
remainder := new(big.Int)
|
||||
for index := len(result) - 1; index >= 0; index-- {
|
||||
number.QuoRem(number, base, remainder)
|
||||
result[index] = alphabet[remainder.Int64()]
|
||||
}
|
||||
return string(result), nil
|
||||
}
|
||||
|
||||
func newScanID(now time.Time) (string, error) {
|
||||
value := make([]byte, 16)
|
||||
milliseconds := uint64(now.UTC().UnixMilli())
|
||||
value[0], value[1], value[2] = byte(milliseconds>>40), byte(milliseconds>>32), byte(milliseconds>>24)
|
||||
value[3], value[4], value[5] = byte(milliseconds>>16), byte(milliseconds>>8), byte(milliseconds)
|
||||
if _, err := rand.Read(value[6:]); err != nil {
|
||||
return "", errors.New("generate orphan scan identifier")
|
||||
}
|
||||
number := new(big.Int).SetBytes(value)
|
||||
base := big.NewInt(32)
|
||||
remainder := new(big.Int)
|
||||
encoded := make([]byte, 26)
|
||||
for index := len(encoded) - 1; index >= 0; index-- {
|
||||
number.QuoRem(number, base, remainder)
|
||||
encoded[index] = crockford[remainder.Int64()]
|
||||
}
|
||||
return "scan_" + string(encoded), nil
|
||||
}
|
||||
|
||||
func Confirmation(scanID string) string {
|
||||
return strings.Join([]string{"DELETE", scanID}, " ")
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
package orphan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"yovision/sense/internal/store"
|
||||
)
|
||||
|
||||
type fakeLease struct {
|
||||
owner, token string
|
||||
expires time.Time
|
||||
}
|
||||
|
||||
type fakeRepository struct {
|
||||
mu sync.Mutex
|
||||
leases map[string]fakeLease
|
||||
ownership []store.MediaPathOwnership
|
||||
scans map[string]store.OrphanScan
|
||||
deleted map[string]map[string]bool
|
||||
}
|
||||
|
||||
func newFakeRepository() *fakeRepository {
|
||||
return &fakeRepository{
|
||||
leases: make(map[string]fakeLease), scans: make(map[string]store.OrphanScan),
|
||||
deleted: make(map[string]map[string]bool),
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeRepository) AcquireOperationalLease(
|
||||
_ context.Context,
|
||||
name, owner, token string,
|
||||
now time.Time,
|
||||
duration time.Duration,
|
||||
) (bool, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
current, exists := f.leases[name]
|
||||
if exists && current.expires.After(now) && (current.owner != owner || current.token != token) {
|
||||
return false, nil
|
||||
}
|
||||
f.leases[name] = fakeLease{owner: owner, token: token, expires: now.Add(duration)}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (f *fakeRepository) ReleaseOperationalLease(
|
||||
_ context.Context,
|
||||
name, owner, token string,
|
||||
now time.Time,
|
||||
) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
current := f.leases[name]
|
||||
if current.owner == owner && current.token == token {
|
||||
current.expires = now
|
||||
f.leases[name] = current
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeRepository) ListMediaPathOwnership(context.Context) ([]store.MediaPathOwnership, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]store.MediaPathOwnership(nil), f.ownership...), nil
|
||||
}
|
||||
|
||||
func (f *fakeRepository) SaveOrphanScan(
|
||||
_ context.Context,
|
||||
scan store.OrphanScan,
|
||||
owner, token string,
|
||||
) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
lease := f.leases[store.OperationalLeaseOrphanScan]
|
||||
if lease.owner != owner || lease.token != token || !lease.expires.After(scan.CompletedAt) {
|
||||
return store.ErrOperationalLeaseLost
|
||||
}
|
||||
f.scans[scan.ID] = cloneScan(scan)
|
||||
lease.expires = scan.CompletedAt
|
||||
f.leases[store.OperationalLeaseOrphanScan] = lease
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeRepository) GetOrphanScan(_ context.Context, id string) (store.OrphanScan, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
scan, exists := f.scans[id]
|
||||
if !exists {
|
||||
return store.OrphanScan{}, store.ErrOrphanScanNotFound
|
||||
}
|
||||
result := cloneScan(scan)
|
||||
for index := range result.Findings {
|
||||
result.Findings[index].Deleted = f.deleted[id][result.Findings[index].PathName]
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (f *fakeRepository) RecordOrphanCleanup(
|
||||
_ context.Context,
|
||||
scanID, pathName, _, status, _ string,
|
||||
_ time.Time,
|
||||
) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.deleted[scanID] == nil {
|
||||
f.deleted[scanID] = make(map[string]bool)
|
||||
}
|
||||
if status == "deleted" {
|
||||
f.deleted[scanID][pathName] = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cloneScan(value store.OrphanScan) store.OrphanScan {
|
||||
value.Findings = append([]store.OrphanFinding(nil), value.Findings...)
|
||||
return value
|
||||
}
|
||||
|
||||
type fakeRuntime struct {
|
||||
paths map[string]bool
|
||||
fail map[string]bool
|
||||
deleted []string
|
||||
}
|
||||
|
||||
func (f *fakeRuntime) ListPathNames(context.Context) ([]string, error) {
|
||||
values := make([]string, 0, len(f.paths))
|
||||
for path, present := range f.paths {
|
||||
if present {
|
||||
values = append(values, path)
|
||||
}
|
||||
}
|
||||
sort.Strings(values)
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func (f *fakeRuntime) DeletePath(_ context.Context, path string) error {
|
||||
if f.fail[path] {
|
||||
return errors.New("redacted media failure")
|
||||
}
|
||||
delete(f.paths, path)
|
||||
f.deleted = append(f.deleted, path)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestSafetyGateUsesExactTenPercentBoundary(t *testing.T) {
|
||||
t.Parallel()
|
||||
if allowed, reason := safetyGate(1, 10); !allowed || reason != "allowed" {
|
||||
t.Fatalf("exact 10%% boundary was rejected: %v %s", allowed, reason)
|
||||
}
|
||||
if allowed, reason := safetyGate(1, 9); allowed || reason != "ratio_exceeded" {
|
||||
t.Fatalf("more than 10%% was accepted: %v %s", allowed, reason)
|
||||
}
|
||||
if allowed, reason := safetyGate(129, 2000); allowed || reason != "scope_exceeded" {
|
||||
t.Fatalf("129-item scope was accepted: %v %s", allowed, reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportClassifiesOwnershipAndApplyNeverDeletesUnowned(t *testing.T) {
|
||||
repository := newFakeRepository()
|
||||
repository.ownership = []store.MediaPathOwnership{
|
||||
{PathName: "owned-stale", DeviceID: "old-device"},
|
||||
{PathName: "owned-current", DeviceID: "live-device", CurrentClaim: true},
|
||||
}
|
||||
runtime := &fakeRuntime{paths: map[string]bool{
|
||||
"owned-stale": true, "owned-current": true,
|
||||
"unowned-1": true, "unowned-2": true, "unowned-3": true, "unowned-4": true,
|
||||
"unowned-5": true, "unowned-6": true, "unowned-7": true, "unowned-8": true,
|
||||
}, fail: make(map[string]bool)}
|
||||
now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC)
|
||||
manager := New(repository, runtime, "ins_test", nil)
|
||||
manager.now = func() time.Time { return now }
|
||||
scan, err := manager.Report(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scan.ObservedCount != 10 || scan.OwnedStaleCount != 1 || scan.UnownedCount != 8 || !scan.SafetyAllowed {
|
||||
t.Fatalf("unexpected report: %+v", scan)
|
||||
}
|
||||
result, err := manager.Apply(
|
||||
context.Background(), scan.ID, "operator-1", Confirmation(scan.ID),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Deleted != 1 || len(runtime.deleted) != 1 || runtime.deleted[0] != "owned-stale" {
|
||||
t.Fatalf("cleanup escaped owned stale set: result=%+v deleted=%v", result, runtime.deleted)
|
||||
}
|
||||
for _, path := range []string{"owned-current", "unowned-1", "unowned-8"} {
|
||||
if !runtime.paths[path] {
|
||||
t.Fatalf("cleanup deleted protected path %q", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyBlocksExpiredOrOversizedSnapshotBeforeDelete(t *testing.T) {
|
||||
repository := newFakeRepository()
|
||||
repository.ownership = []store.MediaPathOwnership{{PathName: "stale", DeviceID: "old"}}
|
||||
runtime := &fakeRuntime{paths: map[string]bool{"stale": true}, fail: make(map[string]bool)}
|
||||
now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC)
|
||||
manager := New(repository, runtime, "ins_test", nil)
|
||||
manager.now = func() time.Time { return now }
|
||||
scan, err := manager.Report(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scan.SafetyAllowed || scan.SafetyReason != "ratio_exceeded" {
|
||||
t.Fatalf("single-path 100%% cleanup was not blocked: %+v", scan)
|
||||
}
|
||||
if _, err := manager.Apply(context.Background(), scan.ID, "operator", Confirmation(scan.ID)); !errors.Is(err, ErrSafetyBlocked) {
|
||||
t.Fatalf("expected ratio gate, got %v", err)
|
||||
}
|
||||
if len(runtime.deleted) != 0 {
|
||||
t.Fatal("ratio-blocked cleanup mutated MediaMTX")
|
||||
}
|
||||
|
||||
for index := 0; index < 9; index++ {
|
||||
runtime.paths[string(rune('a'+index))] = true
|
||||
}
|
||||
scan, err = manager.Report(context.Background())
|
||||
if err != nil || !scan.SafetyAllowed {
|
||||
t.Fatalf("expected a fresh 1/10 executable report: %+v %v", scan, err)
|
||||
}
|
||||
now = now.Add(reportTTL)
|
||||
if _, err := manager.Apply(context.Background(), scan.ID, "operator", Confirmation(scan.ID)); !errors.Is(err, ErrSnapshotExpired) {
|
||||
t.Fatalf("expected expired snapshot, got %v", err)
|
||||
}
|
||||
if len(runtime.deleted) != 0 {
|
||||
t.Fatal("expired cleanup mutated MediaMTX")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyIsRetryableAndCannotExpandBeyondSnapshot(t *testing.T) {
|
||||
repository := newFakeRepository()
|
||||
repository.ownership = []store.MediaPathOwnership{
|
||||
{PathName: "stale-a", DeviceID: "old-a"},
|
||||
{PathName: "stale-b", DeviceID: "old-b"},
|
||||
}
|
||||
runtime := &fakeRuntime{paths: make(map[string]bool), fail: map[string]bool{"stale-b": true}}
|
||||
runtime.paths["stale-a"], runtime.paths["stale-b"] = true, true
|
||||
for index := 0; index < 18; index++ {
|
||||
runtime.paths[string(rune(0x100+index))] = true
|
||||
}
|
||||
now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC)
|
||||
manager := New(repository, runtime, "ins_test", nil)
|
||||
manager.now = func() time.Time { return now }
|
||||
scan, err := manager.Report(context.Background())
|
||||
if err != nil || !scan.SafetyAllowed {
|
||||
t.Fatalf("expected 2/20 report: %+v %v", scan, err)
|
||||
}
|
||||
repository.ownership = append(repository.ownership,
|
||||
store.MediaPathOwnership{PathName: "new-stale", DeviceID: "new-old"})
|
||||
runtime.paths["new-stale"] = true
|
||||
result, err := manager.Apply(context.Background(), scan.ID, "operator", Confirmation(scan.ID))
|
||||
if err == nil || result.Deleted != 1 || result.Failed != 1 {
|
||||
t.Fatalf("expected one partial failure: %+v %v", result, err)
|
||||
}
|
||||
if !runtime.paths["new-stale"] {
|
||||
t.Fatal("cleanup expanded beyond the approved snapshot")
|
||||
}
|
||||
runtime.fail["stale-b"] = false
|
||||
result, err = manager.Apply(context.Background(), scan.ID, "operator", Confirmation(scan.ID))
|
||||
if err != nil || result.Deleted != 1 || result.Failed != 0 {
|
||||
t.Fatalf("failed item was not retryable: %+v %v", result, err)
|
||||
}
|
||||
if !runtime.paths["new-stale"] {
|
||||
t.Fatal("retry expanded beyond the approved snapshot")
|
||||
}
|
||||
}
|
||||
@@ -3,12 +3,15 @@ package reconcile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"yovision/sense/internal/device"
|
||||
"yovision/sense/internal/metrics"
|
||||
"yovision/sense/internal/onvif"
|
||||
"yovision/sense/internal/store"
|
||||
)
|
||||
@@ -16,9 +19,11 @@ import (
|
||||
const defaultBatchSize = 128
|
||||
|
||||
type Repository interface {
|
||||
ListDueReconcile(ctx context.Context, now time.Time, limit int) ([]store.ReconcileCandidate, error)
|
||||
MarkReconciled(ctx context.Context, id string, generation int64, now time.Time) error
|
||||
MarkReconcileFailure(ctx context.Context, id string, failureCount int, nextAttempt time.Time, errorCode string, now time.Time) error
|
||||
ClaimDueReconcile(context.Context, store.ReconcileClaim) ([]store.ReconcileCandidate, error)
|
||||
RenewReconcileLease(context.Context, string, string, string, time.Time, time.Duration) (bool, error)
|
||||
CompleteReconcile(context.Context, string, int64, string, string, time.Time) error
|
||||
FailReconcile(context.Context, string, int, time.Time, string, string, string, time.Time) error
|
||||
ConvergenceSnapshot(context.Context) (store.ConvergenceSnapshot, error)
|
||||
}
|
||||
|
||||
type MediaPaths interface {
|
||||
@@ -27,48 +32,134 @@ type MediaPaths interface {
|
||||
}
|
||||
|
||||
type Reconciler struct {
|
||||
repository Repository
|
||||
discovery onvif.Adapter
|
||||
media MediaPaths
|
||||
now func() time.Time
|
||||
baseBackoff time.Duration
|
||||
maxBackoff time.Duration
|
||||
batchSize int
|
||||
repository Repository
|
||||
discovery onvif.Adapter
|
||||
media MediaPaths
|
||||
now func() time.Time
|
||||
baseBackoff time.Duration
|
||||
maxBackoff time.Duration
|
||||
batchSize int
|
||||
instanceID string
|
||||
leaseDuration time.Duration
|
||||
operationTimeout time.Duration
|
||||
metrics *metrics.Registry
|
||||
}
|
||||
|
||||
type Options struct {
|
||||
InstanceID string
|
||||
LeaseDuration time.Duration
|
||||
OperationTimeout time.Duration
|
||||
Metrics *metrics.Registry
|
||||
}
|
||||
|
||||
func New(repository Repository, discovery onvif.Adapter, media MediaPaths) *Reconciler {
|
||||
return NewWithOptions(repository, discovery, media, Options{})
|
||||
}
|
||||
|
||||
func NewWithOptions(
|
||||
repository Repository,
|
||||
discovery onvif.Adapter,
|
||||
media MediaPaths,
|
||||
options Options,
|
||||
) *Reconciler {
|
||||
if options.InstanceID == "" {
|
||||
options.InstanceID = "single"
|
||||
}
|
||||
if options.LeaseDuration <= 0 {
|
||||
options.LeaseDuration = 30 * time.Second
|
||||
}
|
||||
if options.OperationTimeout <= 0 {
|
||||
options.OperationTimeout = 20 * time.Second
|
||||
}
|
||||
if options.OperationTimeout >= options.LeaseDuration {
|
||||
options.OperationTimeout = options.LeaseDuration / 2
|
||||
}
|
||||
return &Reconciler{
|
||||
repository: repository, discovery: discovery, media: media,
|
||||
now: time.Now, baseBackoff: time.Second, maxBackoff: time.Minute, batchSize: defaultBatchSize,
|
||||
instanceID: options.InstanceID, leaseDuration: options.LeaseDuration,
|
||||
operationTimeout: options.OperationTimeout, metrics: options.Metrics,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Reconciler) RunOnce(ctx context.Context) error {
|
||||
func (r *Reconciler) RunOnce(ctx context.Context) (runErr error) {
|
||||
started := time.Now()
|
||||
defer func() {
|
||||
if r.metrics != nil {
|
||||
r.metrics.ObserveReconcileRun(runErr, time.Since(started))
|
||||
}
|
||||
}()
|
||||
now := r.now().UTC()
|
||||
candidates, err := r.repository.ListDueReconcile(ctx, now, r.batchSize)
|
||||
token, err := newClaimToken()
|
||||
if err != nil {
|
||||
return fmt.Errorf("list reconciliation candidates: %w", err)
|
||||
return err
|
||||
}
|
||||
candidates, err := r.repository.ClaimDueReconcile(ctx, store.ReconcileClaim{
|
||||
Owner: r.instanceID, Token: token, Now: now,
|
||||
LeaseDuration: r.leaseDuration, Limit: r.batchSize,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("claim reconciliation candidates: %w", err)
|
||||
}
|
||||
var runErrors []error
|
||||
for _, candidate := range candidates {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.reconcileOne(ctx, candidate, now); err != nil {
|
||||
itemNow := r.now().UTC()
|
||||
renewed, err := r.repository.RenewReconcileLease(
|
||||
ctx, candidate.Device.ID, r.instanceID, token, itemNow, r.leaseDuration,
|
||||
)
|
||||
if err != nil {
|
||||
r.observeItem("error")
|
||||
runErrors = append(runErrors, fmt.Errorf("renew reconcile device lease: %w", err))
|
||||
continue
|
||||
}
|
||||
if !renewed {
|
||||
r.observeItem("lease_lost")
|
||||
continue
|
||||
}
|
||||
itemCtx, cancel := context.WithTimeout(ctx, r.operationTimeout)
|
||||
err = r.reconcileOne(itemCtx, ctx, candidate, itemNow, r.instanceID, token)
|
||||
cancel()
|
||||
if errors.Is(err, store.ErrReconcileLeaseLost) {
|
||||
r.observeItem("lease_lost")
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
r.observeItem("error")
|
||||
runErrors = append(runErrors, fmt.Errorf("reconcile device %s: %w", candidate.Device.ID, err))
|
||||
} else {
|
||||
r.observeItem("success")
|
||||
}
|
||||
}
|
||||
if r.metrics != nil {
|
||||
snapshot, err := r.repository.ConvergenceSnapshot(ctx)
|
||||
if err != nil {
|
||||
runErrors = append(runErrors, fmt.Errorf("read convergence metrics: %w", err))
|
||||
} else {
|
||||
r.metrics.SetConvergence(snapshot.Total, snapshot.Unconverged)
|
||||
}
|
||||
}
|
||||
return errors.Join(runErrors...)
|
||||
}
|
||||
|
||||
func (r *Reconciler) reconcileOne(ctx context.Context, candidate store.ReconcileCandidate, now time.Time) error {
|
||||
func (r *Reconciler) reconcileOne(
|
||||
ctx context.Context,
|
||||
persistCtx context.Context,
|
||||
candidate store.ReconcileCandidate,
|
||||
now time.Time,
|
||||
owner, token string,
|
||||
) error {
|
||||
if candidate.Device.DesiredState == device.DesiredDisabled {
|
||||
if err := r.media.DeletePath(ctx, candidate.Device.PathName); err == nil {
|
||||
return r.repository.MarkReconciled(ctx, candidate.Device.ID, candidate.Device.Generation, now)
|
||||
} else if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
return r.repository.CompleteReconcile(
|
||||
persistCtx, candidate.Device.ID, candidate.Device.Generation, owner, token, now,
|
||||
)
|
||||
} else if persistCtx.Err() != nil {
|
||||
return persistCtx.Err()
|
||||
} else {
|
||||
return r.persistFailure(ctx, candidate, now, err)
|
||||
return r.persistFailure(persistCtx, candidate, now, owner, token, err)
|
||||
}
|
||||
}
|
||||
result, err := r.discovery.Probe(ctx, onvif.Target{
|
||||
@@ -81,16 +172,22 @@ func (r *Reconciler) reconcileOne(ctx context.Context, candidate store.Reconcile
|
||||
_, err = r.media.EnsurePath(ctx, candidate.Device.PathName, result.StreamURI)
|
||||
}
|
||||
if err == nil {
|
||||
return r.repository.MarkReconciled(ctx, candidate.Device.ID, candidate.Device.Generation, now)
|
||||
return r.repository.CompleteReconcile(
|
||||
persistCtx, candidate.Device.ID, candidate.Device.Generation, owner, token, now,
|
||||
)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
if persistCtx.Err() != nil {
|
||||
return persistCtx.Err()
|
||||
}
|
||||
return r.persistFailure(ctx, candidate, now, err)
|
||||
return r.persistFailure(persistCtx, candidate, now, owner, token, err)
|
||||
}
|
||||
|
||||
func (r *Reconciler) persistFailure(
|
||||
ctx context.Context, candidate store.ReconcileCandidate, now time.Time, err error,
|
||||
ctx context.Context,
|
||||
candidate store.ReconcileCandidate,
|
||||
now time.Time,
|
||||
owner, token string,
|
||||
err error,
|
||||
) error {
|
||||
failureCount := candidate.FailureCount + 1
|
||||
nextAttempt := now.Add(r.backoff(failureCount))
|
||||
@@ -99,14 +196,28 @@ func (r *Reconciler) persistFailure(
|
||||
if !errors.As(err, &onvifError) {
|
||||
errorCode = "media_error"
|
||||
}
|
||||
if markErr := r.repository.MarkReconcileFailure(
|
||||
ctx, candidate.Device.ID, failureCount, nextAttempt, errorCode, now,
|
||||
if markErr := r.repository.FailReconcile(
|
||||
ctx, candidate.Device.ID, failureCount, nextAttempt, errorCode, owner, token, now,
|
||||
); markErr != nil {
|
||||
return errors.Join(err, fmt.Errorf("persist reconcile failure: %w", markErr))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *Reconciler) observeItem(result string) {
|
||||
if r.metrics != nil {
|
||||
r.metrics.ObserveReconcileItem(result)
|
||||
}
|
||||
}
|
||||
|
||||
func newClaimToken() (string, error) {
|
||||
value := make([]byte, 18)
|
||||
if _, err := rand.Read(value); err != nil {
|
||||
return "", errors.New("generate reconcile claim token")
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString(value), nil
|
||||
}
|
||||
|
||||
func validateStreamURI(value string) error {
|
||||
parsed, err := url.Parse(value)
|
||||
if err != nil || parsed.Host == "" || (parsed.Scheme != "rtsp" && parsed.Scheme != "rtsps") {
|
||||
|
||||
@@ -162,6 +162,86 @@ func TestDisabledDeviceDeletesOnlyItsExactPath(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
type lostLeaseRepository struct {
|
||||
completed bool
|
||||
failed bool
|
||||
}
|
||||
|
||||
func (r *lostLeaseRepository) ClaimDueReconcile(
|
||||
context.Context,
|
||||
store.ReconcileClaim,
|
||||
) ([]store.ReconcileCandidate, error) {
|
||||
return []store.ReconcileCandidate{{Device: device.Device{
|
||||
ID: "camera-lost", DesiredState: device.DesiredEnabled,
|
||||
EndpointRef: "onvif://camera-lost", PathName: "camera-lost", Generation: 1,
|
||||
}}}, nil
|
||||
}
|
||||
|
||||
func (r *lostLeaseRepository) RenewReconcileLease(
|
||||
context.Context, string, string, string, time.Time, time.Duration,
|
||||
) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (r *lostLeaseRepository) CompleteReconcile(
|
||||
context.Context, string, int64, string, string, time.Time,
|
||||
) error {
|
||||
r.completed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *lostLeaseRepository) FailReconcile(
|
||||
context.Context, string, int, time.Time, string, string, string, time.Time,
|
||||
) error {
|
||||
r.failed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *lostLeaseRepository) ConvergenceSnapshot(context.Context) (store.ConvergenceSnapshot, error) {
|
||||
return store.ConvergenceSnapshot{}, nil
|
||||
}
|
||||
|
||||
func TestLostLeaseSkipsAllExternalAndStoreMutations(t *testing.T) {
|
||||
t.Parallel()
|
||||
repository := &lostLeaseRepository{}
|
||||
media := &recordingMedia{}
|
||||
reconciler := New(repository, onvif.NewFake(nil), media)
|
||||
if err := reconciler.RunOnce(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if media.calls != 0 || repository.completed || repository.failed {
|
||||
t.Fatalf("lost lease performed a side effect: media=%d completed=%v failed=%v",
|
||||
media.calls, repository.completed, repository.failed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPerItemTimeoutPersistsRetryWhenParentIsAlive(t *testing.T) {
|
||||
t.Parallel()
|
||||
repository := openRepository(t, filepath.Join(t.TempDir(), "sense.db"))
|
||||
createReconcileDevice(t, repository)
|
||||
discovery := onvif.NewFake(map[string]onvif.FakeScenario{
|
||||
"onvif://camera-1": {
|
||||
DelayMillis: 100,
|
||||
Result: onvif.ProbeResult{StreamURI: "rtsp://media.invalid/camera-1"},
|
||||
},
|
||||
})
|
||||
reconciler := NewWithOptions(repository, discovery, &recordingMedia{}, Options{
|
||||
InstanceID: "ins_test", LeaseDuration: time.Second, OperationTimeout: 5 * time.Millisecond,
|
||||
})
|
||||
now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC)
|
||||
reconciler.now = func() time.Time { return now }
|
||||
if err := reconciler.RunOnce(context.Background()); err == nil {
|
||||
t.Fatal("expected bounded operation timeout")
|
||||
}
|
||||
candidates, err := repository.ListDueReconcile(context.Background(), now.Add(time.Hour), 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(candidates) != 1 || candidates[0].FailureCount != 1 {
|
||||
t.Fatalf("operation timeout did not persist retry state: %+v", candidates)
|
||||
}
|
||||
}
|
||||
|
||||
func openRepository(t *testing.T, path string) *store.SQLite {
|
||||
t.Helper()
|
||||
repository, err := store.OpenSQLite(context.Background(), "file:"+filepath.ToSlash(path))
|
||||
|
||||
@@ -514,6 +514,7 @@ func (s *Postgres) PatchControlDevice(
|
||||
if reconcileChanged {
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE sense.reconcile_state SET
|
||||
failure_count = 0, next_attempt_at = NULL, last_error_code = NULL,
|
||||
lease_owner = NULL, lease_token = NULL, lease_until = NULL,
|
||||
updated_at = $1 WHERE device_id = $2`, now, deviceID); err != nil {
|
||||
return ControlMutationResult{}, errors.New("reset postgres Control API reconcile state")
|
||||
}
|
||||
@@ -631,6 +632,7 @@ func setControlDesiredStateTx(
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE sense.reconcile_state SET
|
||||
failure_count = 0, next_attempt_at = NULL, last_error_code = NULL,
|
||||
lease_owner = NULL, lease_token = NULL, lease_until = NULL,
|
||||
updated_at = $1 WHERE device_id = $2`, now, deviceID); err != nil {
|
||||
return ControlDevice{}, errors.New("reset postgres Control API desired-state reconciliation")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
OperationalLeaseOrphanScan = "mediamtx-orphan-scan"
|
||||
OperationalLeaseOrphanCleanup = "mediamtx-orphan-cleanup"
|
||||
|
||||
OrphanOwnedStale = "owned_stale"
|
||||
OrphanUnowned = "unowned"
|
||||
)
|
||||
|
||||
var ErrOrphanScanNotFound = errors.New("orphan scan not found")
|
||||
|
||||
type MediaPathOwnership struct {
|
||||
PathName string
|
||||
DeviceID string
|
||||
CurrentClaim bool
|
||||
}
|
||||
|
||||
type OrphanFinding struct {
|
||||
PathName string
|
||||
Classification string
|
||||
DeviceID string
|
||||
Deleted bool
|
||||
}
|
||||
|
||||
type OrphanScan struct {
|
||||
ID string
|
||||
InstanceID string
|
||||
ObservedCount int
|
||||
OwnedStaleCount int
|
||||
UnownedCount int
|
||||
SafetyAllowed bool
|
||||
SafetyReason string
|
||||
CompletedAt time.Time
|
||||
ExpiresAt time.Time
|
||||
Findings []OrphanFinding
|
||||
}
|
||||
|
||||
type OrphanRepository interface {
|
||||
AcquireOperationalLease(
|
||||
context.Context, string, string, string, time.Time, time.Duration,
|
||||
) (bool, error)
|
||||
ReleaseOperationalLease(context.Context, string, string, string, time.Time) error
|
||||
ListMediaPathOwnership(context.Context) ([]MediaPathOwnership, error)
|
||||
SaveOrphanScan(context.Context, OrphanScan, string, string) error
|
||||
GetOrphanScan(context.Context, string) (OrphanScan, error)
|
||||
RecordOrphanCleanup(
|
||||
context.Context, string, string, string, string, string, time.Time,
|
||||
) error
|
||||
}
|
||||
|
||||
var _ OrphanRepository = (*Postgres)(nil)
|
||||
@@ -0,0 +1,268 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const orphanReportRetention = 7 * 24 * time.Hour
|
||||
|
||||
func (s *Postgres) AcquireOperationalLease(
|
||||
ctx context.Context,
|
||||
name, owner, token string,
|
||||
_ time.Time,
|
||||
duration time.Duration,
|
||||
) (bool, error) {
|
||||
if strings.TrimSpace(name) == "" || strings.TrimSpace(owner) == "" ||
|
||||
strings.TrimSpace(token) == "" || duration <= 0 {
|
||||
return false, errors.New("invalid operational lease")
|
||||
}
|
||||
var acquired int
|
||||
err := s.db.QueryRowContext(ctx, `INSERT INTO sense.operational_leases(
|
||||
lease_name, owner_id, fencing_token, lease_until, updated_at
|
||||
) VALUES (
|
||||
$1, $2, $3,
|
||||
clock_timestamp() + ($4 * interval '1 second'), clock_timestamp()
|
||||
)
|
||||
ON CONFLICT (lease_name) DO UPDATE SET
|
||||
owner_id = EXCLUDED.owner_id,
|
||||
fencing_token = EXCLUDED.fencing_token,
|
||||
lease_until = EXCLUDED.lease_until,
|
||||
updated_at = EXCLUDED.updated_at
|
||||
WHERE sense.operational_leases.lease_until <= clock_timestamp()
|
||||
OR (sense.operational_leases.owner_id = $2
|
||||
AND sense.operational_leases.fencing_token = $3)
|
||||
RETURNING 1`, name, owner, token, duration.Seconds()).Scan(&acquired)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, errors.New("acquire postgres operational lease")
|
||||
}
|
||||
return acquired == 1, nil
|
||||
}
|
||||
|
||||
func (s *Postgres) ReleaseOperationalLease(
|
||||
ctx context.Context,
|
||||
name, owner, token string,
|
||||
_ time.Time,
|
||||
) error {
|
||||
_, err := s.db.ExecContext(ctx, `UPDATE sense.operational_leases
|
||||
SET lease_until = clock_timestamp(), updated_at = clock_timestamp()
|
||||
WHERE lease_name = $1 AND owner_id = $2 AND fencing_token = $3`,
|
||||
name, owner, token)
|
||||
if err != nil {
|
||||
return errors.New("release postgres operational lease")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Postgres) ListMediaPathOwnership(ctx context.Context) ([]MediaPathOwnership, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT
|
||||
o.path_name, o.device_id,
|
||||
EXISTS (
|
||||
SELECT 1 FROM sense.devices d
|
||||
WHERE d.id = o.device_id AND d.path_name = o.path_name
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM sense.device_capabilities c
|
||||
WHERE c.device_id = d.id AND c.capability = 'video_capture'
|
||||
)
|
||||
) AS current_claim
|
||||
FROM sense.media_path_ownership o
|
||||
ORDER BY o.path_name`)
|
||||
if err != nil {
|
||||
return nil, errors.New("list postgres MediaMTX path ownership")
|
||||
}
|
||||
defer rows.Close()
|
||||
values := make([]MediaPathOwnership, 0)
|
||||
for rows.Next() {
|
||||
var value MediaPathOwnership
|
||||
if err := rows.Scan(&value.PathName, &value.DeviceID, &value.CurrentClaim); err != nil {
|
||||
return nil, errors.New("scan postgres MediaMTX path ownership")
|
||||
}
|
||||
values = append(values, value)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, errors.New("iterate postgres MediaMTX path ownership")
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func (s *Postgres) SaveOrphanScan(
|
||||
ctx context.Context,
|
||||
scan OrphanScan,
|
||||
owner, token string,
|
||||
) error {
|
||||
if err := validateOrphanScan(scan); err != nil {
|
||||
return err
|
||||
}
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return errors.New("begin postgres orphan scan save")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var lease int
|
||||
err = tx.QueryRowContext(ctx, `SELECT 1 FROM sense.operational_leases
|
||||
WHERE lease_name = $1 AND owner_id = $2 AND fencing_token = $3
|
||||
AND lease_until > clock_timestamp()
|
||||
FOR UPDATE`, OperationalLeaseOrphanScan, owner, token).Scan(&lease)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return ErrOperationalLeaseLost
|
||||
}
|
||||
if err != nil {
|
||||
return errors.New("verify postgres orphan scan lease")
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO sense.orphan_scan_runs(
|
||||
id, instance_id, observed_count, owned_stale_count, unowned_count,
|
||||
safety_allowed, safety_reason, completed_at, expires_at
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`,
|
||||
scan.ID, scan.InstanceID, scan.ObservedCount, scan.OwnedStaleCount, scan.UnownedCount,
|
||||
scan.SafetyAllowed, scan.SafetyReason, scan.CompletedAt, scan.ExpiresAt,
|
||||
); err != nil {
|
||||
return errors.New("insert postgres orphan scan")
|
||||
}
|
||||
for _, finding := range scan.Findings {
|
||||
var deviceID any
|
||||
if finding.DeviceID != "" {
|
||||
deviceID = finding.DeviceID
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO sense.orphan_scan_findings(
|
||||
scan_id, path_name, classification, device_id
|
||||
) VALUES ($1,$2,$3,$4)`, scan.ID, finding.PathName, finding.Classification, deviceID); err != nil {
|
||||
return errors.New("insert postgres orphan scan finding")
|
||||
}
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE sense.operational_leases
|
||||
SET lease_until = clock_timestamp(), updated_at = clock_timestamp()
|
||||
WHERE lease_name = $1 AND owner_id = $2 AND fencing_token = $3`,
|
||||
OperationalLeaseOrphanScan, owner, token); err != nil {
|
||||
return errors.New("release postgres orphan scan lease")
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM sense.orphan_scan_runs r
|
||||
WHERE r.completed_at < $1
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM sense.orphan_cleanup_actions a WHERE a.scan_id = r.id
|
||||
)`, scan.CompletedAt.Add(-orphanReportRetention)); err != nil {
|
||||
return errors.New("expire postgres orphan scan reports")
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return errors.New("commit postgres orphan scan")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateOrphanScan(scan OrphanScan) error {
|
||||
if strings.TrimSpace(scan.ID) == "" || strings.TrimSpace(scan.InstanceID) == "" ||
|
||||
scan.ObservedCount < 0 || scan.OwnedStaleCount < 0 || scan.UnownedCount < 0 ||
|
||||
scan.OwnedStaleCount+scan.UnownedCount > scan.ObservedCount ||
|
||||
strings.TrimSpace(scan.SafetyReason) == "" || !scan.ExpiresAt.After(scan.CompletedAt) {
|
||||
return errors.New("invalid orphan scan")
|
||||
}
|
||||
seen := make(map[string]struct{}, len(scan.Findings))
|
||||
staleCount, unownedCount := 0, 0
|
||||
for _, finding := range scan.Findings {
|
||||
if strings.TrimSpace(finding.PathName) == "" {
|
||||
return errors.New("invalid orphan finding path")
|
||||
}
|
||||
if _, duplicate := seen[finding.PathName]; duplicate {
|
||||
return errors.New("duplicate orphan finding path")
|
||||
}
|
||||
seen[finding.PathName] = struct{}{}
|
||||
switch finding.Classification {
|
||||
case OrphanOwnedStale:
|
||||
staleCount++
|
||||
if strings.TrimSpace(finding.DeviceID) == "" {
|
||||
return errors.New("owned stale finding lacks device")
|
||||
}
|
||||
case OrphanUnowned:
|
||||
unownedCount++
|
||||
if finding.DeviceID != "" {
|
||||
return errors.New("unowned finding has device")
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("invalid orphan finding classification %q", finding.Classification)
|
||||
}
|
||||
}
|
||||
if staleCount != scan.OwnedStaleCount || unownedCount != scan.UnownedCount {
|
||||
return errors.New("orphan scan counts do not match findings")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Postgres) GetOrphanScan(ctx context.Context, id string) (OrphanScan, error) {
|
||||
var scan OrphanScan
|
||||
err := s.db.QueryRowContext(ctx, `SELECT
|
||||
id, instance_id, observed_count, owned_stale_count, unowned_count,
|
||||
safety_allowed, safety_reason, completed_at, expires_at
|
||||
FROM sense.orphan_scan_runs WHERE id = $1`, id).Scan(
|
||||
&scan.ID, &scan.InstanceID, &scan.ObservedCount, &scan.OwnedStaleCount,
|
||||
&scan.UnownedCount, &scan.SafetyAllowed, &scan.SafetyReason,
|
||||
&scan.CompletedAt, &scan.ExpiresAt,
|
||||
)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return OrphanScan{}, ErrOrphanScanNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return OrphanScan{}, errors.New("read postgres orphan scan")
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT
|
||||
f.path_name, f.classification, COALESCE(f.device_id, ''),
|
||||
COALESCE(a.status = 'deleted', false)
|
||||
FROM sense.orphan_scan_findings f
|
||||
LEFT JOIN sense.orphan_cleanup_actions a
|
||||
ON a.scan_id = f.scan_id AND a.path_name = f.path_name
|
||||
WHERE f.scan_id = $1 ORDER BY f.path_name`, id)
|
||||
if err != nil {
|
||||
return OrphanScan{}, errors.New("list postgres orphan scan findings")
|
||||
}
|
||||
defer rows.Close()
|
||||
scan.Findings = make([]OrphanFinding, 0)
|
||||
for rows.Next() {
|
||||
var finding OrphanFinding
|
||||
if err := rows.Scan(
|
||||
&finding.PathName, &finding.Classification, &finding.DeviceID, &finding.Deleted,
|
||||
); err != nil {
|
||||
return OrphanScan{}, errors.New("scan postgres orphan finding")
|
||||
}
|
||||
scan.Findings = append(scan.Findings, finding)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return OrphanScan{}, errors.New("iterate postgres orphan findings")
|
||||
}
|
||||
return scan, nil
|
||||
}
|
||||
|
||||
func (s *Postgres) RecordOrphanCleanup(
|
||||
ctx context.Context,
|
||||
scanID, pathName, actorID, status, errorCode string,
|
||||
now time.Time,
|
||||
) error {
|
||||
if status != "deleted" && status != "failed" {
|
||||
return errors.New("invalid orphan cleanup status")
|
||||
}
|
||||
var storedError any
|
||||
if status == "failed" {
|
||||
if strings.TrimSpace(errorCode) == "" {
|
||||
return errors.New("failed orphan cleanup requires an error code")
|
||||
}
|
||||
storedError = errorCode
|
||||
}
|
||||
_, err := s.db.ExecContext(ctx, `INSERT INTO sense.orphan_cleanup_actions(
|
||||
scan_id, path_name, classification, actor_id, status, error_code, attempted_at
|
||||
) VALUES ($1,$2,'owned_stale',$3,$4,$5,$6)
|
||||
ON CONFLICT (scan_id, path_name) DO UPDATE SET
|
||||
actor_id = EXCLUDED.actor_id,
|
||||
status = EXCLUDED.status,
|
||||
error_code = EXCLUDED.error_code,
|
||||
attempted_at = EXCLUDED.attempted_at
|
||||
WHERE sense.orphan_cleanup_actions.status <> 'deleted'`,
|
||||
scanID, pathName, actorID, status, storedError, now)
|
||||
if err != nil {
|
||||
return errors.New("record postgres orphan cleanup result")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -53,8 +53,8 @@ func (s *Postgres) Close() error {
|
||||
func (s *Postgres) verifySchemaAndPrivileges(ctx context.Context) error {
|
||||
var version sql.NullInt64
|
||||
if err := s.db.QueryRowContext(ctx,
|
||||
`SELECT MAX(version) FROM sense.schema_migrations`).Scan(&version); err != nil || !version.Valid || version.Int64 < 4 {
|
||||
return errors.New("postgres sense schema migration v4 is required")
|
||||
`SELECT MAX(version) FROM sense.schema_migrations`).Scan(&version); err != nil || !version.Valid || version.Int64 < 5 {
|
||||
return errors.New("postgres sense schema migration v5 is required")
|
||||
}
|
||||
var canReadQuotaView, canWriteQuotaView, canReadSiteSource, canWriteSiteSource bool
|
||||
var canReadAreaView, canWriteAreaView, canReadAreaSource, canWriteAreaSource bool
|
||||
@@ -95,6 +95,28 @@ func (s *Postgres) verifySchemaAndPrivileges(ctx context.Context) error {
|
||||
publicReceipts || publicOperations || publicOperationItems {
|
||||
return errors.New("postgres role violates Control API state privilege boundary")
|
||||
}
|
||||
var canUseOwnership, canUseLeases, canUseScans, canUseFindings, canUseActions bool
|
||||
var publicOwnership, publicLeases, publicScans, publicFindings, publicActions bool
|
||||
if err := s.db.QueryRowContext(ctx, `SELECT
|
||||
has_table_privilege(current_user, 'sense.media_path_ownership', 'SELECT,INSERT,UPDATE,DELETE'),
|
||||
has_table_privilege(current_user, 'sense.operational_leases', 'SELECT,INSERT,UPDATE,DELETE'),
|
||||
has_table_privilege(current_user, 'sense.orphan_scan_runs', 'SELECT,INSERT,UPDATE,DELETE'),
|
||||
has_table_privilege(current_user, 'sense.orphan_scan_findings', 'SELECT,INSERT,UPDATE,DELETE'),
|
||||
has_table_privilege(current_user, 'sense.orphan_cleanup_actions', 'SELECT,INSERT,UPDATE,DELETE'),
|
||||
has_table_privilege('public', 'sense.media_path_ownership', 'SELECT,INSERT,UPDATE,DELETE'),
|
||||
has_table_privilege('public', 'sense.operational_leases', 'SELECT,INSERT,UPDATE,DELETE'),
|
||||
has_table_privilege('public', 'sense.orphan_scan_runs', 'SELECT,INSERT,UPDATE,DELETE'),
|
||||
has_table_privilege('public', 'sense.orphan_scan_findings', 'SELECT,INSERT,UPDATE,DELETE'),
|
||||
has_table_privilege('public', 'sense.orphan_cleanup_actions', 'SELECT,INSERT,UPDATE,DELETE')`).Scan(
|
||||
&canUseOwnership, &canUseLeases, &canUseScans, &canUseFindings, &canUseActions,
|
||||
&publicOwnership, &publicLeases, &publicScans, &publicFindings, &publicActions,
|
||||
); err != nil {
|
||||
return errors.New("verify postgres reconciliation safety privileges")
|
||||
}
|
||||
if !canUseOwnership || !canUseLeases || !canUseScans || !canUseFindings || !canUseActions ||
|
||||
publicOwnership || publicLeases || publicScans || publicFindings || publicActions {
|
||||
return errors.New("postgres role violates reconciliation safety privilege boundary")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -391,7 +413,8 @@ func (s *Postgres) SetDesiredState(ctx context.Context, id string, desired devic
|
||||
return errors.New("update postgres desired state")
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE sense.reconcile_state SET
|
||||
failure_count = 0, next_attempt_at = NULL, last_error_code = NULL, updated_at = $1
|
||||
failure_count = 0, next_attempt_at = NULL, last_error_code = NULL,
|
||||
lease_owner = NULL, lease_token = NULL, lease_until = NULL, updated_at = $1
|
||||
WHERE device_id = $2`, now, id); err != nil {
|
||||
return errors.New("reset postgres reconcile state")
|
||||
}
|
||||
@@ -486,6 +509,217 @@ func (s *Postgres) ListDueReconcile(ctx context.Context, now time.Time, limit in
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func (s *Postgres) ClaimDueReconcile(
|
||||
ctx context.Context,
|
||||
claim ReconcileClaim,
|
||||
) ([]ReconcileCandidate, error) {
|
||||
if claim.Limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if strings.TrimSpace(claim.Owner) == "" || strings.TrimSpace(claim.Token) == "" ||
|
||||
claim.LeaseDuration <= 0 {
|
||||
return nil, errors.New("invalid postgres reconcile claim")
|
||||
}
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, errors.New("begin postgres reconcile claim")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
rows, err := tx.QueryContext(ctx, `SELECT `+postgresDeviceColumns+`, r.failure_count, r.next_attempt_at
|
||||
FROM sense.devices d
|
||||
JOIN sense.reconcile_state r ON r.device_id = d.id
|
||||
WHERE EXISTS (SELECT 1 FROM sense.device_capabilities c
|
||||
WHERE c.device_id = d.id AND c.capability = 'video_capture')
|
||||
AND (r.observed_generation < d.generation
|
||||
OR (d.desired_state = 'enabled' AND r.failure_count > 0))
|
||||
AND (r.next_attempt_at IS NULL OR r.next_attempt_at <= clock_timestamp())
|
||||
AND (r.lease_until IS NULL OR r.lease_until <= clock_timestamp())
|
||||
ORDER BY d.updated_at, d.id
|
||||
FOR UPDATE OF r SKIP LOCKED
|
||||
LIMIT $1`, claim.Limit)
|
||||
if err != nil {
|
||||
return nil, errors.New("select postgres reconcile claims")
|
||||
}
|
||||
values := make([]ReconcileCandidate, 0)
|
||||
for rows.Next() {
|
||||
var candidate ReconcileCandidate
|
||||
var areaID sql.NullString
|
||||
var quotaVersion, areaVersion sql.NullInt64
|
||||
var nextAttempt sql.NullTime
|
||||
if err := rows.Scan(
|
||||
&candidate.Device.ID, &candidate.Device.TenantID, &candidate.Device.SiteID,
|
||||
&areaID, &candidate.Device.SerialNumber, &candidate.Device.Name, &candidate.Device.Modality,
|
||||
&candidate.Device.DesiredState, &candidate.Device.ActualState,
|
||||
&candidate.Device.EndpointRef, &candidate.Device.CredentialRef,
|
||||
&candidate.Device.ProfileToken, &candidate.Device.PathName,
|
||||
&candidate.Device.Generation, &candidate.Device.ResourceVersion,
|
||||
"aVersion, &areaVersion,
|
||||
&candidate.Device.CreatedAt, &candidate.Device.UpdatedAt,
|
||||
&candidate.FailureCount, &nextAttempt,
|
||||
); err != nil {
|
||||
rows.Close()
|
||||
return nil, errors.New("scan postgres reconcile claim")
|
||||
}
|
||||
candidate.Device.AreaID = areaID.String
|
||||
candidate.Device.QuotaSourceVersion = quotaVersion.Int64
|
||||
candidate.Device.AreaPolicySourceVersion = areaVersion.Int64
|
||||
if nextAttempt.Valid {
|
||||
value := nextAttempt.Time
|
||||
candidate.NextAttempt = &value
|
||||
}
|
||||
values = append(values, candidate)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, errors.New("close postgres reconcile claims")
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, errors.New("iterate postgres reconcile claims")
|
||||
}
|
||||
for _, candidate := range values {
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE sense.reconcile_state SET
|
||||
lease_owner = $1, lease_token = $2,
|
||||
lease_until = clock_timestamp() + ($3 * interval '1 second'),
|
||||
updated_at = clock_timestamp()
|
||||
WHERE device_id = $4`,
|
||||
claim.Owner, claim.Token, claim.LeaseDuration.Seconds(), candidate.Device.ID,
|
||||
); err != nil {
|
||||
return nil, errors.New("persist postgres reconcile claim")
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return nil, errors.New("commit postgres reconcile claim")
|
||||
}
|
||||
for index := range values {
|
||||
values[index].Device.Capabilities, err = s.capabilities(ctx, values[index].Device.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func (s *Postgres) RenewReconcileLease(
|
||||
ctx context.Context,
|
||||
id, owner, token string,
|
||||
now time.Time,
|
||||
duration time.Duration,
|
||||
) (bool, error) {
|
||||
if strings.TrimSpace(owner) == "" || strings.TrimSpace(token) == "" || duration <= 0 {
|
||||
return false, errors.New("invalid postgres reconcile lease renewal")
|
||||
}
|
||||
result, err := s.db.ExecContext(ctx, `UPDATE sense.reconcile_state SET
|
||||
lease_until = clock_timestamp() + ($1 * interval '1 second'),
|
||||
updated_at = clock_timestamp()
|
||||
WHERE device_id = $2 AND lease_owner = $3 AND lease_token = $4
|
||||
AND lease_until > clock_timestamp()`, duration.Seconds(), id, owner, token)
|
||||
if err != nil {
|
||||
return false, errors.New("renew postgres reconcile lease")
|
||||
}
|
||||
affected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return false, errors.New("read postgres reconcile lease renewal")
|
||||
}
|
||||
return affected == 1, nil
|
||||
}
|
||||
|
||||
func (s *Postgres) CompleteReconcile(
|
||||
ctx context.Context,
|
||||
id string,
|
||||
generation int64,
|
||||
owner, token string,
|
||||
now time.Time,
|
||||
) error {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return errors.New("begin postgres fenced reconciliation completion")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
result, err := tx.ExecContext(ctx, `UPDATE sense.reconcile_state SET
|
||||
failure_count = 0, next_attempt_at = NULL, last_error_code = NULL,
|
||||
observed_generation = $1, lease_owner = NULL, lease_token = NULL,
|
||||
lease_until = NULL, updated_at = $2
|
||||
WHERE device_id = $3 AND lease_owner = $4 AND lease_token = $5
|
||||
AND lease_until > clock_timestamp()`, generation, now, id, owner, token)
|
||||
if err != nil {
|
||||
return errors.New("complete postgres fenced reconciliation")
|
||||
}
|
||||
affected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return errors.New("read postgres fenced reconciliation completion")
|
||||
}
|
||||
if affected != 1 {
|
||||
return ErrReconcileLeaseLost
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO sense.media_path_ownership(
|
||||
path_name, device_id, tenant_id, site_id, first_claimed_at, last_confirmed_at
|
||||
)
|
||||
SELECT path_name, id, tenant_id, site_id, $1, $1
|
||||
FROM sense.devices
|
||||
WHERE id = $2 AND desired_state = 'enabled' AND btrim(path_name) <> ''
|
||||
ON CONFLICT (path_name) DO UPDATE SET
|
||||
device_id = EXCLUDED.device_id,
|
||||
tenant_id = EXCLUDED.tenant_id,
|
||||
site_id = EXCLUDED.site_id,
|
||||
last_confirmed_at = EXCLUDED.last_confirmed_at`, now, id); err != nil {
|
||||
return errors.New("record postgres MediaMTX path ownership")
|
||||
}
|
||||
result, err = tx.ExecContext(ctx, `UPDATE sense.devices
|
||||
SET actual_state = CASE WHEN desired_state = 'disabled' THEN 'offline' ELSE 'pending' END,
|
||||
updated_at = $1 WHERE id = $2`, now, id)
|
||||
if err != nil {
|
||||
return errors.New("mark postgres fenced device state")
|
||||
}
|
||||
if affected, _ := result.RowsAffected(); affected != 1 {
|
||||
return ErrNotFound
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return errors.New("commit postgres fenced reconciliation completion")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Postgres) FailReconcile(
|
||||
ctx context.Context,
|
||||
id string,
|
||||
failureCount int,
|
||||
nextAttempt time.Time,
|
||||
errorCode, owner, token string,
|
||||
now time.Time,
|
||||
) error {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return errors.New("begin postgres fenced reconciliation failure")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
result, err := tx.ExecContext(ctx, `UPDATE sense.reconcile_state SET
|
||||
failure_count = $1, next_attempt_at = $2, last_error_code = $3,
|
||||
lease_owner = NULL, lease_token = NULL, lease_until = NULL, updated_at = $4
|
||||
WHERE device_id = $5 AND lease_owner = $6 AND lease_token = $7
|
||||
AND lease_until > clock_timestamp()`, failureCount, nextAttempt, errorCode, now, id, owner, token)
|
||||
if err != nil {
|
||||
return errors.New("persist postgres fenced reconciliation failure")
|
||||
}
|
||||
affected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return errors.New("read postgres fenced reconciliation failure")
|
||||
}
|
||||
if affected != 1 {
|
||||
return ErrReconcileLeaseLost
|
||||
}
|
||||
result, err = tx.ExecContext(ctx, `UPDATE sense.devices
|
||||
SET actual_state = 'failed', updated_at = $1 WHERE id = $2`, now, id)
|
||||
if err != nil {
|
||||
return errors.New("mark postgres fenced failed device")
|
||||
}
|
||||
if affected, _ := result.RowsAffected(); affected != 1 {
|
||||
return ErrNotFound
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return errors.New("commit postgres fenced reconciliation failure")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Postgres) ListEnabledVideoDevices(ctx context.Context, limit int) ([]device.Device, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
|
||||
@@ -287,12 +287,12 @@ func TestPostgresOpenRejectsOverprivilegedRuntimeRole(t *testing.T) {
|
||||
_, admin := openPostgresTestStore(t)
|
||||
ctx := context.Background()
|
||||
if _, err := admin.ExecContext(ctx,
|
||||
`GRANT UPDATE ON bell.site_quota_v1 TO yovision_t011_sense`); err != nil {
|
||||
`GRANT UPDATE ON bell.site_quota_v1 TO yovision_t012_sense`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_, _ = admin.ExecContext(context.Background(),
|
||||
`REVOKE UPDATE ON bell.site_quota_v1 FROM yovision_t011_sense`)
|
||||
`REVOKE UPDATE ON bell.site_quota_v1 FROM yovision_t012_sense`)
|
||||
}()
|
||||
value, err := OpenPostgres(ctx, os.Getenv(postgresTestDSNEnv))
|
||||
if value != nil {
|
||||
@@ -304,6 +304,27 @@ func TestPostgresOpenRejectsOverprivilegedRuntimeRole(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresOpenRejectsPublicReconciliationStatePrivilege(t *testing.T) {
|
||||
_, admin := openPostgresTestStore(t)
|
||||
ctx := context.Background()
|
||||
if _, err := admin.ExecContext(ctx,
|
||||
`GRANT SELECT ON sense.orphan_scan_runs TO PUBLIC`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_, _ = admin.ExecContext(context.Background(),
|
||||
`REVOKE SELECT ON sense.orphan_scan_runs FROM PUBLIC`)
|
||||
}()
|
||||
value, err := OpenPostgres(ctx, os.Getenv(postgresTestDSNEnv))
|
||||
if value != nil {
|
||||
_ = value.Close()
|
||||
t.Fatal("PUBLIC reconciliation state privilege was accepted")
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), "reconciliation safety privilege boundary") {
|
||||
t.Fatalf("expected reconciliation privilege-boundary error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresAreaPolicyAllowsNonImagingAndDeniesImagingCreate(t *testing.T) {
|
||||
store, admin := openPostgresTestStore(t)
|
||||
ctx := context.Background()
|
||||
@@ -597,12 +618,12 @@ func TestPostgresOpenRejectsAreaSourcePrivilege(t *testing.T) {
|
||||
_, admin := openPostgresTestStore(t)
|
||||
ctx := context.Background()
|
||||
if _, err := admin.ExecContext(ctx,
|
||||
`GRANT SELECT ON bell.areas TO yovision_t011_sense`); err != nil {
|
||||
`GRANT SELECT ON bell.areas TO yovision_t012_sense`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_, _ = admin.ExecContext(context.Background(),
|
||||
`REVOKE SELECT ON bell.areas FROM yovision_t011_sense`)
|
||||
`REVOKE SELECT ON bell.areas FROM yovision_t012_sense`)
|
||||
}()
|
||||
value, err := OpenPostgres(ctx, os.Getenv(postgresTestDSNEnv))
|
||||
if value != nil {
|
||||
@@ -969,6 +990,180 @@ func TestPostgresConcurrentControlBatchesUseStableDeviceLockOrder(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresConcurrentReconcileClaimHasOneWinner(t *testing.T) {
|
||||
first, admin := openPostgresTestStore(t)
|
||||
insertBellSite(t, admin, "tenant", "site", 2)
|
||||
if err := first.CreateDevice(context.Background(), videoDevice(1, "tenant", "site")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := OpenPostgres(context.Background(), os.Getenv(postgresTestDSNEnv))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = second.Close() })
|
||||
now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC)
|
||||
start := make(chan struct{})
|
||||
counts := make(chan int, 2)
|
||||
errorsFound := make(chan error, 2)
|
||||
var wait sync.WaitGroup
|
||||
for index, repository := range []*Postgres{first, second} {
|
||||
wait.Add(1)
|
||||
go func(index int, repository *Postgres) {
|
||||
defer wait.Done()
|
||||
<-start
|
||||
values, err := repository.ClaimDueReconcile(context.Background(), ReconcileClaim{
|
||||
Owner: fmt.Sprintf("ins-%d", index), Token: fmt.Sprintf("token-%d", index),
|
||||
Now: now, LeaseDuration: 30 * time.Second, Limit: 1,
|
||||
})
|
||||
if err != nil {
|
||||
errorsFound <- err
|
||||
return
|
||||
}
|
||||
counts <- len(values)
|
||||
}(index, repository)
|
||||
}
|
||||
close(start)
|
||||
wait.Wait()
|
||||
close(counts)
|
||||
close(errorsFound)
|
||||
for err := range errorsFound {
|
||||
t.Fatal(err)
|
||||
}
|
||||
total, winners := 0, 0
|
||||
for count := range counts {
|
||||
total += count
|
||||
if count == 1 {
|
||||
winners++
|
||||
}
|
||||
}
|
||||
if total != 1 || winners != 1 {
|
||||
t.Fatalf("due row was not exclusively claimed: total=%d winners=%d", total, winners)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresExpiredReconcileLeaseFencesOldWorkerAndRecordsOwnership(t *testing.T) {
|
||||
postgres, admin := openPostgresTestStore(t)
|
||||
insertBellSite(t, admin, "tenant", "site", 2)
|
||||
value := videoDevice(1, "tenant", "site")
|
||||
if err := postgres.CreateDevice(context.Background(), value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC)
|
||||
first, err := postgres.ClaimDueReconcile(context.Background(), ReconcileClaim{
|
||||
Owner: "ins-a", Token: "token-a", Now: now, LeaseDuration: 30 * time.Second, Limit: 1,
|
||||
})
|
||||
if err != nil || len(first) != 1 {
|
||||
t.Fatalf("first claim failed: %+v %v", first, err)
|
||||
}
|
||||
early, err := postgres.ClaimDueReconcile(context.Background(), ReconcileClaim{
|
||||
Owner: "ins-b", Token: "token-b", Now: now.Add(10 * time.Second),
|
||||
LeaseDuration: 30 * time.Second, Limit: 1,
|
||||
})
|
||||
if err != nil || len(early) != 0 {
|
||||
t.Fatalf("live lease was stolen: %+v %v", early, err)
|
||||
}
|
||||
if _, err := admin.Exec(`UPDATE sense.reconcile_state
|
||||
SET lease_until = clock_timestamp() - interval '1 second'
|
||||
WHERE device_id = $1`, value.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := postgres.ClaimDueReconcile(context.Background(), ReconcileClaim{
|
||||
Owner: "ins-b", Token: "token-b", Now: now.Add(31 * time.Second),
|
||||
LeaseDuration: 30 * time.Second, Limit: 1,
|
||||
})
|
||||
if err != nil || len(second) != 1 {
|
||||
t.Fatalf("expired lease was not recoverable: %+v %v", second, err)
|
||||
}
|
||||
if err := postgres.CompleteReconcile(
|
||||
context.Background(), value.ID, value.Generation, "ins-a", "token-a", now.Add(32*time.Second),
|
||||
); !errors.Is(err, ErrReconcileLeaseLost) {
|
||||
t.Fatalf("old worker was not fenced: %v", err)
|
||||
}
|
||||
if err := postgres.CompleteReconcile(
|
||||
context.Background(), value.ID, value.Generation, "ins-b", "token-b", now.Add(32*time.Second),
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var ownershipDevice string
|
||||
if err := admin.QueryRow(`SELECT device_id FROM sense.media_path_ownership WHERE path_name = $1`,
|
||||
value.PathName).Scan(&ownershipDevice); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ownershipDevice != value.ID {
|
||||
t.Fatalf("wrong ownership was recorded: %q", ownershipDevice)
|
||||
}
|
||||
var leaseToken sql.NullString
|
||||
if err := admin.QueryRow(`SELECT lease_token FROM sense.reconcile_state WHERE device_id = $1`,
|
||||
value.ID).Scan(&leaseToken); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if leaseToken.Valid {
|
||||
t.Fatal("completion did not release the reconcile lease")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresOrphanReportLeaseAndCleanupAuditAreFencedAndIdempotent(t *testing.T) {
|
||||
postgres, _ := openPostgresTestStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC)
|
||||
acquired, err := postgres.AcquireOperationalLease(
|
||||
ctx, OperationalLeaseOrphanScan, "ins-a", "scan-token-a", now, 30*time.Second,
|
||||
)
|
||||
if err != nil || !acquired {
|
||||
t.Fatalf("scan lease failed: %v %v", acquired, err)
|
||||
}
|
||||
scan := OrphanScan{
|
||||
ID: "scan_" + strings.Repeat("0", 26), InstanceID: "ins-a",
|
||||
ObservedCount: 10, OwnedStaleCount: 1, UnownedCount: 1,
|
||||
SafetyAllowed: true, SafetyReason: "allowed",
|
||||
CompletedAt: now.Add(time.Second), ExpiresAt: now.Add(15 * time.Minute),
|
||||
Findings: []OrphanFinding{
|
||||
{PathName: "stale", Classification: OrphanOwnedStale, DeviceID: "old-device"},
|
||||
{PathName: "unknown", Classification: OrphanUnowned},
|
||||
},
|
||||
}
|
||||
if err := postgres.SaveOrphanScan(ctx, scan, "ins-a", "scan-token-a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := postgres.GetOrphanScan(ctx, scan.ID)
|
||||
if err != nil || len(loaded.Findings) != 2 || !loaded.SafetyAllowed {
|
||||
t.Fatalf("stored scan mismatch: %+v %v", loaded, err)
|
||||
}
|
||||
if err := postgres.RecordOrphanCleanup(
|
||||
ctx, scan.ID, "stale", "operator", "deleted", "", now.Add(2*time.Second),
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := postgres.RecordOrphanCleanup(
|
||||
ctx, scan.ID, "unknown", "operator", "deleted", "", now.Add(2*time.Second),
|
||||
); err == nil {
|
||||
t.Fatal("unowned path accepted a cleanup audit record")
|
||||
}
|
||||
if err := postgres.RecordOrphanCleanup(
|
||||
ctx, scan.ID, "stale", "operator-2", "failed", "media_error", now.Add(3*time.Second),
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err = postgres.GetOrphanScan(ctx, scan.ID)
|
||||
if err != nil || !loaded.Findings[0].Deleted {
|
||||
t.Fatalf("successful cleanup was downgraded: %+v %v", loaded, err)
|
||||
}
|
||||
|
||||
acquired, err = postgres.AcquireOperationalLease(
|
||||
ctx, OperationalLeaseOrphanScan, "ins-b", "scan-token-b", now.Add(31*time.Second), 30*time.Second,
|
||||
)
|
||||
if err != nil || !acquired {
|
||||
t.Fatalf("expired scan lease was not recoverable: %v %v", acquired, err)
|
||||
}
|
||||
staleScan := scan
|
||||
staleScan.ID = "scan_" + strings.Repeat("1", 26)
|
||||
staleScan.CompletedAt = now.Add(32 * time.Second)
|
||||
staleScan.ExpiresAt = staleScan.CompletedAt.Add(15 * time.Minute)
|
||||
if err := postgres.SaveOrphanScan(ctx, staleScan, "ins-a", "scan-token-a"); !errors.Is(err, ErrOperationalLeaseLost) {
|
||||
t.Fatalf("stale scan worker was not fenced: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func openPostgresTestStore(t *testing.T) (*Postgres, *sql.DB) {
|
||||
t.Helper()
|
||||
dsn := os.Getenv(postgresTestDSNEnv)
|
||||
@@ -985,6 +1180,11 @@ func openPostgresTestStore(t *testing.T) (*Postgres, *sql.DB) {
|
||||
t.Fatal("connect PostgreSQL test administrator")
|
||||
}
|
||||
if _, err := admin.ExecContext(context.Background(), `TRUNCATE
|
||||
sense.orphan_cleanup_actions,
|
||||
sense.orphan_scan_findings,
|
||||
sense.orphan_scan_runs,
|
||||
sense.operational_leases,
|
||||
sense.media_path_ownership,
|
||||
sense.control_idempotency_receipts,
|
||||
sense.batch_operation_items,
|
||||
sense.batch_operations,
|
||||
|
||||
@@ -21,6 +21,8 @@ var (
|
||||
ErrAreaPolicyUnavailable = errors.New("area policy unavailable")
|
||||
ErrAreaPolicyInvalid = errors.New("area policy invalid")
|
||||
ErrAreaPolicyDenied = errors.New("area policy denies imaging device")
|
||||
ErrReconcileLeaseLost = errors.New("reconcile lease lost")
|
||||
ErrOperationalLeaseLost = errors.New("operational lease lost")
|
||||
)
|
||||
|
||||
// Repository is the storage boundary used by the Sense process. SQLite stays
|
||||
@@ -31,6 +33,10 @@ type Repository interface {
|
||||
SetDesiredState(context.Context, string, device.DesiredState) error
|
||||
GetDevice(context.Context, string) (device.Device, error)
|
||||
ListDueReconcile(context.Context, time.Time, int) ([]ReconcileCandidate, error)
|
||||
ClaimDueReconcile(context.Context, ReconcileClaim) ([]ReconcileCandidate, error)
|
||||
RenewReconcileLease(context.Context, string, string, string, time.Time, time.Duration) (bool, error)
|
||||
CompleteReconcile(context.Context, string, int64, string, string, time.Time) error
|
||||
FailReconcile(context.Context, string, int, time.Time, string, string, string, time.Time) error
|
||||
ListEnabledVideoDevices(context.Context, int) ([]device.Device, error)
|
||||
MarkReconciled(context.Context, string, int64, time.Time) error
|
||||
MarkReconcileFailure(context.Context, string, int, time.Time, string, time.Time) error
|
||||
@@ -39,6 +45,17 @@ type Repository interface {
|
||||
ConvergenceSnapshot(context.Context) (ConvergenceSnapshot, error)
|
||||
}
|
||||
|
||||
// ReconcileClaim identifies one short-lived batch claim. Token is unique per
|
||||
// run and fences a worker whose lease expired and was acquired by another
|
||||
// process. SQLite accepts the shape but remains explicitly single-process.
|
||||
type ReconcileClaim struct {
|
||||
Owner string
|
||||
Token string
|
||||
Now time.Time
|
||||
LeaseDuration time.Duration
|
||||
Limit int
|
||||
}
|
||||
|
||||
func OpenRepository(ctx context.Context, driver, dsn string) (Repository, error) {
|
||||
switch strings.ToLower(strings.TrimSpace(driver)) {
|
||||
case "", DriverSQLite:
|
||||
|
||||
@@ -430,6 +430,24 @@ func (s *SQLite) ListDueReconcile(ctx context.Context, now time.Time, limit int)
|
||||
return candidates, nil
|
||||
}
|
||||
|
||||
func (s *SQLite) ClaimDueReconcile(ctx context.Context, claim ReconcileClaim) ([]ReconcileCandidate, error) {
|
||||
// SQLite is retained for one-process M1 development. It deliberately does
|
||||
// not claim cross-process leases; PostgreSQL is the production M2 boundary.
|
||||
return s.ListDueReconcile(ctx, claim.Now, claim.Limit)
|
||||
}
|
||||
|
||||
func (s *SQLite) RenewReconcileLease(
|
||||
ctx context.Context,
|
||||
_, _, _ string,
|
||||
_ time.Time,
|
||||
_ time.Duration,
|
||||
) (bool, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *SQLite) ListEnabledVideoDevices(ctx context.Context, limit int) ([]device.Device, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
@@ -499,6 +517,16 @@ func (s *SQLite) MarkReconciled(ctx context.Context, id string, generation int64
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SQLite) CompleteReconcile(
|
||||
ctx context.Context,
|
||||
id string,
|
||||
generation int64,
|
||||
_, _ string,
|
||||
now time.Time,
|
||||
) error {
|
||||
return s.MarkReconciled(ctx, id, generation, now)
|
||||
}
|
||||
|
||||
func (s *SQLite) MarkReconcileFailure(ctx context.Context, id string, failureCount int, nextAttempt time.Time, errorCode string, now time.Time) error {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
@@ -526,6 +554,17 @@ func (s *SQLite) MarkReconcileFailure(ctx context.Context, id string, failureCou
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SQLite) FailReconcile(
|
||||
ctx context.Context,
|
||||
id string,
|
||||
failureCount int,
|
||||
nextAttempt time.Time,
|
||||
errorCode, _, _ string,
|
||||
now time.Time,
|
||||
) error {
|
||||
return s.MarkReconcileFailure(ctx, id, failureCount, nextAttempt, errorCode, now)
|
||||
}
|
||||
|
||||
func (s *SQLite) UpdateActualState(ctx context.Context, id string, state device.ActualState, now time.Time) error {
|
||||
if state != device.ActualPending && state != device.ActualOnline && state != device.ActualOffline && state != device.ActualFailed {
|
||||
return fmt.Errorf("invalid actual state %q", state)
|
||||
|
||||
Reference in New Issue
Block a user