From 677ed732f7e5057be99c47e6e44462bbca118457 Mon Sep 17 00:00:00 2001 From: QiuSW <105186638@qq.com> Date: Fri, 7 Aug 2026 22:28:16 +0800 Subject: [PATCH 1/2] chore(task): claim T-012 --- docs/tasks/T-012.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/tasks/T-012.md b/docs/tasks/T-012.md index 2109c86..3edf88c 100644 --- a/docs/tasks/T-012.md +++ b/docs/tasks/T-012.md @@ -3,12 +3,12 @@ id: T-012 title: 补齐 Sense 对账安全闸、孤儿受控处置与多实例可观测性 phase: 2 deps: [T-011] -status: TODO +status: DOING created: 2026-08-07 issue: 43 -context_ref: null -claim_branch: null -work_branch: null +context_ref: 25357723a09218bae58626a9934309abcda34899 +claim_branch: claims/T-012 +work_branch: agent/codex/T-012 write_paths: - docs/tasks/T-012.md - Sense/README.md @@ -88,6 +88,11 @@ T-003 已实现幂等调和与持久化退避，T-011 已完成 Control API、 ## 执行记录 +### 2026-08-07 领取任务 + +- dispatcher `ila` 将 Issue #43 分配给 `codex`；`context_ref` 为 `25357723a09218bae58626a9934309abcda34899`，claim 为 `claims/T-012`，工作分支为 `agent/codex/T-012`。 +- 已读回 Issue `status/doing`、assignee、dispatcher 发布的完整 CLAIM 与两个分支 SHA；接受 frontmatter 全部写路径。T-007 继续 waiting，本任务期间不恢复，当前无活跃写路径冲突。 + ### 2026-08-07 Gitea 映射 - 任务规格先通过 PR #42 合入默认分支，再创建唯一主 Issue #43；本次只回填双向映射，映射合入并读回前不领取任务。 From 12857fdf32a466476784793ff9873475c0580446 Mon Sep 17 00:00:00 2001 From: QiuSW <105186638@qq.com> Date: Fri, 7 Aug 2026 23:00:03 +0800 Subject: [PATCH 2/2] feat(sense): add reconciliation safety controls [T-012] --- Sense/README.md | 27 +- Sense/cmd/sense-api/main.go | 48 ++- Sense/cmd/sense-orphan/main.go | 94 ++++ Sense/internal/config/config.go | 150 +++++-- Sense/internal/config/config_test.go | 30 ++ Sense/internal/metrics/metrics.go | 189 +++++++++ Sense/internal/metrics/metrics_test.go | 84 ++++ Sense/internal/mtx/client.go | 55 +++ Sense/internal/mtx/client_test.go | 82 ++++ Sense/internal/orphan/orphan.go | 400 ++++++++++++++++++ Sense/internal/orphan/orphan_test.go | 272 ++++++++++++ Sense/internal/reconcile/reconciler.go | 163 +++++-- Sense/internal/reconcile/reconciler_test.go | 80 ++++ Sense/internal/store/control_postgres.go | 2 + Sense/internal/store/orphan.go | 58 +++ Sense/internal/store/orphan_postgres.go | 268 ++++++++++++ Sense/internal/store/postgres.go | 240 ++++++++++- Sense/internal/store/postgres_test.go | 208 ++++++++- Sense/internal/store/repository.go | 17 + Sense/internal/store/sqlite.go | 39 ++ deploy/postgres/010_reconcile_safety.sql | 147 +++++++ .../011_privileges_reconcile_safety.sql | 17 + deploy/postgres/README.md | 5 +- deploy/postgres/tests/assertions.sql | 36 +- docs/00-ai-start-here.md | 4 +- docs/03-tech-stack.md | 4 +- docs/04-architecture.md | 12 +- docs/06-tasks.md | 2 +- docs/api.md | 8 +- docs/current-state.md | 19 +- docs/runbooks/sense-reconciliation.md | 65 +++ docs/tasks/T-012.md | 12 +- scripts/test_postgres.ps1 | 14 +- tests/test_postgres_contract.py | 4 +- tests/test_sense_reconcile_safety.py | 86 ++++ 35 files changed, 2817 insertions(+), 124 deletions(-) create mode 100644 Sense/cmd/sense-orphan/main.go create mode 100644 Sense/internal/metrics/metrics.go create mode 100644 Sense/internal/metrics/metrics_test.go create mode 100644 Sense/internal/orphan/orphan.go create mode 100644 Sense/internal/orphan/orphan_test.go create mode 100644 Sense/internal/store/orphan.go create mode 100644 Sense/internal/store/orphan_postgres.go create mode 100644 deploy/postgres/010_reconcile_safety.sql create mode 100644 deploy/postgres/011_privileges_reconcile_safety.sql create mode 100644 docs/runbooks/sense-reconciliation.md create mode 100644 tests/test_sense_reconcile_safety.py diff --git a/Sense/README.md b/Sense/README.md index 5f45216..bf675da 100644 --- a/Sense/README.md +++ b/Sense/README.md @@ -1,6 +1,6 @@ # Sense M1/M2 接入骨架 -本目录是 YoVision Sense 的 M1/M2 接入骨架。数据库保存期望态，ONVIF 和 MediaMTX 通过端口隔离；M1 默认使用 SQLite，T-009～T-011 增加 PostgreSQL 双 schema、Area 准入、本地审计 Outbox 和 Control API v1。默认关闭真实 ONVIF 与公共业务路由；T-006 的真实样机结论仅覆盖已批准的精确海康基线，不能据此宣称多品牌兼容。 +本目录是 YoVision Sense 的 M1/M2 接入骨架。数据库保存期望态，ONVIF 和 MediaMTX 通过端口隔离；M1 默认使用 SQLite，T-009～T-012 增加 PostgreSQL 双 schema、Area 准入、本地审计 Outbox、Control API v1、多实例调和 fencing 和孤儿受控处置。默认关闭真实 ONVIF 与公共业务路由；T-006 的真实样机结论仅覆盖已批准的精确海康基线，不能据此宣称多品牌兼容。 ## 常用命令 @@ -14,7 +14,7 @@ go build -o bin/sense-api.exe ./cmd/sense-api go run ./cmd/sense-api ``` -Unix 将构建产物改为 `bin/sense-api`。服务默认监听 `127.0.0.1:8080`，SQLite 默认写入 `Sense/data/sense.db`，MediaMTX 控制 API 默认是 `http://127.0.0.1:9997`。默认运行只暴露 `/healthz` 与 `/readyz`；只有显式选择 PostgreSQL 并完成安全配置后才注册 7 个 `/api/v1` Control API 路由。 +Unix 将构建产物改为 `bin/sense-api`。服务默认监听 `127.0.0.1:8080`，SQLite 默认写入 `Sense/data/sense.db`，MediaMTX 控制 API 默认是 `http://127.0.0.1:9997`。默认运行暴露 `/healthz`、`/readyz` 和不含租户/设备标签的 `/metrics`；只有显式选择 PostgreSQL 并完成安全配置后才注册 7 个 `/api/v1` Control API 路由。 常用环境变量： @@ -26,7 +26,13 @@ Unix 将构建产物改为 `bin/sense-api`。服务默认监听 `127.0.0.1:8080` | `SENSE_DB_DSN` | `file:data/sense.db` | 所选 driver 的私有 DSN；不得写入日志、文档或仓库，PostgreSQL 密码优先由 `PGPASSFILE`/环境密钥提供 | | `SENSE_MEDIAMTX_URL` | `http://127.0.0.1:9997` | MediaMTX 控制 API；不得包含 userinfo | | `SENSE_RECONCILE_INTERVAL` | `5s` | 对账周期 | +| `SENSE_RECONCILE_LEASE_DURATION` | `30s` | PostgreSQL due-row 租期；最大 5 分钟 | +| `SENSE_RECONCILE_OPERATION_TIMEOUT` | `20s` | 单项 ONVIF/MediaMTX deadline；必须严格短于租期 | | `SENSE_PROBE_INTERVAL` | `10s` | path 探活周期 | +| `SENSE_INSTANCE_ID` | 随进程随机生成 | 最多 64 位低基数字符串；多实例部署建议显式注入唯一实例 ID | +| `SENSE_METRICS_ENABLED` | `true` | 是否注册低基数 Prometheus 文本 `/metrics` | +| `SENSE_ORPHAN_SCAN_ENABLED` | PostgreSQL 为 `true`，SQLite 为 `false` | 周期执行只读 MediaMTX Path 差异报告；从不自动删除 | +| `SENSE_ORPHAN_SCAN_INTERVAL` | `1m` | 孤儿只读扫描周期，最短 10 秒 | | `SENSE_ONVIF_MODE` | `disabled` | `disabled` 或 `standard`；默认不访问真实摄像头 | | `SENSE_ONVIF_RTSP_REWRITE_HOST` | 空 | NAT 或故障代理场景下重写 ONVIF 返回的 RTSP 主机 | | `SENSE_ONVIF_RTSP_REWRITE_PORT` | `0` | 非零时重写 ONVIF 返回的 RTSP 端口 | @@ -50,9 +56,9 @@ SENSE_CREDENTIAL__RTSP_PASSWORD MediaMTX `v1.19.3` 应作为独立二进制启动并只在可信网络开放 API。获取与 SHA-256 校验值见 `docs/03-tech-stack.md`。生成客户端使用固定版本工具和 vendored 官方 OpenAPI；`internal/mtx/generated/client.gen.go` 不可手改。 -## T-009～T-011 PostgreSQL 17.10 +## T-009～T-012 PostgreSQL 17.10 -初始化与增量 SQL 位于 `deploy/postgres/`，由高权限部署步骤按文件名前缀执行；Sense 进程不会自动创建角色、schema 或 Bell 对象。`bell_app` 拥有 Site/Area、配额、`capture_policy` 及两个版本化视图，`sense_app` 只能读取两个视图，不能读取或写入 Bell 源表。T-011 的 v4 schema 另有资源版本、24 小时幂等收据和 batch operation 表；收据只保存 scope/body 摘要与脱敏响应快照。应用登录角色和密码由部署环境创建，不进入仓库。 +初始化与增量 SQL 位于 `deploy/postgres/`，由高权限部署步骤按文件名前缀执行；Sense 进程不会自动创建角色、schema 或 Bell 对象。`bell_app` 拥有 Site/Area、配额、`capture_policy` 及两个版本化视图，`sense_app` 只能读取两个视图，不能读取或写入 Bell 源表。T-011 的 v4 schema 增加资源版本、24 小时幂等收据和 batch operation；T-012 的 v5 schema 增加数据库时钟租约、Path 历史归属及脱敏孤儿报告/处置结果。表中不保存 MediaMTX source URI。应用登录角色和密码由部署环境创建，不进入仓库。 PostgreSQL 新建设备必须携带匹配 tenant/Site 的 `area_id`。具有 `video_capture` 能力的设备在创建、移动 Area 和从 disabled 切到 enabled 时执行 Area 准入；`non_imaging_only` 拒绝成像设备但允许非成像设备。投影缺失、非法或版本回退只拒绝新变更，不关闭已有流。创建、配置修改和期望态受理都与对应脱敏 Outbox 事实同事务；停用后调和器只删除该设备的精确 MediaMTX path 并收敛为 offline，不枚举未知 path。Bell relay 尚未实现。 @@ -72,7 +78,18 @@ $env:SENSE_DB_DSN = '由部署环境私下设置' go run ./cmd/sense-api ``` -PostgreSQL 启动会检查 Sense migration 版本及当前角色对两个 Bell 投影视图的最小权限；权限过宽、视图不可读或 schema 未安装时 readiness 初始化失败。默认 SQLite 路径和 `cmd/sense-lab` 保持不变，但 SQLite 不实现生产 Area/Outbox 一致性，Control API feature flag 在 SQLite 下会直接拒绝启动。 +PostgreSQL 启动会检查 Sense v5 migration、当前角色对两个 Bell 投影视图和本地控制/对账表的最小权限；权限过宽、视图不可读或 schema 未安装时初始化失败。默认 SQLite 路径和 `cmd/sense-lab` 保持不变，但 SQLite 不实现生产 Area/Outbox、多实例租约或孤儿处置语义，Control API 与孤儿扫描在 SQLite 下不会启动。 + +### 孤儿报告与受控处置 + +`sense-api` 的 PostgreSQL 模式默认每分钟只读枚举一次 MediaMTX 配置 Path，并持久化汇总。`unowned` 表示没有 Sense 历史归属证据，永远不会由 Sense 删除；只有 `owned_stale` 可以进入处置候选。需要人工处置时从 `Sense/` 执行： + +```powershell +go run ./cmd/sense-orphan -mode report +go run ./cmd/sense-orphan -mode apply -scan-id scan_... -actor operator-id -confirm "DELETE scan_..." +``` + +`report` 输出不含 source URI 的 JSON 摘要。`apply` 会再次读取数据库和 MediaMTX，只处理仍属于原快照的 `owned_stale`，并要求快照不超过 15 分钟、候选不超过 128 且候选占当前全部配置 Path 不超过 10%；任何门禁失败均为零删除，没有 `force`。完整流程与恢复方法见 [`../docs/runbooks/sense-reconciliation.md`](../docs/runbooks/sense-reconciliation.md)。 ### 开启 Control API diff --git a/Sense/cmd/sense-api/main.go b/Sense/cmd/sense-api/main.go index 6a8ce20..7699160 100644 --- a/Sense/cmd/sense-api/main.go +++ b/Sense/cmd/sense-api/main.go @@ -15,8 +15,10 @@ import ( "yovision/sense/internal/auth" "yovision/sense/internal/config" "yovision/sense/internal/controlapi" + "yovision/sense/internal/metrics" "yovision/sense/internal/mtx" "yovision/sense/internal/onvif" + "yovision/sense/internal/orphan" "yovision/sense/internal/probe" "yovision/sense/internal/reconcile" "yovision/sense/internal/store" @@ -37,6 +39,14 @@ func run(logger *slog.Logger) error { if err != nil { return fmt.Errorf("load configuration: %w", err) } + instanceID := cfg.InstanceID + if instanceID == "" { + instanceID, err = metrics.GenerateInstanceID() + if err != nil { + return err + } + } + registry := metrics.New(instanceID, version) ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() @@ -76,22 +86,36 @@ func run(logger *slog.Logger) error { }) } discovery := onvif.NewRouter(cameraAdapter, credentials) - reconciler := reconcile.New(repository, discovery, mediaClient) + reconciler := reconcile.NewWithOptions(repository, discovery, mediaClient, reconcile.Options{ + InstanceID: instanceID, LeaseDuration: cfg.ReconcileLeaseDuration, + OperationTimeout: cfg.ReconcileOperationTimeout, Metrics: registry, + }) checker := probe.New(repository, mediaClient) + var orphanScanner *orphan.Manager + if cfg.OrphanScanEnabled { + orphanStore, ok := repository.(store.OrphanRepository) + if !ok { + return errors.New("selected repository does not support orphan scanning") + } + orphanScanner = orphan.New(orphanStore, mediaClient, instanceID, registry) + } report := func(err error) { // Domain and MediaMTX errors intentionally omit stream URIs and credentials. logger.Warn("background convergence error", "error", err) } var background sync.WaitGroup - background.Add(2) - go func() { - defer background.Done() - reconciler.Run(ctx, cfg.ReconcileInterval, report) - }() - go func() { - defer background.Done() - checker.Run(ctx, cfg.ProbeInterval, report) - }() + startBackground := func(run func()) { + background.Add(1) + go func() { + defer background.Done() + run() + }() + } + startBackground(func() { reconciler.Run(ctx, cfg.ReconcileInterval, report) }) + startBackground(func() { checker.Run(ctx, cfg.ProbeInterval, report) }) + if orphanScanner != nil { + startBackground(func() { orphanScanner.Run(ctx, cfg.OrphanScanInterval, report) }) + } mux := http.NewServeMux() mux.HandleFunc("GET /healthz", func(writer http.ResponseWriter, _ *http.Request) { @@ -104,6 +128,9 @@ func run(logger *slog.Logger) error { writer.WriteHeader(http.StatusOK) _, _ = writer.Write([]byte(`{"status":"ready"}`)) }) + if cfg.MetricsEnabled { + mux.Handle("GET /metrics", registry.Handler()) + } if cfg.ControlAPIEnabled { mux.Handle("/api/v1/", controlHandler) } @@ -118,6 +145,7 @@ func run(logger *slog.Logger) error { serverErrors := make(chan error, 1) go func() { logger.Info("Sense listening", "address", cfg.HTTPAddress, "version", version, + "instance_id", instanceID, "control_api_enabled", cfg.ControlAPIEnabled) serverErrors <- server.ListenAndServe() }() diff --git a/Sense/cmd/sense-orphan/main.go b/Sense/cmd/sense-orphan/main.go new file mode 100644 index 0000000..fb95706 --- /dev/null +++ b/Sense/cmd/sense-orphan/main.go @@ -0,0 +1,94 @@ +// sense-orphan is a local, PostgreSQL-backed operational command. Report is +// read-only; apply requires a fresh scan ID, actor and exact confirmation. +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "net/http" + "os" + "os/signal" + "strings" + "syscall" + + "yovision/sense/internal/config" + "yovision/sense/internal/metrics" + "yovision/sense/internal/mtx" + "yovision/sense/internal/orphan" + "yovision/sense/internal/store" +) + +func main() { + if err := run(); err != nil { + _, _ = fmt.Fprintln(os.Stderr, "sense-orphan failed:", err) + os.Exit(1) + } +} + +func run() error { + mode := flag.String("mode", "report", "report or apply") + scanID := flag.String("scan-id", "", "fresh scan ID required by apply") + actor := flag.String("actor", "", "operator identifier required by apply") + confirm := flag.String("confirm", "", "exact confirmation: DELETE ") + flag.Parse() + + cfg, err := config.Load() + if err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(cfg.DatabaseDriver), store.DriverPostgres) { + return errors.New("sense-orphan requires SENSE_DB_DRIVER=postgres") + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + repository, err := store.OpenPostgres(ctx, cfg.DatabaseDSN) + if err != nil { + return err + } + defer repository.Close() + media, err := mtx.NewClient(cfg.MediaMTXURL, &http.Client{Timeout: cfg.ReconcileOperationTimeout}) + if err != nil { + return err + } + instanceID := cfg.InstanceID + if instanceID == "" { + instanceID, err = metrics.GenerateInstanceID() + if err != nil { + return err + } + } + manager := orphan.New(repository, media, instanceID, metrics.New(instanceID, "operator")) + encoder := json.NewEncoder(os.Stdout) + switch *mode { + case "report": + scan, err := manager.Report(ctx) + if err != nil { + return err + } + return encoder.Encode(struct { + ScanID string `json:"scan_id"` + Observed int `json:"observed"` + OwnedStale int `json:"owned_stale"` + Unowned int `json:"unowned"` + SafetyAllowed bool `json:"safety_allowed"` + SafetyReason string `json:"safety_reason"` + ExpiresAt string `json:"expires_at"` + }{ + ScanID: scan.ID, Observed: scan.ObservedCount, + OwnedStale: scan.OwnedStaleCount, Unowned: scan.UnownedCount, + SafetyAllowed: scan.SafetyAllowed, SafetyReason: scan.SafetyReason, + ExpiresAt: scan.ExpiresAt.UTC().Format("2006-01-02T15:04:05Z07:00"), + }) + case "apply": + result, err := manager.Apply(ctx, *scanID, *actor, *confirm) + if encodeErr := encoder.Encode(result); encodeErr != nil { + return encodeErr + } + return err + default: + return errors.New("mode must be report or apply") + } +} diff --git a/Sense/internal/config/config.go b/Sense/internal/config/config.go index 860a323..a5ec521 100644 --- a/Sense/internal/config/config.go +++ b/Sense/internal/config/config.go @@ -7,39 +7,52 @@ import ( "net/url" "os" "path/filepath" + "regexp" "strconv" "strings" "time" ) const ( - defaultHTTPAddress = "127.0.0.1:8080" - defaultDatabaseDriver = "sqlite" - defaultDatabaseDSN = "file:data/sense.db" - defaultMediaMTXURL = "http://127.0.0.1:9997" - defaultReconcilePeriod = 5 * time.Second - defaultProbePeriod = 10 * time.Second - defaultONVIFMode = "disabled" - defaultControlAuthMode = "static-sha256" + defaultHTTPAddress = "127.0.0.1:8080" + defaultDatabaseDriver = "sqlite" + postgresDatabaseDriver = "postgres" + defaultDatabaseDSN = "file:data/sense.db" + defaultMediaMTXURL = "http://127.0.0.1:9997" + defaultReconcilePeriod = 5 * time.Second + defaultReconcileLease = 30 * time.Second + defaultOperationTimeout = 20 * time.Second + defaultProbePeriod = 10 * time.Second + defaultOrphanScanPeriod = time.Minute + defaultONVIFMode = "disabled" + defaultControlAuthMode = "static-sha256" ) +var instanceIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`) + type Config struct { - HTTPAddress string - AllowNonLoopback bool - DatabaseDriver string - DatabaseDSN string - MediaMTXURL string - ReconcileInterval time.Duration - ProbeInterval time.Duration - ONVIFMode string - RTSPRewriteHost string - RTSPRewritePort int - RTSPStripQuery bool - ControlAPIEnabled bool - ControlAuthMode string - ControlAuthFile string - ControlCursorKeyFile string - ControlAllowInsecureHTTP bool + HTTPAddress string + AllowNonLoopback bool + DatabaseDriver string + DatabaseDSN string + MediaMTXURL string + ReconcileInterval time.Duration + ReconcileLeaseDuration time.Duration + ReconcileOperationTimeout time.Duration + ProbeInterval time.Duration + InstanceID string + MetricsEnabled bool + OrphanScanEnabled bool + OrphanScanInterval time.Duration + ONVIFMode string + RTSPRewriteHost string + RTSPRewritePort int + RTSPStripQuery bool + ControlAPIEnabled bool + ControlAuthMode string + ControlAuthFile string + ControlCursorKeyFile string + ControlAllowInsecureHTTP bool } func Load() (Config, error) { @@ -51,6 +64,14 @@ func Load() (Config, error) { if err != nil { return Config{}, err } + reconcileLease, err := durationEnv("SENSE_RECONCILE_LEASE_DURATION", defaultReconcileLease) + if err != nil { + return Config{}, err + } + operationTimeout, err := durationEnv("SENSE_RECONCILE_OPERATION_TIMEOUT", defaultOperationTimeout) + if err != nil { + return Config{}, err + } probePeriod, err := durationEnv("SENSE_PROBE_INTERVAL", defaultProbePeriod) if err != nil { return Config{}, err @@ -71,24 +92,44 @@ func Load() (Config, error) { if err != nil { return Config{}, err } + metricsEnabled, err := boolEnv("SENSE_METRICS_ENABLED", true) + if err != nil { + return Config{}, err + } + databaseDriver := stringEnv("SENSE_DB_DRIVER", defaultDatabaseDriver) + orphanDefault := strings.EqualFold(strings.TrimSpace(databaseDriver), postgresDatabaseDriver) + orphanEnabled, err := boolEnv("SENSE_ORPHAN_SCAN_ENABLED", orphanDefault) + if err != nil { + return Config{}, err + } + orphanPeriod, err := durationEnv("SENSE_ORPHAN_SCAN_INTERVAL", defaultOrphanScanPeriod) + if err != nil { + return Config{}, err + } cfg := Config{ - HTTPAddress: stringEnv("SENSE_HTTP_ADDR", defaultHTTPAddress), - AllowNonLoopback: allow, - DatabaseDriver: stringEnv("SENSE_DB_DRIVER", defaultDatabaseDriver), - DatabaseDSN: stringEnv("SENSE_DB_DSN", defaultDatabaseDSN), - MediaMTXURL: stringEnv("SENSE_MEDIAMTX_URL", defaultMediaMTXURL), - ReconcileInterval: reconcilePeriod, - ProbeInterval: probePeriod, - ONVIFMode: stringEnv("SENSE_ONVIF_MODE", defaultONVIFMode), - RTSPRewriteHost: stringEnv("SENSE_ONVIF_RTSP_REWRITE_HOST", ""), - RTSPRewritePort: rewritePort, - RTSPStripQuery: stripQuery, - ControlAPIEnabled: controlEnabled, - ControlAuthMode: stringEnv("SENSE_CONTROL_AUTH_MODE", defaultControlAuthMode), - ControlAuthFile: stringEnv("SENSE_CONTROL_AUTH_FILE", ""), - ControlCursorKeyFile: stringEnv("SENSE_CONTROL_CURSOR_KEY_FILE", ""), - ControlAllowInsecureHTTP: controlAllowInsecure, + HTTPAddress: stringEnv("SENSE_HTTP_ADDR", defaultHTTPAddress), + AllowNonLoopback: allow, + DatabaseDriver: databaseDriver, + DatabaseDSN: stringEnv("SENSE_DB_DSN", defaultDatabaseDSN), + MediaMTXURL: stringEnv("SENSE_MEDIAMTX_URL", defaultMediaMTXURL), + ReconcileInterval: reconcilePeriod, + ReconcileLeaseDuration: reconcileLease, + ReconcileOperationTimeout: operationTimeout, + ProbeInterval: probePeriod, + InstanceID: stringEnv("SENSE_INSTANCE_ID", ""), + MetricsEnabled: metricsEnabled, + OrphanScanEnabled: orphanEnabled, + OrphanScanInterval: orphanPeriod, + ONVIFMode: stringEnv("SENSE_ONVIF_MODE", defaultONVIFMode), + RTSPRewriteHost: stringEnv("SENSE_ONVIF_RTSP_REWRITE_HOST", ""), + RTSPRewritePort: rewritePort, + RTSPStripQuery: stripQuery, + ControlAPIEnabled: controlEnabled, + ControlAuthMode: stringEnv("SENSE_CONTROL_AUTH_MODE", defaultControlAuthMode), + ControlAuthFile: stringEnv("SENSE_CONTROL_AUTH_FILE", ""), + ControlCursorKeyFile: stringEnv("SENSE_CONTROL_CURSOR_KEY_FILE", ""), + ControlAllowInsecureHTTP: controlAllowInsecure, } if err := cfg.Validate(); err != nil { return Config{}, err @@ -129,6 +170,35 @@ func (c Config) Validate() error { if c.ReconcileInterval <= 0 || c.ProbeInterval <= 0 { return fmt.Errorf("loop intervals must be positive") } + leaseDuration := c.ReconcileLeaseDuration + if leaseDuration == 0 { + leaseDuration = defaultReconcileLease + } + operationTimeout := c.ReconcileOperationTimeout + if operationTimeout == 0 { + operationTimeout = defaultOperationTimeout + } + if leaseDuration <= 0 || leaseDuration > 5*time.Minute { + return fmt.Errorf("SENSE_RECONCILE_LEASE_DURATION must be positive and at most 5m") + } + if operationTimeout <= 0 || operationTimeout >= leaseDuration { + return fmt.Errorf("SENSE_RECONCILE_OPERATION_TIMEOUT must be positive and shorter than the lease") + } + if c.InstanceID != "" && !instanceIDPattern.MatchString(c.InstanceID) { + return fmt.Errorf("invalid SENSE_INSTANCE_ID") + } + if c.OrphanScanEnabled { + if databaseDriver != postgresDatabaseDriver { + return fmt.Errorf("orphan scanning requires SENSE_DB_DRIVER=postgres") + } + orphanInterval := c.OrphanScanInterval + if orphanInterval == 0 { + orphanInterval = defaultOrphanScanPeriod + } + if orphanInterval < 10*time.Second { + return fmt.Errorf("SENSE_ORPHAN_SCAN_INTERVAL must be at least 10s") + } + } if c.ONVIFMode != "" && c.ONVIFMode != "disabled" && c.ONVIFMode != "standard" { return fmt.Errorf("SENSE_ONVIF_MODE must be disabled or standard") } diff --git a/Sense/internal/config/config_test.go b/Sense/internal/config/config_test.go index 9555a15..719bd93 100644 --- a/Sense/internal/config/config_test.go +++ b/Sense/internal/config/config_test.go @@ -3,6 +3,7 @@ package config import ( "path/filepath" "testing" + "time" ) func TestValidateRejectsNonLoopbackByDefault(t *testing.T) { @@ -23,6 +24,35 @@ func TestValidateRejectsNonLoopbackByDefault(t *testing.T) { } } +func TestValidateReconcileLeaseAndOrphanProductionBoundary(t *testing.T) { + base := Config{ + HTTPAddress: "127.0.0.1:8080", DatabaseDriver: "postgres", + DatabaseDSN: "postgres://sense-runtime@127.0.0.1/yovision?sslmode=disable", + MediaMTXURL: "http://127.0.0.1:9997", ReconcileInterval: time.Second, + ProbeInterval: time.Second, ReconcileLeaseDuration: 30 * time.Second, + ReconcileOperationTimeout: 20 * time.Second, OrphanScanEnabled: true, + OrphanScanInterval: time.Minute, InstanceID: "ins_edge-01", + } + if err := base.Validate(); err != nil { + t.Fatalf("valid multi-instance configuration failed: %v", err) + } + invalidTimeout := base + invalidTimeout.ReconcileOperationTimeout = invalidTimeout.ReconcileLeaseDuration + if err := invalidTimeout.Validate(); err == nil { + t.Fatal("operation timeout equal to the lease was accepted") + } + invalidInstance := base + invalidInstance.InstanceID = "tenant/site" + if err := invalidInstance.Validate(); err == nil { + t.Fatal("unbounded instance label was accepted") + } + sqlite := base + sqlite.DatabaseDriver, sqlite.DatabaseDSN = "sqlite", "file:test.db" + if err := sqlite.Validate(); err == nil { + t.Fatal("orphan scanner was accepted on SQLite") + } +} + func TestValidateRejectsCredentialsInMediaMTXURL(t *testing.T) { t.Parallel() cfg := Config{ diff --git a/Sense/internal/metrics/metrics.go b/Sense/internal/metrics/metrics.go new file mode 100644 index 0000000..0e75c89 --- /dev/null +++ b/Sense/internal/metrics/metrics.go @@ -0,0 +1,189 @@ +// Package metrics exposes a deliberately small, low-cardinality Prometheus +// surface without adding a runtime dependency. Tenant, Site, device and Path +// identifiers never become labels. +package metrics + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "sync/atomic" + "time" +) + +func GenerateInstanceID() (string, error) { + value := make([]byte, 12) + if _, err := rand.Read(value); err != nil { + return "", errors.New("generate Sense instance identifier") + } + return "ins_" + hex.EncodeToString(value), nil +} + +type Registry struct { + instanceID string + version string + + reconcileRunOK atomic.Uint64 + reconcileRunError atomic.Uint64 + reconcileItemOK atomic.Uint64 + reconcileItemError atomic.Uint64 + reconcileItemLost atomic.Uint64 + reconcileDurationNS atomic.Uint64 + reconcileTotal atomic.Int64 + reconcileUnconverged atomic.Int64 + + orphanScanOK atomic.Uint64 + orphanScanError atomic.Uint64 + orphanObserved atomic.Int64 + orphanOwnedStale atomic.Int64 + orphanUnowned atomic.Int64 + orphanBlockedRatio atomic.Uint64 + orphanBlockedAge atomic.Uint64 + orphanBlockedScope atomic.Uint64 + orphanDeleted atomic.Uint64 + orphanDeleteFailed atomic.Uint64 +} + +func New(instanceID, version string) *Registry { + return &Registry{instanceID: instanceID, version: version} +} + +func (r *Registry) ObserveReconcileRun(err error, duration time.Duration) { + if err == nil { + r.reconcileRunOK.Add(1) + } else { + r.reconcileRunError.Add(1) + } + if duration < 0 { + duration = 0 + } + r.reconcileDurationNS.Store(uint64(duration)) +} + +func (r *Registry) ObserveReconcileItem(result string) { + switch result { + case "success": + r.reconcileItemOK.Add(1) + case "lease_lost": + r.reconcileItemLost.Add(1) + default: + r.reconcileItemError.Add(1) + } +} + +func (r *Registry) SetConvergence(total, unconverged int) { + r.reconcileTotal.Store(int64(total)) + r.reconcileUnconverged.Store(int64(unconverged)) +} + +func (r *Registry) ObserveOrphanScan(observed, ownedStale, unowned int, err error) { + if err == nil { + r.orphanScanOK.Add(1) + r.orphanObserved.Store(int64(observed)) + r.orphanOwnedStale.Store(int64(ownedStale)) + r.orphanUnowned.Store(int64(unowned)) + } else { + r.orphanScanError.Add(1) + } +} + +func (r *Registry) ObserveOrphanCleanupBlocked(reason string) { + switch reason { + case "snapshot_expired": + r.orphanBlockedAge.Add(1) + case "ratio_exceeded": + r.orphanBlockedRatio.Add(1) + default: + r.orphanBlockedScope.Add(1) + } +} + +func (r *Registry) ObserveOrphanCleanup(deleted, failed int) { + if deleted > 0 { + r.orphanDeleted.Add(uint64(deleted)) + } + if failed > 0 { + r.orphanDeleteFailed.Add(uint64(failed)) + } +} + +func (r *Registry) Handler() http.Handler { + return http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writer.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + writer.Header().Set("Cache-Control", "no-store") + _, _ = writer.Write([]byte(r.render())) + }) +} + +func (r *Registry) render() string { + var output strings.Builder + writeMetric(&output, "sense_build_info", "Sense process build and instance information.", + fmt.Sprintf("{instance_id=%s,version=%s}", quoteLabel(r.instanceID), quoteLabel(r.version)), 1) + writeMetric(&output, "sense_reconcile_runs_total", "Reconciliation runs by result.", + `{result="success"}`, r.reconcileRunOK.Load()) + writeSample(&output, "sense_reconcile_runs_total", `{result="error"}`, r.reconcileRunError.Load()) + writeMetric(&output, "sense_reconcile_items_total", "Reconciliation items by fixed result.", + `{result="success"}`, r.reconcileItemOK.Load()) + writeSample(&output, "sense_reconcile_items_total", `{result="error"}`, r.reconcileItemError.Load()) + writeSample(&output, "sense_reconcile_items_total", `{result="lease_lost"}`, r.reconcileItemLost.Load()) + writeMetric(&output, "sense_reconcile_last_duration_seconds", "Duration of the last reconciliation run.", + "", float64(r.reconcileDurationNS.Load())/float64(time.Second)) + writeMetric(&output, "sense_reconcile_devices", "Enabled video devices by convergence state.", + `{state="total"}`, r.reconcileTotal.Load()) + writeSample(&output, "sense_reconcile_devices", `{state="unconverged"}`, r.reconcileUnconverged.Load()) + writeMetric(&output, "sense_orphan_scan_runs_total", "MediaMTX orphan scans by result.", + `{result="success"}`, r.orphanScanOK.Load()) + writeSample(&output, "sense_orphan_scan_runs_total", `{result="error"}`, r.orphanScanError.Load()) + writeMetric(&output, "sense_orphan_paths", "Path counts from the last successful orphan scan.", + `{classification="observed"}`, r.orphanObserved.Load()) + writeSample(&output, "sense_orphan_paths", `{classification="owned_stale"}`, r.orphanOwnedStale.Load()) + writeSample(&output, "sense_orphan_paths", `{classification="unowned"}`, r.orphanUnowned.Load()) + writeMetric(&output, "sense_orphan_cleanup_blocked_total", "Orphan cleanups blocked by a fixed safety reason.", + `{reason="ratio_exceeded"}`, r.orphanBlockedRatio.Load()) + writeSample(&output, "sense_orphan_cleanup_blocked_total", `{reason="snapshot_expired"}`, r.orphanBlockedAge.Load()) + writeSample(&output, "sense_orphan_cleanup_blocked_total", `{reason="scope_changed"}`, r.orphanBlockedScope.Load()) + writeMetric(&output, "sense_orphan_cleanup_items_total", "Owned stale path cleanup results.", + `{result="deleted"}`, r.orphanDeleted.Load()) + writeSample(&output, "sense_orphan_cleanup_items_total", `{result="failed"}`, r.orphanDeleteFailed.Load()) + return output.String() +} + +func writeMetric(builder *strings.Builder, name, help, labels string, value any) { + metricType := "gauge" + if strings.HasSuffix(name, "_total") { + metricType = "counter" + } + builder.WriteString("# HELP ") + builder.WriteString(name) + builder.WriteByte(' ') + builder.WriteString(help) + builder.WriteByte('\n') + builder.WriteString("# TYPE ") + builder.WriteString(name) + builder.WriteByte(' ') + builder.WriteString(metricType) + builder.WriteByte('\n') + writeSample(builder, name, labels, value) +} + +func writeSample(builder *strings.Builder, name, labels string, value any) { + builder.WriteString(name) + builder.WriteString(labels) + builder.WriteByte(' ') + switch typed := value.(type) { + case float64: + builder.WriteString(strconv.FormatFloat(typed, 'f', 6, 64)) + default: + builder.WriteString(fmt.Sprint(typed)) + } + builder.WriteByte('\n') +} + +func quoteLabel(value string) string { + replacer := strings.NewReplacer(`\`, `\\`, "\n", `\n`, `"`, `\"`) + return `"` + replacer.Replace(value) + `"` +} diff --git a/Sense/internal/metrics/metrics_test.go b/Sense/internal/metrics/metrics_test.go new file mode 100644 index 0000000..5015a45 --- /dev/null +++ b/Sense/internal/metrics/metrics_test.go @@ -0,0 +1,84 @@ +package metrics + +import ( + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +func TestHandlerUsesOnlyFixedLowCardinalityLabels(t *testing.T) { + t.Parallel() + registry := New("ins_test-1", "v-test") + registry.ObserveReconcileRun(nil, 1500*time.Millisecond) + registry.ObserveReconcileItem("success") + registry.ObserveReconcileItem("lease_lost") + registry.SetConvergence(16, 2) + registry.ObserveOrphanScan(20, 1, 3, nil) + registry.ObserveOrphanCleanupBlocked("ratio_exceeded") + registry.ObserveOrphanCleanup(1, 1) + + request := httptest.NewRequest("GET", "/metrics", nil) + response := httptest.NewRecorder() + registry.Handler().ServeHTTP(response, request) + body := response.Body.String() + for _, expected := range []string{ + `sense_build_info{instance_id="ins_test-1",version="v-test"} 1`, + `sense_reconcile_items_total{result="lease_lost"} 1`, + `sense_reconcile_devices{state="unconverged"} 2`, + `sense_orphan_paths{classification="owned_stale"} 1`, + `sense_orphan_cleanup_blocked_total{reason="ratio_exceeded"} 1`, + } { + if !strings.Contains(body, expected) { + t.Fatalf("metrics output lacks %q:\n%s", expected, body) + } + } + for _, forbidden := range []string{ + "tenant_id", "site_id", "device_id", "path_name", "rtsp://", "credential", + } { + if strings.Contains(body, forbidden) { + t.Fatalf("high-cardinality or secret-bearing label appeared: %q", forbidden) + } + } + if response.Header().Get("Cache-Control") != "no-store" { + t.Fatal("metrics responses must not be cached") + } +} + +func TestRegistryIsSafeForConcurrentObservers(t *testing.T) { + t.Parallel() + registry := New("ins_concurrent", "dev") + var wait sync.WaitGroup + for index := 0; index < 32; index++ { + wait.Add(1) + go func() { + defer wait.Done() + for count := 0; count < 100; count++ { + registry.ObserveReconcileItem("success") + registry.SetConvergence(128, count%17) + } + }() + } + wait.Wait() + response := httptest.NewRecorder() + registry.Handler().ServeHTTP(response, httptest.NewRequest("GET", "/metrics", nil)) + if !strings.Contains(response.Body.String(), `sense_reconcile_items_total{result="success"} 3200`) { + t.Fatalf("concurrent counter lost updates: %s", response.Body.String()) + } +} + +func TestGenerateInstanceIDShape(t *testing.T) { + t.Parallel() + first, err := GenerateInstanceID() + if err != nil { + t.Fatal(err) + } + second, err := GenerateInstanceID() + if err != nil { + t.Fatal(err) + } + if len(first) != 28 || !strings.HasPrefix(first, "ins_") || first == second { + t.Fatalf("unexpected instance identifiers: %q %q", first, second) + } +} diff --git a/Sense/internal/mtx/client.go b/Sense/internal/mtx/client.go index 9e683f0..29be53e 100644 --- a/Sense/internal/mtx/client.go +++ b/Sense/internal/mtx/client.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "sort" "strings" mediamtxapi "yovision/sense/internal/mtx/generated" @@ -31,9 +32,63 @@ type pathAPI interface { ConfigPathsGetWithResponse(context.Context, string, ...mediamtxapi.RequestEditorFn) (*mediamtxapi.ConfigPathsGetResponse, error) ConfigPathsPatchWithResponse(context.Context, string, mediamtxapi.ConfigPathsPatchJSONRequestBody, ...mediamtxapi.RequestEditorFn) (*mediamtxapi.ConfigPathsPatchResponse, error) ConfigPathsDeleteWithResponse(context.Context, string, ...mediamtxapi.RequestEditorFn) (*mediamtxapi.ConfigPathsDeleteResponse, error) + ConfigPathsListWithResponse(context.Context, *mediamtxapi.ConfigPathsListParams, ...mediamtxapi.RequestEditorFn) (*mediamtxapi.ConfigPathsListResponse, error) PathsGetWithResponse(context.Context, string, ...mediamtxapi.RequestEditorFn) (*mediamtxapi.PathsGetResponse, error) } +// ListPathNames enumerates only configuration names. Sources are deliberately +// discarded so inventory and orphan reports cannot expose stream URIs. +func (c *Client) ListPathNames(ctx context.Context) ([]string, error) { + const ( + itemsPerPage = 100 + maxPages = 1000 + ) + result := make(map[string]struct{}) + seenPages := make(map[string]struct{}) + for page := 0; page < maxPages; page++ { + pageValue, limitValue := page, itemsPerPage + response, err := c.api.ConfigPathsListWithResponse(ctx, &mediamtxapi.ConfigPathsListParams{ + Page: &pageValue, ItemsPerPage: &limitValue, + }) + if err != nil { + return nil, fmt.Errorf("MediaMTX list paths transport: %w", err) + } + if response.StatusCode() != http.StatusOK || response.JSON200 == nil || + response.JSON200.PageCount == nil || response.JSON200.Items == nil { + return nil, &APIError{Operation: "list paths", StatusCode: response.StatusCode()} + } + pageCount := *response.JSON200.PageCount + if pageCount < 0 || pageCount > maxPages { + return nil, &APIError{Operation: "list paths pagination", StatusCode: response.StatusCode()} + } + pageNames := make([]string, 0, len(*response.JSON200.Items)) + for _, item := range *response.JSON200.Items { + if item.Name == nil || strings.TrimSpace(*item.Name) == "" { + return nil, &APIError{Operation: "list paths response", StatusCode: response.StatusCode()} + } + pageNames = append(pageNames, *item.Name) + result[*item.Name] = struct{}{} + } + sort.Strings(pageNames) + signature := strings.Join(pageNames, "\x00") + if page > 0 && signature != "" { + if _, duplicate := seenPages[signature]; duplicate { + return nil, &APIError{Operation: "list paths repeated page", StatusCode: response.StatusCode()} + } + } + seenPages[signature] = struct{}{} + if int64(page+1) >= pageCount { + values := make([]string, 0, len(result)) + for name := range result { + values = append(values, name) + } + sort.Strings(values) + return values, nil + } + } + return nil, &APIError{Operation: "list paths page limit", StatusCode: http.StatusOK} +} + type Client struct { api pathAPI } diff --git a/Sense/internal/mtx/client_test.go b/Sense/internal/mtx/client_test.go index 23faeec..8eeaf05 100644 --- a/Sense/internal/mtx/client_test.go +++ b/Sense/internal/mtx/client_test.go @@ -6,6 +6,8 @@ import ( "fmt" "net/http" "net/http/httptest" + "sort" + "strconv" "strings" "sync" "testing" @@ -21,6 +23,38 @@ func (f *fakeMediaMTX) ServeHTTP(writer http.ResponseWriter, request *http.Reque f.mu.Lock() defer f.mu.Unlock() writer.Header().Set("Content-Type", "application/json") + if request.URL.Path == "/v3/config/paths/list" { + page, _ := strconv.Atoi(request.URL.Query().Get("page")) + limit, _ := strconv.Atoi(request.URL.Query().Get("itemsPerPage")) + if limit <= 0 { + limit = 100 + } + names := make([]string, 0, len(f.paths)) + for name := range f.paths { + names = append(names, name) + } + sort.Strings(names) + start := page * limit + if start > len(names) { + start = len(names) + } + end := start + limit + if end > len(names) { + end = len(names) + } + items := make([]map[string]any, 0, end-start) + for _, name := range names[start:end] { + items = append(items, map[string]any{"name": name}) + } + pageCount := 0 + if len(names) > 0 { + pageCount = (len(names) + limit - 1) / limit + } + _ = json.NewEncoder(writer).Encode(map[string]any{ + "itemCount": len(names), "pageCount": pageCount, "items": items, + }) + return + } prefixes := map[string]string{ "/v3/config/paths/get/": "get", "/v3/config/paths/add/": "add", @@ -72,6 +106,54 @@ func (f *fakeMediaMTX) ServeHTTP(writer http.ResponseWriter, request *http.Reque http.NotFound(writer, request) } +func TestListPathNamesUsesPaginationAndDoesNotReturnSources(t *testing.T) { + t.Parallel() + paths := make(map[string]string) + for index := 0; index < 205; index++ { + paths[fmt.Sprintf("camera-%03d", index)] = fmt.Sprintf("rtsp://secret.invalid/%d", index) + } + fake := &fakeMediaMTX{paths: paths} + server := httptest.NewServer(fake) + defer server.Close() + client, err := NewClient(server.URL, server.Client()) + if err != nil { + t.Fatal(err) + } + values, err := client.ListPathNames(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(values) != 205 || values[0] != "camera-000" || values[204] != "camera-204" { + t.Fatalf("unexpected path inventory: len=%d first=%q last=%q", len(values), values[0], values[len(values)-1]) + } + for _, value := range values { + if strings.Contains(value, "rtsp") || strings.Contains(value, "secret") { + t.Fatalf("source leaked from path inventory: %q", value) + } + } +} + +func TestListPathNamesRejectsRepeatedPage(t *testing.T) { + t.Parallel() + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + writer.Header().Set("Content-Type", "application/json") + if request.URL.Path != "/v3/config/paths/list" { + http.NotFound(writer, request) + return + } + _, _ = writer.Write([]byte(`{"itemCount":2,"pageCount":2,"items":[{"name":"same"}]}`)) + })) + defer server.Close() + client, err := NewClient(server.URL, server.Client()) + if err != nil { + t.Fatal(err) + } + if _, err := client.ListPathNames(context.Background()); err == nil || + !strings.Contains(err.Error(), "repeated page") { + t.Fatalf("repeated pagination was accepted: %v", err) + } +} + func TestGeneratedClientCreateReadDeleteMapping(t *testing.T) { t.Parallel() fake := &fakeMediaMTX{paths: make(map[string]string)} diff --git a/Sense/internal/orphan/orphan.go b/Sense/internal/orphan/orphan.go new file mode 100644 index 0000000..f7973d7 --- /dev/null +++ b/Sense/internal/orphan/orphan.go @@ -0,0 +1,400 @@ +// Package orphan reports MediaMTX configuration paths that do not match the +// current Sense ledger. Only paths with durable Sense ownership evidence can +// ever enter the controlled cleanup set. +package orphan + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "math/big" + "regexp" + "sort" + "strings" + "time" + + "yovision/sense/internal/metrics" + "yovision/sense/internal/store" +) + +const ( + defaultLeaseDuration = 30 * time.Second + defaultOperationTimeout = 20 * time.Second + reportTTL = 15 * time.Minute + maxCleanupItems = 128 + crockford = "0123456789ABCDEFGHJKMNPQRSTVWXYZ" +) + +var ( + ErrLeaseHeld = errors.New("orphan operation lease held") + ErrConfirmation = errors.New("orphan cleanup confirmation invalid") + ErrSnapshotExpired = errors.New("orphan scan snapshot expired") + ErrSafetyBlocked = errors.New("orphan cleanup safety gate blocked") + operatorIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`) +) + +type Runtime interface { + ListPathNames(context.Context) ([]string, error) + DeletePath(context.Context, string) error +} + +type Manager struct { + repository store.OrphanRepository + runtime Runtime + instanceID string + metrics *metrics.Registry + now func() time.Time + leaseDuration time.Duration + operationTimeout time.Duration +} + +type CleanupResult struct { + ScanID string `json:"scan_id"` + Deleted int `json:"deleted"` + Failed int `json:"failed"` +} + +func New( + repository store.OrphanRepository, + runtime Runtime, + instanceID string, + registry *metrics.Registry, +) *Manager { + return &Manager{ + repository: repository, runtime: runtime, instanceID: instanceID, metrics: registry, + now: time.Now, leaseDuration: defaultLeaseDuration, operationTimeout: defaultOperationTimeout, + } +} + +func (m *Manager) Report(ctx context.Context) (store.OrphanScan, error) { + now := m.now().UTC() + token, err := randomToken() + if err != nil { + return store.OrphanScan{}, err + } + acquired, err := m.repository.AcquireOperationalLease( + ctx, store.OperationalLeaseOrphanScan, m.instanceID, token, now, m.leaseDuration, + ) + if err != nil { + m.observeScan(0, 0, 0, err) + return store.OrphanScan{}, err + } + if !acquired { + return store.OrphanScan{}, ErrLeaseHeld + } + saved := false + defer func() { + if saved { + return + } + releaseCtx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = m.repository.ReleaseOperationalLease( + releaseCtx, store.OperationalLeaseOrphanScan, m.instanceID, token, m.now().UTC(), + ) + }() + + operationCtx, cancel := context.WithTimeout(ctx, m.operationTimeout) + paths, err := m.runtime.ListPathNames(operationCtx) + cancel() + if err != nil { + m.observeScan(0, 0, 0, err) + return store.OrphanScan{}, fmt.Errorf("list MediaMTX paths: %w", err) + } + ownership, err := m.repository.ListMediaPathOwnership(ctx) + if err != nil { + m.observeScan(0, 0, 0, err) + return store.OrphanScan{}, err + } + findings, stale, unowned := classify(paths, ownership) + allowed, reason := safetyGate(stale, len(paths)) + completedAt := m.now().UTC() + acquired, err = m.repository.AcquireOperationalLease( + ctx, store.OperationalLeaseOrphanScan, m.instanceID, token, completedAt, m.leaseDuration, + ) + if err != nil || !acquired { + if err == nil { + err = store.ErrOperationalLeaseLost + } + m.observeScan(len(paths), stale, unowned, err) + return store.OrphanScan{}, err + } + id, err := newScanID(completedAt) + if err != nil { + m.observeScan(0, 0, 0, err) + return store.OrphanScan{}, err + } + scan := store.OrphanScan{ + ID: id, InstanceID: m.instanceID, ObservedCount: len(paths), + OwnedStaleCount: stale, UnownedCount: unowned, + SafetyAllowed: allowed, SafetyReason: reason, + CompletedAt: completedAt, ExpiresAt: completedAt.Add(reportTTL), Findings: findings, + } + if err := m.repository.SaveOrphanScan(ctx, scan, m.instanceID, token); err != nil { + m.observeScan(len(paths), stale, unowned, err) + return store.OrphanScan{}, err + } + saved = true + m.observeScan(len(paths), stale, unowned, nil) + return scan, nil +} + +func classify( + paths []string, + ownership []store.MediaPathOwnership, +) ([]store.OrphanFinding, int, int) { + history := make(map[string]store.MediaPathOwnership, len(ownership)) + for _, value := range ownership { + history[value.PathName] = value + } + unique := make(map[string]struct{}, len(paths)) + for _, path := range paths { + unique[path] = struct{}{} + } + ordered := make([]string, 0, len(unique)) + for path := range unique { + ordered = append(ordered, path) + } + sort.Strings(ordered) + findings := make([]store.OrphanFinding, 0) + stale, unowned := 0, 0 + for _, path := range ordered { + value, known := history[path] + switch { + case known && value.CurrentClaim: + continue + case known: + stale++ + findings = append(findings, store.OrphanFinding{ + PathName: path, Classification: store.OrphanOwnedStale, DeviceID: value.DeviceID, + }) + default: + unowned++ + findings = append(findings, store.OrphanFinding{ + PathName: path, Classification: store.OrphanUnowned, + }) + } + } + return findings, stale, unowned +} + +func safetyGate(candidates, observed int) (bool, string) { + switch { + case candidates <= 0: + return false, "no_candidates" + case observed <= 0: + return false, "empty_inventory" + case candidates > maxCleanupItems: + return false, "scope_exceeded" + case candidates*100 > observed*10: + return false, "ratio_exceeded" + default: + return true, "allowed" + } +} + +func (m *Manager) Apply( + ctx context.Context, + scanID, actorID, confirmation string, +) (CleanupResult, error) { + result := CleanupResult{ScanID: scanID} + if !operatorIDPattern.MatchString(actorID) || confirmation != "DELETE "+scanID { + m.blocked("scope_changed") + return result, ErrConfirmation + } + now := m.now().UTC() + token, err := randomToken() + if err != nil { + return result, err + } + acquired, err := m.repository.AcquireOperationalLease( + ctx, store.OperationalLeaseOrphanCleanup, m.instanceID, token, now, m.leaseDuration, + ) + if err != nil { + return result, err + } + if !acquired { + return result, ErrLeaseHeld + } + defer func() { + releaseCtx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = m.repository.ReleaseOperationalLease( + releaseCtx, store.OperationalLeaseOrphanCleanup, m.instanceID, token, m.now().UTC(), + ) + }() + + scan, err := m.repository.GetOrphanScan(ctx, scanID) + if err != nil { + m.blocked("scope_changed") + return result, err + } + if !scan.ExpiresAt.After(now) { + m.blocked("snapshot_expired") + return result, ErrSnapshotExpired + } + if !scan.SafetyAllowed { + m.blocked(scan.SafetyReason) + return result, fmt.Errorf("%w: %s", ErrSafetyBlocked, scan.SafetyReason) + } + operationCtx, cancel := context.WithTimeout(ctx, m.operationTimeout) + paths, err := m.runtime.ListPathNames(operationCtx) + cancel() + if err != nil { + return result, fmt.Errorf("refresh MediaMTX path inventory: %w", err) + } + ownership, err := m.repository.ListMediaPathOwnership(ctx) + if err != nil { + return result, err + } + candidates := cleanupCandidates(scan, paths, ownership) + if len(candidates) == 0 { + return result, nil + } + allowed, reason := safetyGate(len(candidates), len(uniqueNames(paths))) + if !allowed { + m.blocked(reason) + return result, fmt.Errorf("%w: %s", ErrSafetyBlocked, reason) + } + var failures []error + for _, path := range candidates { + now = m.now().UTC() + acquired, err = m.repository.AcquireOperationalLease( + ctx, store.OperationalLeaseOrphanCleanup, m.instanceID, token, now, m.leaseDuration, + ) + if err != nil || !acquired { + if err == nil { + err = ErrLeaseHeld + } + failures = append(failures, err) + break + } + itemCtx, itemCancel := context.WithTimeout(ctx, m.operationTimeout) + deleteErr := m.runtime.DeletePath(itemCtx, path) + itemCancel() + status, code := "deleted", "" + if deleteErr != nil { + status, code = "failed", "media_error" + result.Failed++ + failures = append(failures, fmt.Errorf("delete owned stale path: %w", deleteErr)) + } else { + result.Deleted++ + } + if err := m.repository.RecordOrphanCleanup( + ctx, scanID, path, actorID, status, code, m.now().UTC(), + ); err != nil { + failures = append(failures, err) + break + } + } + if m.metrics != nil { + m.metrics.ObserveOrphanCleanup(result.Deleted, result.Failed) + } + return result, errors.Join(failures...) +} + +func cleanupCandidates( + scan store.OrphanScan, + paths []string, + ownership []store.MediaPathOwnership, +) []string { + runtime := uniqueNames(paths) + history := make(map[string]store.MediaPathOwnership, len(ownership)) + for _, value := range ownership { + history[value.PathName] = value + } + values := make([]string, 0) + for _, finding := range scan.Findings { + if finding.Classification != store.OrphanOwnedStale || finding.Deleted { + continue + } + _, present := runtime[finding.PathName] + owner, known := history[finding.PathName] + if present && known && !owner.CurrentClaim && owner.DeviceID == finding.DeviceID { + values = append(values, finding.PathName) + } + } + sort.Strings(values) + return values +} + +func uniqueNames(values []string) map[string]struct{} { + result := make(map[string]struct{}, len(values)) + for _, value := range values { + result[value] = struct{}{} + } + return result +} + +func (m *Manager) Run(ctx context.Context, interval time.Duration, report func(error)) { + run := func() { + _, err := m.Report(ctx) + if err != nil && !errors.Is(err, ErrLeaseHeld) && ctx.Err() == nil && report != nil { + report(err) + } + } + run() + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + run() + } + } +} + +func (m *Manager) observeScan(observed, stale, unowned int, err error) { + if m.metrics != nil { + m.metrics.ObserveOrphanScan(observed, stale, unowned, err) + } +} + +func (m *Manager) blocked(reason string) { + if m.metrics != nil { + m.metrics.ObserveOrphanCleanupBlocked(reason) + } +} + +func randomToken() (string, error) { + value := make([]byte, 16) + if _, err := rand.Read(value); err != nil { + return "", errors.New("generate fencing token") + } + const alphabet = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + result := make([]byte, 22) + number := new(big.Int).SetBytes(value) + base := big.NewInt(int64(len(alphabet))) + remainder := new(big.Int) + for index := len(result) - 1; index >= 0; index-- { + number.QuoRem(number, base, remainder) + result[index] = alphabet[remainder.Int64()] + } + return string(result), nil +} + +func newScanID(now time.Time) (string, error) { + value := make([]byte, 16) + milliseconds := uint64(now.UTC().UnixMilli()) + value[0], value[1], value[2] = byte(milliseconds>>40), byte(milliseconds>>32), byte(milliseconds>>24) + value[3], value[4], value[5] = byte(milliseconds>>16), byte(milliseconds>>8), byte(milliseconds) + if _, err := rand.Read(value[6:]); err != nil { + return "", errors.New("generate orphan scan identifier") + } + number := new(big.Int).SetBytes(value) + base := big.NewInt(32) + remainder := new(big.Int) + encoded := make([]byte, 26) + for index := len(encoded) - 1; index >= 0; index-- { + number.QuoRem(number, base, remainder) + encoded[index] = crockford[remainder.Int64()] + } + return "scan_" + string(encoded), nil +} + +func Confirmation(scanID string) string { + return strings.Join([]string{"DELETE", scanID}, " ") +} diff --git a/Sense/internal/orphan/orphan_test.go b/Sense/internal/orphan/orphan_test.go new file mode 100644 index 0000000..7bf8b37 --- /dev/null +++ b/Sense/internal/orphan/orphan_test.go @@ -0,0 +1,272 @@ +package orphan + +import ( + "context" + "errors" + "sort" + "sync" + "testing" + "time" + + "yovision/sense/internal/store" +) + +type fakeLease struct { + owner, token string + expires time.Time +} + +type fakeRepository struct { + mu sync.Mutex + leases map[string]fakeLease + ownership []store.MediaPathOwnership + scans map[string]store.OrphanScan + deleted map[string]map[string]bool +} + +func newFakeRepository() *fakeRepository { + return &fakeRepository{ + leases: make(map[string]fakeLease), scans: make(map[string]store.OrphanScan), + deleted: make(map[string]map[string]bool), + } +} + +func (f *fakeRepository) AcquireOperationalLease( + _ context.Context, + name, owner, token string, + now time.Time, + duration time.Duration, +) (bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + current, exists := f.leases[name] + if exists && current.expires.After(now) && (current.owner != owner || current.token != token) { + return false, nil + } + f.leases[name] = fakeLease{owner: owner, token: token, expires: now.Add(duration)} + return true, nil +} + +func (f *fakeRepository) ReleaseOperationalLease( + _ context.Context, + name, owner, token string, + now time.Time, +) error { + f.mu.Lock() + defer f.mu.Unlock() + current := f.leases[name] + if current.owner == owner && current.token == token { + current.expires = now + f.leases[name] = current + } + return nil +} + +func (f *fakeRepository) ListMediaPathOwnership(context.Context) ([]store.MediaPathOwnership, error) { + f.mu.Lock() + defer f.mu.Unlock() + return append([]store.MediaPathOwnership(nil), f.ownership...), nil +} + +func (f *fakeRepository) SaveOrphanScan( + _ context.Context, + scan store.OrphanScan, + owner, token string, +) error { + f.mu.Lock() + defer f.mu.Unlock() + lease := f.leases[store.OperationalLeaseOrphanScan] + if lease.owner != owner || lease.token != token || !lease.expires.After(scan.CompletedAt) { + return store.ErrOperationalLeaseLost + } + f.scans[scan.ID] = cloneScan(scan) + lease.expires = scan.CompletedAt + f.leases[store.OperationalLeaseOrphanScan] = lease + return nil +} + +func (f *fakeRepository) GetOrphanScan(_ context.Context, id string) (store.OrphanScan, error) { + f.mu.Lock() + defer f.mu.Unlock() + scan, exists := f.scans[id] + if !exists { + return store.OrphanScan{}, store.ErrOrphanScanNotFound + } + result := cloneScan(scan) + for index := range result.Findings { + result.Findings[index].Deleted = f.deleted[id][result.Findings[index].PathName] + } + return result, nil +} + +func (f *fakeRepository) RecordOrphanCleanup( + _ context.Context, + scanID, pathName, _, status, _ string, + _ time.Time, +) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.deleted[scanID] == nil { + f.deleted[scanID] = make(map[string]bool) + } + if status == "deleted" { + f.deleted[scanID][pathName] = true + } + return nil +} + +func cloneScan(value store.OrphanScan) store.OrphanScan { + value.Findings = append([]store.OrphanFinding(nil), value.Findings...) + return value +} + +type fakeRuntime struct { + paths map[string]bool + fail map[string]bool + deleted []string +} + +func (f *fakeRuntime) ListPathNames(context.Context) ([]string, error) { + values := make([]string, 0, len(f.paths)) + for path, present := range f.paths { + if present { + values = append(values, path) + } + } + sort.Strings(values) + return values, nil +} + +func (f *fakeRuntime) DeletePath(_ context.Context, path string) error { + if f.fail[path] { + return errors.New("redacted media failure") + } + delete(f.paths, path) + f.deleted = append(f.deleted, path) + return nil +} + +func TestSafetyGateUsesExactTenPercentBoundary(t *testing.T) { + t.Parallel() + if allowed, reason := safetyGate(1, 10); !allowed || reason != "allowed" { + t.Fatalf("exact 10%% boundary was rejected: %v %s", allowed, reason) + } + if allowed, reason := safetyGate(1, 9); allowed || reason != "ratio_exceeded" { + t.Fatalf("more than 10%% was accepted: %v %s", allowed, reason) + } + if allowed, reason := safetyGate(129, 2000); allowed || reason != "scope_exceeded" { + t.Fatalf("129-item scope was accepted: %v %s", allowed, reason) + } +} + +func TestReportClassifiesOwnershipAndApplyNeverDeletesUnowned(t *testing.T) { + repository := newFakeRepository() + repository.ownership = []store.MediaPathOwnership{ + {PathName: "owned-stale", DeviceID: "old-device"}, + {PathName: "owned-current", DeviceID: "live-device", CurrentClaim: true}, + } + runtime := &fakeRuntime{paths: map[string]bool{ + "owned-stale": true, "owned-current": true, + "unowned-1": true, "unowned-2": true, "unowned-3": true, "unowned-4": true, + "unowned-5": true, "unowned-6": true, "unowned-7": true, "unowned-8": true, + }, fail: make(map[string]bool)} + now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC) + manager := New(repository, runtime, "ins_test", nil) + manager.now = func() time.Time { return now } + scan, err := manager.Report(context.Background()) + if err != nil { + t.Fatal(err) + } + if scan.ObservedCount != 10 || scan.OwnedStaleCount != 1 || scan.UnownedCount != 8 || !scan.SafetyAllowed { + t.Fatalf("unexpected report: %+v", scan) + } + result, err := manager.Apply( + context.Background(), scan.ID, "operator-1", Confirmation(scan.ID), + ) + if err != nil { + t.Fatal(err) + } + if result.Deleted != 1 || len(runtime.deleted) != 1 || runtime.deleted[0] != "owned-stale" { + t.Fatalf("cleanup escaped owned stale set: result=%+v deleted=%v", result, runtime.deleted) + } + for _, path := range []string{"owned-current", "unowned-1", "unowned-8"} { + if !runtime.paths[path] { + t.Fatalf("cleanup deleted protected path %q", path) + } + } +} + +func TestApplyBlocksExpiredOrOversizedSnapshotBeforeDelete(t *testing.T) { + repository := newFakeRepository() + repository.ownership = []store.MediaPathOwnership{{PathName: "stale", DeviceID: "old"}} + runtime := &fakeRuntime{paths: map[string]bool{"stale": true}, fail: make(map[string]bool)} + now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC) + manager := New(repository, runtime, "ins_test", nil) + manager.now = func() time.Time { return now } + scan, err := manager.Report(context.Background()) + if err != nil { + t.Fatal(err) + } + if scan.SafetyAllowed || scan.SafetyReason != "ratio_exceeded" { + t.Fatalf("single-path 100%% cleanup was not blocked: %+v", scan) + } + if _, err := manager.Apply(context.Background(), scan.ID, "operator", Confirmation(scan.ID)); !errors.Is(err, ErrSafetyBlocked) { + t.Fatalf("expected ratio gate, got %v", err) + } + if len(runtime.deleted) != 0 { + t.Fatal("ratio-blocked cleanup mutated MediaMTX") + } + + for index := 0; index < 9; index++ { + runtime.paths[string(rune('a'+index))] = true + } + scan, err = manager.Report(context.Background()) + if err != nil || !scan.SafetyAllowed { + t.Fatalf("expected a fresh 1/10 executable report: %+v %v", scan, err) + } + now = now.Add(reportTTL) + if _, err := manager.Apply(context.Background(), scan.ID, "operator", Confirmation(scan.ID)); !errors.Is(err, ErrSnapshotExpired) { + t.Fatalf("expected expired snapshot, got %v", err) + } + if len(runtime.deleted) != 0 { + t.Fatal("expired cleanup mutated MediaMTX") + } +} + +func TestApplyIsRetryableAndCannotExpandBeyondSnapshot(t *testing.T) { + repository := newFakeRepository() + repository.ownership = []store.MediaPathOwnership{ + {PathName: "stale-a", DeviceID: "old-a"}, + {PathName: "stale-b", DeviceID: "old-b"}, + } + runtime := &fakeRuntime{paths: make(map[string]bool), fail: map[string]bool{"stale-b": true}} + runtime.paths["stale-a"], runtime.paths["stale-b"] = true, true + for index := 0; index < 18; index++ { + runtime.paths[string(rune(0x100+index))] = true + } + now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC) + manager := New(repository, runtime, "ins_test", nil) + manager.now = func() time.Time { return now } + scan, err := manager.Report(context.Background()) + if err != nil || !scan.SafetyAllowed { + t.Fatalf("expected 2/20 report: %+v %v", scan, err) + } + repository.ownership = append(repository.ownership, + store.MediaPathOwnership{PathName: "new-stale", DeviceID: "new-old"}) + runtime.paths["new-stale"] = true + result, err := manager.Apply(context.Background(), scan.ID, "operator", Confirmation(scan.ID)) + if err == nil || result.Deleted != 1 || result.Failed != 1 { + t.Fatalf("expected one partial failure: %+v %v", result, err) + } + if !runtime.paths["new-stale"] { + t.Fatal("cleanup expanded beyond the approved snapshot") + } + runtime.fail["stale-b"] = false + result, err = manager.Apply(context.Background(), scan.ID, "operator", Confirmation(scan.ID)) + if err != nil || result.Deleted != 1 || result.Failed != 0 { + t.Fatalf("failed item was not retryable: %+v %v", result, err) + } + if !runtime.paths["new-stale"] { + t.Fatal("retry expanded beyond the approved snapshot") + } +} diff --git a/Sense/internal/reconcile/reconciler.go b/Sense/internal/reconcile/reconciler.go index 8af902b..35c1b68 100644 --- a/Sense/internal/reconcile/reconciler.go +++ b/Sense/internal/reconcile/reconciler.go @@ -3,12 +3,15 @@ package reconcile import ( "context" + "crypto/rand" + "encoding/base64" "errors" "fmt" "net/url" "time" "yovision/sense/internal/device" + "yovision/sense/internal/metrics" "yovision/sense/internal/onvif" "yovision/sense/internal/store" ) @@ -16,9 +19,11 @@ import ( const defaultBatchSize = 128 type Repository interface { - ListDueReconcile(ctx context.Context, now time.Time, limit int) ([]store.ReconcileCandidate, error) - MarkReconciled(ctx context.Context, id string, generation int64, now time.Time) error - MarkReconcileFailure(ctx context.Context, id string, failureCount int, nextAttempt time.Time, errorCode string, now time.Time) error + ClaimDueReconcile(context.Context, store.ReconcileClaim) ([]store.ReconcileCandidate, error) + RenewReconcileLease(context.Context, string, string, string, time.Time, time.Duration) (bool, error) + CompleteReconcile(context.Context, string, int64, string, string, time.Time) error + FailReconcile(context.Context, string, int, time.Time, string, string, string, time.Time) error + ConvergenceSnapshot(context.Context) (store.ConvergenceSnapshot, error) } type MediaPaths interface { @@ -27,48 +32,134 @@ type MediaPaths interface { } type Reconciler struct { - repository Repository - discovery onvif.Adapter - media MediaPaths - now func() time.Time - baseBackoff time.Duration - maxBackoff time.Duration - batchSize int + repository Repository + discovery onvif.Adapter + media MediaPaths + now func() time.Time + baseBackoff time.Duration + maxBackoff time.Duration + batchSize int + instanceID string + leaseDuration time.Duration + operationTimeout time.Duration + metrics *metrics.Registry +} + +type Options struct { + InstanceID string + LeaseDuration time.Duration + OperationTimeout time.Duration + Metrics *metrics.Registry } func New(repository Repository, discovery onvif.Adapter, media MediaPaths) *Reconciler { + return NewWithOptions(repository, discovery, media, Options{}) +} + +func NewWithOptions( + repository Repository, + discovery onvif.Adapter, + media MediaPaths, + options Options, +) *Reconciler { + if options.InstanceID == "" { + options.InstanceID = "single" + } + if options.LeaseDuration <= 0 { + options.LeaseDuration = 30 * time.Second + } + if options.OperationTimeout <= 0 { + options.OperationTimeout = 20 * time.Second + } + if options.OperationTimeout >= options.LeaseDuration { + options.OperationTimeout = options.LeaseDuration / 2 + } return &Reconciler{ repository: repository, discovery: discovery, media: media, now: time.Now, baseBackoff: time.Second, maxBackoff: time.Minute, batchSize: defaultBatchSize, + instanceID: options.InstanceID, leaseDuration: options.LeaseDuration, + operationTimeout: options.OperationTimeout, metrics: options.Metrics, } } -func (r *Reconciler) RunOnce(ctx context.Context) error { +func (r *Reconciler) RunOnce(ctx context.Context) (runErr error) { + started := time.Now() + defer func() { + if r.metrics != nil { + r.metrics.ObserveReconcileRun(runErr, time.Since(started)) + } + }() now := r.now().UTC() - candidates, err := r.repository.ListDueReconcile(ctx, now, r.batchSize) + token, err := newClaimToken() if err != nil { - return fmt.Errorf("list reconciliation candidates: %w", err) + return err + } + candidates, err := r.repository.ClaimDueReconcile(ctx, store.ReconcileClaim{ + Owner: r.instanceID, Token: token, Now: now, + LeaseDuration: r.leaseDuration, Limit: r.batchSize, + }) + if err != nil { + return fmt.Errorf("claim reconciliation candidates: %w", err) } var runErrors []error for _, candidate := range candidates { if err := ctx.Err(); err != nil { return err } - if err := r.reconcileOne(ctx, candidate, now); err != nil { + itemNow := r.now().UTC() + renewed, err := r.repository.RenewReconcileLease( + ctx, candidate.Device.ID, r.instanceID, token, itemNow, r.leaseDuration, + ) + if err != nil { + r.observeItem("error") + runErrors = append(runErrors, fmt.Errorf("renew reconcile device lease: %w", err)) + continue + } + if !renewed { + r.observeItem("lease_lost") + continue + } + itemCtx, cancel := context.WithTimeout(ctx, r.operationTimeout) + err = r.reconcileOne(itemCtx, ctx, candidate, itemNow, r.instanceID, token) + cancel() + if errors.Is(err, store.ErrReconcileLeaseLost) { + r.observeItem("lease_lost") + continue + } + if err != nil { + r.observeItem("error") runErrors = append(runErrors, fmt.Errorf("reconcile device %s: %w", candidate.Device.ID, err)) + } else { + r.observeItem("success") + } + } + if r.metrics != nil { + snapshot, err := r.repository.ConvergenceSnapshot(ctx) + if err != nil { + runErrors = append(runErrors, fmt.Errorf("read convergence metrics: %w", err)) + } else { + r.metrics.SetConvergence(snapshot.Total, snapshot.Unconverged) } } return errors.Join(runErrors...) } -func (r *Reconciler) reconcileOne(ctx context.Context, candidate store.ReconcileCandidate, now time.Time) error { +func (r *Reconciler) reconcileOne( + ctx context.Context, + persistCtx context.Context, + candidate store.ReconcileCandidate, + now time.Time, + owner, token string, +) error { if candidate.Device.DesiredState == device.DesiredDisabled { if err := r.media.DeletePath(ctx, candidate.Device.PathName); err == nil { - return r.repository.MarkReconciled(ctx, candidate.Device.ID, candidate.Device.Generation, now) - } else if ctx.Err() != nil { - return ctx.Err() + return r.repository.CompleteReconcile( + persistCtx, candidate.Device.ID, candidate.Device.Generation, owner, token, now, + ) + } else if persistCtx.Err() != nil { + return persistCtx.Err() } else { - return r.persistFailure(ctx, candidate, now, err) + return r.persistFailure(persistCtx, candidate, now, owner, token, err) } } result, err := r.discovery.Probe(ctx, onvif.Target{ @@ -81,16 +172,22 @@ func (r *Reconciler) reconcileOne(ctx context.Context, candidate store.Reconcile _, err = r.media.EnsurePath(ctx, candidate.Device.PathName, result.StreamURI) } if err == nil { - return r.repository.MarkReconciled(ctx, candidate.Device.ID, candidate.Device.Generation, now) + return r.repository.CompleteReconcile( + persistCtx, candidate.Device.ID, candidate.Device.Generation, owner, token, now, + ) } - if ctx.Err() != nil { - return ctx.Err() + if persistCtx.Err() != nil { + return persistCtx.Err() } - return r.persistFailure(ctx, candidate, now, err) + return r.persistFailure(persistCtx, candidate, now, owner, token, err) } func (r *Reconciler) persistFailure( - ctx context.Context, candidate store.ReconcileCandidate, now time.Time, err error, + ctx context.Context, + candidate store.ReconcileCandidate, + now time.Time, + owner, token string, + err error, ) error { failureCount := candidate.FailureCount + 1 nextAttempt := now.Add(r.backoff(failureCount)) @@ -99,14 +196,28 @@ func (r *Reconciler) persistFailure( if !errors.As(err, &onvifError) { errorCode = "media_error" } - if markErr := r.repository.MarkReconcileFailure( - ctx, candidate.Device.ID, failureCount, nextAttempt, errorCode, now, + if markErr := r.repository.FailReconcile( + ctx, candidate.Device.ID, failureCount, nextAttempt, errorCode, owner, token, now, ); markErr != nil { return errors.Join(err, fmt.Errorf("persist reconcile failure: %w", markErr)) } return err } +func (r *Reconciler) observeItem(result string) { + if r.metrics != nil { + r.metrics.ObserveReconcileItem(result) + } +} + +func newClaimToken() (string, error) { + value := make([]byte, 18) + if _, err := rand.Read(value); err != nil { + return "", errors.New("generate reconcile claim token") + } + return base64.RawURLEncoding.EncodeToString(value), nil +} + func validateStreamURI(value string) error { parsed, err := url.Parse(value) if err != nil || parsed.Host == "" || (parsed.Scheme != "rtsp" && parsed.Scheme != "rtsps") { diff --git a/Sense/internal/reconcile/reconciler_test.go b/Sense/internal/reconcile/reconciler_test.go index a06abad..20526f9 100644 --- a/Sense/internal/reconcile/reconciler_test.go +++ b/Sense/internal/reconcile/reconciler_test.go @@ -162,6 +162,86 @@ func TestDisabledDeviceDeletesOnlyItsExactPath(t *testing.T) { } } +type lostLeaseRepository struct { + completed bool + failed bool +} + +func (r *lostLeaseRepository) ClaimDueReconcile( + context.Context, + store.ReconcileClaim, +) ([]store.ReconcileCandidate, error) { + return []store.ReconcileCandidate{{Device: device.Device{ + ID: "camera-lost", DesiredState: device.DesiredEnabled, + EndpointRef: "onvif://camera-lost", PathName: "camera-lost", Generation: 1, + }}}, nil +} + +func (r *lostLeaseRepository) RenewReconcileLease( + context.Context, string, string, string, time.Time, time.Duration, +) (bool, error) { + return false, nil +} + +func (r *lostLeaseRepository) CompleteReconcile( + context.Context, string, int64, string, string, time.Time, +) error { + r.completed = true + return nil +} + +func (r *lostLeaseRepository) FailReconcile( + context.Context, string, int, time.Time, string, string, string, time.Time, +) error { + r.failed = true + return nil +} + +func (r *lostLeaseRepository) ConvergenceSnapshot(context.Context) (store.ConvergenceSnapshot, error) { + return store.ConvergenceSnapshot{}, nil +} + +func TestLostLeaseSkipsAllExternalAndStoreMutations(t *testing.T) { + t.Parallel() + repository := &lostLeaseRepository{} + media := &recordingMedia{} + reconciler := New(repository, onvif.NewFake(nil), media) + if err := reconciler.RunOnce(context.Background()); err != nil { + t.Fatal(err) + } + if media.calls != 0 || repository.completed || repository.failed { + t.Fatalf("lost lease performed a side effect: media=%d completed=%v failed=%v", + media.calls, repository.completed, repository.failed) + } +} + +func TestPerItemTimeoutPersistsRetryWhenParentIsAlive(t *testing.T) { + t.Parallel() + repository := openRepository(t, filepath.Join(t.TempDir(), "sense.db")) + createReconcileDevice(t, repository) + discovery := onvif.NewFake(map[string]onvif.FakeScenario{ + "onvif://camera-1": { + DelayMillis: 100, + Result: onvif.ProbeResult{StreamURI: "rtsp://media.invalid/camera-1"}, + }, + }) + reconciler := NewWithOptions(repository, discovery, &recordingMedia{}, Options{ + InstanceID: "ins_test", LeaseDuration: time.Second, OperationTimeout: 5 * time.Millisecond, + }) + now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC) + reconciler.now = func() time.Time { return now } + if err := reconciler.RunOnce(context.Background()); err == nil { + t.Fatal("expected bounded operation timeout") + } + candidates, err := repository.ListDueReconcile(context.Background(), now.Add(time.Hour), 1) + if err != nil { + t.Fatal(err) + } + if len(candidates) != 1 || candidates[0].FailureCount != 1 { + t.Fatalf("operation timeout did not persist retry state: %+v", candidates) + } +} + func openRepository(t *testing.T, path string) *store.SQLite { t.Helper() repository, err := store.OpenSQLite(context.Background(), "file:"+filepath.ToSlash(path)) diff --git a/Sense/internal/store/control_postgres.go b/Sense/internal/store/control_postgres.go index 8457351..3c40ca1 100644 --- a/Sense/internal/store/control_postgres.go +++ b/Sense/internal/store/control_postgres.go @@ -514,6 +514,7 @@ func (s *Postgres) PatchControlDevice( if reconcileChanged { if _, err := tx.ExecContext(ctx, `UPDATE sense.reconcile_state SET failure_count = 0, next_attempt_at = NULL, last_error_code = NULL, + lease_owner = NULL, lease_token = NULL, lease_until = NULL, updated_at = $1 WHERE device_id = $2`, now, deviceID); err != nil { return ControlMutationResult{}, errors.New("reset postgres Control API reconcile state") } @@ -631,6 +632,7 @@ func setControlDesiredStateTx( } if _, err := tx.ExecContext(ctx, `UPDATE sense.reconcile_state SET failure_count = 0, next_attempt_at = NULL, last_error_code = NULL, + lease_owner = NULL, lease_token = NULL, lease_until = NULL, updated_at = $1 WHERE device_id = $2`, now, deviceID); err != nil { return ControlDevice{}, errors.New("reset postgres Control API desired-state reconciliation") } diff --git a/Sense/internal/store/orphan.go b/Sense/internal/store/orphan.go new file mode 100644 index 0000000..3f1b807 --- /dev/null +++ b/Sense/internal/store/orphan.go @@ -0,0 +1,58 @@ +package store + +import ( + "context" + "errors" + "time" +) + +const ( + OperationalLeaseOrphanScan = "mediamtx-orphan-scan" + OperationalLeaseOrphanCleanup = "mediamtx-orphan-cleanup" + + OrphanOwnedStale = "owned_stale" + OrphanUnowned = "unowned" +) + +var ErrOrphanScanNotFound = errors.New("orphan scan not found") + +type MediaPathOwnership struct { + PathName string + DeviceID string + CurrentClaim bool +} + +type OrphanFinding struct { + PathName string + Classification string + DeviceID string + Deleted bool +} + +type OrphanScan struct { + ID string + InstanceID string + ObservedCount int + OwnedStaleCount int + UnownedCount int + SafetyAllowed bool + SafetyReason string + CompletedAt time.Time + ExpiresAt time.Time + Findings []OrphanFinding +} + +type OrphanRepository interface { + AcquireOperationalLease( + context.Context, string, string, string, time.Time, time.Duration, + ) (bool, error) + ReleaseOperationalLease(context.Context, string, string, string, time.Time) error + ListMediaPathOwnership(context.Context) ([]MediaPathOwnership, error) + SaveOrphanScan(context.Context, OrphanScan, string, string) error + GetOrphanScan(context.Context, string) (OrphanScan, error) + RecordOrphanCleanup( + context.Context, string, string, string, string, string, time.Time, + ) error +} + +var _ OrphanRepository = (*Postgres)(nil) diff --git a/Sense/internal/store/orphan_postgres.go b/Sense/internal/store/orphan_postgres.go new file mode 100644 index 0000000..1626959 --- /dev/null +++ b/Sense/internal/store/orphan_postgres.go @@ -0,0 +1,268 @@ +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "time" +) + +const orphanReportRetention = 7 * 24 * time.Hour + +func (s *Postgres) AcquireOperationalLease( + ctx context.Context, + name, owner, token string, + _ time.Time, + duration time.Duration, +) (bool, error) { + if strings.TrimSpace(name) == "" || strings.TrimSpace(owner) == "" || + strings.TrimSpace(token) == "" || duration <= 0 { + return false, errors.New("invalid operational lease") + } + var acquired int + err := s.db.QueryRowContext(ctx, `INSERT INTO sense.operational_leases( + lease_name, owner_id, fencing_token, lease_until, updated_at + ) VALUES ( + $1, $2, $3, + clock_timestamp() + ($4 * interval '1 second'), clock_timestamp() + ) + ON CONFLICT (lease_name) DO UPDATE SET + owner_id = EXCLUDED.owner_id, + fencing_token = EXCLUDED.fencing_token, + lease_until = EXCLUDED.lease_until, + updated_at = EXCLUDED.updated_at + WHERE sense.operational_leases.lease_until <= clock_timestamp() + OR (sense.operational_leases.owner_id = $2 + AND sense.operational_leases.fencing_token = $3) + RETURNING 1`, name, owner, token, duration.Seconds()).Scan(&acquired) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, errors.New("acquire postgres operational lease") + } + return acquired == 1, nil +} + +func (s *Postgres) ReleaseOperationalLease( + ctx context.Context, + name, owner, token string, + _ time.Time, +) error { + _, err := s.db.ExecContext(ctx, `UPDATE sense.operational_leases + SET lease_until = clock_timestamp(), updated_at = clock_timestamp() + WHERE lease_name = $1 AND owner_id = $2 AND fencing_token = $3`, + name, owner, token) + if err != nil { + return errors.New("release postgres operational lease") + } + return nil +} + +func (s *Postgres) ListMediaPathOwnership(ctx context.Context) ([]MediaPathOwnership, error) { + rows, err := s.db.QueryContext(ctx, `SELECT + o.path_name, o.device_id, + EXISTS ( + SELECT 1 FROM sense.devices d + WHERE d.id = o.device_id AND d.path_name = o.path_name + AND EXISTS ( + SELECT 1 FROM sense.device_capabilities c + WHERE c.device_id = d.id AND c.capability = 'video_capture' + ) + ) AS current_claim + FROM sense.media_path_ownership o + ORDER BY o.path_name`) + if err != nil { + return nil, errors.New("list postgres MediaMTX path ownership") + } + defer rows.Close() + values := make([]MediaPathOwnership, 0) + for rows.Next() { + var value MediaPathOwnership + if err := rows.Scan(&value.PathName, &value.DeviceID, &value.CurrentClaim); err != nil { + return nil, errors.New("scan postgres MediaMTX path ownership") + } + values = append(values, value) + } + if err := rows.Err(); err != nil { + return nil, errors.New("iterate postgres MediaMTX path ownership") + } + return values, nil +} + +func (s *Postgres) SaveOrphanScan( + ctx context.Context, + scan OrphanScan, + owner, token string, +) error { + if err := validateOrphanScan(scan); err != nil { + return err + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return errors.New("begin postgres orphan scan save") + } + defer tx.Rollback() + var lease int + err = tx.QueryRowContext(ctx, `SELECT 1 FROM sense.operational_leases + WHERE lease_name = $1 AND owner_id = $2 AND fencing_token = $3 + AND lease_until > clock_timestamp() + FOR UPDATE`, OperationalLeaseOrphanScan, owner, token).Scan(&lease) + if errors.Is(err, sql.ErrNoRows) { + return ErrOperationalLeaseLost + } + if err != nil { + return errors.New("verify postgres orphan scan lease") + } + if _, err := tx.ExecContext(ctx, `INSERT INTO sense.orphan_scan_runs( + id, instance_id, observed_count, owned_stale_count, unowned_count, + safety_allowed, safety_reason, completed_at, expires_at + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`, + scan.ID, scan.InstanceID, scan.ObservedCount, scan.OwnedStaleCount, scan.UnownedCount, + scan.SafetyAllowed, scan.SafetyReason, scan.CompletedAt, scan.ExpiresAt, + ); err != nil { + return errors.New("insert postgres orphan scan") + } + for _, finding := range scan.Findings { + var deviceID any + if finding.DeviceID != "" { + deviceID = finding.DeviceID + } + if _, err := tx.ExecContext(ctx, `INSERT INTO sense.orphan_scan_findings( + scan_id, path_name, classification, device_id + ) VALUES ($1,$2,$3,$4)`, scan.ID, finding.PathName, finding.Classification, deviceID); err != nil { + return errors.New("insert postgres orphan scan finding") + } + } + if _, err := tx.ExecContext(ctx, `UPDATE sense.operational_leases + SET lease_until = clock_timestamp(), updated_at = clock_timestamp() + WHERE lease_name = $1 AND owner_id = $2 AND fencing_token = $3`, + OperationalLeaseOrphanScan, owner, token); err != nil { + return errors.New("release postgres orphan scan lease") + } + if _, err := tx.ExecContext(ctx, `DELETE FROM sense.orphan_scan_runs r + WHERE r.completed_at < $1 + AND NOT EXISTS ( + SELECT 1 FROM sense.orphan_cleanup_actions a WHERE a.scan_id = r.id + )`, scan.CompletedAt.Add(-orphanReportRetention)); err != nil { + return errors.New("expire postgres orphan scan reports") + } + if err := tx.Commit(); err != nil { + return errors.New("commit postgres orphan scan") + } + return nil +} + +func validateOrphanScan(scan OrphanScan) error { + if strings.TrimSpace(scan.ID) == "" || strings.TrimSpace(scan.InstanceID) == "" || + scan.ObservedCount < 0 || scan.OwnedStaleCount < 0 || scan.UnownedCount < 0 || + scan.OwnedStaleCount+scan.UnownedCount > scan.ObservedCount || + strings.TrimSpace(scan.SafetyReason) == "" || !scan.ExpiresAt.After(scan.CompletedAt) { + return errors.New("invalid orphan scan") + } + seen := make(map[string]struct{}, len(scan.Findings)) + staleCount, unownedCount := 0, 0 + for _, finding := range scan.Findings { + if strings.TrimSpace(finding.PathName) == "" { + return errors.New("invalid orphan finding path") + } + if _, duplicate := seen[finding.PathName]; duplicate { + return errors.New("duplicate orphan finding path") + } + seen[finding.PathName] = struct{}{} + switch finding.Classification { + case OrphanOwnedStale: + staleCount++ + if strings.TrimSpace(finding.DeviceID) == "" { + return errors.New("owned stale finding lacks device") + } + case OrphanUnowned: + unownedCount++ + if finding.DeviceID != "" { + return errors.New("unowned finding has device") + } + default: + return fmt.Errorf("invalid orphan finding classification %q", finding.Classification) + } + } + if staleCount != scan.OwnedStaleCount || unownedCount != scan.UnownedCount { + return errors.New("orphan scan counts do not match findings") + } + return nil +} + +func (s *Postgres) GetOrphanScan(ctx context.Context, id string) (OrphanScan, error) { + var scan OrphanScan + err := s.db.QueryRowContext(ctx, `SELECT + id, instance_id, observed_count, owned_stale_count, unowned_count, + safety_allowed, safety_reason, completed_at, expires_at + FROM sense.orphan_scan_runs WHERE id = $1`, id).Scan( + &scan.ID, &scan.InstanceID, &scan.ObservedCount, &scan.OwnedStaleCount, + &scan.UnownedCount, &scan.SafetyAllowed, &scan.SafetyReason, + &scan.CompletedAt, &scan.ExpiresAt, + ) + if errors.Is(err, sql.ErrNoRows) { + return OrphanScan{}, ErrOrphanScanNotFound + } + if err != nil { + return OrphanScan{}, errors.New("read postgres orphan scan") + } + rows, err := s.db.QueryContext(ctx, `SELECT + f.path_name, f.classification, COALESCE(f.device_id, ''), + COALESCE(a.status = 'deleted', false) + FROM sense.orphan_scan_findings f + LEFT JOIN sense.orphan_cleanup_actions a + ON a.scan_id = f.scan_id AND a.path_name = f.path_name + WHERE f.scan_id = $1 ORDER BY f.path_name`, id) + if err != nil { + return OrphanScan{}, errors.New("list postgres orphan scan findings") + } + defer rows.Close() + scan.Findings = make([]OrphanFinding, 0) + for rows.Next() { + var finding OrphanFinding + if err := rows.Scan( + &finding.PathName, &finding.Classification, &finding.DeviceID, &finding.Deleted, + ); err != nil { + return OrphanScan{}, errors.New("scan postgres orphan finding") + } + scan.Findings = append(scan.Findings, finding) + } + if err := rows.Err(); err != nil { + return OrphanScan{}, errors.New("iterate postgres orphan findings") + } + return scan, nil +} + +func (s *Postgres) RecordOrphanCleanup( + ctx context.Context, + scanID, pathName, actorID, status, errorCode string, + now time.Time, +) error { + if status != "deleted" && status != "failed" { + return errors.New("invalid orphan cleanup status") + } + var storedError any + if status == "failed" { + if strings.TrimSpace(errorCode) == "" { + return errors.New("failed orphan cleanup requires an error code") + } + storedError = errorCode + } + _, err := s.db.ExecContext(ctx, `INSERT INTO sense.orphan_cleanup_actions( + scan_id, path_name, classification, actor_id, status, error_code, attempted_at + ) VALUES ($1,$2,'owned_stale',$3,$4,$5,$6) + ON CONFLICT (scan_id, path_name) DO UPDATE SET + actor_id = EXCLUDED.actor_id, + status = EXCLUDED.status, + error_code = EXCLUDED.error_code, + attempted_at = EXCLUDED.attempted_at + WHERE sense.orphan_cleanup_actions.status <> 'deleted'`, + scanID, pathName, actorID, status, storedError, now) + if err != nil { + return errors.New("record postgres orphan cleanup result") + } + return nil +} diff --git a/Sense/internal/store/postgres.go b/Sense/internal/store/postgres.go index 980c59b..6ac06c8 100644 --- a/Sense/internal/store/postgres.go +++ b/Sense/internal/store/postgres.go @@ -53,8 +53,8 @@ func (s *Postgres) Close() error { func (s *Postgres) verifySchemaAndPrivileges(ctx context.Context) error { var version sql.NullInt64 if err := s.db.QueryRowContext(ctx, - `SELECT MAX(version) FROM sense.schema_migrations`).Scan(&version); err != nil || !version.Valid || version.Int64 < 4 { - return errors.New("postgres sense schema migration v4 is required") + `SELECT MAX(version) FROM sense.schema_migrations`).Scan(&version); err != nil || !version.Valid || version.Int64 < 5 { + return errors.New("postgres sense schema migration v5 is required") } var canReadQuotaView, canWriteQuotaView, canReadSiteSource, canWriteSiteSource bool var canReadAreaView, canWriteAreaView, canReadAreaSource, canWriteAreaSource bool @@ -95,6 +95,28 @@ func (s *Postgres) verifySchemaAndPrivileges(ctx context.Context) error { publicReceipts || publicOperations || publicOperationItems { return errors.New("postgres role violates Control API state privilege boundary") } + var canUseOwnership, canUseLeases, canUseScans, canUseFindings, canUseActions bool + var publicOwnership, publicLeases, publicScans, publicFindings, publicActions bool + if err := s.db.QueryRowContext(ctx, `SELECT + has_table_privilege(current_user, 'sense.media_path_ownership', 'SELECT,INSERT,UPDATE,DELETE'), + has_table_privilege(current_user, 'sense.operational_leases', 'SELECT,INSERT,UPDATE,DELETE'), + has_table_privilege(current_user, 'sense.orphan_scan_runs', 'SELECT,INSERT,UPDATE,DELETE'), + has_table_privilege(current_user, 'sense.orphan_scan_findings', 'SELECT,INSERT,UPDATE,DELETE'), + has_table_privilege(current_user, 'sense.orphan_cleanup_actions', 'SELECT,INSERT,UPDATE,DELETE'), + has_table_privilege('public', 'sense.media_path_ownership', 'SELECT,INSERT,UPDATE,DELETE'), + has_table_privilege('public', 'sense.operational_leases', 'SELECT,INSERT,UPDATE,DELETE'), + has_table_privilege('public', 'sense.orphan_scan_runs', 'SELECT,INSERT,UPDATE,DELETE'), + has_table_privilege('public', 'sense.orphan_scan_findings', 'SELECT,INSERT,UPDATE,DELETE'), + has_table_privilege('public', 'sense.orphan_cleanup_actions', 'SELECT,INSERT,UPDATE,DELETE')`).Scan( + &canUseOwnership, &canUseLeases, &canUseScans, &canUseFindings, &canUseActions, + &publicOwnership, &publicLeases, &publicScans, &publicFindings, &publicActions, + ); err != nil { + return errors.New("verify postgres reconciliation safety privileges") + } + if !canUseOwnership || !canUseLeases || !canUseScans || !canUseFindings || !canUseActions || + publicOwnership || publicLeases || publicScans || publicFindings || publicActions { + return errors.New("postgres role violates reconciliation safety privilege boundary") + } return nil } @@ -391,7 +413,8 @@ func (s *Postgres) SetDesiredState(ctx context.Context, id string, desired devic return errors.New("update postgres desired state") } if _, err := tx.ExecContext(ctx, `UPDATE sense.reconcile_state SET - failure_count = 0, next_attempt_at = NULL, last_error_code = NULL, updated_at = $1 + failure_count = 0, next_attempt_at = NULL, last_error_code = NULL, + lease_owner = NULL, lease_token = NULL, lease_until = NULL, updated_at = $1 WHERE device_id = $2`, now, id); err != nil { return errors.New("reset postgres reconcile state") } @@ -486,6 +509,217 @@ func (s *Postgres) ListDueReconcile(ctx context.Context, now time.Time, limit in return values, nil } +func (s *Postgres) ClaimDueReconcile( + ctx context.Context, + claim ReconcileClaim, +) ([]ReconcileCandidate, error) { + if claim.Limit <= 0 { + return nil, nil + } + if strings.TrimSpace(claim.Owner) == "" || strings.TrimSpace(claim.Token) == "" || + claim.LeaseDuration <= 0 { + return nil, errors.New("invalid postgres reconcile claim") + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return nil, errors.New("begin postgres reconcile claim") + } + defer tx.Rollback() + rows, err := tx.QueryContext(ctx, `SELECT `+postgresDeviceColumns+`, r.failure_count, r.next_attempt_at + FROM sense.devices d + JOIN sense.reconcile_state r ON r.device_id = d.id + WHERE EXISTS (SELECT 1 FROM sense.device_capabilities c + WHERE c.device_id = d.id AND c.capability = 'video_capture') + AND (r.observed_generation < d.generation + OR (d.desired_state = 'enabled' AND r.failure_count > 0)) + AND (r.next_attempt_at IS NULL OR r.next_attempt_at <= clock_timestamp()) + AND (r.lease_until IS NULL OR r.lease_until <= clock_timestamp()) + ORDER BY d.updated_at, d.id + FOR UPDATE OF r SKIP LOCKED + LIMIT $1`, claim.Limit) + if err != nil { + return nil, errors.New("select postgres reconcile claims") + } + values := make([]ReconcileCandidate, 0) + for rows.Next() { + var candidate ReconcileCandidate + var areaID sql.NullString + var quotaVersion, areaVersion sql.NullInt64 + var nextAttempt sql.NullTime + if err := rows.Scan( + &candidate.Device.ID, &candidate.Device.TenantID, &candidate.Device.SiteID, + &areaID, &candidate.Device.SerialNumber, &candidate.Device.Name, &candidate.Device.Modality, + &candidate.Device.DesiredState, &candidate.Device.ActualState, + &candidate.Device.EndpointRef, &candidate.Device.CredentialRef, + &candidate.Device.ProfileToken, &candidate.Device.PathName, + &candidate.Device.Generation, &candidate.Device.ResourceVersion, + "aVersion, &areaVersion, + &candidate.Device.CreatedAt, &candidate.Device.UpdatedAt, + &candidate.FailureCount, &nextAttempt, + ); err != nil { + rows.Close() + return nil, errors.New("scan postgres reconcile claim") + } + candidate.Device.AreaID = areaID.String + candidate.Device.QuotaSourceVersion = quotaVersion.Int64 + candidate.Device.AreaPolicySourceVersion = areaVersion.Int64 + if nextAttempt.Valid { + value := nextAttempt.Time + candidate.NextAttempt = &value + } + values = append(values, candidate) + } + if err := rows.Close(); err != nil { + return nil, errors.New("close postgres reconcile claims") + } + if err := rows.Err(); err != nil { + return nil, errors.New("iterate postgres reconcile claims") + } + for _, candidate := range values { + if _, err := tx.ExecContext(ctx, `UPDATE sense.reconcile_state SET + lease_owner = $1, lease_token = $2, + lease_until = clock_timestamp() + ($3 * interval '1 second'), + updated_at = clock_timestamp() + WHERE device_id = $4`, + claim.Owner, claim.Token, claim.LeaseDuration.Seconds(), candidate.Device.ID, + ); err != nil { + return nil, errors.New("persist postgres reconcile claim") + } + } + if err := tx.Commit(); err != nil { + return nil, errors.New("commit postgres reconcile claim") + } + for index := range values { + values[index].Device.Capabilities, err = s.capabilities(ctx, values[index].Device.ID) + if err != nil { + return nil, err + } + } + return values, nil +} + +func (s *Postgres) RenewReconcileLease( + ctx context.Context, + id, owner, token string, + now time.Time, + duration time.Duration, +) (bool, error) { + if strings.TrimSpace(owner) == "" || strings.TrimSpace(token) == "" || duration <= 0 { + return false, errors.New("invalid postgres reconcile lease renewal") + } + result, err := s.db.ExecContext(ctx, `UPDATE sense.reconcile_state SET + lease_until = clock_timestamp() + ($1 * interval '1 second'), + updated_at = clock_timestamp() + WHERE device_id = $2 AND lease_owner = $3 AND lease_token = $4 + AND lease_until > clock_timestamp()`, duration.Seconds(), id, owner, token) + if err != nil { + return false, errors.New("renew postgres reconcile lease") + } + affected, err := result.RowsAffected() + if err != nil { + return false, errors.New("read postgres reconcile lease renewal") + } + return affected == 1, nil +} + +func (s *Postgres) CompleteReconcile( + ctx context.Context, + id string, + generation int64, + owner, token string, + now time.Time, +) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return errors.New("begin postgres fenced reconciliation completion") + } + defer tx.Rollback() + result, err := tx.ExecContext(ctx, `UPDATE sense.reconcile_state SET + failure_count = 0, next_attempt_at = NULL, last_error_code = NULL, + observed_generation = $1, lease_owner = NULL, lease_token = NULL, + lease_until = NULL, updated_at = $2 + WHERE device_id = $3 AND lease_owner = $4 AND lease_token = $5 + AND lease_until > clock_timestamp()`, generation, now, id, owner, token) + if err != nil { + return errors.New("complete postgres fenced reconciliation") + } + affected, err := result.RowsAffected() + if err != nil { + return errors.New("read postgres fenced reconciliation completion") + } + if affected != 1 { + return ErrReconcileLeaseLost + } + if _, err := tx.ExecContext(ctx, `INSERT INTO sense.media_path_ownership( + path_name, device_id, tenant_id, site_id, first_claimed_at, last_confirmed_at + ) + SELECT path_name, id, tenant_id, site_id, $1, $1 + FROM sense.devices + WHERE id = $2 AND desired_state = 'enabled' AND btrim(path_name) <> '' + ON CONFLICT (path_name) DO UPDATE SET + device_id = EXCLUDED.device_id, + tenant_id = EXCLUDED.tenant_id, + site_id = EXCLUDED.site_id, + last_confirmed_at = EXCLUDED.last_confirmed_at`, now, id); err != nil { + return errors.New("record postgres MediaMTX path ownership") + } + result, err = tx.ExecContext(ctx, `UPDATE sense.devices + SET actual_state = CASE WHEN desired_state = 'disabled' THEN 'offline' ELSE 'pending' END, + updated_at = $1 WHERE id = $2`, now, id) + if err != nil { + return errors.New("mark postgres fenced device state") + } + if affected, _ := result.RowsAffected(); affected != 1 { + return ErrNotFound + } + if err := tx.Commit(); err != nil { + return errors.New("commit postgres fenced reconciliation completion") + } + return nil +} + +func (s *Postgres) FailReconcile( + ctx context.Context, + id string, + failureCount int, + nextAttempt time.Time, + errorCode, owner, token string, + now time.Time, +) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return errors.New("begin postgres fenced reconciliation failure") + } + defer tx.Rollback() + result, err := tx.ExecContext(ctx, `UPDATE sense.reconcile_state SET + failure_count = $1, next_attempt_at = $2, last_error_code = $3, + lease_owner = NULL, lease_token = NULL, lease_until = NULL, updated_at = $4 + WHERE device_id = $5 AND lease_owner = $6 AND lease_token = $7 + AND lease_until > clock_timestamp()`, failureCount, nextAttempt, errorCode, now, id, owner, token) + if err != nil { + return errors.New("persist postgres fenced reconciliation failure") + } + affected, err := result.RowsAffected() + if err != nil { + return errors.New("read postgres fenced reconciliation failure") + } + if affected != 1 { + return ErrReconcileLeaseLost + } + result, err = tx.ExecContext(ctx, `UPDATE sense.devices + SET actual_state = 'failed', updated_at = $1 WHERE id = $2`, now, id) + if err != nil { + return errors.New("mark postgres fenced failed device") + } + if affected, _ := result.RowsAffected(); affected != 1 { + return ErrNotFound + } + if err := tx.Commit(); err != nil { + return errors.New("commit postgres fenced reconciliation failure") + } + return nil +} + func (s *Postgres) ListEnabledVideoDevices(ctx context.Context, limit int) ([]device.Device, error) { if limit <= 0 { return nil, nil diff --git a/Sense/internal/store/postgres_test.go b/Sense/internal/store/postgres_test.go index ac7fa45..6b0069f 100644 --- a/Sense/internal/store/postgres_test.go +++ b/Sense/internal/store/postgres_test.go @@ -287,12 +287,12 @@ func TestPostgresOpenRejectsOverprivilegedRuntimeRole(t *testing.T) { _, admin := openPostgresTestStore(t) ctx := context.Background() if _, err := admin.ExecContext(ctx, - `GRANT UPDATE ON bell.site_quota_v1 TO yovision_t011_sense`); err != nil { + `GRANT UPDATE ON bell.site_quota_v1 TO yovision_t012_sense`); err != nil { t.Fatal(err) } defer func() { _, _ = admin.ExecContext(context.Background(), - `REVOKE UPDATE ON bell.site_quota_v1 FROM yovision_t011_sense`) + `REVOKE UPDATE ON bell.site_quota_v1 FROM yovision_t012_sense`) }() value, err := OpenPostgres(ctx, os.Getenv(postgresTestDSNEnv)) if value != nil { @@ -304,6 +304,27 @@ func TestPostgresOpenRejectsOverprivilegedRuntimeRole(t *testing.T) { } } +func TestPostgresOpenRejectsPublicReconciliationStatePrivilege(t *testing.T) { + _, admin := openPostgresTestStore(t) + ctx := context.Background() + if _, err := admin.ExecContext(ctx, + `GRANT SELECT ON sense.orphan_scan_runs TO PUBLIC`); err != nil { + t.Fatal(err) + } + defer func() { + _, _ = admin.ExecContext(context.Background(), + `REVOKE SELECT ON sense.orphan_scan_runs FROM PUBLIC`) + }() + value, err := OpenPostgres(ctx, os.Getenv(postgresTestDSNEnv)) + if value != nil { + _ = value.Close() + t.Fatal("PUBLIC reconciliation state privilege was accepted") + } + if err == nil || !strings.Contains(err.Error(), "reconciliation safety privilege boundary") { + t.Fatalf("expected reconciliation privilege-boundary error, got %v", err) + } +} + func TestPostgresAreaPolicyAllowsNonImagingAndDeniesImagingCreate(t *testing.T) { store, admin := openPostgresTestStore(t) ctx := context.Background() @@ -597,12 +618,12 @@ func TestPostgresOpenRejectsAreaSourcePrivilege(t *testing.T) { _, admin := openPostgresTestStore(t) ctx := context.Background() if _, err := admin.ExecContext(ctx, - `GRANT SELECT ON bell.areas TO yovision_t011_sense`); err != nil { + `GRANT SELECT ON bell.areas TO yovision_t012_sense`); err != nil { t.Fatal(err) } defer func() { _, _ = admin.ExecContext(context.Background(), - `REVOKE SELECT ON bell.areas FROM yovision_t011_sense`) + `REVOKE SELECT ON bell.areas FROM yovision_t012_sense`) }() value, err := OpenPostgres(ctx, os.Getenv(postgresTestDSNEnv)) if value != nil { @@ -969,6 +990,180 @@ func TestPostgresConcurrentControlBatchesUseStableDeviceLockOrder(t *testing.T) } } +func TestPostgresConcurrentReconcileClaimHasOneWinner(t *testing.T) { + first, admin := openPostgresTestStore(t) + insertBellSite(t, admin, "tenant", "site", 2) + if err := first.CreateDevice(context.Background(), videoDevice(1, "tenant", "site")); err != nil { + t.Fatal(err) + } + second, err := OpenPostgres(context.Background(), os.Getenv(postgresTestDSNEnv)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = second.Close() }) + now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC) + start := make(chan struct{}) + counts := make(chan int, 2) + errorsFound := make(chan error, 2) + var wait sync.WaitGroup + for index, repository := range []*Postgres{first, second} { + wait.Add(1) + go func(index int, repository *Postgres) { + defer wait.Done() + <-start + values, err := repository.ClaimDueReconcile(context.Background(), ReconcileClaim{ + Owner: fmt.Sprintf("ins-%d", index), Token: fmt.Sprintf("token-%d", index), + Now: now, LeaseDuration: 30 * time.Second, Limit: 1, + }) + if err != nil { + errorsFound <- err + return + } + counts <- len(values) + }(index, repository) + } + close(start) + wait.Wait() + close(counts) + close(errorsFound) + for err := range errorsFound { + t.Fatal(err) + } + total, winners := 0, 0 + for count := range counts { + total += count + if count == 1 { + winners++ + } + } + if total != 1 || winners != 1 { + t.Fatalf("due row was not exclusively claimed: total=%d winners=%d", total, winners) + } +} + +func TestPostgresExpiredReconcileLeaseFencesOldWorkerAndRecordsOwnership(t *testing.T) { + postgres, admin := openPostgresTestStore(t) + insertBellSite(t, admin, "tenant", "site", 2) + value := videoDevice(1, "tenant", "site") + if err := postgres.CreateDevice(context.Background(), value); err != nil { + t.Fatal(err) + } + now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC) + first, err := postgres.ClaimDueReconcile(context.Background(), ReconcileClaim{ + Owner: "ins-a", Token: "token-a", Now: now, LeaseDuration: 30 * time.Second, Limit: 1, + }) + if err != nil || len(first) != 1 { + t.Fatalf("first claim failed: %+v %v", first, err) + } + early, err := postgres.ClaimDueReconcile(context.Background(), ReconcileClaim{ + Owner: "ins-b", Token: "token-b", Now: now.Add(10 * time.Second), + LeaseDuration: 30 * time.Second, Limit: 1, + }) + if err != nil || len(early) != 0 { + t.Fatalf("live lease was stolen: %+v %v", early, err) + } + if _, err := admin.Exec(`UPDATE sense.reconcile_state + SET lease_until = clock_timestamp() - interval '1 second' + WHERE device_id = $1`, value.ID); err != nil { + t.Fatal(err) + } + second, err := postgres.ClaimDueReconcile(context.Background(), ReconcileClaim{ + Owner: "ins-b", Token: "token-b", Now: now.Add(31 * time.Second), + LeaseDuration: 30 * time.Second, Limit: 1, + }) + if err != nil || len(second) != 1 { + t.Fatalf("expired lease was not recoverable: %+v %v", second, err) + } + if err := postgres.CompleteReconcile( + context.Background(), value.ID, value.Generation, "ins-a", "token-a", now.Add(32*time.Second), + ); !errors.Is(err, ErrReconcileLeaseLost) { + t.Fatalf("old worker was not fenced: %v", err) + } + if err := postgres.CompleteReconcile( + context.Background(), value.ID, value.Generation, "ins-b", "token-b", now.Add(32*time.Second), + ); err != nil { + t.Fatal(err) + } + var ownershipDevice string + if err := admin.QueryRow(`SELECT device_id FROM sense.media_path_ownership WHERE path_name = $1`, + value.PathName).Scan(&ownershipDevice); err != nil { + t.Fatal(err) + } + if ownershipDevice != value.ID { + t.Fatalf("wrong ownership was recorded: %q", ownershipDevice) + } + var leaseToken sql.NullString + if err := admin.QueryRow(`SELECT lease_token FROM sense.reconcile_state WHERE device_id = $1`, + value.ID).Scan(&leaseToken); err != nil { + t.Fatal(err) + } + if leaseToken.Valid { + t.Fatal("completion did not release the reconcile lease") + } +} + +func TestPostgresOrphanReportLeaseAndCleanupAuditAreFencedAndIdempotent(t *testing.T) { + postgres, _ := openPostgresTestStore(t) + ctx := context.Background() + now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC) + acquired, err := postgres.AcquireOperationalLease( + ctx, OperationalLeaseOrphanScan, "ins-a", "scan-token-a", now, 30*time.Second, + ) + if err != nil || !acquired { + t.Fatalf("scan lease failed: %v %v", acquired, err) + } + scan := OrphanScan{ + ID: "scan_" + strings.Repeat("0", 26), InstanceID: "ins-a", + ObservedCount: 10, OwnedStaleCount: 1, UnownedCount: 1, + SafetyAllowed: true, SafetyReason: "allowed", + CompletedAt: now.Add(time.Second), ExpiresAt: now.Add(15 * time.Minute), + Findings: []OrphanFinding{ + {PathName: "stale", Classification: OrphanOwnedStale, DeviceID: "old-device"}, + {PathName: "unknown", Classification: OrphanUnowned}, + }, + } + if err := postgres.SaveOrphanScan(ctx, scan, "ins-a", "scan-token-a"); err != nil { + t.Fatal(err) + } + loaded, err := postgres.GetOrphanScan(ctx, scan.ID) + if err != nil || len(loaded.Findings) != 2 || !loaded.SafetyAllowed { + t.Fatalf("stored scan mismatch: %+v %v", loaded, err) + } + if err := postgres.RecordOrphanCleanup( + ctx, scan.ID, "stale", "operator", "deleted", "", now.Add(2*time.Second), + ); err != nil { + t.Fatal(err) + } + if err := postgres.RecordOrphanCleanup( + ctx, scan.ID, "unknown", "operator", "deleted", "", now.Add(2*time.Second), + ); err == nil { + t.Fatal("unowned path accepted a cleanup audit record") + } + if err := postgres.RecordOrphanCleanup( + ctx, scan.ID, "stale", "operator-2", "failed", "media_error", now.Add(3*time.Second), + ); err != nil { + t.Fatal(err) + } + loaded, err = postgres.GetOrphanScan(ctx, scan.ID) + if err != nil || !loaded.Findings[0].Deleted { + t.Fatalf("successful cleanup was downgraded: %+v %v", loaded, err) + } + + acquired, err = postgres.AcquireOperationalLease( + ctx, OperationalLeaseOrphanScan, "ins-b", "scan-token-b", now.Add(31*time.Second), 30*time.Second, + ) + if err != nil || !acquired { + t.Fatalf("expired scan lease was not recoverable: %v %v", acquired, err) + } + staleScan := scan + staleScan.ID = "scan_" + strings.Repeat("1", 26) + staleScan.CompletedAt = now.Add(32 * time.Second) + staleScan.ExpiresAt = staleScan.CompletedAt.Add(15 * time.Minute) + if err := postgres.SaveOrphanScan(ctx, staleScan, "ins-a", "scan-token-a"); !errors.Is(err, ErrOperationalLeaseLost) { + t.Fatalf("stale scan worker was not fenced: %v", err) + } +} + func openPostgresTestStore(t *testing.T) (*Postgres, *sql.DB) { t.Helper() dsn := os.Getenv(postgresTestDSNEnv) @@ -985,6 +1180,11 @@ func openPostgresTestStore(t *testing.T) (*Postgres, *sql.DB) { t.Fatal("connect PostgreSQL test administrator") } if _, err := admin.ExecContext(context.Background(), `TRUNCATE + sense.orphan_cleanup_actions, + sense.orphan_scan_findings, + sense.orphan_scan_runs, + sense.operational_leases, + sense.media_path_ownership, sense.control_idempotency_receipts, sense.batch_operation_items, sense.batch_operations, diff --git a/Sense/internal/store/repository.go b/Sense/internal/store/repository.go index 50d3867..351f5b0 100644 --- a/Sense/internal/store/repository.go +++ b/Sense/internal/store/repository.go @@ -21,6 +21,8 @@ var ( ErrAreaPolicyUnavailable = errors.New("area policy unavailable") ErrAreaPolicyInvalid = errors.New("area policy invalid") ErrAreaPolicyDenied = errors.New("area policy denies imaging device") + ErrReconcileLeaseLost = errors.New("reconcile lease lost") + ErrOperationalLeaseLost = errors.New("operational lease lost") ) // Repository is the storage boundary used by the Sense process. SQLite stays @@ -31,6 +33,10 @@ type Repository interface { SetDesiredState(context.Context, string, device.DesiredState) error GetDevice(context.Context, string) (device.Device, error) ListDueReconcile(context.Context, time.Time, int) ([]ReconcileCandidate, error) + ClaimDueReconcile(context.Context, ReconcileClaim) ([]ReconcileCandidate, error) + RenewReconcileLease(context.Context, string, string, string, time.Time, time.Duration) (bool, error) + CompleteReconcile(context.Context, string, int64, string, string, time.Time) error + FailReconcile(context.Context, string, int, time.Time, string, string, string, time.Time) error ListEnabledVideoDevices(context.Context, int) ([]device.Device, error) MarkReconciled(context.Context, string, int64, time.Time) error MarkReconcileFailure(context.Context, string, int, time.Time, string, time.Time) error @@ -39,6 +45,17 @@ type Repository interface { ConvergenceSnapshot(context.Context) (ConvergenceSnapshot, error) } +// ReconcileClaim identifies one short-lived batch claim. Token is unique per +// run and fences a worker whose lease expired and was acquired by another +// process. SQLite accepts the shape but remains explicitly single-process. +type ReconcileClaim struct { + Owner string + Token string + Now time.Time + LeaseDuration time.Duration + Limit int +} + func OpenRepository(ctx context.Context, driver, dsn string) (Repository, error) { switch strings.ToLower(strings.TrimSpace(driver)) { case "", DriverSQLite: diff --git a/Sense/internal/store/sqlite.go b/Sense/internal/store/sqlite.go index 9c9fc95..9e5e77f 100644 --- a/Sense/internal/store/sqlite.go +++ b/Sense/internal/store/sqlite.go @@ -430,6 +430,24 @@ func (s *SQLite) ListDueReconcile(ctx context.Context, now time.Time, limit int) return candidates, nil } +func (s *SQLite) ClaimDueReconcile(ctx context.Context, claim ReconcileClaim) ([]ReconcileCandidate, error) { + // SQLite is retained for one-process M1 development. It deliberately does + // not claim cross-process leases; PostgreSQL is the production M2 boundary. + return s.ListDueReconcile(ctx, claim.Now, claim.Limit) +} + +func (s *SQLite) RenewReconcileLease( + ctx context.Context, + _, _, _ string, + _ time.Time, + _ time.Duration, +) (bool, error) { + if err := ctx.Err(); err != nil { + return false, err + } + return true, nil +} + func (s *SQLite) ListEnabledVideoDevices(ctx context.Context, limit int) ([]device.Device, error) { if limit <= 0 { return nil, nil @@ -499,6 +517,16 @@ func (s *SQLite) MarkReconciled(ctx context.Context, id string, generation int64 return nil } +func (s *SQLite) CompleteReconcile( + ctx context.Context, + id string, + generation int64, + _, _ string, + now time.Time, +) error { + return s.MarkReconciled(ctx, id, generation, now) +} + func (s *SQLite) MarkReconcileFailure(ctx context.Context, id string, failureCount int, nextAttempt time.Time, errorCode string, now time.Time) error { tx, err := s.db.BeginTx(ctx, nil) if err != nil { @@ -526,6 +554,17 @@ func (s *SQLite) MarkReconcileFailure(ctx context.Context, id string, failureCou return nil } +func (s *SQLite) FailReconcile( + ctx context.Context, + id string, + failureCount int, + nextAttempt time.Time, + errorCode, _, _ string, + now time.Time, +) error { + return s.MarkReconcileFailure(ctx, id, failureCount, nextAttempt, errorCode, now) +} + func (s *SQLite) UpdateActualState(ctx context.Context, id string, state device.ActualState, now time.Time) error { if state != device.ActualPending && state != device.ActualOnline && state != device.ActualOffline && state != device.ActualFailed { return fmt.Errorf("invalid actual state %q", state) diff --git a/deploy/postgres/010_reconcile_safety.sql b/deploy/postgres/010_reconcile_safety.sql new file mode 100644 index 0000000..6b7371e --- /dev/null +++ b/deploy/postgres/010_reconcile_safety.sql @@ -0,0 +1,147 @@ +-- Add fenced reconciliation claims and durable, secret-free MediaMTX path +-- ownership/orphan reports. Runtime migrations are installed out of process. + +ALTER TABLE sense.reconcile_state + ADD COLUMN IF NOT EXISTS lease_owner text, + ADD COLUMN IF NOT EXISTS lease_token text, + ADD COLUMN IF NOT EXISTS lease_until timestamptz; + +DO $constraints$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'sense.reconcile_state'::regclass + AND conname = 'sense_reconcile_lease_all_or_none' + ) THEN + ALTER TABLE sense.reconcile_state + ADD CONSTRAINT sense_reconcile_lease_all_or_none CHECK ( + (lease_owner IS NULL AND lease_token IS NULL AND lease_until IS NULL) + OR (btrim(lease_owner) <> '' AND btrim(lease_token) <> '' AND lease_until IS NOT NULL) + ); + END IF; +END +$constraints$; + +CREATE INDEX IF NOT EXISTS sense_reconcile_lease_due_idx + ON sense.reconcile_state(lease_until, next_attempt_at); + +CREATE TABLE IF NOT EXISTS sense.media_path_ownership ( + path_name text PRIMARY KEY, + device_id text NOT NULL, + tenant_id text NOT NULL, + site_id text NOT NULL, + first_claimed_at timestamptz NOT NULL, + last_confirmed_at timestamptz NOT NULL, + CONSTRAINT sense_media_path_ownership_not_blank CHECK ( + btrim(path_name) <> '' AND btrim(device_id) <> '' + AND btrim(tenant_id) <> '' AND btrim(site_id) <> '' + ) +); +ALTER TABLE sense.media_path_ownership OWNER TO sense_app; + +-- Keep the historical row independent from device lifecycle so a later device +-- deletion cannot erase the evidence needed to classify an owned stale path. +INSERT INTO sense.media_path_ownership( + path_name, device_id, tenant_id, site_id, first_claimed_at, last_confirmed_at +) +SELECT d.path_name, d.id, d.tenant_id, d.site_id, d.created_at, d.updated_at +FROM sense.devices d +WHERE btrim(d.path_name) <> '' + AND EXISTS ( + SELECT 1 FROM sense.device_capabilities c + WHERE c.device_id = d.id AND c.capability = 'video_capture' + ) +ON CONFLICT (path_name) DO UPDATE SET + device_id = EXCLUDED.device_id, + tenant_id = EXCLUDED.tenant_id, + site_id = EXCLUDED.site_id, + last_confirmed_at = GREATEST( + sense.media_path_ownership.last_confirmed_at, + EXCLUDED.last_confirmed_at + ); + +CREATE TABLE IF NOT EXISTS sense.operational_leases ( + lease_name text PRIMARY KEY, + owner_id text NOT NULL, + fencing_token text NOT NULL, + lease_until timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + CONSTRAINT sense_operational_lease_not_blank CHECK ( + btrim(lease_name) <> '' AND btrim(owner_id) <> '' AND btrim(fencing_token) <> '' + ) +); +ALTER TABLE sense.operational_leases OWNER TO sense_app; + +CREATE TABLE IF NOT EXISTS sense.orphan_scan_runs ( + id text PRIMARY KEY, + instance_id text NOT NULL, + observed_count integer NOT NULL, + owned_stale_count integer NOT NULL, + unowned_count integer NOT NULL, + safety_allowed boolean NOT NULL, + safety_reason text NOT NULL, + completed_at timestamptz NOT NULL, + expires_at timestamptz NOT NULL, + CONSTRAINT sense_orphan_scan_id CHECK (id ~ '^scan_[0-9A-HJKMNP-TV-Z]{26}$'), + CONSTRAINT sense_orphan_scan_counts CHECK ( + observed_count >= 0 AND owned_stale_count >= 0 AND unowned_count >= 0 + AND owned_stale_count + unowned_count <= observed_count + ), + CONSTRAINT sense_orphan_scan_not_blank CHECK ( + btrim(instance_id) <> '' AND btrim(safety_reason) <> '' + ), + CONSTRAINT sense_orphan_scan_expiry CHECK (expires_at > completed_at) +); +ALTER TABLE sense.orphan_scan_runs OWNER TO sense_app; + +CREATE TABLE IF NOT EXISTS sense.orphan_scan_findings ( + scan_id text NOT NULL REFERENCES sense.orphan_scan_runs(id) ON DELETE CASCADE, + path_name text NOT NULL, + classification text NOT NULL, + device_id text, + PRIMARY KEY (scan_id, path_name), + UNIQUE (scan_id, path_name, classification), + CONSTRAINT sense_orphan_finding_classification CHECK ( + classification IN ('owned_stale', 'unowned') + ), + CONSTRAINT sense_orphan_finding_not_blank CHECK ( + btrim(path_name) <> '' AND (device_id IS NULL OR btrim(device_id) <> '') + ), + CONSTRAINT sense_orphan_finding_owner_shape CHECK ( + (classification = 'owned_stale' AND device_id IS NOT NULL) + OR (classification = 'unowned' AND device_id IS NULL) + ) +); +ALTER TABLE sense.orphan_scan_findings OWNER TO sense_app; + +CREATE TABLE IF NOT EXISTS sense.orphan_cleanup_actions ( + scan_id text NOT NULL REFERENCES sense.orphan_scan_runs(id), + path_name text NOT NULL, + classification text NOT NULL DEFAULT 'owned_stale', + actor_id text NOT NULL, + status text NOT NULL, + error_code text, + attempted_at timestamptz NOT NULL, + PRIMARY KEY (scan_id, path_name), + FOREIGN KEY (scan_id, path_name, classification) + REFERENCES sense.orphan_scan_findings(scan_id, path_name, classification), + CONSTRAINT sense_orphan_cleanup_owned_only CHECK (classification = 'owned_stale'), + CONSTRAINT sense_orphan_cleanup_status CHECK (status IN ('deleted', 'failed')), + CONSTRAINT sense_orphan_cleanup_not_blank CHECK ( + btrim(path_name) <> '' AND btrim(actor_id) <> '' + AND (error_code IS NULL OR btrim(error_code) <> '') + ), + CONSTRAINT sense_orphan_cleanup_error_shape CHECK ( + (status = 'deleted' AND error_code IS NULL) + OR (status = 'failed' AND error_code IS NOT NULL) + ) +); +ALTER TABLE sense.orphan_cleanup_actions OWNER TO sense_app; + +CREATE INDEX IF NOT EXISTS sense_orphan_scan_completed_idx + ON sense.orphan_scan_runs(completed_at); +CREATE INDEX IF NOT EXISTS sense_orphan_cleanup_status_idx + ON sense.orphan_cleanup_actions(scan_id, status); + +INSERT INTO sense.schema_migrations(version) VALUES (5) +ON CONFLICT (version) DO NOTHING; diff --git a/deploy/postgres/011_privileges_reconcile_safety.sql b/deploy/postgres/011_privileges_reconcile_safety.sql new file mode 100644 index 0000000..5af5a8d --- /dev/null +++ b/deploy/postgres/011_privileges_reconcile_safety.sql @@ -0,0 +1,17 @@ +-- Extend the least-privilege boundary to reconciliation and orphan state. + +REVOKE ALL ON TABLE + sense.media_path_ownership, + sense.operational_leases, + sense.orphan_scan_runs, + sense.orphan_scan_findings, + sense.orphan_cleanup_actions +FROM PUBLIC; + +GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE + sense.media_path_ownership, + sense.operational_leases, + sense.orphan_scan_runs, + sense.orphan_scan_findings, + sense.orphan_cleanup_actions +TO sense_app; diff --git a/deploy/postgres/README.md b/deploy/postgres/README.md index 8f8160b..76c89be 100644 --- a/deploy/postgres/README.md +++ b/deploy/postgres/README.md @@ -1,6 +1,6 @@ # YoVision PostgreSQL 初始化 -本目录实现 T-009～T-011 的 PostgreSQL `17.10` schema。SQL 必须按文件名前缀顺序执行：`001`～`004` 创建 NOLOGIN 权限角色、Bell/Sense 初始对象和配额权限；`005`～`007` 增量增加 Area 与审计；`008`～`009` 增加 Control API 的资源版本、write-only profile token、24 小时幂等收据、batch operation 和对应最小权限。全部 SQL 可重放。应用登录角色及其密码由部署环境或密钥系统创建，并分别加入 `bell_app`、`sense_app`；仓库不保存登录凭据。 +本目录实现 T-009～T-012 的 PostgreSQL `17.10` schema。SQL 必须按文件名前缀顺序执行：`001`～`004` 创建 NOLOGIN 权限角色、Bell/Sense 初始对象和配额权限；`005`～`007` 增量增加 Area 与审计；`008`～`009` 增加 Control API 状态；`010`～`011` 增加调和 fencing、MediaMTX Path 历史归属、孤儿报告/受控处置结果和对应最小权限。全部 SQL 可重放。应用登录角色及其密码由部署环境或密钥系统创建，并分别加入 `bell_app`、`sense_app`；仓库不保存登录凭据。 生产/共享实例必须由管理员先备份并在 YoVision 专用数据库中执行。Sense 进程不会用高权限自动建库或建角色。示例只使用私有环境变量，不把实际 DSN 写入脚本或日志： @@ -22,8 +22,9 @@ Get-ChildItem deploy/postgres/[0-9][0-9][0-9]_*.sql | - `sense_app` 对 Bell 源表、Bell migration 表和 trigger function 没有权限；启动检查发现权限过宽时拒绝运行。 - `sense.device_operation_outbox` 是本地持久化审计事实，不是 Bell 全局审计真相；relay 的传输、签名、确认和留存尚未实现。 - `sense.control_idempotency_receipts` 不保存原始 Idempotency-Key，只保存 scope/request SHA-256 和脱敏响应快照；`batch_operations`/items 只保存逻辑 ID、状态和稳定错误，不保存连接秘密。 +- 调和与孤儿租约使用 PostgreSQL `clock_timestamp()` 和 fencing token；过期 worker 不能提交完成/失败或扫描报告。`media_path_ownership`、扫描和处置表不保存 endpoint、credential 或 source URI；数据库约束禁止为 `unowned` finding 写删除结果。 - PUBLIC 对两个业务 schema 的表和函数没有权限。 ## 恢复 -T-009～T-011 不迁移或删除现有 SQLite 数据，默认运行配置也仍为 SQLite。SQLite 不具备生产 Area/Outbox/Control API 一致性，业务路由不会在 SQLite 上注册。若 PostgreSQL 切换验证失败，先关闭 Control API feature flag；需要实验室回退时停止 PostgreSQL 模式进程并恢复 SQLite DSN，不得误报为生产语义等价。schema 不提供自动 down migration；需要移除时由管理员先备份，再只删除明确确认的 YoVision 专用数据库，禁止对共享实例做模糊或递归清理。 +T-009～T-012 不迁移或删除现有 SQLite 数据，默认运行配置也仍为 SQLite。SQLite 不具备生产 Area/Outbox/Control API、多实例租约或孤儿处置一致性，业务路由不会在 SQLite 上注册。若 PostgreSQL 切换验证失败，先关闭 Control API 和孤儿扫描；需要实验室回退时停止 PostgreSQL 模式进程并恢复 SQLite DSN，不得误报为生产语义等价。schema 不提供自动 down migration；需要移除时由管理员先备份，再只删除明确确认的 YoVision 专用数据库，禁止对共享实例做模糊或递归清理。 diff --git a/deploy/postgres/tests/assertions.sql b/deploy/postgres/tests/assertions.sql index c9e89ab..82fc204 100644 --- a/deploy/postgres/tests/assertions.sql +++ b/deploy/postgres/tests/assertions.sql @@ -72,7 +72,7 @@ BEGIN RAISE EXCEPTION 'sense_app lacks access to its local audit Outbox'; END IF; IF (SELECT max(version) FROM bell.schema_migrations) <> 2 - OR (SELECT max(version) FROM sense.schema_migrations) <> 4 THEN + OR (SELECT max(version) FROM sense.schema_migrations) <> 5 THEN RAISE EXCEPTION 'schema migration version drift'; END IF; @@ -129,9 +129,9 @@ $assertions$; DO $control_api$ BEGIN - IF NOT has_table_privilege('yovision_t011_sense', 'sense.control_idempotency_receipts', 'SELECT,INSERT,UPDATE,DELETE') - OR NOT has_table_privilege('yovision_t011_sense', 'sense.batch_operations', 'SELECT,INSERT,UPDATE,DELETE') - OR NOT has_table_privilege('yovision_t011_sense', 'sense.batch_operation_items', 'SELECT,INSERT,UPDATE,DELETE') THEN + IF NOT has_table_privilege('yovision_t012_sense', 'sense.control_idempotency_receipts', 'SELECT,INSERT,UPDATE,DELETE') + OR NOT has_table_privilege('yovision_t012_sense', 'sense.batch_operations', 'SELECT,INSERT,UPDATE,DELETE') + OR NOT has_table_privilege('yovision_t012_sense', 'sense.batch_operation_items', 'SELECT,INSERT,UPDATE,DELETE') THEN RAISE EXCEPTION 'Sense runtime lacks Control API table privileges'; END IF; IF has_table_privilege('public', 'sense.control_idempotency_receipts', 'SELECT') @@ -151,3 +151,31 @@ BEGIN END IF; END $control_api$; + +DO $reconcile_safety$ +DECLARE + target text; +BEGIN + FOREACH target IN ARRAY ARRAY[ + 'sense.media_path_ownership', + 'sense.operational_leases', + 'sense.orphan_scan_runs', + 'sense.orphan_scan_findings', + 'sense.orphan_cleanup_actions' + ] LOOP + IF NOT has_table_privilege('yovision_t012_sense', target, 'SELECT,INSERT,UPDATE,DELETE') THEN + RAISE EXCEPTION 'Sense runtime lacks reconciliation safety privilege on %', target; + END IF; + IF has_table_privilege('public', target, 'SELECT,INSERT,UPDATE,DELETE') THEN + RAISE EXCEPTION 'Reconciliation safety state leaked to PUBLIC on %', target; + END IF; + END LOOP; + IF NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'sense' AND table_name = 'reconcile_state' + AND column_name = 'lease_token' + ) THEN + RAISE EXCEPTION 'reconcile fencing columns are missing'; + END IF; +END +$reconcile_safety$; diff --git a/docs/00-ai-start-here.md b/docs/00-ai-start-here.md index 7c08673..0d43d96 100644 --- a/docs/00-ai-start-here.md +++ b/docs/00-ai-start-here.md @@ -40,13 +40,13 @@ MVP 以默认 16 路跑通一个场景的端到端闭环；架构、数据和 UI ## 当前阶段 -当前为 **M0 指定型号实机准入已完成，M1 Sense 五路混合源集成待执行**。后续本地开发统一使用已准入的一台海康样机，多路软件闭环使用独立合成 RTSP 源补足；真实多设备证据延后到客户/借用/租赁条件具备时执行。 +当前为 **M0 指定型号实机准入与 M1 Sense 五路混合源集成均已完成，M2 正在补齐生产接入边界**。后续本地开发统一使用已准入的一台海康样机，多路软件闭环使用独立合成 RTSP 源补足；真实多设备证据延后到客户/借用/租赁条件具备时执行。 优先路径： 1. M0：已用一台真实样机完成首期指定“型号 + 硬件版本 + 固件”ONVIF/RTSP 准入；结论不外推为多品牌兼容，真实断网恢复证据按负责人豁免留痕。 2. M1：只在 `Sense/` 建立 MediaMTX 生产接入骨架，以 1 路准入实机 + 至少 4 路独立合成源完成五路自动建 path、探活和断线重建。 -3. M2：对账、多租户、隧道和至少一个站点的 16 路全流程。 +3. M2：Control API、多租户投影、调和 fencing、孤儿安全闸已完成；下一步是隧道和至少一个站点的 16 路全流程。 4. M3：Brain + Bell 起步，默认 16 路端到端事件、预警、ack 与误报反馈。 5. M4–M5：64/128 路分片、管理端和第二/第三场景包。 diff --git a/docs/03-tech-stack.md b/docs/03-tech-stack.md index 5d23684..807a976 100644 --- a/docs/03-tech-stack.md +++ b/docs/03-tech-stack.md @@ -50,7 +50,9 @@ MediaMTX 官方 `v1.19.3` OpenAPI 固定保存于 `Sense/api/vendor/mediamtx-v1. M2 仍保留 SQLite 作为单机开发与 T-006 回归入口，但生产真相源切换目标是 PostgreSQL。运行时必须显式设置 `SENSE_DB_DRIVER=postgres` 和私有 `SENSE_DB_DSN` 才使用 PostgreSQL；仓库不提供带密码 DSN。初始 schema 由 `deploy/postgres/` 的顺序 migration 创建，不在 Sense 进程启动时使用高权限自动迁移。 -T-011 复用同一套冻结的 Go、PostgreSQL、pgx 与 `oapi-codegen`，未增加第三方依赖。Sense Control API v1 从仓库 OpenAPI 生成 `net/http` server glue；首版认证是可替换 port 下的外部静态 SHA-256 注册表，公共路由默认关闭且只在 PostgreSQL v4 schema 上启用。静态适配器是私有部署启动方案，不替代后续 Bell/JWT/OIDC 身份服务。 +T-011 复用同一套冻结的 Go、PostgreSQL、pgx 与 `oapi-codegen`，未增加第三方依赖。Sense Control API v1 从仓库 OpenAPI 生成 `net/http` server glue；首版认证是可替换 port 下的外部静态 SHA-256 注册表，公共路由默认关闭且只在 PostgreSQL 生产路径启用。T-011 首次落成 v4 schema，T-012 后进程统一要求 v5。静态适配器是私有部署启动方案，不替代后续 Bell/JWT/OIDC 身份服务。 + +T-012 同样未增加第三方依赖：PostgreSQL v5 使用数据库时钟租约和 fencing token 协调多实例调和/孤儿扫描；MediaMTX Path 枚举继续使用同版生成客户端。`/metrics` 由 Go 标准库输出 Prometheus 文本格式，只包含固定结果枚举、构建/实例信息和汇总 gauge/counter，不使用 tenant、Site、device 或 Path 标签。孤儿处置是 PostgreSQL 专用本地命令，默认周期任务只报告、不自动删除。 ## 2. 外部项目边界 diff --git a/docs/04-architecture.md b/docs/04-architecture.md index 3173dd8..d5dd393 100644 --- a/docs/04-architecture.md +++ b/docs/04-architecture.md @@ -46,7 +46,7 @@ Sense ── 视频流/触发信号 ──> Brain 主流程： 1. Bell 持有站点、Area、配额与 `capture_policy`；首期在同一 PostgreSQL 实例内发布 `bell.site_quota_v1` 和 `bell.area_policy_v1` 两个版本化只读视图。T-009/T-010 已实现 Bell 源表/视图、最小权限和 Sense PostgreSQL repository；Sense 按 Area→Site 的固定 advisory-lock 顺序执行策略与配额准入并记录所用版本。未来分库必须发布新版本契约，不能静默改变 v1 语义。 -2. Sense 维护设备期望态，通过 MediaMTX API 和对账器收敛实际态。 +2. Sense 维护设备期望态，通过 MediaMTX API 和对账器收敛实际态；PostgreSQL 多实例以数据库时钟短租约和 fencing token 领取 due row，过期 worker 不得提交结果。 3. Brain 消费视频与触发信号，产生符合 v0.1 的事件。 4. Bell 做 schema 与代码级断言，生成平台 ULID，保存不可变事件。 5. 规则命中后创建独立 Alert，先落库再投递，等待 ack 并按策略升级。 @@ -82,8 +82,8 @@ Sense ── 视频流/触发信号 ──> Brain ## 7. 一致性与失败处理 - PostgreSQL `sense` schema 是生产期望态真相源；SQLite 只保留为 M1 本地开发/回归路径。MediaMTX、推理 worker 和对象存储是可对账的实际态。 -- 对账器水平触发、幂等、指数退避、限制并发；部分失败不做跨系统回滚，只持续收敛。 -- 孤儿删除必须有 10% 安全闸和人工可观察指标。 +- 对账器水平触发、幂等、指数退避、限制并发；PostgreSQL 使用 `FOR UPDATE SKIP LOCKED`、每项续租和 fencing token，SQLite 只保留单进程开发语义。部分失败不做跨系统回滚，只持续收敛。 +- MediaMTX Path 扫描把“Sense 历史拥有但当前失配”和“从未归属 Sense”分开；未知归属永不自动删除。历史拥有项也只允许在 15 分钟二次快照、1～128 项和 `候选 × 100 <= 当前 Path 总数 × 10` 全部通过时由本地运维命令逐项处置，不提供绕过。 - `bell.site_quota_v1` 行缺失、数值越界、版本回退或读取失败只阻止视频设备新增/启用，不中断已有流；降低配额导致超限时不自动停用，后续准入返回稳定错误并产生运维信号。多 Sense 实例使用 PostgreSQL transaction-scoped advisory lock 串行化同 tenant/site 的计数与写入，不能用进程内锁替代。 - `bell.area_policy_v1` 缺失、非法、版本回退或读取失败时，PostgreSQL repository 拒绝相关新增/启用；`non_imaging_only` 允许非成像设备但拒绝具有 `video_capture` 的设备。已有设备保持原状态，策略冲突由 Bell 管理端显式迁移或取消。同库实时视图不以源记录年龄误判 freshness。 - 设备创建和期望态受理在本地事务内同时写脱敏 `sense.device_operation_outbox`；Outbox 失败回滚业务写入，相同期望态不增加 generation 但仍审计。异步 relay 尚未实现。 @@ -107,14 +107,14 @@ Sense ── 视频流/触发信号 ──> Brain ## 9. 目录目标 ```text -Sense/cmd + Sense/internal/{device,onvif,mtx,reconcile,probe,trigger,tunnel,authcb,store} +Sense/cmd + Sense/internal/{device,onvif,mtx,reconcile,orphan,metrics,probe,trigger,tunnel,auth,store} Brain/{pipeline,models,judge,emit,trigger,contracts} Bell/cmd + Bell/internal/{ingest,event,rule,alert,deliver,feedback,tenant,audit,store} Bell/{web,packs,contracts} -deploy/postgres/{001_roles.sql,...,007_privileges_area_audit.sql,tests} +deploy/postgres/{001_roles.sql,...,011_privileges_reconcile_safety.sql,tests} ``` -Sense 脚手架和 PostgreSQL `001`～`007` 已实现；Brain/Bell 应用目录仍是占位，真实服务必须由对应任务创建。 +Sense 脚手架和 PostgreSQL `001`～`011` 已实现；Brain/Bell 应用目录仍是占位，真实服务必须由对应任务创建。 ## 10. 开发顺序 diff --git a/docs/06-tasks.md b/docs/06-tasks.md index bbac273..fd1fb17 100644 --- a/docs/06-tasks.md +++ b/docs/06-tasks.md @@ -26,7 +26,7 @@ - T-009：使用 PostgreSQL 17.10 建立 `sense`/`bell` schema、Bell 配额源表/只读视图、最小权限和 Sense PostgreSQL repository；SQLite 保留为 M1 本地路径。 - T-010：建立 Bell Area/`capture_policy` 只读投影、Sense 成像准入与设备操作审计 Outbox 原子写入；relay 传输协议留给后续任务。 - T-011：按 T-008 契约实现认证 tenant 上下文、7 个设备/operation handler、PostgreSQL 幂等收据、ETag/HMAC cursor、最多 128 项批量操作和停用精确 path 收敛；公共控制 API 默认关闭且只在 PostgreSQL 路径启用。 -- 对账器幂等/退避/并发/10% 安全闸。 +- T-012：以 PostgreSQL 数据库时钟租约和 fencing token 防止多实例重复调和；增加可证明的 Path 历史归属、默认只读孤儿报告、15 分钟二次确认、10% 不可绕过安全闸和低基数 `/metrics`。 - WireGuard 边缘隧道与断网恢复。 - 16 路批量开通、停用和容量基准。 diff --git a/docs/api.md b/docs/api.md index 02b4d9b..b31f3ea 100644 --- a/docs/api.md +++ b/docs/api.md @@ -69,10 +69,10 @@ Sense 使用 MediaMTX 官方 OpenAPI 生成客户端并加薄封装。业务代 | Port | 操作 | 数据所有者 / 失败语义 | | --- | --- | --- | | ONVIF adapter | `Probe(target)`、`SetSystemDateAndTime(target, time)` | 设备是外部来源；`target` 只含 endpoint ref 与不透明 credential ref。错误稳定映射为认证失败、超时、不可用、响应无效，不记录凭据或完整流地址 | -| MediaMTX paths | `CreatePath`、`GetPath`、`EnsurePath`、`DeletePath`、`PathReady` | SQLite 设备台账持有期望态，MediaMTX 只持有运行配置；`EnsurePath` 相同 source 不写、不同 source patch、缺失时 add；当前调和器绝不枚举或删除孤儿 | +| MediaMTX paths | `CreatePath`、`GetPath`、`EnsurePath`、`DeletePath`、`PathReady`、`ListPathNames` | 设备台账持有期望态，MediaMTX 只持有运行配置；分页枚举只返回名称、不返回 source。普通调和只操作精确台账 Path；独立孤儿流程只报告未知归属，受控处置只接受有历史归属的 stale Path | | Device repository | 设备、期望态、实际态、调和进度 | SQLite 是 M1 默认开发路径；PostgreSQL 是 M2 生产路径并只读 Site/Area 两个 Bell 视图。调和退避持久化；配额/Area 失败时拒绝相关新增或启用，不关闭已有流；高风险写入与脱敏 Outbox 同事务 | -MediaMTX 薄封装调用同版官方 OpenAPI 的 `/v3/config/paths/get|add|patch|delete/{name}` 与 `/v3/paths/get/{name}`。生成源、版本和 SHA-256 见 `docs/03-tech-stack.md`；业务包不得直接 import 生成包。 +MediaMTX 薄封装调用同版官方 OpenAPI 的 `/v3/config/paths/get|add|patch|delete/{name}`、`/v3/config/paths/list` 与 `/v3/paths/get/{name}`。列表有最大页数和重复页保护，source URI 在薄封装内丢弃。生成源、版本和 SHA-256 见 `docs/03-tech-stack.md`；业务包不得直接 import 生成包。 ### 5.2 设备台账语义 @@ -83,7 +83,9 @@ MediaMTX 薄封装调用同版官方 OpenAPI 的 `/v3/config/paths/get|add|patch ### 5.3 Sense 进程 HTTP 面 -`GET /healthz` 表示进程存活，`GET /readyz` 表示所选数据库已打开且 schema/权限前置检查完成；两者不要求认证，也不等价于摄像头、MediaMTX path 或里程碑健康。T-011 在 `SENSE_CONTROL_API_ENABLED=true`、PostgreSQL v4 schema 和外部安全文件全部有效时注册冻结的 `/api/v1` 路由；默认 SQLite 运行仍只暴露探针。业务路由不提供无 Site 边界的 `/api/v1/devices` 临时接口。 +`GET /healthz` 表示进程存活，`GET /readyz` 表示所选数据库已打开且 schema/权限前置检查完成；两者不要求认证，也不等价于摄像头、MediaMTX path 或里程碑健康。`GET /metrics` 默认输出无租户/设备/Path 标签的 Prometheus 汇总，可通过 `SENSE_METRICS_ENABLED=false` 关闭。T-011 在 `SENSE_CONTROL_API_ENABLED=true`、PostgreSQL v5 schema 和外部安全文件全部有效时注册冻结的 `/api/v1` 路由；默认 SQLite 不注册业务路由。业务路由不提供无 Site 边界的 `/api/v1/devices` 临时接口。 + +孤儿报告/处置不是公共 HTTP API。它由 PostgreSQL 专用 `cmd/sense-orphan` 在受控运维主机执行，使用 15 分钟 scan ID、actor、精确确认文本和运行前二次快照；不改变 Sense Control API v1 的 7 个 endpoint。 ## 6. 变更流程 diff --git a/docs/current-state.md b/docs/current-state.md index 822c922..45e31e9 100644 --- a/docs/current-state.md +++ b/docs/current-state.md @@ -4,8 +4,8 @@ ## 当前阶段 -- 阶段：M0 指定摄像头型号准入已完成；M1 的“一实机 + 四合成源”实验室软件闭环已通过；M2 已完成控制面契约、Site/Area 准入与本地审计事务基础，但五条独立真实上游和生产 SLA 尚未验收。 -- 生产代码：Sense 已包含可构建进程、SQLite/PostgreSQL repository、Site/Area 准入、设备操作 Outbox、标准 ONVIF SOAP/WS-Security adapter、凭据引用、MediaMTX 生成客户端、对账、探活和 MediaMTX 重启重建；公共设备管理 API、认证、Outbox relay 和完整生产部署仍未实现。 +- 阶段：M0 指定摄像头型号准入已完成；M1 的“一实机 + 四合成源”实验室软件闭环已通过；M2 已完成 Control API、Site/Area 准入、本地审计事务、多实例调和 fencing 和孤儿受控处置基础，但 WireGuard、16 路容量出口、五条独立真实上游和生产 SLA 尚未验收。 +- 生产代码：Sense 已包含可构建进程、SQLite/PostgreSQL repository、Site/Area 准入、设备操作 Outbox、标准 ONVIF SOAP/WS-Security adapter、凭据引用、MediaMTX 生成客户端、Control API v1、对账/探活、数据库租约、孤儿只读扫描/受控命令和低基数指标；Bell 管理服务/JWT、Outbox relay、WireGuard 和完整生产部署仍未实现。 - 默认容量：16 路；单站点本阶段上限 128 路，必须横向分片。 ## 仓库现实 @@ -14,13 +14,14 @@ - Sense 设备模型使用 `modality + capabilities`，SQLite 执行 v1 migration；视频配额默认 16、允许 1～128，17/128/129、新增/启用和“降低配额不关闭已有流”均有测试。 - T-009 冻结 PostgreSQL `17.10` 和 `pgx/v5 v5.10.0`，实现 `bell`/`sense` schema、NOLOGIN 权限角色、Bell Site 版本 trigger、`bell.site_quota_v1` 和 Sense PostgreSQL repository；同站点并发准入用事务级 advisory lock，配额缺失/越界/版本回退时失败关闭且不改变已有流。 - T-010 增量实现 `bell.areas`、`bell.area_policy_v1`、Area 版本观察和 `sense.device_operation_outbox`；`non_imaging_only` 拒绝成像设备创建/启用，失败不改变已有设备。设备创建/期望态受理与脱敏 Outbox 同事务，相同期望态不增加 generation 但仍审计。 -- Windows 隔离测试使用 `D:\pgsql17\bin` 启动随机回环端口临时集群，`001`～`007` migration 可重放、权限断言和 18 个真实 PostgreSQL 测试通过后自动清理；现有 `D:\pgsql17\data` 和 5432 服务未被读取、停止或修改。 -- MediaMTX 固定为独立二进制 `v1.19.3`，官方 OpenAPI 已按 SHA-256 vendoring，并由固定 `oapi-codegen v2.8.0` 生成客户端；手写薄封装有 create/read/delete、幂等 ensure 与探活假 HTTP 测试。 -- T-003 对账进度与指数退避持久化，覆盖取消和 SQLite 重启恢复；T-006 增加真实 ONVIF adapter、RTSP router、实验室播种/状态工具、故障代理和五路自动验收。当前仍不枚举/删除孤儿。 +- Windows 隔离测试使用 `D:\pgsql17\bin` 启动随机回环端口临时集群，`001`～`011` migration 可重放、权限断言和 29 个真实 PostgreSQL 测试通过后自动清理；现有 `D:\pgsql17\data` 和 5432 服务未被读取、停止或修改。 +- MediaMTX 固定为独立二进制 `v1.19.3`，官方 OpenAPI 已按 SHA-256 vendoring，并由固定 `oapi-codegen v2.8.0` 生成客户端；手写薄封装有 create/read/delete、幂等 ensure、探活和只返回名称的受限分页枚举测试。 +- T-003 对账进度与指数退避持久化，覆盖取消和 SQLite 重启恢复；T-006 增加真实 ONVIF adapter、RTSP router、实验室播种/状态工具、故障代理和五路自动验收。T-012 的普通调和不枚举孤儿；独立 PostgreSQL 扫描默认只报告，未知归属永不删除。 - T-006 正式使用 1 台准入实机和 4 个独立合成 publisher 连续观察 `1806.6 s` / 180 次采样，四类恢复均通过，最大与最终 `unconverged` 均为 0；详细证据见 `docs/research/sense-5-stream-integration.md`。 - `docs/raw/01`～`08` 已记录需求、分析、方案、客户场景、事件比对和三系统职责。 - `docs/raw/contracts/event-v0.1.schema.json` 已冻结，并有多份示例与语义说明。 -- `docs/contracts/sense-control-v1.openapi.json` 的 7 个站点作用域/operation endpoint 已由 T-011 实现：外部静态 SHA-256 主体注册表、tenant/Site scope、HMAC cursor、ETag、PostgreSQL 24 小时幂等收据和最多 128 项 batch operation 均有代码与隔离集成测试。默认 SQLite 仍只暴露探针；Bell 管理服务、JWT/OIDC 和 Outbox relay 尚未实现。 +- `docs/contracts/sense-control-v1.openapi.json` 的 7 个站点作用域/operation endpoint 已由 T-011 实现：外部静态 SHA-256 主体注册表、tenant/Site scope、HMAC cursor、ETag、PostgreSQL 24 小时幂等收据和最多 128 项 batch operation 均有代码与隔离集成测试。默认 SQLite 只暴露运维探针与低基数 `/metrics`，不注册业务路由；Bell 管理服务、JWT/OIDC 和 Outbox relay 尚未实现。 +- T-012 把 PostgreSQL schema 提升到 v5：due row 用数据库时钟、`FOR UPDATE SKIP LOCKED`、逐项续租和 fencing token 协调；MediaMTX Path 历史归属、15 分钟孤儿快照、最多 10%/128 项安全闸、无 bypass 的本地处置命令及 `/metrics` 已实现。SQLite 明确保留单实例开发语义。 - harness coding 文档、上下文清单、Gitea Issue/PR 模板和治理脚本已接入。 - Gitea 已初始化 12 个协作标签；`status/waiting` 用于依赖或外部条件未满足的未领取任务，实时可领取状态必须从 Gitea 查询，不在本文复制。 - T-002 已关闭架构影响型问题：首期 S2 民办寄宿学校、客户侧私有部署、ONVIF/RTSP、NVIDIA 主路径、自研 Bell、Web/H5 与客户侧证据存储等边界已批准。 @@ -55,7 +56,7 @@ python scripts/validate_harness_governance.py ./scripts/test_postgres.ps1 -PgRoot D:\pgsql17 ``` -Sense 默认监听 `127.0.0.1:8080`，提供 `/healthz` 与 `/readyz` 运维探针；它们不代表摄像头或 M1 里程碑健康。MediaMTX 获取、校验和独立启动方法见 `Sense/README.md`。 +Sense 默认监听 `127.0.0.1:8080`，提供 `/healthz`、`/readyz` 运维探针和低基数 `/metrics`；探针不代表摄像头或 M1 里程碑健康。MediaMTX 获取、校验和独立启动方法见 `Sense/README.md`。 ## 当前 blocker / 待确认 @@ -66,12 +67,12 @@ Sense 默认监听 `127.0.0.1:8080`，提供 `/healthz` 与 `/readyz` 运维探 - 人脸方向已延后至 M5 的 S4 成人园区候选试点；必要性/PIP 影响评估、单独同意与替代方式、合法底库来源和删除流程未完成，阻塞人脸能力上线。 - 短信/语音具体供应商未选；生产前必须选定两条独立投递路径并验证故障切换。 - Python/Savant 的精确版本、目标硬件和 Bell 前端栈尚未冻结；Sense M1 的 Go、SQLite driver、MediaMTX、生成器及生成运行时版本已在 T-003 冻结，PostgreSQL/pgx 版本已在 T-009 冻结。 -- 本机现有 PostgreSQL 5432 实例使用 SCRAM 且当前开发进程没有管理员密码；T-009～T-011 不绕过认证，自动验收使用隔离临时集群。向共享/生产实例安装 migration 前仍需管理员私下提供专用数据库、登录角色、外部 Control API 安全文件与备份方案。 +- 本机现有 PostgreSQL 5432 实例使用 SCRAM 且当前开发进程没有管理员密码；T-009～T-012 不绕过认证，自动验收使用隔离临时集群。向共享/生产实例安装 migration 前仍需管理员私下提供专用数据库、登录角色、外部 Control API 安全文件与备份方案。 - 代码知识图谱在无业务代码阶段可能为空；工具不可用时使用 `rg` 处理文档与配置。 ## 下一步 -下一项建议创建 T-012，补齐 M2 对账安全闸、孤儿检测（只报告/受控处置）与多实例可观测性，再进入 WireGuard 和 16 路批量开通基准；不要把 Control API 完成误报为 M2 容量出口。客户授权、借用或租赁条件具备后再执行 T-007 五条独立真实上游现场门禁。T-006 的合成结果不解除 T-007，也不形成容量或生产 SLA 承诺。 +下一项建议先冻结并实施 WireGuard 边缘隧道与断网恢复，再创建 16 路批量开通/停用和分维度容量基准任务；T-012 完成不等于 M2 容量出口。客户授权、借用或租赁条件具备后再执行 T-007 五条独立真实上游现场门禁。T-006 的合成结果不解除 T-007，也不形成容量或生产 SLA 承诺。 ## 已知风险 diff --git a/docs/runbooks/sense-reconciliation.md b/docs/runbooks/sense-reconciliation.md new file mode 100644 index 0000000..e32dfc8 --- /dev/null +++ b/docs/runbooks/sense-reconciliation.md @@ -0,0 +1,65 @@ +# Sense 对账与孤儿处置 Runbook + +本文适用于 PostgreSQL v5（migration `001`～`011`）与 MediaMTX `v1.19.3`。SQLite 仅用于单进程开发，不具备本 runbook 的多实例 fencing 或孤儿处置语义。 + +## 观察入口 + +- `GET /healthz`：进程存活。 +- `GET /readyz`：数据库 schema 和权限前置检查已完成。 +- `GET /metrics`：低基数进程指标；不包含 tenant、Site、device、Path 或 URI 标签。 +- 日志中的 `background convergence error`：只输出稳定、脱敏错误，不输出 stream URI 或凭据。 + +重点指标： + +- `sense_reconcile_devices{state="unconverged"}`：未收敛的 enabled 视频设备数。 +- `sense_reconcile_items_total{result="lease_lost"}`：worker 在执行前或提交时失去 fencing token 的次数；失去租约的 worker 不得继续产生外部变更。 +- `sense_orphan_paths{classification="owned_stale|unowned"}`：最近一次成功扫描的两类差异。 +- `sense_orphan_cleanup_blocked_total`、`sense_orphan_cleanup_items_total`：安全闸阻断和人工处置结果。 + +## 多实例调和 + +每个实例应注入唯一、稳定且不含客户信息的 `SENSE_INSTANCE_ID`。PostgreSQL 使用数据库时钟、短事务、`FOR UPDATE SKIP LOCKED` 和随机 fencing token 领取 due row；批量等待中的每一项在调用 ONVIF/MediaMTX 前续租。默认租期 30 秒、单项 deadline 20 秒；operation timeout 必须严格短于租期。 + +发现 `lease_lost` 时先检查实例 ID 是否重复、数据库时钟和外部调用延迟。不要通过延长到超过 5 分钟或关闭 fencing 规避问题；先定位超时，再在变更评审后同时调整租期和单项 deadline。 + +## 生成孤儿报告 + +在已安装 v5 migration、能访问同一 PostgreSQL 和目标 MediaMTX 的受控运维主机执行： + +```powershell +$env:SENSE_DB_DRIVER = 'postgres' +$env:SENSE_DB_DSN = '由部署环境私下设置' +$env:SENSE_MEDIAMTX_URL = '受控 MediaMTX API 地址' +go -C Sense run ./cmd/sense-orphan -mode report +``` + +命令只输出 `scan_id`、计数、安全闸结论和过期时间。分类含义： + +- `owned_stale`：Sense 有历史 Path 归属记录，但当前没有同一设备继续声明该 Path；可进入受控候选。 +- `unowned`：没有 Sense 归属证据；可能属于人工或其他系统，永远只报告。 +- 当前设备仍声明的 Path 不计入 finding。 + +不要因为名称相似把 `unowned` 手工改成 owned。先调查其创建者和用途；需要接管时应走独立、可审计的迁移任务。 + +## 执行受控处置 + +仅当报告显示 `safety_allowed=true`，并在 15 分钟有效期内由授权运维人员执行： + +```powershell +go -C Sense run ./cmd/sense-orphan -mode apply ` + -scan-id 'scan_...' ` + -actor 'operator-id' ` + -confirm 'DELETE scan_...' +``` + +执行前程序会重新枚举 Path、重新读取归属并重新计算：候选必须仍是报告集合的子集、数量 1～128，且满足 `候选数 × 100 <= 当前 Path 总数 × 10`。过期、比例超限、归属变化、确认不匹配或租约冲突都会整批零删除；没有 `force`/bypass。删除逐项记入 PostgreSQL，成功项重复执行会跳过，失败项在快照仍有效且重新计算仍通过时可重试。 + +## 故障与恢复 + +- 扫描失败：不产生可执行快照；修复数据库/MediaMTX 连通性后重新 `report`。 +- `ratio_exceeded`：停止自动化，核对 MediaMTX 实例/分片是否选错、台账是否缺失或发生大面积配置漂移；不得拆小批次规避 10% 闸。 +- 部分删除失败：保留输出与数据库处置事实，修复 MediaMTX 后用同一 scan ID 重试；超过 15 分钟必须重新报告和审批。 +- 误删怀疑：立即停止 apply。数据库期望态仍是来源；当前设备声明的 Path 会由调和器重建。无当前设备声明的历史 Path 不自动恢复，应根据变更记录人工确认来源。 +- 回滚 Sense 二进制：先停用 `SENSE_ORPHAN_SCAN_ENABLED` 并停止所有新实例，再回滚；v5 表和已记录事实保留，不执行破坏性 down migration。 + +本流程只处置 MediaMTX 配置 Path，不删除设备、录像、证据对象或 Bell 数据。 diff --git a/docs/tasks/T-012.md b/docs/tasks/T-012.md index 3edf88c..560ad70 100644 --- a/docs/tasks/T-012.md +++ b/docs/tasks/T-012.md @@ -3,7 +3,7 @@ id: T-012 title: 补齐 Sense 对账安全闸、孤儿受控处置与多实例可观测性 phase: 2 deps: [T-011] -status: DOING +status: DONE created: 2026-08-07 issue: 43 context_ref: 25357723a09218bae58626a9934309abcda34899 @@ -88,6 +88,16 @@ T-003 已实现幂等调和与持久化退避，T-011 已完成 Control API、 ## 执行记录 +### 2026-08-07 完成多实例对账安全与孤儿受控处置 + +- PostgreSQL schema 提升到 v5：`010`/`011` 为 due row 增加 owner/token/expiry fencing，新增不含 source URI 的 MediaMTX Path 历史归属、孤儿扫描租约、15 分钟报告、发现项和处置审计，并补齐 `sense_app` 最小权限与 PUBLIC 拒绝；启动会拒绝旧 schema 或越权运行角色。 +- 调和器改为 `FOR UPDATE SKIP LOCKED` 批量领取、逐项续租、短于租期的外部调用 deadline 和带 token 的完成/失败；数据库时钟判定租约，旧 worker 不能覆盖接管者结果。成功启用在同一完成事务中刷新 Path 归属；SQLite 明确保留单进程开发语义。 +- MediaMTX 薄客户端新增只返回排序去重 Path 名称的有界分页枚举，包含重复页保护且不读取/返回 source；独立孤儿扫描把差异分为 `owned_stale` 与永不删除的 `unowned`。周期任务默认只报告，人工 `sense-orphan apply` 必须使用 15 分钟内 scan ID、合法 actor 和精确确认文本，并在删除前重取库存/归属。 +- 处置安全闸固定为 1～128 项且不超过当前 Path 的 10%，使用整数交叉相乘，无 force/bypass；目标只能从原快照收窄。逐项结果可审计和幂等重试，数据库外键/约束也拒绝为 `unowned` 写入删除记录。 +- 新增无第三方运行时依赖的 `/metrics`：只使用构建/实例及固定结果枚举标签，覆盖调和 run/item/lease-lost/时长/未收敛和孤儿扫描/阻断/删除汇总；tenant、Site、device、Path、URI 与错误正文均不进入标签。 +- `./scripts/test_postgres.ps1 -PgRoot D:\pgsql17` 通过：隔离 PostgreSQL 17.10 临时集群连续重放 `001`～`011`，SQL 权限/约束断言及 29 个 `TestPostgres*` 全绿，覆盖两个 store 并发唯一领取、租约接管/旧 token fencing、归属记录、扫描租约、未知 Path 数据库拒删和处置幂等；随机端口实例已停止并清理，现有 `D:\pgsql17\data`/5432 未被读取、停止或修改。 +- `./init.ps1` 通过；Python 治理/契约测试共 52 项通过，T-012 定向静态契约 12 项通过；全部 Go 测试、生成漂移、`go vet`、`go build` 与 `git diff --check` 通过；`go test -race ./internal/metrics ./internal/mtx ./internal/orphan ./internal/reconcile ./internal/store` 通过。不需要摄像头/GPU，本结果不解除 T-007，也不形成 16/128 路容量承诺。 + ### 2026-08-07 领取任务 - dispatcher `ila` 将 Issue #43 分配给 `codex`；`context_ref` 为 `25357723a09218bae58626a9934309abcda34899`，claim 为 `claims/T-012`，工作分支为 `agent/codex/T-012`。 diff --git a/scripts/test_postgres.ps1 b/scripts/test_postgres.ps1 index 376ae00..83cdeb8 100644 --- a/scripts/test_postgres.ps1 +++ b/scripts/test_postgres.ps1 @@ -40,7 +40,7 @@ $port = ([System.Net.IPEndPoint]$listener.LocalEndpoint).Port $listener.Stop() $systemTemp = [IO.Path]::GetFullPath([IO.Path]::GetTempPath()) -$clusterName = 'yovision-t011-pg-' + [guid]::NewGuid().ToString('N') +$clusterName = 'yovision-t012-pg-' + [guid]::NewGuid().ToString('N') $dataDir = Join-Path $systemTemp $clusterName $logPath = Join-Path $systemTemp ($clusterName + '.log') $started = $false @@ -69,9 +69,9 @@ try { $started = $true $adminRootDSN = "postgres://postgres@127.0.0.1:$port/postgres?sslmode=disable" - $databaseName = 'yovision_t011' + $databaseName = 'yovision_t012' $adminDatabaseDSN = "postgres://postgres@127.0.0.1:$port/${databaseName}?sslmode=disable" - $senseDSN = "postgres://yovision_t011_sense@127.0.0.1:$port/${databaseName}?sslmode=disable" + $senseDSN = "postgres://yovision_t012_sense@127.0.0.1:$port/${databaseName}?sslmode=disable" Invoke-Checked $psql '-X' '-v' 'ON_ERROR_STOP=1' '-d' $adminRootDSN '-f' (Join-Path $repoRoot 'deploy\postgres\001_roles.sql') Invoke-Checked $createdb '-h' '127.0.0.1' '-p' ([string]$port) '-U' 'postgres' $databaseName @@ -85,12 +85,14 @@ try { '006_device_operation_outbox.sql', '007_privileges_area_audit.sql', '008_control_api.sql', - '009_privileges_control_api.sql' + '009_privileges_control_api.sql', + '010_reconcile_safety.sql', + '011_privileges_reconcile_safety.sql' )) { Invoke-Checked $psql '-X' '-v' 'ON_ERROR_STOP=1' '-d' $adminDatabaseDSN '-f' (Join-Path $repoRoot "deploy\postgres\$name") } } - Invoke-Checked $psql '-X' '-v' 'ON_ERROR_STOP=1' '-d' $adminRootDSN '-c' 'CREATE ROLE yovision_t011_sense LOGIN IN ROLE sense_app' + Invoke-Checked $psql '-X' '-v' 'ON_ERROR_STOP=1' '-d' $adminRootDSN '-c' 'CREATE ROLE yovision_t012_sense LOGIN IN ROLE sense_app' Invoke-Checked $psql '-X' '-v' 'ON_ERROR_STOP=1' '-d' $adminDatabaseDSN '-f' (Join-Path $repoRoot 'deploy\postgres\tests\assertions.sql') $env:YOVISION_TEST_POSTGRES_DSN = $senseDSN @@ -105,7 +107,7 @@ finally { if (-not $started -or $stopped) { $resolvedData = [IO.Path]::GetFullPath($dataDir) if (-not $resolvedData.StartsWith($systemTemp, [StringComparison]::OrdinalIgnoreCase) -or - [IO.Path]::GetFileName($resolvedData) -notlike 'yovision-t011-pg-*') { + [IO.Path]::GetFileName($resolvedData) -notlike 'yovision-t012-pg-*') { throw "Refusing to clean unexpected temporary path." } if (Test-Path -LiteralPath $resolvedData) { diff --git a/tests/test_postgres_contract.py b/tests/test_postgres_contract.py index ef84dc2..9806f99 100644 --- a/tests/test_postgres_contract.py +++ b/tests/test_postgres_contract.py @@ -56,6 +56,8 @@ class PostgresContractTests(unittest.TestCase): "007_privileges_area_audit.sql", "008_control_api.sql", "009_privileges_control_api.sql", + "010_reconcile_safety.sql", + "011_privileges_reconcile_safety.sql", ], names, ) @@ -109,7 +111,7 @@ class PostgresContractTests(unittest.TestCase): "initdb.exe", "pg_ctl.exe", "127.0.0.1", - "yovision-t011-pg-", + "yovision-t012-pg-", "YOVISION_TEST_POSTGRES_DSN", "Get-NetTCPConnection", ): diff --git a/tests/test_sense_reconcile_safety.py b/tests/test_sense_reconcile_safety.py new file mode 100644 index 0000000..01771fe --- /dev/null +++ b/tests/test_sense_reconcile_safety.py @@ -0,0 +1,86 @@ +"""Static safety checks for the T-012 reconciliation/orphan boundary.""" + +from __future__ import annotations + +import pathlib +import re +import unittest + + +ROOT = pathlib.Path(__file__).resolve().parents[1] + + +def text(relative: str) -> str: + return (ROOT / relative).read_text(encoding="utf-8") + + +def normalized(value: str) -> str: + return re.sub(r"\s+", " ", value.lower()) + + +class SenseReconcileSafetyTests(unittest.TestCase): + def test_postgres_v5_has_fencing_ownership_and_private_reports(self) -> None: + schema = normalized(text("deploy/postgres/010_reconcile_safety.sql")) + privileges = normalized( + text("deploy/postgres/011_privileges_reconcile_safety.sql") + ) + implementation = normalized(text("Sense/internal/store/postgres.go")) + for table in ( + "media_path_ownership", + "operational_leases", + "orphan_scan_runs", + "orphan_scan_findings", + "orphan_cleanup_actions", + ): + self.assertIn(f"sense.{table}", schema) + self.assertIn(f"sense.{table}", privileges) + self.assertIn("insert into sense.schema_migrations(version) values (5)", schema) + self.assertIn("clock_timestamp()", implementation) + self.assertIn("for update of r skip locked", implementation) + self.assertIn("and lease_token = $5", implementation) + self.assertNotIn("endpoint_ref", schema) + self.assertNotIn("credential_ref", schema) + self.assertNotIn("source_uri", schema) + + def test_cleanup_gate_has_no_force_or_unowned_delete_path(self) -> None: + source = text("Sense/internal/orphan/orphan.go") + normalized_source = normalized(source) + self.assertIn("candidates*100 > observed*10", normalized_source) + self.assertIn("candidates > maxcleanupitems", normalized_source) + self.assertIn('confirmation != "delete "+scanid', normalized_source) + self.assertIn( + "finding.classification != store.orphanownedstale", normalized_source + ) + self.assertNotRegex(normalized_source, r"\bforce\b|bypass") + migration = normalized(text("deploy/postgres/010_reconcile_safety.sql")) + self.assertIn("classification = 'owned_stale'", migration) + + def test_mediamtx_inventory_discards_sources_and_bounds_pagination(self) -> None: + source = text("Sense/internal/mtx/client.go") + self.assertIn("ListPathNames", source) + self.assertIn("maxPages", source) + self.assertIn("list paths repeated page", source) + inventory = source[source.index("func (c *Client) ListPathNames") :] + inventory = inventory[: inventory.index("func (c *Client) CreatePath")] + self.assertNotIn("item.Source", inventory) + + def test_metrics_are_fixed_and_control_contract_is_untouched(self) -> None: + metrics = text("Sense/internal/metrics/metrics.go") + main = text("Sense/cmd/sense-api/main.go") + self.assertIn('mux.Handle("GET /metrics", registry.Handler())', main) + for forbidden in ("tenant_id", "site_id", "device_id", "path_name"): + self.assertNotIn(forbidden, metrics) + control = text("docs/contracts/sense-control-v1.openapi.json") + self.assertEqual(7, len(re.findall(r'"operationId"\s*:', control))) + + def test_operator_command_requires_fresh_scan_confirmation(self) -> None: + command = text("Sense/cmd/sense-orphan/main.go") + manager = text("Sense/internal/orphan/orphan.go") + self.assertIn("exact confirmation: DELETE ", command) + self.assertIn("scan.ExpiresAt.After(now)", manager) + self.assertIn("cleanupCandidates(scan, paths, ownership)", manager) + self.assertIn("RecordOrphanCleanup", manager) + + +if __name__ == "__main__": + unittest.main()