feat(sense): add reconciliation safety controls [T-012]
This commit is contained in:
+22
-5
@@ -1,6 +1,6 @@
|
||||
# Sense M1/M2 接入骨架
|
||||
|
||||
本目录是 YoVision Sense 的 M1/M2 接入骨架。数据库保存期望态,ONVIF 和 MediaMTX 通过端口隔离;M1 默认使用 SQLite,T-009~T-011 增加 PostgreSQL 双 schema、Area 准入、本地审计 Outbox 和 Control API v1。默认关闭真实 ONVIF 与公共业务路由;T-006 的真实样机结论仅覆盖已批准的精确海康基线,不能据此宣称多品牌兼容。
|
||||
本目录是 YoVision Sense 的 M1/M2 接入骨架。数据库保存期望态,ONVIF 和 MediaMTX 通过端口隔离;M1 默认使用 SQLite,T-009~T-012 增加 PostgreSQL 双 schema、Area 准入、本地审计 Outbox、Control API v1、多实例调和 fencing 和孤儿受控处置。默认关闭真实 ONVIF 与公共业务路由;T-006 的真实样机结论仅覆盖已批准的精确海康基线,不能据此宣称多品牌兼容。
|
||||
|
||||
## 常用命令
|
||||
|
||||
@@ -14,7 +14,7 @@ go build -o bin/sense-api.exe ./cmd/sense-api
|
||||
go run ./cmd/sense-api
|
||||
```
|
||||
|
||||
Unix 将构建产物改为 `bin/sense-api`。服务默认监听 `127.0.0.1:8080`,SQLite 默认写入 `Sense/data/sense.db`,MediaMTX 控制 API 默认是 `http://127.0.0.1:9997`。默认运行只暴露 `/healthz` 与 `/readyz`;只有显式选择 PostgreSQL 并完成安全配置后才注册 7 个 `/api/v1` Control API 路由。
|
||||
Unix 将构建产物改为 `bin/sense-api`。服务默认监听 `127.0.0.1:8080`,SQLite 默认写入 `Sense/data/sense.db`,MediaMTX 控制 API 默认是 `http://127.0.0.1:9997`。默认运行暴露 `/healthz`、`/readyz` 和不含租户/设备标签的 `/metrics`;只有显式选择 PostgreSQL 并完成安全配置后才注册 7 个 `/api/v1` Control API 路由。
|
||||
|
||||
常用环境变量:
|
||||
|
||||
@@ -26,7 +26,13 @@ Unix 将构建产物改为 `bin/sense-api`。服务默认监听 `127.0.0.1:8080`
|
||||
| `SENSE_DB_DSN` | `file:data/sense.db` | 所选 driver 的私有 DSN;不得写入日志、文档或仓库,PostgreSQL 密码优先由 `PGPASSFILE`/环境密钥提供 |
|
||||
| `SENSE_MEDIAMTX_URL` | `http://127.0.0.1:9997` | MediaMTX 控制 API;不得包含 userinfo |
|
||||
| `SENSE_RECONCILE_INTERVAL` | `5s` | 对账周期 |
|
||||
| `SENSE_RECONCILE_LEASE_DURATION` | `30s` | PostgreSQL due-row 租期;最大 5 分钟 |
|
||||
| `SENSE_RECONCILE_OPERATION_TIMEOUT` | `20s` | 单项 ONVIF/MediaMTX deadline;必须严格短于租期 |
|
||||
| `SENSE_PROBE_INTERVAL` | `10s` | path 探活周期 |
|
||||
| `SENSE_INSTANCE_ID` | 随进程随机生成 | 最多 64 位低基数字符串;多实例部署建议显式注入唯一实例 ID |
|
||||
| `SENSE_METRICS_ENABLED` | `true` | 是否注册低基数 Prometheus 文本 `/metrics` |
|
||||
| `SENSE_ORPHAN_SCAN_ENABLED` | PostgreSQL 为 `true`,SQLite 为 `false` | 周期执行只读 MediaMTX Path 差异报告;从不自动删除 |
|
||||
| `SENSE_ORPHAN_SCAN_INTERVAL` | `1m` | 孤儿只读扫描周期,最短 10 秒 |
|
||||
| `SENSE_ONVIF_MODE` | `disabled` | `disabled` 或 `standard`;默认不访问真实摄像头 |
|
||||
| `SENSE_ONVIF_RTSP_REWRITE_HOST` | 空 | NAT 或故障代理场景下重写 ONVIF 返回的 RTSP 主机 |
|
||||
| `SENSE_ONVIF_RTSP_REWRITE_PORT` | `0` | 非零时重写 ONVIF 返回的 RTSP 端口 |
|
||||
@@ -50,9 +56,9 @@ SENSE_CREDENTIAL_<KEY>_RTSP_PASSWORD
|
||||
|
||||
MediaMTX `v1.19.3` 应作为独立二进制启动并只在可信网络开放 API。获取与 SHA-256 校验值见 `docs/03-tech-stack.md`。生成客户端使用固定版本工具和 vendored 官方 OpenAPI;`internal/mtx/generated/client.gen.go` 不可手改。
|
||||
|
||||
## T-009~T-011 PostgreSQL 17.10
|
||||
## T-009~T-012 PostgreSQL 17.10
|
||||
|
||||
初始化与增量 SQL 位于 `deploy/postgres/`,由高权限部署步骤按文件名前缀执行;Sense 进程不会自动创建角色、schema 或 Bell 对象。`bell_app` 拥有 Site/Area、配额、`capture_policy` 及两个版本化视图,`sense_app` 只能读取两个视图,不能读取或写入 Bell 源表。T-011 的 v4 schema 另有资源版本、24 小时幂等收据和 batch operation 表;收据只保存 scope/body 摘要与脱敏响应快照。应用登录角色和密码由部署环境创建,不进入仓库。
|
||||
初始化与增量 SQL 位于 `deploy/postgres/`,由高权限部署步骤按文件名前缀执行;Sense 进程不会自动创建角色、schema 或 Bell 对象。`bell_app` 拥有 Site/Area、配额、`capture_policy` 及两个版本化视图,`sense_app` 只能读取两个视图,不能读取或写入 Bell 源表。T-011 的 v4 schema 增加资源版本、24 小时幂等收据和 batch operation;T-012 的 v5 schema 增加数据库时钟租约、Path 历史归属及脱敏孤儿报告/处置结果。表中不保存 MediaMTX source URI。应用登录角色和密码由部署环境创建,不进入仓库。
|
||||
|
||||
PostgreSQL 新建设备必须携带匹配 tenant/Site 的 `area_id`。具有 `video_capture` 能力的设备在创建、移动 Area 和从 disabled 切到 enabled 时执行 Area 准入;`non_imaging_only` 拒绝成像设备但允许非成像设备。投影缺失、非法或版本回退只拒绝新变更,不关闭已有流。创建、配置修改和期望态受理都与对应脱敏 Outbox 事实同事务;停用后调和器只删除该设备的精确 MediaMTX path 并收敛为 offline,不枚举未知 path。Bell relay 尚未实现。
|
||||
|
||||
@@ -72,7 +78,18 @@ $env:SENSE_DB_DSN = '由部署环境私下设置'
|
||||
go run ./cmd/sense-api
|
||||
```
|
||||
|
||||
PostgreSQL 启动会检查 Sense migration 版本及当前角色对两个 Bell 投影视图的最小权限;权限过宽、视图不可读或 schema 未安装时 readiness 初始化失败。默认 SQLite 路径和 `cmd/sense-lab` 保持不变,但 SQLite 不实现生产 Area/Outbox 一致性,Control API feature flag 在 SQLite 下会直接拒绝启动。
|
||||
PostgreSQL 启动会检查 Sense v5 migration、当前角色对两个 Bell 投影视图和本地控制/对账表的最小权限;权限过宽、视图不可读或 schema 未安装时初始化失败。默认 SQLite 路径和 `cmd/sense-lab` 保持不变,但 SQLite 不实现生产 Area/Outbox、多实例租约或孤儿处置语义,Control API 与孤儿扫描在 SQLite 下不会启动。
|
||||
|
||||
### 孤儿报告与受控处置
|
||||
|
||||
`sense-api` 的 PostgreSQL 模式默认每分钟只读枚举一次 MediaMTX 配置 Path,并持久化汇总。`unowned` 表示没有 Sense 历史归属证据,永远不会由 Sense 删除;只有 `owned_stale` 可以进入处置候选。需要人工处置时从 `Sense/` 执行:
|
||||
|
||||
```powershell
|
||||
go run ./cmd/sense-orphan -mode report
|
||||
go run ./cmd/sense-orphan -mode apply -scan-id scan_... -actor operator-id -confirm "DELETE scan_..."
|
||||
```
|
||||
|
||||
`report` 输出不含 source URI 的 JSON 摘要。`apply` 会再次读取数据库和 MediaMTX,只处理仍属于原快照的 `owned_stale`,并要求快照不超过 15 分钟、候选不超过 128 且候选占当前全部配置 Path 不超过 10%;任何门禁失败均为零删除,没有 `force`。完整流程与恢复方法见 [`../docs/runbooks/sense-reconciliation.md`](../docs/runbooks/sense-reconciliation.md)。
|
||||
|
||||
### 开启 Control API
|
||||
|
||||
|
||||
+38
-10
@@ -15,8 +15,10 @@ import (
|
||||
"yovision/sense/internal/auth"
|
||||
"yovision/sense/internal/config"
|
||||
"yovision/sense/internal/controlapi"
|
||||
"yovision/sense/internal/metrics"
|
||||
"yovision/sense/internal/mtx"
|
||||
"yovision/sense/internal/onvif"
|
||||
"yovision/sense/internal/orphan"
|
||||
"yovision/sense/internal/probe"
|
||||
"yovision/sense/internal/reconcile"
|
||||
"yovision/sense/internal/store"
|
||||
@@ -37,6 +39,14 @@ func run(logger *slog.Logger) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("load configuration: %w", err)
|
||||
}
|
||||
instanceID := cfg.InstanceID
|
||||
if instanceID == "" {
|
||||
instanceID, err = metrics.GenerateInstanceID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
registry := metrics.New(instanceID, version)
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
@@ -76,22 +86,36 @@ func run(logger *slog.Logger) error {
|
||||
})
|
||||
}
|
||||
discovery := onvif.NewRouter(cameraAdapter, credentials)
|
||||
reconciler := reconcile.New(repository, discovery, mediaClient)
|
||||
reconciler := reconcile.NewWithOptions(repository, discovery, mediaClient, reconcile.Options{
|
||||
InstanceID: instanceID, LeaseDuration: cfg.ReconcileLeaseDuration,
|
||||
OperationTimeout: cfg.ReconcileOperationTimeout, Metrics: registry,
|
||||
})
|
||||
checker := probe.New(repository, mediaClient)
|
||||
var orphanScanner *orphan.Manager
|
||||
if cfg.OrphanScanEnabled {
|
||||
orphanStore, ok := repository.(store.OrphanRepository)
|
||||
if !ok {
|
||||
return errors.New("selected repository does not support orphan scanning")
|
||||
}
|
||||
orphanScanner = orphan.New(orphanStore, mediaClient, instanceID, registry)
|
||||
}
|
||||
report := func(err error) {
|
||||
// Domain and MediaMTX errors intentionally omit stream URIs and credentials.
|
||||
logger.Warn("background convergence error", "error", err)
|
||||
}
|
||||
var background sync.WaitGroup
|
||||
background.Add(2)
|
||||
go func() {
|
||||
defer background.Done()
|
||||
reconciler.Run(ctx, cfg.ReconcileInterval, report)
|
||||
}()
|
||||
go func() {
|
||||
defer background.Done()
|
||||
checker.Run(ctx, cfg.ProbeInterval, report)
|
||||
}()
|
||||
startBackground := func(run func()) {
|
||||
background.Add(1)
|
||||
go func() {
|
||||
defer background.Done()
|
||||
run()
|
||||
}()
|
||||
}
|
||||
startBackground(func() { reconciler.Run(ctx, cfg.ReconcileInterval, report) })
|
||||
startBackground(func() { checker.Run(ctx, cfg.ProbeInterval, report) })
|
||||
if orphanScanner != nil {
|
||||
startBackground(func() { orphanScanner.Run(ctx, cfg.OrphanScanInterval, report) })
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /healthz", func(writer http.ResponseWriter, _ *http.Request) {
|
||||
@@ -104,6 +128,9 @@ func run(logger *slog.Logger) error {
|
||||
writer.WriteHeader(http.StatusOK)
|
||||
_, _ = writer.Write([]byte(`{"status":"ready"}`))
|
||||
})
|
||||
if cfg.MetricsEnabled {
|
||||
mux.Handle("GET /metrics", registry.Handler())
|
||||
}
|
||||
if cfg.ControlAPIEnabled {
|
||||
mux.Handle("/api/v1/", controlHandler)
|
||||
}
|
||||
@@ -118,6 +145,7 @@ func run(logger *slog.Logger) error {
|
||||
serverErrors := make(chan error, 1)
|
||||
go func() {
|
||||
logger.Info("Sense listening", "address", cfg.HTTPAddress, "version", version,
|
||||
"instance_id", instanceID,
|
||||
"control_api_enabled", cfg.ControlAPIEnabled)
|
||||
serverErrors <- server.ListenAndServe()
|
||||
}()
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
// sense-orphan is a local, PostgreSQL-backed operational command. Report is
|
||||
// read-only; apply requires a fresh scan ID, actor and exact confirmation.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"yovision/sense/internal/config"
|
||||
"yovision/sense/internal/metrics"
|
||||
"yovision/sense/internal/mtx"
|
||||
"yovision/sense/internal/orphan"
|
||||
"yovision/sense/internal/store"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
_, _ = fmt.Fprintln(os.Stderr, "sense-orphan failed:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run() error {
|
||||
mode := flag.String("mode", "report", "report or apply")
|
||||
scanID := flag.String("scan-id", "", "fresh scan ID required by apply")
|
||||
actor := flag.String("actor", "", "operator identifier required by apply")
|
||||
confirm := flag.String("confirm", "", "exact confirmation: DELETE <scan-id>")
|
||||
flag.Parse()
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !strings.EqualFold(strings.TrimSpace(cfg.DatabaseDriver), store.DriverPostgres) {
|
||||
return errors.New("sense-orphan requires SENSE_DB_DRIVER=postgres")
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
repository, err := store.OpenPostgres(ctx, cfg.DatabaseDSN)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer repository.Close()
|
||||
media, err := mtx.NewClient(cfg.MediaMTXURL, &http.Client{Timeout: cfg.ReconcileOperationTimeout})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
instanceID := cfg.InstanceID
|
||||
if instanceID == "" {
|
||||
instanceID, err = metrics.GenerateInstanceID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
manager := orphan.New(repository, media, instanceID, metrics.New(instanceID, "operator"))
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
switch *mode {
|
||||
case "report":
|
||||
scan, err := manager.Report(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return encoder.Encode(struct {
|
||||
ScanID string `json:"scan_id"`
|
||||
Observed int `json:"observed"`
|
||||
OwnedStale int `json:"owned_stale"`
|
||||
Unowned int `json:"unowned"`
|
||||
SafetyAllowed bool `json:"safety_allowed"`
|
||||
SafetyReason string `json:"safety_reason"`
|
||||
ExpiresAt string `json:"expires_at"`
|
||||
}{
|
||||
ScanID: scan.ID, Observed: scan.ObservedCount,
|
||||
OwnedStale: scan.OwnedStaleCount, Unowned: scan.UnownedCount,
|
||||
SafetyAllowed: scan.SafetyAllowed, SafetyReason: scan.SafetyReason,
|
||||
ExpiresAt: scan.ExpiresAt.UTC().Format("2006-01-02T15:04:05Z07:00"),
|
||||
})
|
||||
case "apply":
|
||||
result, err := manager.Apply(ctx, *scanID, *actor, *confirm)
|
||||
if encodeErr := encoder.Encode(result); encodeErr != nil {
|
||||
return encodeErr
|
||||
}
|
||||
return err
|
||||
default:
|
||||
return errors.New("mode must be report or apply")
|
||||
}
|
||||
}
|
||||
+110
-40
@@ -7,39 +7,52 @@ import (
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultHTTPAddress = "127.0.0.1:8080"
|
||||
defaultDatabaseDriver = "sqlite"
|
||||
defaultDatabaseDSN = "file:data/sense.db"
|
||||
defaultMediaMTXURL = "http://127.0.0.1:9997"
|
||||
defaultReconcilePeriod = 5 * time.Second
|
||||
defaultProbePeriod = 10 * time.Second
|
||||
defaultONVIFMode = "disabled"
|
||||
defaultControlAuthMode = "static-sha256"
|
||||
defaultHTTPAddress = "127.0.0.1:8080"
|
||||
defaultDatabaseDriver = "sqlite"
|
||||
postgresDatabaseDriver = "postgres"
|
||||
defaultDatabaseDSN = "file:data/sense.db"
|
||||
defaultMediaMTXURL = "http://127.0.0.1:9997"
|
||||
defaultReconcilePeriod = 5 * time.Second
|
||||
defaultReconcileLease = 30 * time.Second
|
||||
defaultOperationTimeout = 20 * time.Second
|
||||
defaultProbePeriod = 10 * time.Second
|
||||
defaultOrphanScanPeriod = time.Minute
|
||||
defaultONVIFMode = "disabled"
|
||||
defaultControlAuthMode = "static-sha256"
|
||||
)
|
||||
|
||||
var instanceIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`)
|
||||
|
||||
type Config struct {
|
||||
HTTPAddress string
|
||||
AllowNonLoopback bool
|
||||
DatabaseDriver string
|
||||
DatabaseDSN string
|
||||
MediaMTXURL string
|
||||
ReconcileInterval time.Duration
|
||||
ProbeInterval time.Duration
|
||||
ONVIFMode string
|
||||
RTSPRewriteHost string
|
||||
RTSPRewritePort int
|
||||
RTSPStripQuery bool
|
||||
ControlAPIEnabled bool
|
||||
ControlAuthMode string
|
||||
ControlAuthFile string
|
||||
ControlCursorKeyFile string
|
||||
ControlAllowInsecureHTTP bool
|
||||
HTTPAddress string
|
||||
AllowNonLoopback bool
|
||||
DatabaseDriver string
|
||||
DatabaseDSN string
|
||||
MediaMTXURL string
|
||||
ReconcileInterval time.Duration
|
||||
ReconcileLeaseDuration time.Duration
|
||||
ReconcileOperationTimeout time.Duration
|
||||
ProbeInterval time.Duration
|
||||
InstanceID string
|
||||
MetricsEnabled bool
|
||||
OrphanScanEnabled bool
|
||||
OrphanScanInterval time.Duration
|
||||
ONVIFMode string
|
||||
RTSPRewriteHost string
|
||||
RTSPRewritePort int
|
||||
RTSPStripQuery bool
|
||||
ControlAPIEnabled bool
|
||||
ControlAuthMode string
|
||||
ControlAuthFile string
|
||||
ControlCursorKeyFile string
|
||||
ControlAllowInsecureHTTP bool
|
||||
}
|
||||
|
||||
func Load() (Config, error) {
|
||||
@@ -51,6 +64,14 @@ func Load() (Config, error) {
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
reconcileLease, err := durationEnv("SENSE_RECONCILE_LEASE_DURATION", defaultReconcileLease)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
operationTimeout, err := durationEnv("SENSE_RECONCILE_OPERATION_TIMEOUT", defaultOperationTimeout)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
probePeriod, err := durationEnv("SENSE_PROBE_INTERVAL", defaultProbePeriod)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
@@ -71,24 +92,44 @@ func Load() (Config, error) {
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
metricsEnabled, err := boolEnv("SENSE_METRICS_ENABLED", true)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
databaseDriver := stringEnv("SENSE_DB_DRIVER", defaultDatabaseDriver)
|
||||
orphanDefault := strings.EqualFold(strings.TrimSpace(databaseDriver), postgresDatabaseDriver)
|
||||
orphanEnabled, err := boolEnv("SENSE_ORPHAN_SCAN_ENABLED", orphanDefault)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
orphanPeriod, err := durationEnv("SENSE_ORPHAN_SCAN_INTERVAL", defaultOrphanScanPeriod)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
|
||||
cfg := Config{
|
||||
HTTPAddress: stringEnv("SENSE_HTTP_ADDR", defaultHTTPAddress),
|
||||
AllowNonLoopback: allow,
|
||||
DatabaseDriver: stringEnv("SENSE_DB_DRIVER", defaultDatabaseDriver),
|
||||
DatabaseDSN: stringEnv("SENSE_DB_DSN", defaultDatabaseDSN),
|
||||
MediaMTXURL: stringEnv("SENSE_MEDIAMTX_URL", defaultMediaMTXURL),
|
||||
ReconcileInterval: reconcilePeriod,
|
||||
ProbeInterval: probePeriod,
|
||||
ONVIFMode: stringEnv("SENSE_ONVIF_MODE", defaultONVIFMode),
|
||||
RTSPRewriteHost: stringEnv("SENSE_ONVIF_RTSP_REWRITE_HOST", ""),
|
||||
RTSPRewritePort: rewritePort,
|
||||
RTSPStripQuery: stripQuery,
|
||||
ControlAPIEnabled: controlEnabled,
|
||||
ControlAuthMode: stringEnv("SENSE_CONTROL_AUTH_MODE", defaultControlAuthMode),
|
||||
ControlAuthFile: stringEnv("SENSE_CONTROL_AUTH_FILE", ""),
|
||||
ControlCursorKeyFile: stringEnv("SENSE_CONTROL_CURSOR_KEY_FILE", ""),
|
||||
ControlAllowInsecureHTTP: controlAllowInsecure,
|
||||
HTTPAddress: stringEnv("SENSE_HTTP_ADDR", defaultHTTPAddress),
|
||||
AllowNonLoopback: allow,
|
||||
DatabaseDriver: databaseDriver,
|
||||
DatabaseDSN: stringEnv("SENSE_DB_DSN", defaultDatabaseDSN),
|
||||
MediaMTXURL: stringEnv("SENSE_MEDIAMTX_URL", defaultMediaMTXURL),
|
||||
ReconcileInterval: reconcilePeriod,
|
||||
ReconcileLeaseDuration: reconcileLease,
|
||||
ReconcileOperationTimeout: operationTimeout,
|
||||
ProbeInterval: probePeriod,
|
||||
InstanceID: stringEnv("SENSE_INSTANCE_ID", ""),
|
||||
MetricsEnabled: metricsEnabled,
|
||||
OrphanScanEnabled: orphanEnabled,
|
||||
OrphanScanInterval: orphanPeriod,
|
||||
ONVIFMode: stringEnv("SENSE_ONVIF_MODE", defaultONVIFMode),
|
||||
RTSPRewriteHost: stringEnv("SENSE_ONVIF_RTSP_REWRITE_HOST", ""),
|
||||
RTSPRewritePort: rewritePort,
|
||||
RTSPStripQuery: stripQuery,
|
||||
ControlAPIEnabled: controlEnabled,
|
||||
ControlAuthMode: stringEnv("SENSE_CONTROL_AUTH_MODE", defaultControlAuthMode),
|
||||
ControlAuthFile: stringEnv("SENSE_CONTROL_AUTH_FILE", ""),
|
||||
ControlCursorKeyFile: stringEnv("SENSE_CONTROL_CURSOR_KEY_FILE", ""),
|
||||
ControlAllowInsecureHTTP: controlAllowInsecure,
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return Config{}, err
|
||||
@@ -129,6 +170,35 @@ func (c Config) Validate() error {
|
||||
if c.ReconcileInterval <= 0 || c.ProbeInterval <= 0 {
|
||||
return fmt.Errorf("loop intervals must be positive")
|
||||
}
|
||||
leaseDuration := c.ReconcileLeaseDuration
|
||||
if leaseDuration == 0 {
|
||||
leaseDuration = defaultReconcileLease
|
||||
}
|
||||
operationTimeout := c.ReconcileOperationTimeout
|
||||
if operationTimeout == 0 {
|
||||
operationTimeout = defaultOperationTimeout
|
||||
}
|
||||
if leaseDuration <= 0 || leaseDuration > 5*time.Minute {
|
||||
return fmt.Errorf("SENSE_RECONCILE_LEASE_DURATION must be positive and at most 5m")
|
||||
}
|
||||
if operationTimeout <= 0 || operationTimeout >= leaseDuration {
|
||||
return fmt.Errorf("SENSE_RECONCILE_OPERATION_TIMEOUT must be positive and shorter than the lease")
|
||||
}
|
||||
if c.InstanceID != "" && !instanceIDPattern.MatchString(c.InstanceID) {
|
||||
return fmt.Errorf("invalid SENSE_INSTANCE_ID")
|
||||
}
|
||||
if c.OrphanScanEnabled {
|
||||
if databaseDriver != postgresDatabaseDriver {
|
||||
return fmt.Errorf("orphan scanning requires SENSE_DB_DRIVER=postgres")
|
||||
}
|
||||
orphanInterval := c.OrphanScanInterval
|
||||
if orphanInterval == 0 {
|
||||
orphanInterval = defaultOrphanScanPeriod
|
||||
}
|
||||
if orphanInterval < 10*time.Second {
|
||||
return fmt.Errorf("SENSE_ORPHAN_SCAN_INTERVAL must be at least 10s")
|
||||
}
|
||||
}
|
||||
if c.ONVIFMode != "" && c.ONVIFMode != "disabled" && c.ONVIFMode != "standard" {
|
||||
return fmt.Errorf("SENSE_ONVIF_MODE must be disabled or standard")
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package config
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestValidateRejectsNonLoopbackByDefault(t *testing.T) {
|
||||
@@ -23,6 +24,35 @@ func TestValidateRejectsNonLoopbackByDefault(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateReconcileLeaseAndOrphanProductionBoundary(t *testing.T) {
|
||||
base := Config{
|
||||
HTTPAddress: "127.0.0.1:8080", DatabaseDriver: "postgres",
|
||||
DatabaseDSN: "postgres://sense-runtime@127.0.0.1/yovision?sslmode=disable",
|
||||
MediaMTXURL: "http://127.0.0.1:9997", ReconcileInterval: time.Second,
|
||||
ProbeInterval: time.Second, ReconcileLeaseDuration: 30 * time.Second,
|
||||
ReconcileOperationTimeout: 20 * time.Second, OrphanScanEnabled: true,
|
||||
OrphanScanInterval: time.Minute, InstanceID: "ins_edge-01",
|
||||
}
|
||||
if err := base.Validate(); err != nil {
|
||||
t.Fatalf("valid multi-instance configuration failed: %v", err)
|
||||
}
|
||||
invalidTimeout := base
|
||||
invalidTimeout.ReconcileOperationTimeout = invalidTimeout.ReconcileLeaseDuration
|
||||
if err := invalidTimeout.Validate(); err == nil {
|
||||
t.Fatal("operation timeout equal to the lease was accepted")
|
||||
}
|
||||
invalidInstance := base
|
||||
invalidInstance.InstanceID = "tenant/site"
|
||||
if err := invalidInstance.Validate(); err == nil {
|
||||
t.Fatal("unbounded instance label was accepted")
|
||||
}
|
||||
sqlite := base
|
||||
sqlite.DatabaseDriver, sqlite.DatabaseDSN = "sqlite", "file:test.db"
|
||||
if err := sqlite.Validate(); err == nil {
|
||||
t.Fatal("orphan scanner was accepted on SQLite")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsCredentialsInMediaMTXURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := Config{
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
// Package metrics exposes a deliberately small, low-cardinality Prometheus
|
||||
// surface without adding a runtime dependency. Tenant, Site, device and Path
|
||||
// identifiers never become labels.
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
func GenerateInstanceID() (string, error) {
|
||||
value := make([]byte, 12)
|
||||
if _, err := rand.Read(value); err != nil {
|
||||
return "", errors.New("generate Sense instance identifier")
|
||||
}
|
||||
return "ins_" + hex.EncodeToString(value), nil
|
||||
}
|
||||
|
||||
type Registry struct {
|
||||
instanceID string
|
||||
version string
|
||||
|
||||
reconcileRunOK atomic.Uint64
|
||||
reconcileRunError atomic.Uint64
|
||||
reconcileItemOK atomic.Uint64
|
||||
reconcileItemError atomic.Uint64
|
||||
reconcileItemLost atomic.Uint64
|
||||
reconcileDurationNS atomic.Uint64
|
||||
reconcileTotal atomic.Int64
|
||||
reconcileUnconverged atomic.Int64
|
||||
|
||||
orphanScanOK atomic.Uint64
|
||||
orphanScanError atomic.Uint64
|
||||
orphanObserved atomic.Int64
|
||||
orphanOwnedStale atomic.Int64
|
||||
orphanUnowned atomic.Int64
|
||||
orphanBlockedRatio atomic.Uint64
|
||||
orphanBlockedAge atomic.Uint64
|
||||
orphanBlockedScope atomic.Uint64
|
||||
orphanDeleted atomic.Uint64
|
||||
orphanDeleteFailed atomic.Uint64
|
||||
}
|
||||
|
||||
func New(instanceID, version string) *Registry {
|
||||
return &Registry{instanceID: instanceID, version: version}
|
||||
}
|
||||
|
||||
func (r *Registry) ObserveReconcileRun(err error, duration time.Duration) {
|
||||
if err == nil {
|
||||
r.reconcileRunOK.Add(1)
|
||||
} else {
|
||||
r.reconcileRunError.Add(1)
|
||||
}
|
||||
if duration < 0 {
|
||||
duration = 0
|
||||
}
|
||||
r.reconcileDurationNS.Store(uint64(duration))
|
||||
}
|
||||
|
||||
func (r *Registry) ObserveReconcileItem(result string) {
|
||||
switch result {
|
||||
case "success":
|
||||
r.reconcileItemOK.Add(1)
|
||||
case "lease_lost":
|
||||
r.reconcileItemLost.Add(1)
|
||||
default:
|
||||
r.reconcileItemError.Add(1)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Registry) SetConvergence(total, unconverged int) {
|
||||
r.reconcileTotal.Store(int64(total))
|
||||
r.reconcileUnconverged.Store(int64(unconverged))
|
||||
}
|
||||
|
||||
func (r *Registry) ObserveOrphanScan(observed, ownedStale, unowned int, err error) {
|
||||
if err == nil {
|
||||
r.orphanScanOK.Add(1)
|
||||
r.orphanObserved.Store(int64(observed))
|
||||
r.orphanOwnedStale.Store(int64(ownedStale))
|
||||
r.orphanUnowned.Store(int64(unowned))
|
||||
} else {
|
||||
r.orphanScanError.Add(1)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Registry) ObserveOrphanCleanupBlocked(reason string) {
|
||||
switch reason {
|
||||
case "snapshot_expired":
|
||||
r.orphanBlockedAge.Add(1)
|
||||
case "ratio_exceeded":
|
||||
r.orphanBlockedRatio.Add(1)
|
||||
default:
|
||||
r.orphanBlockedScope.Add(1)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Registry) ObserveOrphanCleanup(deleted, failed int) {
|
||||
if deleted > 0 {
|
||||
r.orphanDeleted.Add(uint64(deleted))
|
||||
}
|
||||
if failed > 0 {
|
||||
r.orphanDeleteFailed.Add(uint64(failed))
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Registry) Handler() http.Handler {
|
||||
return http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
writer.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
||||
writer.Header().Set("Cache-Control", "no-store")
|
||||
_, _ = writer.Write([]byte(r.render()))
|
||||
})
|
||||
}
|
||||
|
||||
func (r *Registry) render() string {
|
||||
var output strings.Builder
|
||||
writeMetric(&output, "sense_build_info", "Sense process build and instance information.",
|
||||
fmt.Sprintf("{instance_id=%s,version=%s}", quoteLabel(r.instanceID), quoteLabel(r.version)), 1)
|
||||
writeMetric(&output, "sense_reconcile_runs_total", "Reconciliation runs by result.",
|
||||
`{result="success"}`, r.reconcileRunOK.Load())
|
||||
writeSample(&output, "sense_reconcile_runs_total", `{result="error"}`, r.reconcileRunError.Load())
|
||||
writeMetric(&output, "sense_reconcile_items_total", "Reconciliation items by fixed result.",
|
||||
`{result="success"}`, r.reconcileItemOK.Load())
|
||||
writeSample(&output, "sense_reconcile_items_total", `{result="error"}`, r.reconcileItemError.Load())
|
||||
writeSample(&output, "sense_reconcile_items_total", `{result="lease_lost"}`, r.reconcileItemLost.Load())
|
||||
writeMetric(&output, "sense_reconcile_last_duration_seconds", "Duration of the last reconciliation run.",
|
||||
"", float64(r.reconcileDurationNS.Load())/float64(time.Second))
|
||||
writeMetric(&output, "sense_reconcile_devices", "Enabled video devices by convergence state.",
|
||||
`{state="total"}`, r.reconcileTotal.Load())
|
||||
writeSample(&output, "sense_reconcile_devices", `{state="unconverged"}`, r.reconcileUnconverged.Load())
|
||||
writeMetric(&output, "sense_orphan_scan_runs_total", "MediaMTX orphan scans by result.",
|
||||
`{result="success"}`, r.orphanScanOK.Load())
|
||||
writeSample(&output, "sense_orphan_scan_runs_total", `{result="error"}`, r.orphanScanError.Load())
|
||||
writeMetric(&output, "sense_orphan_paths", "Path counts from the last successful orphan scan.",
|
||||
`{classification="observed"}`, r.orphanObserved.Load())
|
||||
writeSample(&output, "sense_orphan_paths", `{classification="owned_stale"}`, r.orphanOwnedStale.Load())
|
||||
writeSample(&output, "sense_orphan_paths", `{classification="unowned"}`, r.orphanUnowned.Load())
|
||||
writeMetric(&output, "sense_orphan_cleanup_blocked_total", "Orphan cleanups blocked by a fixed safety reason.",
|
||||
`{reason="ratio_exceeded"}`, r.orphanBlockedRatio.Load())
|
||||
writeSample(&output, "sense_orphan_cleanup_blocked_total", `{reason="snapshot_expired"}`, r.orphanBlockedAge.Load())
|
||||
writeSample(&output, "sense_orphan_cleanup_blocked_total", `{reason="scope_changed"}`, r.orphanBlockedScope.Load())
|
||||
writeMetric(&output, "sense_orphan_cleanup_items_total", "Owned stale path cleanup results.",
|
||||
`{result="deleted"}`, r.orphanDeleted.Load())
|
||||
writeSample(&output, "sense_orphan_cleanup_items_total", `{result="failed"}`, r.orphanDeleteFailed.Load())
|
||||
return output.String()
|
||||
}
|
||||
|
||||
func writeMetric(builder *strings.Builder, name, help, labels string, value any) {
|
||||
metricType := "gauge"
|
||||
if strings.HasSuffix(name, "_total") {
|
||||
metricType = "counter"
|
||||
}
|
||||
builder.WriteString("# HELP ")
|
||||
builder.WriteString(name)
|
||||
builder.WriteByte(' ')
|
||||
builder.WriteString(help)
|
||||
builder.WriteByte('\n')
|
||||
builder.WriteString("# TYPE ")
|
||||
builder.WriteString(name)
|
||||
builder.WriteByte(' ')
|
||||
builder.WriteString(metricType)
|
||||
builder.WriteByte('\n')
|
||||
writeSample(builder, name, labels, value)
|
||||
}
|
||||
|
||||
func writeSample(builder *strings.Builder, name, labels string, value any) {
|
||||
builder.WriteString(name)
|
||||
builder.WriteString(labels)
|
||||
builder.WriteByte(' ')
|
||||
switch typed := value.(type) {
|
||||
case float64:
|
||||
builder.WriteString(strconv.FormatFloat(typed, 'f', 6, 64))
|
||||
default:
|
||||
builder.WriteString(fmt.Sprint(typed))
|
||||
}
|
||||
builder.WriteByte('\n')
|
||||
}
|
||||
|
||||
func quoteLabel(value string) string {
|
||||
replacer := strings.NewReplacer(`\`, `\\`, "\n", `\n`, `"`, `\"`)
|
||||
return `"` + replacer.Replace(value) + `"`
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestHandlerUsesOnlyFixedLowCardinalityLabels(t *testing.T) {
|
||||
t.Parallel()
|
||||
registry := New("ins_test-1", "v-test")
|
||||
registry.ObserveReconcileRun(nil, 1500*time.Millisecond)
|
||||
registry.ObserveReconcileItem("success")
|
||||
registry.ObserveReconcileItem("lease_lost")
|
||||
registry.SetConvergence(16, 2)
|
||||
registry.ObserveOrphanScan(20, 1, 3, nil)
|
||||
registry.ObserveOrphanCleanupBlocked("ratio_exceeded")
|
||||
registry.ObserveOrphanCleanup(1, 1)
|
||||
|
||||
request := httptest.NewRequest("GET", "/metrics", nil)
|
||||
response := httptest.NewRecorder()
|
||||
registry.Handler().ServeHTTP(response, request)
|
||||
body := response.Body.String()
|
||||
for _, expected := range []string{
|
||||
`sense_build_info{instance_id="ins_test-1",version="v-test"} 1`,
|
||||
`sense_reconcile_items_total{result="lease_lost"} 1`,
|
||||
`sense_reconcile_devices{state="unconverged"} 2`,
|
||||
`sense_orphan_paths{classification="owned_stale"} 1`,
|
||||
`sense_orphan_cleanup_blocked_total{reason="ratio_exceeded"} 1`,
|
||||
} {
|
||||
if !strings.Contains(body, expected) {
|
||||
t.Fatalf("metrics output lacks %q:\n%s", expected, body)
|
||||
}
|
||||
}
|
||||
for _, forbidden := range []string{
|
||||
"tenant_id", "site_id", "device_id", "path_name", "rtsp://", "credential",
|
||||
} {
|
||||
if strings.Contains(body, forbidden) {
|
||||
t.Fatalf("high-cardinality or secret-bearing label appeared: %q", forbidden)
|
||||
}
|
||||
}
|
||||
if response.Header().Get("Cache-Control") != "no-store" {
|
||||
t.Fatal("metrics responses must not be cached")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryIsSafeForConcurrentObservers(t *testing.T) {
|
||||
t.Parallel()
|
||||
registry := New("ins_concurrent", "dev")
|
||||
var wait sync.WaitGroup
|
||||
for index := 0; index < 32; index++ {
|
||||
wait.Add(1)
|
||||
go func() {
|
||||
defer wait.Done()
|
||||
for count := 0; count < 100; count++ {
|
||||
registry.ObserveReconcileItem("success")
|
||||
registry.SetConvergence(128, count%17)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wait.Wait()
|
||||
response := httptest.NewRecorder()
|
||||
registry.Handler().ServeHTTP(response, httptest.NewRequest("GET", "/metrics", nil))
|
||||
if !strings.Contains(response.Body.String(), `sense_reconcile_items_total{result="success"} 3200`) {
|
||||
t.Fatalf("concurrent counter lost updates: %s", response.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateInstanceIDShape(t *testing.T) {
|
||||
t.Parallel()
|
||||
first, err := GenerateInstanceID()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := GenerateInstanceID()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(first) != 28 || !strings.HasPrefix(first, "ins_") || first == second {
|
||||
t.Fatalf("unexpected instance identifiers: %q %q", first, second)
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
mediamtxapi "yovision/sense/internal/mtx/generated"
|
||||
@@ -31,9 +32,63 @@ type pathAPI interface {
|
||||
ConfigPathsGetWithResponse(context.Context, string, ...mediamtxapi.RequestEditorFn) (*mediamtxapi.ConfigPathsGetResponse, error)
|
||||
ConfigPathsPatchWithResponse(context.Context, string, mediamtxapi.ConfigPathsPatchJSONRequestBody, ...mediamtxapi.RequestEditorFn) (*mediamtxapi.ConfigPathsPatchResponse, error)
|
||||
ConfigPathsDeleteWithResponse(context.Context, string, ...mediamtxapi.RequestEditorFn) (*mediamtxapi.ConfigPathsDeleteResponse, error)
|
||||
ConfigPathsListWithResponse(context.Context, *mediamtxapi.ConfigPathsListParams, ...mediamtxapi.RequestEditorFn) (*mediamtxapi.ConfigPathsListResponse, error)
|
||||
PathsGetWithResponse(context.Context, string, ...mediamtxapi.RequestEditorFn) (*mediamtxapi.PathsGetResponse, error)
|
||||
}
|
||||
|
||||
// ListPathNames enumerates only configuration names. Sources are deliberately
|
||||
// discarded so inventory and orphan reports cannot expose stream URIs.
|
||||
func (c *Client) ListPathNames(ctx context.Context) ([]string, error) {
|
||||
const (
|
||||
itemsPerPage = 100
|
||||
maxPages = 1000
|
||||
)
|
||||
result := make(map[string]struct{})
|
||||
seenPages := make(map[string]struct{})
|
||||
for page := 0; page < maxPages; page++ {
|
||||
pageValue, limitValue := page, itemsPerPage
|
||||
response, err := c.api.ConfigPathsListWithResponse(ctx, &mediamtxapi.ConfigPathsListParams{
|
||||
Page: &pageValue, ItemsPerPage: &limitValue,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("MediaMTX list paths transport: %w", err)
|
||||
}
|
||||
if response.StatusCode() != http.StatusOK || response.JSON200 == nil ||
|
||||
response.JSON200.PageCount == nil || response.JSON200.Items == nil {
|
||||
return nil, &APIError{Operation: "list paths", StatusCode: response.StatusCode()}
|
||||
}
|
||||
pageCount := *response.JSON200.PageCount
|
||||
if pageCount < 0 || pageCount > maxPages {
|
||||
return nil, &APIError{Operation: "list paths pagination", StatusCode: response.StatusCode()}
|
||||
}
|
||||
pageNames := make([]string, 0, len(*response.JSON200.Items))
|
||||
for _, item := range *response.JSON200.Items {
|
||||
if item.Name == nil || strings.TrimSpace(*item.Name) == "" {
|
||||
return nil, &APIError{Operation: "list paths response", StatusCode: response.StatusCode()}
|
||||
}
|
||||
pageNames = append(pageNames, *item.Name)
|
||||
result[*item.Name] = struct{}{}
|
||||
}
|
||||
sort.Strings(pageNames)
|
||||
signature := strings.Join(pageNames, "\x00")
|
||||
if page > 0 && signature != "" {
|
||||
if _, duplicate := seenPages[signature]; duplicate {
|
||||
return nil, &APIError{Operation: "list paths repeated page", StatusCode: response.StatusCode()}
|
||||
}
|
||||
}
|
||||
seenPages[signature] = struct{}{}
|
||||
if int64(page+1) >= pageCount {
|
||||
values := make([]string, 0, len(result))
|
||||
for name := range result {
|
||||
values = append(values, name)
|
||||
}
|
||||
sort.Strings(values)
|
||||
return values, nil
|
||||
}
|
||||
}
|
||||
return nil, &APIError{Operation: "list paths page limit", StatusCode: http.StatusOK}
|
||||
}
|
||||
|
||||
type Client struct {
|
||||
api pathAPI
|
||||
}
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -21,6 +23,38 @@ func (f *fakeMediaMTX) ServeHTTP(writer http.ResponseWriter, request *http.Reque
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
if request.URL.Path == "/v3/config/paths/list" {
|
||||
page, _ := strconv.Atoi(request.URL.Query().Get("page"))
|
||||
limit, _ := strconv.Atoi(request.URL.Query().Get("itemsPerPage"))
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
names := make([]string, 0, len(f.paths))
|
||||
for name := range f.paths {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
start := page * limit
|
||||
if start > len(names) {
|
||||
start = len(names)
|
||||
}
|
||||
end := start + limit
|
||||
if end > len(names) {
|
||||
end = len(names)
|
||||
}
|
||||
items := make([]map[string]any, 0, end-start)
|
||||
for _, name := range names[start:end] {
|
||||
items = append(items, map[string]any{"name": name})
|
||||
}
|
||||
pageCount := 0
|
||||
if len(names) > 0 {
|
||||
pageCount = (len(names) + limit - 1) / limit
|
||||
}
|
||||
_ = json.NewEncoder(writer).Encode(map[string]any{
|
||||
"itemCount": len(names), "pageCount": pageCount, "items": items,
|
||||
})
|
||||
return
|
||||
}
|
||||
prefixes := map[string]string{
|
||||
"/v3/config/paths/get/": "get",
|
||||
"/v3/config/paths/add/": "add",
|
||||
@@ -72,6 +106,54 @@ func (f *fakeMediaMTX) ServeHTTP(writer http.ResponseWriter, request *http.Reque
|
||||
http.NotFound(writer, request)
|
||||
}
|
||||
|
||||
func TestListPathNamesUsesPaginationAndDoesNotReturnSources(t *testing.T) {
|
||||
t.Parallel()
|
||||
paths := make(map[string]string)
|
||||
for index := 0; index < 205; index++ {
|
||||
paths[fmt.Sprintf("camera-%03d", index)] = fmt.Sprintf("rtsp://secret.invalid/%d", index)
|
||||
}
|
||||
fake := &fakeMediaMTX{paths: paths}
|
||||
server := httptest.NewServer(fake)
|
||||
defer server.Close()
|
||||
client, err := NewClient(server.URL, server.Client())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
values, err := client.ListPathNames(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(values) != 205 || values[0] != "camera-000" || values[204] != "camera-204" {
|
||||
t.Fatalf("unexpected path inventory: len=%d first=%q last=%q", len(values), values[0], values[len(values)-1])
|
||||
}
|
||||
for _, value := range values {
|
||||
if strings.Contains(value, "rtsp") || strings.Contains(value, "secret") {
|
||||
t.Fatalf("source leaked from path inventory: %q", value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListPathNamesRejectsRepeatedPage(t *testing.T) {
|
||||
t.Parallel()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
if request.URL.Path != "/v3/config/paths/list" {
|
||||
http.NotFound(writer, request)
|
||||
return
|
||||
}
|
||||
_, _ = writer.Write([]byte(`{"itemCount":2,"pageCount":2,"items":[{"name":"same"}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := NewClient(server.URL, server.Client())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := client.ListPathNames(context.Background()); err == nil ||
|
||||
!strings.Contains(err.Error(), "repeated page") {
|
||||
t.Fatalf("repeated pagination was accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratedClientCreateReadDeleteMapping(t *testing.T) {
|
||||
t.Parallel()
|
||||
fake := &fakeMediaMTX{paths: make(map[string]string)}
|
||||
|
||||
@@ -0,0 +1,400 @@
|
||||
// Package orphan reports MediaMTX configuration paths that do not match the
|
||||
// current Sense ledger. Only paths with durable Sense ownership evidence can
|
||||
// ever enter the controlled cleanup set.
|
||||
package orphan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"yovision/sense/internal/metrics"
|
||||
"yovision/sense/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultLeaseDuration = 30 * time.Second
|
||||
defaultOperationTimeout = 20 * time.Second
|
||||
reportTTL = 15 * time.Minute
|
||||
maxCleanupItems = 128
|
||||
crockford = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrLeaseHeld = errors.New("orphan operation lease held")
|
||||
ErrConfirmation = errors.New("orphan cleanup confirmation invalid")
|
||||
ErrSnapshotExpired = errors.New("orphan scan snapshot expired")
|
||||
ErrSafetyBlocked = errors.New("orphan cleanup safety gate blocked")
|
||||
operatorIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`)
|
||||
)
|
||||
|
||||
type Runtime interface {
|
||||
ListPathNames(context.Context) ([]string, error)
|
||||
DeletePath(context.Context, string) error
|
||||
}
|
||||
|
||||
type Manager struct {
|
||||
repository store.OrphanRepository
|
||||
runtime Runtime
|
||||
instanceID string
|
||||
metrics *metrics.Registry
|
||||
now func() time.Time
|
||||
leaseDuration time.Duration
|
||||
operationTimeout time.Duration
|
||||
}
|
||||
|
||||
type CleanupResult struct {
|
||||
ScanID string `json:"scan_id"`
|
||||
Deleted int `json:"deleted"`
|
||||
Failed int `json:"failed"`
|
||||
}
|
||||
|
||||
func New(
|
||||
repository store.OrphanRepository,
|
||||
runtime Runtime,
|
||||
instanceID string,
|
||||
registry *metrics.Registry,
|
||||
) *Manager {
|
||||
return &Manager{
|
||||
repository: repository, runtime: runtime, instanceID: instanceID, metrics: registry,
|
||||
now: time.Now, leaseDuration: defaultLeaseDuration, operationTimeout: defaultOperationTimeout,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) Report(ctx context.Context) (store.OrphanScan, error) {
|
||||
now := m.now().UTC()
|
||||
token, err := randomToken()
|
||||
if err != nil {
|
||||
return store.OrphanScan{}, err
|
||||
}
|
||||
acquired, err := m.repository.AcquireOperationalLease(
|
||||
ctx, store.OperationalLeaseOrphanScan, m.instanceID, token, now, m.leaseDuration,
|
||||
)
|
||||
if err != nil {
|
||||
m.observeScan(0, 0, 0, err)
|
||||
return store.OrphanScan{}, err
|
||||
}
|
||||
if !acquired {
|
||||
return store.OrphanScan{}, ErrLeaseHeld
|
||||
}
|
||||
saved := false
|
||||
defer func() {
|
||||
if saved {
|
||||
return
|
||||
}
|
||||
releaseCtx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
_ = m.repository.ReleaseOperationalLease(
|
||||
releaseCtx, store.OperationalLeaseOrphanScan, m.instanceID, token, m.now().UTC(),
|
||||
)
|
||||
}()
|
||||
|
||||
operationCtx, cancel := context.WithTimeout(ctx, m.operationTimeout)
|
||||
paths, err := m.runtime.ListPathNames(operationCtx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
m.observeScan(0, 0, 0, err)
|
||||
return store.OrphanScan{}, fmt.Errorf("list MediaMTX paths: %w", err)
|
||||
}
|
||||
ownership, err := m.repository.ListMediaPathOwnership(ctx)
|
||||
if err != nil {
|
||||
m.observeScan(0, 0, 0, err)
|
||||
return store.OrphanScan{}, err
|
||||
}
|
||||
findings, stale, unowned := classify(paths, ownership)
|
||||
allowed, reason := safetyGate(stale, len(paths))
|
||||
completedAt := m.now().UTC()
|
||||
acquired, err = m.repository.AcquireOperationalLease(
|
||||
ctx, store.OperationalLeaseOrphanScan, m.instanceID, token, completedAt, m.leaseDuration,
|
||||
)
|
||||
if err != nil || !acquired {
|
||||
if err == nil {
|
||||
err = store.ErrOperationalLeaseLost
|
||||
}
|
||||
m.observeScan(len(paths), stale, unowned, err)
|
||||
return store.OrphanScan{}, err
|
||||
}
|
||||
id, err := newScanID(completedAt)
|
||||
if err != nil {
|
||||
m.observeScan(0, 0, 0, err)
|
||||
return store.OrphanScan{}, err
|
||||
}
|
||||
scan := store.OrphanScan{
|
||||
ID: id, InstanceID: m.instanceID, ObservedCount: len(paths),
|
||||
OwnedStaleCount: stale, UnownedCount: unowned,
|
||||
SafetyAllowed: allowed, SafetyReason: reason,
|
||||
CompletedAt: completedAt, ExpiresAt: completedAt.Add(reportTTL), Findings: findings,
|
||||
}
|
||||
if err := m.repository.SaveOrphanScan(ctx, scan, m.instanceID, token); err != nil {
|
||||
m.observeScan(len(paths), stale, unowned, err)
|
||||
return store.OrphanScan{}, err
|
||||
}
|
||||
saved = true
|
||||
m.observeScan(len(paths), stale, unowned, nil)
|
||||
return scan, nil
|
||||
}
|
||||
|
||||
func classify(
|
||||
paths []string,
|
||||
ownership []store.MediaPathOwnership,
|
||||
) ([]store.OrphanFinding, int, int) {
|
||||
history := make(map[string]store.MediaPathOwnership, len(ownership))
|
||||
for _, value := range ownership {
|
||||
history[value.PathName] = value
|
||||
}
|
||||
unique := make(map[string]struct{}, len(paths))
|
||||
for _, path := range paths {
|
||||
unique[path] = struct{}{}
|
||||
}
|
||||
ordered := make([]string, 0, len(unique))
|
||||
for path := range unique {
|
||||
ordered = append(ordered, path)
|
||||
}
|
||||
sort.Strings(ordered)
|
||||
findings := make([]store.OrphanFinding, 0)
|
||||
stale, unowned := 0, 0
|
||||
for _, path := range ordered {
|
||||
value, known := history[path]
|
||||
switch {
|
||||
case known && value.CurrentClaim:
|
||||
continue
|
||||
case known:
|
||||
stale++
|
||||
findings = append(findings, store.OrphanFinding{
|
||||
PathName: path, Classification: store.OrphanOwnedStale, DeviceID: value.DeviceID,
|
||||
})
|
||||
default:
|
||||
unowned++
|
||||
findings = append(findings, store.OrphanFinding{
|
||||
PathName: path, Classification: store.OrphanUnowned,
|
||||
})
|
||||
}
|
||||
}
|
||||
return findings, stale, unowned
|
||||
}
|
||||
|
||||
func safetyGate(candidates, observed int) (bool, string) {
|
||||
switch {
|
||||
case candidates <= 0:
|
||||
return false, "no_candidates"
|
||||
case observed <= 0:
|
||||
return false, "empty_inventory"
|
||||
case candidates > maxCleanupItems:
|
||||
return false, "scope_exceeded"
|
||||
case candidates*100 > observed*10:
|
||||
return false, "ratio_exceeded"
|
||||
default:
|
||||
return true, "allowed"
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) Apply(
|
||||
ctx context.Context,
|
||||
scanID, actorID, confirmation string,
|
||||
) (CleanupResult, error) {
|
||||
result := CleanupResult{ScanID: scanID}
|
||||
if !operatorIDPattern.MatchString(actorID) || confirmation != "DELETE "+scanID {
|
||||
m.blocked("scope_changed")
|
||||
return result, ErrConfirmation
|
||||
}
|
||||
now := m.now().UTC()
|
||||
token, err := randomToken()
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
acquired, err := m.repository.AcquireOperationalLease(
|
||||
ctx, store.OperationalLeaseOrphanCleanup, m.instanceID, token, now, m.leaseDuration,
|
||||
)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
if !acquired {
|
||||
return result, ErrLeaseHeld
|
||||
}
|
||||
defer func() {
|
||||
releaseCtx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
_ = m.repository.ReleaseOperationalLease(
|
||||
releaseCtx, store.OperationalLeaseOrphanCleanup, m.instanceID, token, m.now().UTC(),
|
||||
)
|
||||
}()
|
||||
|
||||
scan, err := m.repository.GetOrphanScan(ctx, scanID)
|
||||
if err != nil {
|
||||
m.blocked("scope_changed")
|
||||
return result, err
|
||||
}
|
||||
if !scan.ExpiresAt.After(now) {
|
||||
m.blocked("snapshot_expired")
|
||||
return result, ErrSnapshotExpired
|
||||
}
|
||||
if !scan.SafetyAllowed {
|
||||
m.blocked(scan.SafetyReason)
|
||||
return result, fmt.Errorf("%w: %s", ErrSafetyBlocked, scan.SafetyReason)
|
||||
}
|
||||
operationCtx, cancel := context.WithTimeout(ctx, m.operationTimeout)
|
||||
paths, err := m.runtime.ListPathNames(operationCtx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("refresh MediaMTX path inventory: %w", err)
|
||||
}
|
||||
ownership, err := m.repository.ListMediaPathOwnership(ctx)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
candidates := cleanupCandidates(scan, paths, ownership)
|
||||
if len(candidates) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
allowed, reason := safetyGate(len(candidates), len(uniqueNames(paths)))
|
||||
if !allowed {
|
||||
m.blocked(reason)
|
||||
return result, fmt.Errorf("%w: %s", ErrSafetyBlocked, reason)
|
||||
}
|
||||
var failures []error
|
||||
for _, path := range candidates {
|
||||
now = m.now().UTC()
|
||||
acquired, err = m.repository.AcquireOperationalLease(
|
||||
ctx, store.OperationalLeaseOrphanCleanup, m.instanceID, token, now, m.leaseDuration,
|
||||
)
|
||||
if err != nil || !acquired {
|
||||
if err == nil {
|
||||
err = ErrLeaseHeld
|
||||
}
|
||||
failures = append(failures, err)
|
||||
break
|
||||
}
|
||||
itemCtx, itemCancel := context.WithTimeout(ctx, m.operationTimeout)
|
||||
deleteErr := m.runtime.DeletePath(itemCtx, path)
|
||||
itemCancel()
|
||||
status, code := "deleted", ""
|
||||
if deleteErr != nil {
|
||||
status, code = "failed", "media_error"
|
||||
result.Failed++
|
||||
failures = append(failures, fmt.Errorf("delete owned stale path: %w", deleteErr))
|
||||
} else {
|
||||
result.Deleted++
|
||||
}
|
||||
if err := m.repository.RecordOrphanCleanup(
|
||||
ctx, scanID, path, actorID, status, code, m.now().UTC(),
|
||||
); err != nil {
|
||||
failures = append(failures, err)
|
||||
break
|
||||
}
|
||||
}
|
||||
if m.metrics != nil {
|
||||
m.metrics.ObserveOrphanCleanup(result.Deleted, result.Failed)
|
||||
}
|
||||
return result, errors.Join(failures...)
|
||||
}
|
||||
|
||||
func cleanupCandidates(
|
||||
scan store.OrphanScan,
|
||||
paths []string,
|
||||
ownership []store.MediaPathOwnership,
|
||||
) []string {
|
||||
runtime := uniqueNames(paths)
|
||||
history := make(map[string]store.MediaPathOwnership, len(ownership))
|
||||
for _, value := range ownership {
|
||||
history[value.PathName] = value
|
||||
}
|
||||
values := make([]string, 0)
|
||||
for _, finding := range scan.Findings {
|
||||
if finding.Classification != store.OrphanOwnedStale || finding.Deleted {
|
||||
continue
|
||||
}
|
||||
_, present := runtime[finding.PathName]
|
||||
owner, known := history[finding.PathName]
|
||||
if present && known && !owner.CurrentClaim && owner.DeviceID == finding.DeviceID {
|
||||
values = append(values, finding.PathName)
|
||||
}
|
||||
}
|
||||
sort.Strings(values)
|
||||
return values
|
||||
}
|
||||
|
||||
func uniqueNames(values []string) map[string]struct{} {
|
||||
result := make(map[string]struct{}, len(values))
|
||||
for _, value := range values {
|
||||
result[value] = struct{}{}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (m *Manager) Run(ctx context.Context, interval time.Duration, report func(error)) {
|
||||
run := func() {
|
||||
_, err := m.Report(ctx)
|
||||
if err != nil && !errors.Is(err, ErrLeaseHeld) && ctx.Err() == nil && report != nil {
|
||||
report(err)
|
||||
}
|
||||
}
|
||||
run()
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
run()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) observeScan(observed, stale, unowned int, err error) {
|
||||
if m.metrics != nil {
|
||||
m.metrics.ObserveOrphanScan(observed, stale, unowned, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) blocked(reason string) {
|
||||
if m.metrics != nil {
|
||||
m.metrics.ObserveOrphanCleanupBlocked(reason)
|
||||
}
|
||||
}
|
||||
|
||||
func randomToken() (string, error) {
|
||||
value := make([]byte, 16)
|
||||
if _, err := rand.Read(value); err != nil {
|
||||
return "", errors.New("generate fencing token")
|
||||
}
|
||||
const alphabet = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||
result := make([]byte, 22)
|
||||
number := new(big.Int).SetBytes(value)
|
||||
base := big.NewInt(int64(len(alphabet)))
|
||||
remainder := new(big.Int)
|
||||
for index := len(result) - 1; index >= 0; index-- {
|
||||
number.QuoRem(number, base, remainder)
|
||||
result[index] = alphabet[remainder.Int64()]
|
||||
}
|
||||
return string(result), nil
|
||||
}
|
||||
|
||||
func newScanID(now time.Time) (string, error) {
|
||||
value := make([]byte, 16)
|
||||
milliseconds := uint64(now.UTC().UnixMilli())
|
||||
value[0], value[1], value[2] = byte(milliseconds>>40), byte(milliseconds>>32), byte(milliseconds>>24)
|
||||
value[3], value[4], value[5] = byte(milliseconds>>16), byte(milliseconds>>8), byte(milliseconds)
|
||||
if _, err := rand.Read(value[6:]); err != nil {
|
||||
return "", errors.New("generate orphan scan identifier")
|
||||
}
|
||||
number := new(big.Int).SetBytes(value)
|
||||
base := big.NewInt(32)
|
||||
remainder := new(big.Int)
|
||||
encoded := make([]byte, 26)
|
||||
for index := len(encoded) - 1; index >= 0; index-- {
|
||||
number.QuoRem(number, base, remainder)
|
||||
encoded[index] = crockford[remainder.Int64()]
|
||||
}
|
||||
return "scan_" + string(encoded), nil
|
||||
}
|
||||
|
||||
func Confirmation(scanID string) string {
|
||||
return strings.Join([]string{"DELETE", scanID}, " ")
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
package orphan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"yovision/sense/internal/store"
|
||||
)
|
||||
|
||||
type fakeLease struct {
|
||||
owner, token string
|
||||
expires time.Time
|
||||
}
|
||||
|
||||
type fakeRepository struct {
|
||||
mu sync.Mutex
|
||||
leases map[string]fakeLease
|
||||
ownership []store.MediaPathOwnership
|
||||
scans map[string]store.OrphanScan
|
||||
deleted map[string]map[string]bool
|
||||
}
|
||||
|
||||
func newFakeRepository() *fakeRepository {
|
||||
return &fakeRepository{
|
||||
leases: make(map[string]fakeLease), scans: make(map[string]store.OrphanScan),
|
||||
deleted: make(map[string]map[string]bool),
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeRepository) AcquireOperationalLease(
|
||||
_ context.Context,
|
||||
name, owner, token string,
|
||||
now time.Time,
|
||||
duration time.Duration,
|
||||
) (bool, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
current, exists := f.leases[name]
|
||||
if exists && current.expires.After(now) && (current.owner != owner || current.token != token) {
|
||||
return false, nil
|
||||
}
|
||||
f.leases[name] = fakeLease{owner: owner, token: token, expires: now.Add(duration)}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (f *fakeRepository) ReleaseOperationalLease(
|
||||
_ context.Context,
|
||||
name, owner, token string,
|
||||
now time.Time,
|
||||
) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
current := f.leases[name]
|
||||
if current.owner == owner && current.token == token {
|
||||
current.expires = now
|
||||
f.leases[name] = current
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeRepository) ListMediaPathOwnership(context.Context) ([]store.MediaPathOwnership, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]store.MediaPathOwnership(nil), f.ownership...), nil
|
||||
}
|
||||
|
||||
func (f *fakeRepository) SaveOrphanScan(
|
||||
_ context.Context,
|
||||
scan store.OrphanScan,
|
||||
owner, token string,
|
||||
) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
lease := f.leases[store.OperationalLeaseOrphanScan]
|
||||
if lease.owner != owner || lease.token != token || !lease.expires.After(scan.CompletedAt) {
|
||||
return store.ErrOperationalLeaseLost
|
||||
}
|
||||
f.scans[scan.ID] = cloneScan(scan)
|
||||
lease.expires = scan.CompletedAt
|
||||
f.leases[store.OperationalLeaseOrphanScan] = lease
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeRepository) GetOrphanScan(_ context.Context, id string) (store.OrphanScan, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
scan, exists := f.scans[id]
|
||||
if !exists {
|
||||
return store.OrphanScan{}, store.ErrOrphanScanNotFound
|
||||
}
|
||||
result := cloneScan(scan)
|
||||
for index := range result.Findings {
|
||||
result.Findings[index].Deleted = f.deleted[id][result.Findings[index].PathName]
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (f *fakeRepository) RecordOrphanCleanup(
|
||||
_ context.Context,
|
||||
scanID, pathName, _, status, _ string,
|
||||
_ time.Time,
|
||||
) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.deleted[scanID] == nil {
|
||||
f.deleted[scanID] = make(map[string]bool)
|
||||
}
|
||||
if status == "deleted" {
|
||||
f.deleted[scanID][pathName] = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cloneScan(value store.OrphanScan) store.OrphanScan {
|
||||
value.Findings = append([]store.OrphanFinding(nil), value.Findings...)
|
||||
return value
|
||||
}
|
||||
|
||||
type fakeRuntime struct {
|
||||
paths map[string]bool
|
||||
fail map[string]bool
|
||||
deleted []string
|
||||
}
|
||||
|
||||
func (f *fakeRuntime) ListPathNames(context.Context) ([]string, error) {
|
||||
values := make([]string, 0, len(f.paths))
|
||||
for path, present := range f.paths {
|
||||
if present {
|
||||
values = append(values, path)
|
||||
}
|
||||
}
|
||||
sort.Strings(values)
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func (f *fakeRuntime) DeletePath(_ context.Context, path string) error {
|
||||
if f.fail[path] {
|
||||
return errors.New("redacted media failure")
|
||||
}
|
||||
delete(f.paths, path)
|
||||
f.deleted = append(f.deleted, path)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestSafetyGateUsesExactTenPercentBoundary(t *testing.T) {
|
||||
t.Parallel()
|
||||
if allowed, reason := safetyGate(1, 10); !allowed || reason != "allowed" {
|
||||
t.Fatalf("exact 10%% boundary was rejected: %v %s", allowed, reason)
|
||||
}
|
||||
if allowed, reason := safetyGate(1, 9); allowed || reason != "ratio_exceeded" {
|
||||
t.Fatalf("more than 10%% was accepted: %v %s", allowed, reason)
|
||||
}
|
||||
if allowed, reason := safetyGate(129, 2000); allowed || reason != "scope_exceeded" {
|
||||
t.Fatalf("129-item scope was accepted: %v %s", allowed, reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportClassifiesOwnershipAndApplyNeverDeletesUnowned(t *testing.T) {
|
||||
repository := newFakeRepository()
|
||||
repository.ownership = []store.MediaPathOwnership{
|
||||
{PathName: "owned-stale", DeviceID: "old-device"},
|
||||
{PathName: "owned-current", DeviceID: "live-device", CurrentClaim: true},
|
||||
}
|
||||
runtime := &fakeRuntime{paths: map[string]bool{
|
||||
"owned-stale": true, "owned-current": true,
|
||||
"unowned-1": true, "unowned-2": true, "unowned-3": true, "unowned-4": true,
|
||||
"unowned-5": true, "unowned-6": true, "unowned-7": true, "unowned-8": true,
|
||||
}, fail: make(map[string]bool)}
|
||||
now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC)
|
||||
manager := New(repository, runtime, "ins_test", nil)
|
||||
manager.now = func() time.Time { return now }
|
||||
scan, err := manager.Report(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scan.ObservedCount != 10 || scan.OwnedStaleCount != 1 || scan.UnownedCount != 8 || !scan.SafetyAllowed {
|
||||
t.Fatalf("unexpected report: %+v", scan)
|
||||
}
|
||||
result, err := manager.Apply(
|
||||
context.Background(), scan.ID, "operator-1", Confirmation(scan.ID),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Deleted != 1 || len(runtime.deleted) != 1 || runtime.deleted[0] != "owned-stale" {
|
||||
t.Fatalf("cleanup escaped owned stale set: result=%+v deleted=%v", result, runtime.deleted)
|
||||
}
|
||||
for _, path := range []string{"owned-current", "unowned-1", "unowned-8"} {
|
||||
if !runtime.paths[path] {
|
||||
t.Fatalf("cleanup deleted protected path %q", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyBlocksExpiredOrOversizedSnapshotBeforeDelete(t *testing.T) {
|
||||
repository := newFakeRepository()
|
||||
repository.ownership = []store.MediaPathOwnership{{PathName: "stale", DeviceID: "old"}}
|
||||
runtime := &fakeRuntime{paths: map[string]bool{"stale": true}, fail: make(map[string]bool)}
|
||||
now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC)
|
||||
manager := New(repository, runtime, "ins_test", nil)
|
||||
manager.now = func() time.Time { return now }
|
||||
scan, err := manager.Report(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if scan.SafetyAllowed || scan.SafetyReason != "ratio_exceeded" {
|
||||
t.Fatalf("single-path 100%% cleanup was not blocked: %+v", scan)
|
||||
}
|
||||
if _, err := manager.Apply(context.Background(), scan.ID, "operator", Confirmation(scan.ID)); !errors.Is(err, ErrSafetyBlocked) {
|
||||
t.Fatalf("expected ratio gate, got %v", err)
|
||||
}
|
||||
if len(runtime.deleted) != 0 {
|
||||
t.Fatal("ratio-blocked cleanup mutated MediaMTX")
|
||||
}
|
||||
|
||||
for index := 0; index < 9; index++ {
|
||||
runtime.paths[string(rune('a'+index))] = true
|
||||
}
|
||||
scan, err = manager.Report(context.Background())
|
||||
if err != nil || !scan.SafetyAllowed {
|
||||
t.Fatalf("expected a fresh 1/10 executable report: %+v %v", scan, err)
|
||||
}
|
||||
now = now.Add(reportTTL)
|
||||
if _, err := manager.Apply(context.Background(), scan.ID, "operator", Confirmation(scan.ID)); !errors.Is(err, ErrSnapshotExpired) {
|
||||
t.Fatalf("expected expired snapshot, got %v", err)
|
||||
}
|
||||
if len(runtime.deleted) != 0 {
|
||||
t.Fatal("expired cleanup mutated MediaMTX")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyIsRetryableAndCannotExpandBeyondSnapshot(t *testing.T) {
|
||||
repository := newFakeRepository()
|
||||
repository.ownership = []store.MediaPathOwnership{
|
||||
{PathName: "stale-a", DeviceID: "old-a"},
|
||||
{PathName: "stale-b", DeviceID: "old-b"},
|
||||
}
|
||||
runtime := &fakeRuntime{paths: make(map[string]bool), fail: map[string]bool{"stale-b": true}}
|
||||
runtime.paths["stale-a"], runtime.paths["stale-b"] = true, true
|
||||
for index := 0; index < 18; index++ {
|
||||
runtime.paths[string(rune(0x100+index))] = true
|
||||
}
|
||||
now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC)
|
||||
manager := New(repository, runtime, "ins_test", nil)
|
||||
manager.now = func() time.Time { return now }
|
||||
scan, err := manager.Report(context.Background())
|
||||
if err != nil || !scan.SafetyAllowed {
|
||||
t.Fatalf("expected 2/20 report: %+v %v", scan, err)
|
||||
}
|
||||
repository.ownership = append(repository.ownership,
|
||||
store.MediaPathOwnership{PathName: "new-stale", DeviceID: "new-old"})
|
||||
runtime.paths["new-stale"] = true
|
||||
result, err := manager.Apply(context.Background(), scan.ID, "operator", Confirmation(scan.ID))
|
||||
if err == nil || result.Deleted != 1 || result.Failed != 1 {
|
||||
t.Fatalf("expected one partial failure: %+v %v", result, err)
|
||||
}
|
||||
if !runtime.paths["new-stale"] {
|
||||
t.Fatal("cleanup expanded beyond the approved snapshot")
|
||||
}
|
||||
runtime.fail["stale-b"] = false
|
||||
result, err = manager.Apply(context.Background(), scan.ID, "operator", Confirmation(scan.ID))
|
||||
if err != nil || result.Deleted != 1 || result.Failed != 0 {
|
||||
t.Fatalf("failed item was not retryable: %+v %v", result, err)
|
||||
}
|
||||
if !runtime.paths["new-stale"] {
|
||||
t.Fatal("retry expanded beyond the approved snapshot")
|
||||
}
|
||||
}
|
||||
@@ -3,12 +3,15 @@ package reconcile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"yovision/sense/internal/device"
|
||||
"yovision/sense/internal/metrics"
|
||||
"yovision/sense/internal/onvif"
|
||||
"yovision/sense/internal/store"
|
||||
)
|
||||
@@ -16,9 +19,11 @@ import (
|
||||
const defaultBatchSize = 128
|
||||
|
||||
type Repository interface {
|
||||
ListDueReconcile(ctx context.Context, now time.Time, limit int) ([]store.ReconcileCandidate, error)
|
||||
MarkReconciled(ctx context.Context, id string, generation int64, now time.Time) error
|
||||
MarkReconcileFailure(ctx context.Context, id string, failureCount int, nextAttempt time.Time, errorCode string, now time.Time) error
|
||||
ClaimDueReconcile(context.Context, store.ReconcileClaim) ([]store.ReconcileCandidate, error)
|
||||
RenewReconcileLease(context.Context, string, string, string, time.Time, time.Duration) (bool, error)
|
||||
CompleteReconcile(context.Context, string, int64, string, string, time.Time) error
|
||||
FailReconcile(context.Context, string, int, time.Time, string, string, string, time.Time) error
|
||||
ConvergenceSnapshot(context.Context) (store.ConvergenceSnapshot, error)
|
||||
}
|
||||
|
||||
type MediaPaths interface {
|
||||
@@ -27,48 +32,134 @@ type MediaPaths interface {
|
||||
}
|
||||
|
||||
type Reconciler struct {
|
||||
repository Repository
|
||||
discovery onvif.Adapter
|
||||
media MediaPaths
|
||||
now func() time.Time
|
||||
baseBackoff time.Duration
|
||||
maxBackoff time.Duration
|
||||
batchSize int
|
||||
repository Repository
|
||||
discovery onvif.Adapter
|
||||
media MediaPaths
|
||||
now func() time.Time
|
||||
baseBackoff time.Duration
|
||||
maxBackoff time.Duration
|
||||
batchSize int
|
||||
instanceID string
|
||||
leaseDuration time.Duration
|
||||
operationTimeout time.Duration
|
||||
metrics *metrics.Registry
|
||||
}
|
||||
|
||||
type Options struct {
|
||||
InstanceID string
|
||||
LeaseDuration time.Duration
|
||||
OperationTimeout time.Duration
|
||||
Metrics *metrics.Registry
|
||||
}
|
||||
|
||||
func New(repository Repository, discovery onvif.Adapter, media MediaPaths) *Reconciler {
|
||||
return NewWithOptions(repository, discovery, media, Options{})
|
||||
}
|
||||
|
||||
func NewWithOptions(
|
||||
repository Repository,
|
||||
discovery onvif.Adapter,
|
||||
media MediaPaths,
|
||||
options Options,
|
||||
) *Reconciler {
|
||||
if options.InstanceID == "" {
|
||||
options.InstanceID = "single"
|
||||
}
|
||||
if options.LeaseDuration <= 0 {
|
||||
options.LeaseDuration = 30 * time.Second
|
||||
}
|
||||
if options.OperationTimeout <= 0 {
|
||||
options.OperationTimeout = 20 * time.Second
|
||||
}
|
||||
if options.OperationTimeout >= options.LeaseDuration {
|
||||
options.OperationTimeout = options.LeaseDuration / 2
|
||||
}
|
||||
return &Reconciler{
|
||||
repository: repository, discovery: discovery, media: media,
|
||||
now: time.Now, baseBackoff: time.Second, maxBackoff: time.Minute, batchSize: defaultBatchSize,
|
||||
instanceID: options.InstanceID, leaseDuration: options.LeaseDuration,
|
||||
operationTimeout: options.OperationTimeout, metrics: options.Metrics,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Reconciler) RunOnce(ctx context.Context) error {
|
||||
func (r *Reconciler) RunOnce(ctx context.Context) (runErr error) {
|
||||
started := time.Now()
|
||||
defer func() {
|
||||
if r.metrics != nil {
|
||||
r.metrics.ObserveReconcileRun(runErr, time.Since(started))
|
||||
}
|
||||
}()
|
||||
now := r.now().UTC()
|
||||
candidates, err := r.repository.ListDueReconcile(ctx, now, r.batchSize)
|
||||
token, err := newClaimToken()
|
||||
if err != nil {
|
||||
return fmt.Errorf("list reconciliation candidates: %w", err)
|
||||
return err
|
||||
}
|
||||
candidates, err := r.repository.ClaimDueReconcile(ctx, store.ReconcileClaim{
|
||||
Owner: r.instanceID, Token: token, Now: now,
|
||||
LeaseDuration: r.leaseDuration, Limit: r.batchSize,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("claim reconciliation candidates: %w", err)
|
||||
}
|
||||
var runErrors []error
|
||||
for _, candidate := range candidates {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.reconcileOne(ctx, candidate, now); err != nil {
|
||||
itemNow := r.now().UTC()
|
||||
renewed, err := r.repository.RenewReconcileLease(
|
||||
ctx, candidate.Device.ID, r.instanceID, token, itemNow, r.leaseDuration,
|
||||
)
|
||||
if err != nil {
|
||||
r.observeItem("error")
|
||||
runErrors = append(runErrors, fmt.Errorf("renew reconcile device lease: %w", err))
|
||||
continue
|
||||
}
|
||||
if !renewed {
|
||||
r.observeItem("lease_lost")
|
||||
continue
|
||||
}
|
||||
itemCtx, cancel := context.WithTimeout(ctx, r.operationTimeout)
|
||||
err = r.reconcileOne(itemCtx, ctx, candidate, itemNow, r.instanceID, token)
|
||||
cancel()
|
||||
if errors.Is(err, store.ErrReconcileLeaseLost) {
|
||||
r.observeItem("lease_lost")
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
r.observeItem("error")
|
||||
runErrors = append(runErrors, fmt.Errorf("reconcile device %s: %w", candidate.Device.ID, err))
|
||||
} else {
|
||||
r.observeItem("success")
|
||||
}
|
||||
}
|
||||
if r.metrics != nil {
|
||||
snapshot, err := r.repository.ConvergenceSnapshot(ctx)
|
||||
if err != nil {
|
||||
runErrors = append(runErrors, fmt.Errorf("read convergence metrics: %w", err))
|
||||
} else {
|
||||
r.metrics.SetConvergence(snapshot.Total, snapshot.Unconverged)
|
||||
}
|
||||
}
|
||||
return errors.Join(runErrors...)
|
||||
}
|
||||
|
||||
func (r *Reconciler) reconcileOne(ctx context.Context, candidate store.ReconcileCandidate, now time.Time) error {
|
||||
func (r *Reconciler) reconcileOne(
|
||||
ctx context.Context,
|
||||
persistCtx context.Context,
|
||||
candidate store.ReconcileCandidate,
|
||||
now time.Time,
|
||||
owner, token string,
|
||||
) error {
|
||||
if candidate.Device.DesiredState == device.DesiredDisabled {
|
||||
if err := r.media.DeletePath(ctx, candidate.Device.PathName); err == nil {
|
||||
return r.repository.MarkReconciled(ctx, candidate.Device.ID, candidate.Device.Generation, now)
|
||||
} else if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
return r.repository.CompleteReconcile(
|
||||
persistCtx, candidate.Device.ID, candidate.Device.Generation, owner, token, now,
|
||||
)
|
||||
} else if persistCtx.Err() != nil {
|
||||
return persistCtx.Err()
|
||||
} else {
|
||||
return r.persistFailure(ctx, candidate, now, err)
|
||||
return r.persistFailure(persistCtx, candidate, now, owner, token, err)
|
||||
}
|
||||
}
|
||||
result, err := r.discovery.Probe(ctx, onvif.Target{
|
||||
@@ -81,16 +172,22 @@ func (r *Reconciler) reconcileOne(ctx context.Context, candidate store.Reconcile
|
||||
_, err = r.media.EnsurePath(ctx, candidate.Device.PathName, result.StreamURI)
|
||||
}
|
||||
if err == nil {
|
||||
return r.repository.MarkReconciled(ctx, candidate.Device.ID, candidate.Device.Generation, now)
|
||||
return r.repository.CompleteReconcile(
|
||||
persistCtx, candidate.Device.ID, candidate.Device.Generation, owner, token, now,
|
||||
)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
if persistCtx.Err() != nil {
|
||||
return persistCtx.Err()
|
||||
}
|
||||
return r.persistFailure(ctx, candidate, now, err)
|
||||
return r.persistFailure(persistCtx, candidate, now, owner, token, err)
|
||||
}
|
||||
|
||||
func (r *Reconciler) persistFailure(
|
||||
ctx context.Context, candidate store.ReconcileCandidate, now time.Time, err error,
|
||||
ctx context.Context,
|
||||
candidate store.ReconcileCandidate,
|
||||
now time.Time,
|
||||
owner, token string,
|
||||
err error,
|
||||
) error {
|
||||
failureCount := candidate.FailureCount + 1
|
||||
nextAttempt := now.Add(r.backoff(failureCount))
|
||||
@@ -99,14 +196,28 @@ func (r *Reconciler) persistFailure(
|
||||
if !errors.As(err, &onvifError) {
|
||||
errorCode = "media_error"
|
||||
}
|
||||
if markErr := r.repository.MarkReconcileFailure(
|
||||
ctx, candidate.Device.ID, failureCount, nextAttempt, errorCode, now,
|
||||
if markErr := r.repository.FailReconcile(
|
||||
ctx, candidate.Device.ID, failureCount, nextAttempt, errorCode, owner, token, now,
|
||||
); markErr != nil {
|
||||
return errors.Join(err, fmt.Errorf("persist reconcile failure: %w", markErr))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *Reconciler) observeItem(result string) {
|
||||
if r.metrics != nil {
|
||||
r.metrics.ObserveReconcileItem(result)
|
||||
}
|
||||
}
|
||||
|
||||
func newClaimToken() (string, error) {
|
||||
value := make([]byte, 18)
|
||||
if _, err := rand.Read(value); err != nil {
|
||||
return "", errors.New("generate reconcile claim token")
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString(value), nil
|
||||
}
|
||||
|
||||
func validateStreamURI(value string) error {
|
||||
parsed, err := url.Parse(value)
|
||||
if err != nil || parsed.Host == "" || (parsed.Scheme != "rtsp" && parsed.Scheme != "rtsps") {
|
||||
|
||||
@@ -162,6 +162,86 @@ func TestDisabledDeviceDeletesOnlyItsExactPath(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
type lostLeaseRepository struct {
|
||||
completed bool
|
||||
failed bool
|
||||
}
|
||||
|
||||
func (r *lostLeaseRepository) ClaimDueReconcile(
|
||||
context.Context,
|
||||
store.ReconcileClaim,
|
||||
) ([]store.ReconcileCandidate, error) {
|
||||
return []store.ReconcileCandidate{{Device: device.Device{
|
||||
ID: "camera-lost", DesiredState: device.DesiredEnabled,
|
||||
EndpointRef: "onvif://camera-lost", PathName: "camera-lost", Generation: 1,
|
||||
}}}, nil
|
||||
}
|
||||
|
||||
func (r *lostLeaseRepository) RenewReconcileLease(
|
||||
context.Context, string, string, string, time.Time, time.Duration,
|
||||
) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (r *lostLeaseRepository) CompleteReconcile(
|
||||
context.Context, string, int64, string, string, time.Time,
|
||||
) error {
|
||||
r.completed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *lostLeaseRepository) FailReconcile(
|
||||
context.Context, string, int, time.Time, string, string, string, time.Time,
|
||||
) error {
|
||||
r.failed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *lostLeaseRepository) ConvergenceSnapshot(context.Context) (store.ConvergenceSnapshot, error) {
|
||||
return store.ConvergenceSnapshot{}, nil
|
||||
}
|
||||
|
||||
func TestLostLeaseSkipsAllExternalAndStoreMutations(t *testing.T) {
|
||||
t.Parallel()
|
||||
repository := &lostLeaseRepository{}
|
||||
media := &recordingMedia{}
|
||||
reconciler := New(repository, onvif.NewFake(nil), media)
|
||||
if err := reconciler.RunOnce(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if media.calls != 0 || repository.completed || repository.failed {
|
||||
t.Fatalf("lost lease performed a side effect: media=%d completed=%v failed=%v",
|
||||
media.calls, repository.completed, repository.failed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPerItemTimeoutPersistsRetryWhenParentIsAlive(t *testing.T) {
|
||||
t.Parallel()
|
||||
repository := openRepository(t, filepath.Join(t.TempDir(), "sense.db"))
|
||||
createReconcileDevice(t, repository)
|
||||
discovery := onvif.NewFake(map[string]onvif.FakeScenario{
|
||||
"onvif://camera-1": {
|
||||
DelayMillis: 100,
|
||||
Result: onvif.ProbeResult{StreamURI: "rtsp://media.invalid/camera-1"},
|
||||
},
|
||||
})
|
||||
reconciler := NewWithOptions(repository, discovery, &recordingMedia{}, Options{
|
||||
InstanceID: "ins_test", LeaseDuration: time.Second, OperationTimeout: 5 * time.Millisecond,
|
||||
})
|
||||
now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC)
|
||||
reconciler.now = func() time.Time { return now }
|
||||
if err := reconciler.RunOnce(context.Background()); err == nil {
|
||||
t.Fatal("expected bounded operation timeout")
|
||||
}
|
||||
candidates, err := repository.ListDueReconcile(context.Background(), now.Add(time.Hour), 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(candidates) != 1 || candidates[0].FailureCount != 1 {
|
||||
t.Fatalf("operation timeout did not persist retry state: %+v", candidates)
|
||||
}
|
||||
}
|
||||
|
||||
func openRepository(t *testing.T, path string) *store.SQLite {
|
||||
t.Helper()
|
||||
repository, err := store.OpenSQLite(context.Background(), "file:"+filepath.ToSlash(path))
|
||||
|
||||
@@ -514,6 +514,7 @@ func (s *Postgres) PatchControlDevice(
|
||||
if reconcileChanged {
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE sense.reconcile_state SET
|
||||
failure_count = 0, next_attempt_at = NULL, last_error_code = NULL,
|
||||
lease_owner = NULL, lease_token = NULL, lease_until = NULL,
|
||||
updated_at = $1 WHERE device_id = $2`, now, deviceID); err != nil {
|
||||
return ControlMutationResult{}, errors.New("reset postgres Control API reconcile state")
|
||||
}
|
||||
@@ -631,6 +632,7 @@ func setControlDesiredStateTx(
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE sense.reconcile_state SET
|
||||
failure_count = 0, next_attempt_at = NULL, last_error_code = NULL,
|
||||
lease_owner = NULL, lease_token = NULL, lease_until = NULL,
|
||||
updated_at = $1 WHERE device_id = $2`, now, deviceID); err != nil {
|
||||
return ControlDevice{}, errors.New("reset postgres Control API desired-state reconciliation")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
OperationalLeaseOrphanScan = "mediamtx-orphan-scan"
|
||||
OperationalLeaseOrphanCleanup = "mediamtx-orphan-cleanup"
|
||||
|
||||
OrphanOwnedStale = "owned_stale"
|
||||
OrphanUnowned = "unowned"
|
||||
)
|
||||
|
||||
var ErrOrphanScanNotFound = errors.New("orphan scan not found")
|
||||
|
||||
type MediaPathOwnership struct {
|
||||
PathName string
|
||||
DeviceID string
|
||||
CurrentClaim bool
|
||||
}
|
||||
|
||||
type OrphanFinding struct {
|
||||
PathName string
|
||||
Classification string
|
||||
DeviceID string
|
||||
Deleted bool
|
||||
}
|
||||
|
||||
type OrphanScan struct {
|
||||
ID string
|
||||
InstanceID string
|
||||
ObservedCount int
|
||||
OwnedStaleCount int
|
||||
UnownedCount int
|
||||
SafetyAllowed bool
|
||||
SafetyReason string
|
||||
CompletedAt time.Time
|
||||
ExpiresAt time.Time
|
||||
Findings []OrphanFinding
|
||||
}
|
||||
|
||||
type OrphanRepository interface {
|
||||
AcquireOperationalLease(
|
||||
context.Context, string, string, string, time.Time, time.Duration,
|
||||
) (bool, error)
|
||||
ReleaseOperationalLease(context.Context, string, string, string, time.Time) error
|
||||
ListMediaPathOwnership(context.Context) ([]MediaPathOwnership, error)
|
||||
SaveOrphanScan(context.Context, OrphanScan, string, string) error
|
||||
GetOrphanScan(context.Context, string) (OrphanScan, error)
|
||||
RecordOrphanCleanup(
|
||||
context.Context, string, string, string, string, string, time.Time,
|
||||
) error
|
||||
}
|
||||
|
||||
var _ OrphanRepository = (*Postgres)(nil)
|
||||
@@ -0,0 +1,268 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const orphanReportRetention = 7 * 24 * time.Hour
|
||||
|
||||
func (s *Postgres) AcquireOperationalLease(
|
||||
ctx context.Context,
|
||||
name, owner, token string,
|
||||
_ time.Time,
|
||||
duration time.Duration,
|
||||
) (bool, error) {
|
||||
if strings.TrimSpace(name) == "" || strings.TrimSpace(owner) == "" ||
|
||||
strings.TrimSpace(token) == "" || duration <= 0 {
|
||||
return false, errors.New("invalid operational lease")
|
||||
}
|
||||
var acquired int
|
||||
err := s.db.QueryRowContext(ctx, `INSERT INTO sense.operational_leases(
|
||||
lease_name, owner_id, fencing_token, lease_until, updated_at
|
||||
) VALUES (
|
||||
$1, $2, $3,
|
||||
clock_timestamp() + ($4 * interval '1 second'), clock_timestamp()
|
||||
)
|
||||
ON CONFLICT (lease_name) DO UPDATE SET
|
||||
owner_id = EXCLUDED.owner_id,
|
||||
fencing_token = EXCLUDED.fencing_token,
|
||||
lease_until = EXCLUDED.lease_until,
|
||||
updated_at = EXCLUDED.updated_at
|
||||
WHERE sense.operational_leases.lease_until <= clock_timestamp()
|
||||
OR (sense.operational_leases.owner_id = $2
|
||||
AND sense.operational_leases.fencing_token = $3)
|
||||
RETURNING 1`, name, owner, token, duration.Seconds()).Scan(&acquired)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, errors.New("acquire postgres operational lease")
|
||||
}
|
||||
return acquired == 1, nil
|
||||
}
|
||||
|
||||
func (s *Postgres) ReleaseOperationalLease(
|
||||
ctx context.Context,
|
||||
name, owner, token string,
|
||||
_ time.Time,
|
||||
) error {
|
||||
_, err := s.db.ExecContext(ctx, `UPDATE sense.operational_leases
|
||||
SET lease_until = clock_timestamp(), updated_at = clock_timestamp()
|
||||
WHERE lease_name = $1 AND owner_id = $2 AND fencing_token = $3`,
|
||||
name, owner, token)
|
||||
if err != nil {
|
||||
return errors.New("release postgres operational lease")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Postgres) ListMediaPathOwnership(ctx context.Context) ([]MediaPathOwnership, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT
|
||||
o.path_name, o.device_id,
|
||||
EXISTS (
|
||||
SELECT 1 FROM sense.devices d
|
||||
WHERE d.id = o.device_id AND d.path_name = o.path_name
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM sense.device_capabilities c
|
||||
WHERE c.device_id = d.id AND c.capability = 'video_capture'
|
||||
)
|
||||
) AS current_claim
|
||||
FROM sense.media_path_ownership o
|
||||
ORDER BY o.path_name`)
|
||||
if err != nil {
|
||||
return nil, errors.New("list postgres MediaMTX path ownership")
|
||||
}
|
||||
defer rows.Close()
|
||||
values := make([]MediaPathOwnership, 0)
|
||||
for rows.Next() {
|
||||
var value MediaPathOwnership
|
||||
if err := rows.Scan(&value.PathName, &value.DeviceID, &value.CurrentClaim); err != nil {
|
||||
return nil, errors.New("scan postgres MediaMTX path ownership")
|
||||
}
|
||||
values = append(values, value)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, errors.New("iterate postgres MediaMTX path ownership")
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func (s *Postgres) SaveOrphanScan(
|
||||
ctx context.Context,
|
||||
scan OrphanScan,
|
||||
owner, token string,
|
||||
) error {
|
||||
if err := validateOrphanScan(scan); err != nil {
|
||||
return err
|
||||
}
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return errors.New("begin postgres orphan scan save")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var lease int
|
||||
err = tx.QueryRowContext(ctx, `SELECT 1 FROM sense.operational_leases
|
||||
WHERE lease_name = $1 AND owner_id = $2 AND fencing_token = $3
|
||||
AND lease_until > clock_timestamp()
|
||||
FOR UPDATE`, OperationalLeaseOrphanScan, owner, token).Scan(&lease)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return ErrOperationalLeaseLost
|
||||
}
|
||||
if err != nil {
|
||||
return errors.New("verify postgres orphan scan lease")
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO sense.orphan_scan_runs(
|
||||
id, instance_id, observed_count, owned_stale_count, unowned_count,
|
||||
safety_allowed, safety_reason, completed_at, expires_at
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`,
|
||||
scan.ID, scan.InstanceID, scan.ObservedCount, scan.OwnedStaleCount, scan.UnownedCount,
|
||||
scan.SafetyAllowed, scan.SafetyReason, scan.CompletedAt, scan.ExpiresAt,
|
||||
); err != nil {
|
||||
return errors.New("insert postgres orphan scan")
|
||||
}
|
||||
for _, finding := range scan.Findings {
|
||||
var deviceID any
|
||||
if finding.DeviceID != "" {
|
||||
deviceID = finding.DeviceID
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO sense.orphan_scan_findings(
|
||||
scan_id, path_name, classification, device_id
|
||||
) VALUES ($1,$2,$3,$4)`, scan.ID, finding.PathName, finding.Classification, deviceID); err != nil {
|
||||
return errors.New("insert postgres orphan scan finding")
|
||||
}
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE sense.operational_leases
|
||||
SET lease_until = clock_timestamp(), updated_at = clock_timestamp()
|
||||
WHERE lease_name = $1 AND owner_id = $2 AND fencing_token = $3`,
|
||||
OperationalLeaseOrphanScan, owner, token); err != nil {
|
||||
return errors.New("release postgres orphan scan lease")
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM sense.orphan_scan_runs r
|
||||
WHERE r.completed_at < $1
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM sense.orphan_cleanup_actions a WHERE a.scan_id = r.id
|
||||
)`, scan.CompletedAt.Add(-orphanReportRetention)); err != nil {
|
||||
return errors.New("expire postgres orphan scan reports")
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return errors.New("commit postgres orphan scan")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateOrphanScan(scan OrphanScan) error {
|
||||
if strings.TrimSpace(scan.ID) == "" || strings.TrimSpace(scan.InstanceID) == "" ||
|
||||
scan.ObservedCount < 0 || scan.OwnedStaleCount < 0 || scan.UnownedCount < 0 ||
|
||||
scan.OwnedStaleCount+scan.UnownedCount > scan.ObservedCount ||
|
||||
strings.TrimSpace(scan.SafetyReason) == "" || !scan.ExpiresAt.After(scan.CompletedAt) {
|
||||
return errors.New("invalid orphan scan")
|
||||
}
|
||||
seen := make(map[string]struct{}, len(scan.Findings))
|
||||
staleCount, unownedCount := 0, 0
|
||||
for _, finding := range scan.Findings {
|
||||
if strings.TrimSpace(finding.PathName) == "" {
|
||||
return errors.New("invalid orphan finding path")
|
||||
}
|
||||
if _, duplicate := seen[finding.PathName]; duplicate {
|
||||
return errors.New("duplicate orphan finding path")
|
||||
}
|
||||
seen[finding.PathName] = struct{}{}
|
||||
switch finding.Classification {
|
||||
case OrphanOwnedStale:
|
||||
staleCount++
|
||||
if strings.TrimSpace(finding.DeviceID) == "" {
|
||||
return errors.New("owned stale finding lacks device")
|
||||
}
|
||||
case OrphanUnowned:
|
||||
unownedCount++
|
||||
if finding.DeviceID != "" {
|
||||
return errors.New("unowned finding has device")
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("invalid orphan finding classification %q", finding.Classification)
|
||||
}
|
||||
}
|
||||
if staleCount != scan.OwnedStaleCount || unownedCount != scan.UnownedCount {
|
||||
return errors.New("orphan scan counts do not match findings")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Postgres) GetOrphanScan(ctx context.Context, id string) (OrphanScan, error) {
|
||||
var scan OrphanScan
|
||||
err := s.db.QueryRowContext(ctx, `SELECT
|
||||
id, instance_id, observed_count, owned_stale_count, unowned_count,
|
||||
safety_allowed, safety_reason, completed_at, expires_at
|
||||
FROM sense.orphan_scan_runs WHERE id = $1`, id).Scan(
|
||||
&scan.ID, &scan.InstanceID, &scan.ObservedCount, &scan.OwnedStaleCount,
|
||||
&scan.UnownedCount, &scan.SafetyAllowed, &scan.SafetyReason,
|
||||
&scan.CompletedAt, &scan.ExpiresAt,
|
||||
)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return OrphanScan{}, ErrOrphanScanNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return OrphanScan{}, errors.New("read postgres orphan scan")
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT
|
||||
f.path_name, f.classification, COALESCE(f.device_id, ''),
|
||||
COALESCE(a.status = 'deleted', false)
|
||||
FROM sense.orphan_scan_findings f
|
||||
LEFT JOIN sense.orphan_cleanup_actions a
|
||||
ON a.scan_id = f.scan_id AND a.path_name = f.path_name
|
||||
WHERE f.scan_id = $1 ORDER BY f.path_name`, id)
|
||||
if err != nil {
|
||||
return OrphanScan{}, errors.New("list postgres orphan scan findings")
|
||||
}
|
||||
defer rows.Close()
|
||||
scan.Findings = make([]OrphanFinding, 0)
|
||||
for rows.Next() {
|
||||
var finding OrphanFinding
|
||||
if err := rows.Scan(
|
||||
&finding.PathName, &finding.Classification, &finding.DeviceID, &finding.Deleted,
|
||||
); err != nil {
|
||||
return OrphanScan{}, errors.New("scan postgres orphan finding")
|
||||
}
|
||||
scan.Findings = append(scan.Findings, finding)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return OrphanScan{}, errors.New("iterate postgres orphan findings")
|
||||
}
|
||||
return scan, nil
|
||||
}
|
||||
|
||||
func (s *Postgres) RecordOrphanCleanup(
|
||||
ctx context.Context,
|
||||
scanID, pathName, actorID, status, errorCode string,
|
||||
now time.Time,
|
||||
) error {
|
||||
if status != "deleted" && status != "failed" {
|
||||
return errors.New("invalid orphan cleanup status")
|
||||
}
|
||||
var storedError any
|
||||
if status == "failed" {
|
||||
if strings.TrimSpace(errorCode) == "" {
|
||||
return errors.New("failed orphan cleanup requires an error code")
|
||||
}
|
||||
storedError = errorCode
|
||||
}
|
||||
_, err := s.db.ExecContext(ctx, `INSERT INTO sense.orphan_cleanup_actions(
|
||||
scan_id, path_name, classification, actor_id, status, error_code, attempted_at
|
||||
) VALUES ($1,$2,'owned_stale',$3,$4,$5,$6)
|
||||
ON CONFLICT (scan_id, path_name) DO UPDATE SET
|
||||
actor_id = EXCLUDED.actor_id,
|
||||
status = EXCLUDED.status,
|
||||
error_code = EXCLUDED.error_code,
|
||||
attempted_at = EXCLUDED.attempted_at
|
||||
WHERE sense.orphan_cleanup_actions.status <> 'deleted'`,
|
||||
scanID, pathName, actorID, status, storedError, now)
|
||||
if err != nil {
|
||||
return errors.New("record postgres orphan cleanup result")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -53,8 +53,8 @@ func (s *Postgres) Close() error {
|
||||
func (s *Postgres) verifySchemaAndPrivileges(ctx context.Context) error {
|
||||
var version sql.NullInt64
|
||||
if err := s.db.QueryRowContext(ctx,
|
||||
`SELECT MAX(version) FROM sense.schema_migrations`).Scan(&version); err != nil || !version.Valid || version.Int64 < 4 {
|
||||
return errors.New("postgres sense schema migration v4 is required")
|
||||
`SELECT MAX(version) FROM sense.schema_migrations`).Scan(&version); err != nil || !version.Valid || version.Int64 < 5 {
|
||||
return errors.New("postgres sense schema migration v5 is required")
|
||||
}
|
||||
var canReadQuotaView, canWriteQuotaView, canReadSiteSource, canWriteSiteSource bool
|
||||
var canReadAreaView, canWriteAreaView, canReadAreaSource, canWriteAreaSource bool
|
||||
@@ -95,6 +95,28 @@ func (s *Postgres) verifySchemaAndPrivileges(ctx context.Context) error {
|
||||
publicReceipts || publicOperations || publicOperationItems {
|
||||
return errors.New("postgres role violates Control API state privilege boundary")
|
||||
}
|
||||
var canUseOwnership, canUseLeases, canUseScans, canUseFindings, canUseActions bool
|
||||
var publicOwnership, publicLeases, publicScans, publicFindings, publicActions bool
|
||||
if err := s.db.QueryRowContext(ctx, `SELECT
|
||||
has_table_privilege(current_user, 'sense.media_path_ownership', 'SELECT,INSERT,UPDATE,DELETE'),
|
||||
has_table_privilege(current_user, 'sense.operational_leases', 'SELECT,INSERT,UPDATE,DELETE'),
|
||||
has_table_privilege(current_user, 'sense.orphan_scan_runs', 'SELECT,INSERT,UPDATE,DELETE'),
|
||||
has_table_privilege(current_user, 'sense.orphan_scan_findings', 'SELECT,INSERT,UPDATE,DELETE'),
|
||||
has_table_privilege(current_user, 'sense.orphan_cleanup_actions', 'SELECT,INSERT,UPDATE,DELETE'),
|
||||
has_table_privilege('public', 'sense.media_path_ownership', 'SELECT,INSERT,UPDATE,DELETE'),
|
||||
has_table_privilege('public', 'sense.operational_leases', 'SELECT,INSERT,UPDATE,DELETE'),
|
||||
has_table_privilege('public', 'sense.orphan_scan_runs', 'SELECT,INSERT,UPDATE,DELETE'),
|
||||
has_table_privilege('public', 'sense.orphan_scan_findings', 'SELECT,INSERT,UPDATE,DELETE'),
|
||||
has_table_privilege('public', 'sense.orphan_cleanup_actions', 'SELECT,INSERT,UPDATE,DELETE')`).Scan(
|
||||
&canUseOwnership, &canUseLeases, &canUseScans, &canUseFindings, &canUseActions,
|
||||
&publicOwnership, &publicLeases, &publicScans, &publicFindings, &publicActions,
|
||||
); err != nil {
|
||||
return errors.New("verify postgres reconciliation safety privileges")
|
||||
}
|
||||
if !canUseOwnership || !canUseLeases || !canUseScans || !canUseFindings || !canUseActions ||
|
||||
publicOwnership || publicLeases || publicScans || publicFindings || publicActions {
|
||||
return errors.New("postgres role violates reconciliation safety privilege boundary")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -391,7 +413,8 @@ func (s *Postgres) SetDesiredState(ctx context.Context, id string, desired devic
|
||||
return errors.New("update postgres desired state")
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE sense.reconcile_state SET
|
||||
failure_count = 0, next_attempt_at = NULL, last_error_code = NULL, updated_at = $1
|
||||
failure_count = 0, next_attempt_at = NULL, last_error_code = NULL,
|
||||
lease_owner = NULL, lease_token = NULL, lease_until = NULL, updated_at = $1
|
||||
WHERE device_id = $2`, now, id); err != nil {
|
||||
return errors.New("reset postgres reconcile state")
|
||||
}
|
||||
@@ -486,6 +509,217 @@ func (s *Postgres) ListDueReconcile(ctx context.Context, now time.Time, limit in
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func (s *Postgres) ClaimDueReconcile(
|
||||
ctx context.Context,
|
||||
claim ReconcileClaim,
|
||||
) ([]ReconcileCandidate, error) {
|
||||
if claim.Limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if strings.TrimSpace(claim.Owner) == "" || strings.TrimSpace(claim.Token) == "" ||
|
||||
claim.LeaseDuration <= 0 {
|
||||
return nil, errors.New("invalid postgres reconcile claim")
|
||||
}
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, errors.New("begin postgres reconcile claim")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
rows, err := tx.QueryContext(ctx, `SELECT `+postgresDeviceColumns+`, r.failure_count, r.next_attempt_at
|
||||
FROM sense.devices d
|
||||
JOIN sense.reconcile_state r ON r.device_id = d.id
|
||||
WHERE EXISTS (SELECT 1 FROM sense.device_capabilities c
|
||||
WHERE c.device_id = d.id AND c.capability = 'video_capture')
|
||||
AND (r.observed_generation < d.generation
|
||||
OR (d.desired_state = 'enabled' AND r.failure_count > 0))
|
||||
AND (r.next_attempt_at IS NULL OR r.next_attempt_at <= clock_timestamp())
|
||||
AND (r.lease_until IS NULL OR r.lease_until <= clock_timestamp())
|
||||
ORDER BY d.updated_at, d.id
|
||||
FOR UPDATE OF r SKIP LOCKED
|
||||
LIMIT $1`, claim.Limit)
|
||||
if err != nil {
|
||||
return nil, errors.New("select postgres reconcile claims")
|
||||
}
|
||||
values := make([]ReconcileCandidate, 0)
|
||||
for rows.Next() {
|
||||
var candidate ReconcileCandidate
|
||||
var areaID sql.NullString
|
||||
var quotaVersion, areaVersion sql.NullInt64
|
||||
var nextAttempt sql.NullTime
|
||||
if err := rows.Scan(
|
||||
&candidate.Device.ID, &candidate.Device.TenantID, &candidate.Device.SiteID,
|
||||
&areaID, &candidate.Device.SerialNumber, &candidate.Device.Name, &candidate.Device.Modality,
|
||||
&candidate.Device.DesiredState, &candidate.Device.ActualState,
|
||||
&candidate.Device.EndpointRef, &candidate.Device.CredentialRef,
|
||||
&candidate.Device.ProfileToken, &candidate.Device.PathName,
|
||||
&candidate.Device.Generation, &candidate.Device.ResourceVersion,
|
||||
"aVersion, &areaVersion,
|
||||
&candidate.Device.CreatedAt, &candidate.Device.UpdatedAt,
|
||||
&candidate.FailureCount, &nextAttempt,
|
||||
); err != nil {
|
||||
rows.Close()
|
||||
return nil, errors.New("scan postgres reconcile claim")
|
||||
}
|
||||
candidate.Device.AreaID = areaID.String
|
||||
candidate.Device.QuotaSourceVersion = quotaVersion.Int64
|
||||
candidate.Device.AreaPolicySourceVersion = areaVersion.Int64
|
||||
if nextAttempt.Valid {
|
||||
value := nextAttempt.Time
|
||||
candidate.NextAttempt = &value
|
||||
}
|
||||
values = append(values, candidate)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, errors.New("close postgres reconcile claims")
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, errors.New("iterate postgres reconcile claims")
|
||||
}
|
||||
for _, candidate := range values {
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE sense.reconcile_state SET
|
||||
lease_owner = $1, lease_token = $2,
|
||||
lease_until = clock_timestamp() + ($3 * interval '1 second'),
|
||||
updated_at = clock_timestamp()
|
||||
WHERE device_id = $4`,
|
||||
claim.Owner, claim.Token, claim.LeaseDuration.Seconds(), candidate.Device.ID,
|
||||
); err != nil {
|
||||
return nil, errors.New("persist postgres reconcile claim")
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return nil, errors.New("commit postgres reconcile claim")
|
||||
}
|
||||
for index := range values {
|
||||
values[index].Device.Capabilities, err = s.capabilities(ctx, values[index].Device.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func (s *Postgres) RenewReconcileLease(
|
||||
ctx context.Context,
|
||||
id, owner, token string,
|
||||
now time.Time,
|
||||
duration time.Duration,
|
||||
) (bool, error) {
|
||||
if strings.TrimSpace(owner) == "" || strings.TrimSpace(token) == "" || duration <= 0 {
|
||||
return false, errors.New("invalid postgres reconcile lease renewal")
|
||||
}
|
||||
result, err := s.db.ExecContext(ctx, `UPDATE sense.reconcile_state SET
|
||||
lease_until = clock_timestamp() + ($1 * interval '1 second'),
|
||||
updated_at = clock_timestamp()
|
||||
WHERE device_id = $2 AND lease_owner = $3 AND lease_token = $4
|
||||
AND lease_until > clock_timestamp()`, duration.Seconds(), id, owner, token)
|
||||
if err != nil {
|
||||
return false, errors.New("renew postgres reconcile lease")
|
||||
}
|
||||
affected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return false, errors.New("read postgres reconcile lease renewal")
|
||||
}
|
||||
return affected == 1, nil
|
||||
}
|
||||
|
||||
func (s *Postgres) CompleteReconcile(
|
||||
ctx context.Context,
|
||||
id string,
|
||||
generation int64,
|
||||
owner, token string,
|
||||
now time.Time,
|
||||
) error {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return errors.New("begin postgres fenced reconciliation completion")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
result, err := tx.ExecContext(ctx, `UPDATE sense.reconcile_state SET
|
||||
failure_count = 0, next_attempt_at = NULL, last_error_code = NULL,
|
||||
observed_generation = $1, lease_owner = NULL, lease_token = NULL,
|
||||
lease_until = NULL, updated_at = $2
|
||||
WHERE device_id = $3 AND lease_owner = $4 AND lease_token = $5
|
||||
AND lease_until > clock_timestamp()`, generation, now, id, owner, token)
|
||||
if err != nil {
|
||||
return errors.New("complete postgres fenced reconciliation")
|
||||
}
|
||||
affected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return errors.New("read postgres fenced reconciliation completion")
|
||||
}
|
||||
if affected != 1 {
|
||||
return ErrReconcileLeaseLost
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO sense.media_path_ownership(
|
||||
path_name, device_id, tenant_id, site_id, first_claimed_at, last_confirmed_at
|
||||
)
|
||||
SELECT path_name, id, tenant_id, site_id, $1, $1
|
||||
FROM sense.devices
|
||||
WHERE id = $2 AND desired_state = 'enabled' AND btrim(path_name) <> ''
|
||||
ON CONFLICT (path_name) DO UPDATE SET
|
||||
device_id = EXCLUDED.device_id,
|
||||
tenant_id = EXCLUDED.tenant_id,
|
||||
site_id = EXCLUDED.site_id,
|
||||
last_confirmed_at = EXCLUDED.last_confirmed_at`, now, id); err != nil {
|
||||
return errors.New("record postgres MediaMTX path ownership")
|
||||
}
|
||||
result, err = tx.ExecContext(ctx, `UPDATE sense.devices
|
||||
SET actual_state = CASE WHEN desired_state = 'disabled' THEN 'offline' ELSE 'pending' END,
|
||||
updated_at = $1 WHERE id = $2`, now, id)
|
||||
if err != nil {
|
||||
return errors.New("mark postgres fenced device state")
|
||||
}
|
||||
if affected, _ := result.RowsAffected(); affected != 1 {
|
||||
return ErrNotFound
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return errors.New("commit postgres fenced reconciliation completion")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Postgres) FailReconcile(
|
||||
ctx context.Context,
|
||||
id string,
|
||||
failureCount int,
|
||||
nextAttempt time.Time,
|
||||
errorCode, owner, token string,
|
||||
now time.Time,
|
||||
) error {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return errors.New("begin postgres fenced reconciliation failure")
|
||||
}
|
||||
defer tx.Rollback()
|
||||
result, err := tx.ExecContext(ctx, `UPDATE sense.reconcile_state SET
|
||||
failure_count = $1, next_attempt_at = $2, last_error_code = $3,
|
||||
lease_owner = NULL, lease_token = NULL, lease_until = NULL, updated_at = $4
|
||||
WHERE device_id = $5 AND lease_owner = $6 AND lease_token = $7
|
||||
AND lease_until > clock_timestamp()`, failureCount, nextAttempt, errorCode, now, id, owner, token)
|
||||
if err != nil {
|
||||
return errors.New("persist postgres fenced reconciliation failure")
|
||||
}
|
||||
affected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return errors.New("read postgres fenced reconciliation failure")
|
||||
}
|
||||
if affected != 1 {
|
||||
return ErrReconcileLeaseLost
|
||||
}
|
||||
result, err = tx.ExecContext(ctx, `UPDATE sense.devices
|
||||
SET actual_state = 'failed', updated_at = $1 WHERE id = $2`, now, id)
|
||||
if err != nil {
|
||||
return errors.New("mark postgres fenced failed device")
|
||||
}
|
||||
if affected, _ := result.RowsAffected(); affected != 1 {
|
||||
return ErrNotFound
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return errors.New("commit postgres fenced reconciliation failure")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Postgres) ListEnabledVideoDevices(ctx context.Context, limit int) ([]device.Device, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
|
||||
@@ -287,12 +287,12 @@ func TestPostgresOpenRejectsOverprivilegedRuntimeRole(t *testing.T) {
|
||||
_, admin := openPostgresTestStore(t)
|
||||
ctx := context.Background()
|
||||
if _, err := admin.ExecContext(ctx,
|
||||
`GRANT UPDATE ON bell.site_quota_v1 TO yovision_t011_sense`); err != nil {
|
||||
`GRANT UPDATE ON bell.site_quota_v1 TO yovision_t012_sense`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_, _ = admin.ExecContext(context.Background(),
|
||||
`REVOKE UPDATE ON bell.site_quota_v1 FROM yovision_t011_sense`)
|
||||
`REVOKE UPDATE ON bell.site_quota_v1 FROM yovision_t012_sense`)
|
||||
}()
|
||||
value, err := OpenPostgres(ctx, os.Getenv(postgresTestDSNEnv))
|
||||
if value != nil {
|
||||
@@ -304,6 +304,27 @@ func TestPostgresOpenRejectsOverprivilegedRuntimeRole(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresOpenRejectsPublicReconciliationStatePrivilege(t *testing.T) {
|
||||
_, admin := openPostgresTestStore(t)
|
||||
ctx := context.Background()
|
||||
if _, err := admin.ExecContext(ctx,
|
||||
`GRANT SELECT ON sense.orphan_scan_runs TO PUBLIC`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_, _ = admin.ExecContext(context.Background(),
|
||||
`REVOKE SELECT ON sense.orphan_scan_runs FROM PUBLIC`)
|
||||
}()
|
||||
value, err := OpenPostgres(ctx, os.Getenv(postgresTestDSNEnv))
|
||||
if value != nil {
|
||||
_ = value.Close()
|
||||
t.Fatal("PUBLIC reconciliation state privilege was accepted")
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), "reconciliation safety privilege boundary") {
|
||||
t.Fatalf("expected reconciliation privilege-boundary error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresAreaPolicyAllowsNonImagingAndDeniesImagingCreate(t *testing.T) {
|
||||
store, admin := openPostgresTestStore(t)
|
||||
ctx := context.Background()
|
||||
@@ -597,12 +618,12 @@ func TestPostgresOpenRejectsAreaSourcePrivilege(t *testing.T) {
|
||||
_, admin := openPostgresTestStore(t)
|
||||
ctx := context.Background()
|
||||
if _, err := admin.ExecContext(ctx,
|
||||
`GRANT SELECT ON bell.areas TO yovision_t011_sense`); err != nil {
|
||||
`GRANT SELECT ON bell.areas TO yovision_t012_sense`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_, _ = admin.ExecContext(context.Background(),
|
||||
`REVOKE SELECT ON bell.areas FROM yovision_t011_sense`)
|
||||
`REVOKE SELECT ON bell.areas FROM yovision_t012_sense`)
|
||||
}()
|
||||
value, err := OpenPostgres(ctx, os.Getenv(postgresTestDSNEnv))
|
||||
if value != nil {
|
||||
@@ -969,6 +990,180 @@ func TestPostgresConcurrentControlBatchesUseStableDeviceLockOrder(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresConcurrentReconcileClaimHasOneWinner(t *testing.T) {
|
||||
first, admin := openPostgresTestStore(t)
|
||||
insertBellSite(t, admin, "tenant", "site", 2)
|
||||
if err := first.CreateDevice(context.Background(), videoDevice(1, "tenant", "site")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := OpenPostgres(context.Background(), os.Getenv(postgresTestDSNEnv))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = second.Close() })
|
||||
now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC)
|
||||
start := make(chan struct{})
|
||||
counts := make(chan int, 2)
|
||||
errorsFound := make(chan error, 2)
|
||||
var wait sync.WaitGroup
|
||||
for index, repository := range []*Postgres{first, second} {
|
||||
wait.Add(1)
|
||||
go func(index int, repository *Postgres) {
|
||||
defer wait.Done()
|
||||
<-start
|
||||
values, err := repository.ClaimDueReconcile(context.Background(), ReconcileClaim{
|
||||
Owner: fmt.Sprintf("ins-%d", index), Token: fmt.Sprintf("token-%d", index),
|
||||
Now: now, LeaseDuration: 30 * time.Second, Limit: 1,
|
||||
})
|
||||
if err != nil {
|
||||
errorsFound <- err
|
||||
return
|
||||
}
|
||||
counts <- len(values)
|
||||
}(index, repository)
|
||||
}
|
||||
close(start)
|
||||
wait.Wait()
|
||||
close(counts)
|
||||
close(errorsFound)
|
||||
for err := range errorsFound {
|
||||
t.Fatal(err)
|
||||
}
|
||||
total, winners := 0, 0
|
||||
for count := range counts {
|
||||
total += count
|
||||
if count == 1 {
|
||||
winners++
|
||||
}
|
||||
}
|
||||
if total != 1 || winners != 1 {
|
||||
t.Fatalf("due row was not exclusively claimed: total=%d winners=%d", total, winners)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresExpiredReconcileLeaseFencesOldWorkerAndRecordsOwnership(t *testing.T) {
|
||||
postgres, admin := openPostgresTestStore(t)
|
||||
insertBellSite(t, admin, "tenant", "site", 2)
|
||||
value := videoDevice(1, "tenant", "site")
|
||||
if err := postgres.CreateDevice(context.Background(), value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC)
|
||||
first, err := postgres.ClaimDueReconcile(context.Background(), ReconcileClaim{
|
||||
Owner: "ins-a", Token: "token-a", Now: now, LeaseDuration: 30 * time.Second, Limit: 1,
|
||||
})
|
||||
if err != nil || len(first) != 1 {
|
||||
t.Fatalf("first claim failed: %+v %v", first, err)
|
||||
}
|
||||
early, err := postgres.ClaimDueReconcile(context.Background(), ReconcileClaim{
|
||||
Owner: "ins-b", Token: "token-b", Now: now.Add(10 * time.Second),
|
||||
LeaseDuration: 30 * time.Second, Limit: 1,
|
||||
})
|
||||
if err != nil || len(early) != 0 {
|
||||
t.Fatalf("live lease was stolen: %+v %v", early, err)
|
||||
}
|
||||
if _, err := admin.Exec(`UPDATE sense.reconcile_state
|
||||
SET lease_until = clock_timestamp() - interval '1 second'
|
||||
WHERE device_id = $1`, value.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := postgres.ClaimDueReconcile(context.Background(), ReconcileClaim{
|
||||
Owner: "ins-b", Token: "token-b", Now: now.Add(31 * time.Second),
|
||||
LeaseDuration: 30 * time.Second, Limit: 1,
|
||||
})
|
||||
if err != nil || len(second) != 1 {
|
||||
t.Fatalf("expired lease was not recoverable: %+v %v", second, err)
|
||||
}
|
||||
if err := postgres.CompleteReconcile(
|
||||
context.Background(), value.ID, value.Generation, "ins-a", "token-a", now.Add(32*time.Second),
|
||||
); !errors.Is(err, ErrReconcileLeaseLost) {
|
||||
t.Fatalf("old worker was not fenced: %v", err)
|
||||
}
|
||||
if err := postgres.CompleteReconcile(
|
||||
context.Background(), value.ID, value.Generation, "ins-b", "token-b", now.Add(32*time.Second),
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var ownershipDevice string
|
||||
if err := admin.QueryRow(`SELECT device_id FROM sense.media_path_ownership WHERE path_name = $1`,
|
||||
value.PathName).Scan(&ownershipDevice); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ownershipDevice != value.ID {
|
||||
t.Fatalf("wrong ownership was recorded: %q", ownershipDevice)
|
||||
}
|
||||
var leaseToken sql.NullString
|
||||
if err := admin.QueryRow(`SELECT lease_token FROM sense.reconcile_state WHERE device_id = $1`,
|
||||
value.ID).Scan(&leaseToken); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if leaseToken.Valid {
|
||||
t.Fatal("completion did not release the reconcile lease")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresOrphanReportLeaseAndCleanupAuditAreFencedAndIdempotent(t *testing.T) {
|
||||
postgres, _ := openPostgresTestStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC)
|
||||
acquired, err := postgres.AcquireOperationalLease(
|
||||
ctx, OperationalLeaseOrphanScan, "ins-a", "scan-token-a", now, 30*time.Second,
|
||||
)
|
||||
if err != nil || !acquired {
|
||||
t.Fatalf("scan lease failed: %v %v", acquired, err)
|
||||
}
|
||||
scan := OrphanScan{
|
||||
ID: "scan_" + strings.Repeat("0", 26), InstanceID: "ins-a",
|
||||
ObservedCount: 10, OwnedStaleCount: 1, UnownedCount: 1,
|
||||
SafetyAllowed: true, SafetyReason: "allowed",
|
||||
CompletedAt: now.Add(time.Second), ExpiresAt: now.Add(15 * time.Minute),
|
||||
Findings: []OrphanFinding{
|
||||
{PathName: "stale", Classification: OrphanOwnedStale, DeviceID: "old-device"},
|
||||
{PathName: "unknown", Classification: OrphanUnowned},
|
||||
},
|
||||
}
|
||||
if err := postgres.SaveOrphanScan(ctx, scan, "ins-a", "scan-token-a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := postgres.GetOrphanScan(ctx, scan.ID)
|
||||
if err != nil || len(loaded.Findings) != 2 || !loaded.SafetyAllowed {
|
||||
t.Fatalf("stored scan mismatch: %+v %v", loaded, err)
|
||||
}
|
||||
if err := postgres.RecordOrphanCleanup(
|
||||
ctx, scan.ID, "stale", "operator", "deleted", "", now.Add(2*time.Second),
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := postgres.RecordOrphanCleanup(
|
||||
ctx, scan.ID, "unknown", "operator", "deleted", "", now.Add(2*time.Second),
|
||||
); err == nil {
|
||||
t.Fatal("unowned path accepted a cleanup audit record")
|
||||
}
|
||||
if err := postgres.RecordOrphanCleanup(
|
||||
ctx, scan.ID, "stale", "operator-2", "failed", "media_error", now.Add(3*time.Second),
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err = postgres.GetOrphanScan(ctx, scan.ID)
|
||||
if err != nil || !loaded.Findings[0].Deleted {
|
||||
t.Fatalf("successful cleanup was downgraded: %+v %v", loaded, err)
|
||||
}
|
||||
|
||||
acquired, err = postgres.AcquireOperationalLease(
|
||||
ctx, OperationalLeaseOrphanScan, "ins-b", "scan-token-b", now.Add(31*time.Second), 30*time.Second,
|
||||
)
|
||||
if err != nil || !acquired {
|
||||
t.Fatalf("expired scan lease was not recoverable: %v %v", acquired, err)
|
||||
}
|
||||
staleScan := scan
|
||||
staleScan.ID = "scan_" + strings.Repeat("1", 26)
|
||||
staleScan.CompletedAt = now.Add(32 * time.Second)
|
||||
staleScan.ExpiresAt = staleScan.CompletedAt.Add(15 * time.Minute)
|
||||
if err := postgres.SaveOrphanScan(ctx, staleScan, "ins-a", "scan-token-a"); !errors.Is(err, ErrOperationalLeaseLost) {
|
||||
t.Fatalf("stale scan worker was not fenced: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func openPostgresTestStore(t *testing.T) (*Postgres, *sql.DB) {
|
||||
t.Helper()
|
||||
dsn := os.Getenv(postgresTestDSNEnv)
|
||||
@@ -985,6 +1180,11 @@ func openPostgresTestStore(t *testing.T) (*Postgres, *sql.DB) {
|
||||
t.Fatal("connect PostgreSQL test administrator")
|
||||
}
|
||||
if _, err := admin.ExecContext(context.Background(), `TRUNCATE
|
||||
sense.orphan_cleanup_actions,
|
||||
sense.orphan_scan_findings,
|
||||
sense.orphan_scan_runs,
|
||||
sense.operational_leases,
|
||||
sense.media_path_ownership,
|
||||
sense.control_idempotency_receipts,
|
||||
sense.batch_operation_items,
|
||||
sense.batch_operations,
|
||||
|
||||
@@ -21,6 +21,8 @@ var (
|
||||
ErrAreaPolicyUnavailable = errors.New("area policy unavailable")
|
||||
ErrAreaPolicyInvalid = errors.New("area policy invalid")
|
||||
ErrAreaPolicyDenied = errors.New("area policy denies imaging device")
|
||||
ErrReconcileLeaseLost = errors.New("reconcile lease lost")
|
||||
ErrOperationalLeaseLost = errors.New("operational lease lost")
|
||||
)
|
||||
|
||||
// Repository is the storage boundary used by the Sense process. SQLite stays
|
||||
@@ -31,6 +33,10 @@ type Repository interface {
|
||||
SetDesiredState(context.Context, string, device.DesiredState) error
|
||||
GetDevice(context.Context, string) (device.Device, error)
|
||||
ListDueReconcile(context.Context, time.Time, int) ([]ReconcileCandidate, error)
|
||||
ClaimDueReconcile(context.Context, ReconcileClaim) ([]ReconcileCandidate, error)
|
||||
RenewReconcileLease(context.Context, string, string, string, time.Time, time.Duration) (bool, error)
|
||||
CompleteReconcile(context.Context, string, int64, string, string, time.Time) error
|
||||
FailReconcile(context.Context, string, int, time.Time, string, string, string, time.Time) error
|
||||
ListEnabledVideoDevices(context.Context, int) ([]device.Device, error)
|
||||
MarkReconciled(context.Context, string, int64, time.Time) error
|
||||
MarkReconcileFailure(context.Context, string, int, time.Time, string, time.Time) error
|
||||
@@ -39,6 +45,17 @@ type Repository interface {
|
||||
ConvergenceSnapshot(context.Context) (ConvergenceSnapshot, error)
|
||||
}
|
||||
|
||||
// ReconcileClaim identifies one short-lived batch claim. Token is unique per
|
||||
// run and fences a worker whose lease expired and was acquired by another
|
||||
// process. SQLite accepts the shape but remains explicitly single-process.
|
||||
type ReconcileClaim struct {
|
||||
Owner string
|
||||
Token string
|
||||
Now time.Time
|
||||
LeaseDuration time.Duration
|
||||
Limit int
|
||||
}
|
||||
|
||||
func OpenRepository(ctx context.Context, driver, dsn string) (Repository, error) {
|
||||
switch strings.ToLower(strings.TrimSpace(driver)) {
|
||||
case "", DriverSQLite:
|
||||
|
||||
@@ -430,6 +430,24 @@ func (s *SQLite) ListDueReconcile(ctx context.Context, now time.Time, limit int)
|
||||
return candidates, nil
|
||||
}
|
||||
|
||||
func (s *SQLite) ClaimDueReconcile(ctx context.Context, claim ReconcileClaim) ([]ReconcileCandidate, error) {
|
||||
// SQLite is retained for one-process M1 development. It deliberately does
|
||||
// not claim cross-process leases; PostgreSQL is the production M2 boundary.
|
||||
return s.ListDueReconcile(ctx, claim.Now, claim.Limit)
|
||||
}
|
||||
|
||||
func (s *SQLite) RenewReconcileLease(
|
||||
ctx context.Context,
|
||||
_, _, _ string,
|
||||
_ time.Time,
|
||||
_ time.Duration,
|
||||
) (bool, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *SQLite) ListEnabledVideoDevices(ctx context.Context, limit int) ([]device.Device, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
@@ -499,6 +517,16 @@ func (s *SQLite) MarkReconciled(ctx context.Context, id string, generation int64
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SQLite) CompleteReconcile(
|
||||
ctx context.Context,
|
||||
id string,
|
||||
generation int64,
|
||||
_, _ string,
|
||||
now time.Time,
|
||||
) error {
|
||||
return s.MarkReconciled(ctx, id, generation, now)
|
||||
}
|
||||
|
||||
func (s *SQLite) MarkReconcileFailure(ctx context.Context, id string, failureCount int, nextAttempt time.Time, errorCode string, now time.Time) error {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
@@ -526,6 +554,17 @@ func (s *SQLite) MarkReconcileFailure(ctx context.Context, id string, failureCou
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SQLite) FailReconcile(
|
||||
ctx context.Context,
|
||||
id string,
|
||||
failureCount int,
|
||||
nextAttempt time.Time,
|
||||
errorCode, _, _ string,
|
||||
now time.Time,
|
||||
) error {
|
||||
return s.MarkReconcileFailure(ctx, id, failureCount, nextAttempt, errorCode, now)
|
||||
}
|
||||
|
||||
func (s *SQLite) UpdateActualState(ctx context.Context, id string, state device.ActualState, now time.Time) error {
|
||||
if state != device.ActualPending && state != device.ActualOnline && state != device.ActualOffline && state != device.ActualFailed {
|
||||
return fmt.Errorf("invalid actual state %q", state)
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
-- Add fenced reconciliation claims and durable, secret-free MediaMTX path
|
||||
-- ownership/orphan reports. Runtime migrations are installed out of process.
|
||||
|
||||
ALTER TABLE sense.reconcile_state
|
||||
ADD COLUMN IF NOT EXISTS lease_owner text,
|
||||
ADD COLUMN IF NOT EXISTS lease_token text,
|
||||
ADD COLUMN IF NOT EXISTS lease_until timestamptz;
|
||||
|
||||
DO $constraints$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint
|
||||
WHERE conrelid = 'sense.reconcile_state'::regclass
|
||||
AND conname = 'sense_reconcile_lease_all_or_none'
|
||||
) THEN
|
||||
ALTER TABLE sense.reconcile_state
|
||||
ADD CONSTRAINT sense_reconcile_lease_all_or_none CHECK (
|
||||
(lease_owner IS NULL AND lease_token IS NULL AND lease_until IS NULL)
|
||||
OR (btrim(lease_owner) <> '' AND btrim(lease_token) <> '' AND lease_until IS NOT NULL)
|
||||
);
|
||||
END IF;
|
||||
END
|
||||
$constraints$;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS sense_reconcile_lease_due_idx
|
||||
ON sense.reconcile_state(lease_until, next_attempt_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sense.media_path_ownership (
|
||||
path_name text PRIMARY KEY,
|
||||
device_id text NOT NULL,
|
||||
tenant_id text NOT NULL,
|
||||
site_id text NOT NULL,
|
||||
first_claimed_at timestamptz NOT NULL,
|
||||
last_confirmed_at timestamptz NOT NULL,
|
||||
CONSTRAINT sense_media_path_ownership_not_blank CHECK (
|
||||
btrim(path_name) <> '' AND btrim(device_id) <> ''
|
||||
AND btrim(tenant_id) <> '' AND btrim(site_id) <> ''
|
||||
)
|
||||
);
|
||||
ALTER TABLE sense.media_path_ownership OWNER TO sense_app;
|
||||
|
||||
-- Keep the historical row independent from device lifecycle so a later device
|
||||
-- deletion cannot erase the evidence needed to classify an owned stale path.
|
||||
INSERT INTO sense.media_path_ownership(
|
||||
path_name, device_id, tenant_id, site_id, first_claimed_at, last_confirmed_at
|
||||
)
|
||||
SELECT d.path_name, d.id, d.tenant_id, d.site_id, d.created_at, d.updated_at
|
||||
FROM sense.devices d
|
||||
WHERE btrim(d.path_name) <> ''
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM sense.device_capabilities c
|
||||
WHERE c.device_id = d.id AND c.capability = 'video_capture'
|
||||
)
|
||||
ON CONFLICT (path_name) DO UPDATE SET
|
||||
device_id = EXCLUDED.device_id,
|
||||
tenant_id = EXCLUDED.tenant_id,
|
||||
site_id = EXCLUDED.site_id,
|
||||
last_confirmed_at = GREATEST(
|
||||
sense.media_path_ownership.last_confirmed_at,
|
||||
EXCLUDED.last_confirmed_at
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sense.operational_leases (
|
||||
lease_name text PRIMARY KEY,
|
||||
owner_id text NOT NULL,
|
||||
fencing_token text NOT NULL,
|
||||
lease_until timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL,
|
||||
CONSTRAINT sense_operational_lease_not_blank CHECK (
|
||||
btrim(lease_name) <> '' AND btrim(owner_id) <> '' AND btrim(fencing_token) <> ''
|
||||
)
|
||||
);
|
||||
ALTER TABLE sense.operational_leases OWNER TO sense_app;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sense.orphan_scan_runs (
|
||||
id text PRIMARY KEY,
|
||||
instance_id text NOT NULL,
|
||||
observed_count integer NOT NULL,
|
||||
owned_stale_count integer NOT NULL,
|
||||
unowned_count integer NOT NULL,
|
||||
safety_allowed boolean NOT NULL,
|
||||
safety_reason text NOT NULL,
|
||||
completed_at timestamptz NOT NULL,
|
||||
expires_at timestamptz NOT NULL,
|
||||
CONSTRAINT sense_orphan_scan_id CHECK (id ~ '^scan_[0-9A-HJKMNP-TV-Z]{26}$'),
|
||||
CONSTRAINT sense_orphan_scan_counts CHECK (
|
||||
observed_count >= 0 AND owned_stale_count >= 0 AND unowned_count >= 0
|
||||
AND owned_stale_count + unowned_count <= observed_count
|
||||
),
|
||||
CONSTRAINT sense_orphan_scan_not_blank CHECK (
|
||||
btrim(instance_id) <> '' AND btrim(safety_reason) <> ''
|
||||
),
|
||||
CONSTRAINT sense_orphan_scan_expiry CHECK (expires_at > completed_at)
|
||||
);
|
||||
ALTER TABLE sense.orphan_scan_runs OWNER TO sense_app;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sense.orphan_scan_findings (
|
||||
scan_id text NOT NULL REFERENCES sense.orphan_scan_runs(id) ON DELETE CASCADE,
|
||||
path_name text NOT NULL,
|
||||
classification text NOT NULL,
|
||||
device_id text,
|
||||
PRIMARY KEY (scan_id, path_name),
|
||||
UNIQUE (scan_id, path_name, classification),
|
||||
CONSTRAINT sense_orphan_finding_classification CHECK (
|
||||
classification IN ('owned_stale', 'unowned')
|
||||
),
|
||||
CONSTRAINT sense_orphan_finding_not_blank CHECK (
|
||||
btrim(path_name) <> '' AND (device_id IS NULL OR btrim(device_id) <> '')
|
||||
),
|
||||
CONSTRAINT sense_orphan_finding_owner_shape CHECK (
|
||||
(classification = 'owned_stale' AND device_id IS NOT NULL)
|
||||
OR (classification = 'unowned' AND device_id IS NULL)
|
||||
)
|
||||
);
|
||||
ALTER TABLE sense.orphan_scan_findings OWNER TO sense_app;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sense.orphan_cleanup_actions (
|
||||
scan_id text NOT NULL REFERENCES sense.orphan_scan_runs(id),
|
||||
path_name text NOT NULL,
|
||||
classification text NOT NULL DEFAULT 'owned_stale',
|
||||
actor_id text NOT NULL,
|
||||
status text NOT NULL,
|
||||
error_code text,
|
||||
attempted_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (scan_id, path_name),
|
||||
FOREIGN KEY (scan_id, path_name, classification)
|
||||
REFERENCES sense.orphan_scan_findings(scan_id, path_name, classification),
|
||||
CONSTRAINT sense_orphan_cleanup_owned_only CHECK (classification = 'owned_stale'),
|
||||
CONSTRAINT sense_orphan_cleanup_status CHECK (status IN ('deleted', 'failed')),
|
||||
CONSTRAINT sense_orphan_cleanup_not_blank CHECK (
|
||||
btrim(path_name) <> '' AND btrim(actor_id) <> ''
|
||||
AND (error_code IS NULL OR btrim(error_code) <> '')
|
||||
),
|
||||
CONSTRAINT sense_orphan_cleanup_error_shape CHECK (
|
||||
(status = 'deleted' AND error_code IS NULL)
|
||||
OR (status = 'failed' AND error_code IS NOT NULL)
|
||||
)
|
||||
);
|
||||
ALTER TABLE sense.orphan_cleanup_actions OWNER TO sense_app;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS sense_orphan_scan_completed_idx
|
||||
ON sense.orphan_scan_runs(completed_at);
|
||||
CREATE INDEX IF NOT EXISTS sense_orphan_cleanup_status_idx
|
||||
ON sense.orphan_cleanup_actions(scan_id, status);
|
||||
|
||||
INSERT INTO sense.schema_migrations(version) VALUES (5)
|
||||
ON CONFLICT (version) DO NOTHING;
|
||||
@@ -0,0 +1,17 @@
|
||||
-- Extend the least-privilege boundary to reconciliation and orphan state.
|
||||
|
||||
REVOKE ALL ON TABLE
|
||||
sense.media_path_ownership,
|
||||
sense.operational_leases,
|
||||
sense.orphan_scan_runs,
|
||||
sense.orphan_scan_findings,
|
||||
sense.orphan_cleanup_actions
|
||||
FROM PUBLIC;
|
||||
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE
|
||||
sense.media_path_ownership,
|
||||
sense.operational_leases,
|
||||
sense.orphan_scan_runs,
|
||||
sense.orphan_scan_findings,
|
||||
sense.orphan_cleanup_actions
|
||||
TO sense_app;
|
||||
@@ -1,6 +1,6 @@
|
||||
# YoVision PostgreSQL 初始化
|
||||
|
||||
本目录实现 T-009~T-011 的 PostgreSQL `17.10` schema。SQL 必须按文件名前缀顺序执行:`001`~`004` 创建 NOLOGIN 权限角色、Bell/Sense 初始对象和配额权限;`005`~`007` 增量增加 Area 与审计;`008`~`009` 增加 Control API 的资源版本、write-only profile token、24 小时幂等收据、batch operation 和对应最小权限。全部 SQL 可重放。应用登录角色及其密码由部署环境或密钥系统创建,并分别加入 `bell_app`、`sense_app`;仓库不保存登录凭据。
|
||||
本目录实现 T-009~T-012 的 PostgreSQL `17.10` schema。SQL 必须按文件名前缀顺序执行:`001`~`004` 创建 NOLOGIN 权限角色、Bell/Sense 初始对象和配额权限;`005`~`007` 增量增加 Area 与审计;`008`~`009` 增加 Control API 状态;`010`~`011` 增加调和 fencing、MediaMTX Path 历史归属、孤儿报告/受控处置结果和对应最小权限。全部 SQL 可重放。应用登录角色及其密码由部署环境或密钥系统创建,并分别加入 `bell_app`、`sense_app`;仓库不保存登录凭据。
|
||||
|
||||
生产/共享实例必须由管理员先备份并在 YoVision 专用数据库中执行。Sense 进程不会用高权限自动建库或建角色。示例只使用私有环境变量,不把实际 DSN 写入脚本或日志:
|
||||
|
||||
@@ -22,8 +22,9 @@ Get-ChildItem deploy/postgres/[0-9][0-9][0-9]_*.sql |
|
||||
- `sense_app` 对 Bell 源表、Bell migration 表和 trigger function 没有权限;启动检查发现权限过宽时拒绝运行。
|
||||
- `sense.device_operation_outbox` 是本地持久化审计事实,不是 Bell 全局审计真相;relay 的传输、签名、确认和留存尚未实现。
|
||||
- `sense.control_idempotency_receipts` 不保存原始 Idempotency-Key,只保存 scope/request SHA-256 和脱敏响应快照;`batch_operations`/items 只保存逻辑 ID、状态和稳定错误,不保存连接秘密。
|
||||
- 调和与孤儿租约使用 PostgreSQL `clock_timestamp()` 和 fencing token;过期 worker 不能提交完成/失败或扫描报告。`media_path_ownership`、扫描和处置表不保存 endpoint、credential 或 source URI;数据库约束禁止为 `unowned` finding 写删除结果。
|
||||
- PUBLIC 对两个业务 schema 的表和函数没有权限。
|
||||
|
||||
## 恢复
|
||||
|
||||
T-009~T-011 不迁移或删除现有 SQLite 数据,默认运行配置也仍为 SQLite。SQLite 不具备生产 Area/Outbox/Control API 一致性,业务路由不会在 SQLite 上注册。若 PostgreSQL 切换验证失败,先关闭 Control API feature flag;需要实验室回退时停止 PostgreSQL 模式进程并恢复 SQLite DSN,不得误报为生产语义等价。schema 不提供自动 down migration;需要移除时由管理员先备份,再只删除明确确认的 YoVision 专用数据库,禁止对共享实例做模糊或递归清理。
|
||||
T-009~T-012 不迁移或删除现有 SQLite 数据,默认运行配置也仍为 SQLite。SQLite 不具备生产 Area/Outbox/Control API、多实例租约或孤儿处置一致性,业务路由不会在 SQLite 上注册。若 PostgreSQL 切换验证失败,先关闭 Control API 和孤儿扫描;需要实验室回退时停止 PostgreSQL 模式进程并恢复 SQLite DSN,不得误报为生产语义等价。schema 不提供自动 down migration;需要移除时由管理员先备份,再只删除明确确认的 YoVision 专用数据库,禁止对共享实例做模糊或递归清理。
|
||||
|
||||
@@ -72,7 +72,7 @@ BEGIN
|
||||
RAISE EXCEPTION 'sense_app lacks access to its local audit Outbox';
|
||||
END IF;
|
||||
IF (SELECT max(version) FROM bell.schema_migrations) <> 2
|
||||
OR (SELECT max(version) FROM sense.schema_migrations) <> 4 THEN
|
||||
OR (SELECT max(version) FROM sense.schema_migrations) <> 5 THEN
|
||||
RAISE EXCEPTION 'schema migration version drift';
|
||||
END IF;
|
||||
|
||||
@@ -129,9 +129,9 @@ $assertions$;
|
||||
|
||||
DO $control_api$
|
||||
BEGIN
|
||||
IF NOT has_table_privilege('yovision_t011_sense', 'sense.control_idempotency_receipts', 'SELECT,INSERT,UPDATE,DELETE')
|
||||
OR NOT has_table_privilege('yovision_t011_sense', 'sense.batch_operations', 'SELECT,INSERT,UPDATE,DELETE')
|
||||
OR NOT has_table_privilege('yovision_t011_sense', 'sense.batch_operation_items', 'SELECT,INSERT,UPDATE,DELETE') THEN
|
||||
IF NOT has_table_privilege('yovision_t012_sense', 'sense.control_idempotency_receipts', 'SELECT,INSERT,UPDATE,DELETE')
|
||||
OR NOT has_table_privilege('yovision_t012_sense', 'sense.batch_operations', 'SELECT,INSERT,UPDATE,DELETE')
|
||||
OR NOT has_table_privilege('yovision_t012_sense', 'sense.batch_operation_items', 'SELECT,INSERT,UPDATE,DELETE') THEN
|
||||
RAISE EXCEPTION 'Sense runtime lacks Control API table privileges';
|
||||
END IF;
|
||||
IF has_table_privilege('public', 'sense.control_idempotency_receipts', 'SELECT')
|
||||
@@ -151,3 +151,31 @@ BEGIN
|
||||
END IF;
|
||||
END
|
||||
$control_api$;
|
||||
|
||||
DO $reconcile_safety$
|
||||
DECLARE
|
||||
target text;
|
||||
BEGIN
|
||||
FOREACH target IN ARRAY ARRAY[
|
||||
'sense.media_path_ownership',
|
||||
'sense.operational_leases',
|
||||
'sense.orphan_scan_runs',
|
||||
'sense.orphan_scan_findings',
|
||||
'sense.orphan_cleanup_actions'
|
||||
] LOOP
|
||||
IF NOT has_table_privilege('yovision_t012_sense', target, 'SELECT,INSERT,UPDATE,DELETE') THEN
|
||||
RAISE EXCEPTION 'Sense runtime lacks reconciliation safety privilege on %', target;
|
||||
END IF;
|
||||
IF has_table_privilege('public', target, 'SELECT,INSERT,UPDATE,DELETE') THEN
|
||||
RAISE EXCEPTION 'Reconciliation safety state leaked to PUBLIC on %', target;
|
||||
END IF;
|
||||
END LOOP;
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = 'sense' AND table_name = 'reconcile_state'
|
||||
AND column_name = 'lease_token'
|
||||
) THEN
|
||||
RAISE EXCEPTION 'reconcile fencing columns are missing';
|
||||
END IF;
|
||||
END
|
||||
$reconcile_safety$;
|
||||
|
||||
@@ -40,13 +40,13 @@ MVP 以默认 16 路跑通一个场景的端到端闭环;架构、数据和 UI
|
||||
|
||||
## 当前阶段
|
||||
|
||||
当前为 **M0 指定型号实机准入已完成,M1 Sense 五路混合源集成待执行**。后续本地开发统一使用已准入的一台海康样机,多路软件闭环使用独立合成 RTSP 源补足;真实多设备证据延后到客户/借用/租赁条件具备时执行。
|
||||
当前为 **M0 指定型号实机准入与 M1 Sense 五路混合源集成均已完成,M2 正在补齐生产接入边界**。后续本地开发统一使用已准入的一台海康样机,多路软件闭环使用独立合成 RTSP 源补足;真实多设备证据延后到客户/借用/租赁条件具备时执行。
|
||||
|
||||
优先路径:
|
||||
|
||||
1. M0:已用一台真实样机完成首期指定“型号 + 硬件版本 + 固件”ONVIF/RTSP 准入;结论不外推为多品牌兼容,真实断网恢复证据按负责人豁免留痕。
|
||||
2. M1:只在 `Sense/` 建立 MediaMTX 生产接入骨架,以 1 路准入实机 + 至少 4 路独立合成源完成五路自动建 path、探活和断线重建。
|
||||
3. M2:对账、多租户、隧道和至少一个站点的 16 路全流程。
|
||||
3. M2:Control API、多租户投影、调和 fencing、孤儿安全闸已完成;下一步是隧道和至少一个站点的 16 路全流程。
|
||||
4. M3:Brain + Bell 起步,默认 16 路端到端事件、预警、ack 与误报反馈。
|
||||
5. M4–M5:64/128 路分片、管理端和第二/第三场景包。
|
||||
|
||||
|
||||
@@ -50,7 +50,9 @@ MediaMTX 官方 `v1.19.3` OpenAPI 固定保存于 `Sense/api/vendor/mediamtx-v1.
|
||||
|
||||
M2 仍保留 SQLite 作为单机开发与 T-006 回归入口,但生产真相源切换目标是 PostgreSQL。运行时必须显式设置 `SENSE_DB_DRIVER=postgres` 和私有 `SENSE_DB_DSN` 才使用 PostgreSQL;仓库不提供带密码 DSN。初始 schema 由 `deploy/postgres/` 的顺序 migration 创建,不在 Sense 进程启动时使用高权限自动迁移。
|
||||
|
||||
T-011 复用同一套冻结的 Go、PostgreSQL、pgx 与 `oapi-codegen`,未增加第三方依赖。Sense Control API v1 从仓库 OpenAPI 生成 `net/http` server glue;首版认证是可替换 port 下的外部静态 SHA-256 注册表,公共路由默认关闭且只在 PostgreSQL v4 schema 上启用。静态适配器是私有部署启动方案,不替代后续 Bell/JWT/OIDC 身份服务。
|
||||
T-011 复用同一套冻结的 Go、PostgreSQL、pgx 与 `oapi-codegen`,未增加第三方依赖。Sense Control API v1 从仓库 OpenAPI 生成 `net/http` server glue;首版认证是可替换 port 下的外部静态 SHA-256 注册表,公共路由默认关闭且只在 PostgreSQL 生产路径启用。T-011 首次落成 v4 schema,T-012 后进程统一要求 v5。静态适配器是私有部署启动方案,不替代后续 Bell/JWT/OIDC 身份服务。
|
||||
|
||||
T-012 同样未增加第三方依赖:PostgreSQL v5 使用数据库时钟租约和 fencing token 协调多实例调和/孤儿扫描;MediaMTX Path 枚举继续使用同版生成客户端。`/metrics` 由 Go 标准库输出 Prometheus 文本格式,只包含固定结果枚举、构建/实例信息和汇总 gauge/counter,不使用 tenant、Site、device 或 Path 标签。孤儿处置是 PostgreSQL 专用本地命令,默认周期任务只报告、不自动删除。
|
||||
|
||||
## 2. 外部项目边界
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ Sense ── 视频流/触发信号 ──> Brain
|
||||
主流程:
|
||||
|
||||
1. Bell 持有站点、Area、配额与 `capture_policy`;首期在同一 PostgreSQL 实例内发布 `bell.site_quota_v1` 和 `bell.area_policy_v1` 两个版本化只读视图。T-009/T-010 已实现 Bell 源表/视图、最小权限和 Sense PostgreSQL repository;Sense 按 Area→Site 的固定 advisory-lock 顺序执行策略与配额准入并记录所用版本。未来分库必须发布新版本契约,不能静默改变 v1 语义。
|
||||
2. Sense 维护设备期望态,通过 MediaMTX API 和对账器收敛实际态。
|
||||
2. Sense 维护设备期望态,通过 MediaMTX API 和对账器收敛实际态;PostgreSQL 多实例以数据库时钟短租约和 fencing token 领取 due row,过期 worker 不得提交结果。
|
||||
3. Brain 消费视频与触发信号,产生符合 v0.1 的事件。
|
||||
4. Bell 做 schema 与代码级断言,生成平台 ULID,保存不可变事件。
|
||||
5. 规则命中后创建独立 Alert,先落库再投递,等待 ack 并按策略升级。
|
||||
@@ -82,8 +82,8 @@ Sense ── 视频流/触发信号 ──> Brain
|
||||
## 7. 一致性与失败处理
|
||||
|
||||
- PostgreSQL `sense` schema 是生产期望态真相源;SQLite 只保留为 M1 本地开发/回归路径。MediaMTX、推理 worker 和对象存储是可对账的实际态。
|
||||
- 对账器水平触发、幂等、指数退避、限制并发;部分失败不做跨系统回滚,只持续收敛。
|
||||
- 孤儿删除必须有 10% 安全闸和人工可观察指标。
|
||||
- 对账器水平触发、幂等、指数退避、限制并发;PostgreSQL 使用 `FOR UPDATE SKIP LOCKED`、每项续租和 fencing token,SQLite 只保留单进程开发语义。部分失败不做跨系统回滚,只持续收敛。
|
||||
- MediaMTX Path 扫描把“Sense 历史拥有但当前失配”和“从未归属 Sense”分开;未知归属永不自动删除。历史拥有项也只允许在 15 分钟二次快照、1~128 项和 `候选 × 100 <= 当前 Path 总数 × 10` 全部通过时由本地运维命令逐项处置,不提供绕过。
|
||||
- `bell.site_quota_v1` 行缺失、数值越界、版本回退或读取失败只阻止视频设备新增/启用,不中断已有流;降低配额导致超限时不自动停用,后续准入返回稳定错误并产生运维信号。多 Sense 实例使用 PostgreSQL transaction-scoped advisory lock 串行化同 tenant/site 的计数与写入,不能用进程内锁替代。
|
||||
- `bell.area_policy_v1` 缺失、非法、版本回退或读取失败时,PostgreSQL repository 拒绝相关新增/启用;`non_imaging_only` 允许非成像设备但拒绝具有 `video_capture` 的设备。已有设备保持原状态,策略冲突由 Bell 管理端显式迁移或取消。同库实时视图不以源记录年龄误判 freshness。
|
||||
- 设备创建和期望态受理在本地事务内同时写脱敏 `sense.device_operation_outbox`;Outbox 失败回滚业务写入,相同期望态不增加 generation 但仍审计。异步 relay 尚未实现。
|
||||
@@ -107,14 +107,14 @@ Sense ── 视频流/触发信号 ──> Brain
|
||||
## 9. 目录目标
|
||||
|
||||
```text
|
||||
Sense/cmd + Sense/internal/{device,onvif,mtx,reconcile,probe,trigger,tunnel,authcb,store}
|
||||
Sense/cmd + Sense/internal/{device,onvif,mtx,reconcile,orphan,metrics,probe,trigger,tunnel,auth,store}
|
||||
Brain/{pipeline,models,judge,emit,trigger,contracts}
|
||||
Bell/cmd + Bell/internal/{ingest,event,rule,alert,deliver,feedback,tenant,audit,store}
|
||||
Bell/{web,packs,contracts}
|
||||
deploy/postgres/{001_roles.sql,...,007_privileges_area_audit.sql,tests}
|
||||
deploy/postgres/{001_roles.sql,...,011_privileges_reconcile_safety.sql,tests}
|
||||
```
|
||||
|
||||
Sense 脚手架和 PostgreSQL `001`~`007` 已实现;Brain/Bell 应用目录仍是占位,真实服务必须由对应任务创建。
|
||||
Sense 脚手架和 PostgreSQL `001`~`011` 已实现;Brain/Bell 应用目录仍是占位,真实服务必须由对应任务创建。
|
||||
|
||||
## 10. 开发顺序
|
||||
|
||||
|
||||
+1
-1
@@ -26,7 +26,7 @@
|
||||
- T-009:使用 PostgreSQL 17.10 建立 `sense`/`bell` schema、Bell 配额源表/只读视图、最小权限和 Sense PostgreSQL repository;SQLite 保留为 M1 本地路径。
|
||||
- T-010:建立 Bell Area/`capture_policy` 只读投影、Sense 成像准入与设备操作审计 Outbox 原子写入;relay 传输协议留给后续任务。
|
||||
- T-011:按 T-008 契约实现认证 tenant 上下文、7 个设备/operation handler、PostgreSQL 幂等收据、ETag/HMAC cursor、最多 128 项批量操作和停用精确 path 收敛;公共控制 API 默认关闭且只在 PostgreSQL 路径启用。
|
||||
- 对账器幂等/退避/并发/10% 安全闸。
|
||||
- T-012:以 PostgreSQL 数据库时钟租约和 fencing token 防止多实例重复调和;增加可证明的 Path 历史归属、默认只读孤儿报告、15 分钟二次确认、10% 不可绕过安全闸和低基数 `/metrics`。
|
||||
- WireGuard 边缘隧道与断网恢复。
|
||||
- 16 路批量开通、停用和容量基准。
|
||||
|
||||
|
||||
+5
-3
@@ -69,10 +69,10 @@ Sense 使用 MediaMTX 官方 OpenAPI 生成客户端并加薄封装。业务代
|
||||
| Port | 操作 | 数据所有者 / 失败语义 |
|
||||
| --- | --- | --- |
|
||||
| ONVIF adapter | `Probe(target)`、`SetSystemDateAndTime(target, time)` | 设备是外部来源;`target` 只含 endpoint ref 与不透明 credential ref。错误稳定映射为认证失败、超时、不可用、响应无效,不记录凭据或完整流地址 |
|
||||
| MediaMTX paths | `CreatePath`、`GetPath`、`EnsurePath`、`DeletePath`、`PathReady` | SQLite 设备台账持有期望态,MediaMTX 只持有运行配置;`EnsurePath` 相同 source 不写、不同 source patch、缺失时 add;当前调和器绝不枚举或删除孤儿 |
|
||||
| MediaMTX paths | `CreatePath`、`GetPath`、`EnsurePath`、`DeletePath`、`PathReady`、`ListPathNames` | 设备台账持有期望态,MediaMTX 只持有运行配置;分页枚举只返回名称、不返回 source。普通调和只操作精确台账 Path;独立孤儿流程只报告未知归属,受控处置只接受有历史归属的 stale Path |
|
||||
| Device repository | 设备、期望态、实际态、调和进度 | SQLite 是 M1 默认开发路径;PostgreSQL 是 M2 生产路径并只读 Site/Area 两个 Bell 视图。调和退避持久化;配额/Area 失败时拒绝相关新增或启用,不关闭已有流;高风险写入与脱敏 Outbox 同事务 |
|
||||
|
||||
MediaMTX 薄封装调用同版官方 OpenAPI 的 `/v3/config/paths/get|add|patch|delete/{name}` 与 `/v3/paths/get/{name}`。生成源、版本和 SHA-256 见 `docs/03-tech-stack.md`;业务包不得直接 import 生成包。
|
||||
MediaMTX 薄封装调用同版官方 OpenAPI 的 `/v3/config/paths/get|add|patch|delete/{name}`、`/v3/config/paths/list` 与 `/v3/paths/get/{name}`。列表有最大页数和重复页保护,source URI 在薄封装内丢弃。生成源、版本和 SHA-256 见 `docs/03-tech-stack.md`;业务包不得直接 import 生成包。
|
||||
|
||||
### 5.2 设备台账语义
|
||||
|
||||
@@ -83,7 +83,9 @@ MediaMTX 薄封装调用同版官方 OpenAPI 的 `/v3/config/paths/get|add|patch
|
||||
|
||||
### 5.3 Sense 进程 HTTP 面
|
||||
|
||||
`GET /healthz` 表示进程存活,`GET /readyz` 表示所选数据库已打开且 schema/权限前置检查完成;两者不要求认证,也不等价于摄像头、MediaMTX path 或里程碑健康。T-011 在 `SENSE_CONTROL_API_ENABLED=true`、PostgreSQL v4 schema 和外部安全文件全部有效时注册冻结的 `/api/v1` 路由;默认 SQLite 运行仍只暴露探针。业务路由不提供无 Site 边界的 `/api/v1/devices` 临时接口。
|
||||
`GET /healthz` 表示进程存活,`GET /readyz` 表示所选数据库已打开且 schema/权限前置检查完成;两者不要求认证,也不等价于摄像头、MediaMTX path 或里程碑健康。`GET /metrics` 默认输出无租户/设备/Path 标签的 Prometheus 汇总,可通过 `SENSE_METRICS_ENABLED=false` 关闭。T-011 在 `SENSE_CONTROL_API_ENABLED=true`、PostgreSQL v5 schema 和外部安全文件全部有效时注册冻结的 `/api/v1` 路由;默认 SQLite 不注册业务路由。业务路由不提供无 Site 边界的 `/api/v1/devices` 临时接口。
|
||||
|
||||
孤儿报告/处置不是公共 HTTP API。它由 PostgreSQL 专用 `cmd/sense-orphan` 在受控运维主机执行,使用 15 分钟 scan ID、actor、精确确认文本和运行前二次快照;不改变 Sense Control API v1 的 7 个 endpoint。
|
||||
|
||||
## 6. 变更流程
|
||||
|
||||
|
||||
+10
-9
@@ -4,8 +4,8 @@
|
||||
|
||||
## 当前阶段
|
||||
|
||||
- 阶段:M0 指定摄像头型号准入已完成;M1 的“一实机 + 四合成源”实验室软件闭环已通过;M2 已完成控制面契约、Site/Area 准入与本地审计事务基础,但五条独立真实上游和生产 SLA 尚未验收。
|
||||
- 生产代码:Sense 已包含可构建进程、SQLite/PostgreSQL repository、Site/Area 准入、设备操作 Outbox、标准 ONVIF SOAP/WS-Security adapter、凭据引用、MediaMTX 生成客户端、对账、探活和 MediaMTX 重启重建;公共设备管理 API、认证、Outbox relay 和完整生产部署仍未实现。
|
||||
- 阶段:M0 指定摄像头型号准入已完成;M1 的“一实机 + 四合成源”实验室软件闭环已通过;M2 已完成 Control API、Site/Area 准入、本地审计事务、多实例调和 fencing 和孤儿受控处置基础,但 WireGuard、16 路容量出口、五条独立真实上游和生产 SLA 尚未验收。
|
||||
- 生产代码:Sense 已包含可构建进程、SQLite/PostgreSQL repository、Site/Area 准入、设备操作 Outbox、标准 ONVIF SOAP/WS-Security adapter、凭据引用、MediaMTX 生成客户端、Control API v1、对账/探活、数据库租约、孤儿只读扫描/受控命令和低基数指标;Bell 管理服务/JWT、Outbox relay、WireGuard 和完整生产部署仍未实现。
|
||||
- 默认容量:16 路;单站点本阶段上限 128 路,必须横向分片。
|
||||
|
||||
## 仓库现实
|
||||
@@ -14,13 +14,14 @@
|
||||
- Sense 设备模型使用 `modality + capabilities`,SQLite 执行 v1 migration;视频配额默认 16、允许 1~128,17/128/129、新增/启用和“降低配额不关闭已有流”均有测试。
|
||||
- T-009 冻结 PostgreSQL `17.10` 和 `pgx/v5 v5.10.0`,实现 `bell`/`sense` schema、NOLOGIN 权限角色、Bell Site 版本 trigger、`bell.site_quota_v1` 和 Sense PostgreSQL repository;同站点并发准入用事务级 advisory lock,配额缺失/越界/版本回退时失败关闭且不改变已有流。
|
||||
- T-010 增量实现 `bell.areas`、`bell.area_policy_v1`、Area 版本观察和 `sense.device_operation_outbox`;`non_imaging_only` 拒绝成像设备创建/启用,失败不改变已有设备。设备创建/期望态受理与脱敏 Outbox 同事务,相同期望态不增加 generation 但仍审计。
|
||||
- Windows 隔离测试使用 `D:\pgsql17\bin` 启动随机回环端口临时集群,`001`~`007` migration 可重放、权限断言和 18 个真实 PostgreSQL 测试通过后自动清理;现有 `D:\pgsql17\data` 和 5432 服务未被读取、停止或修改。
|
||||
- MediaMTX 固定为独立二进制 `v1.19.3`,官方 OpenAPI 已按 SHA-256 vendoring,并由固定 `oapi-codegen v2.8.0` 生成客户端;手写薄封装有 create/read/delete、幂等 ensure 与探活假 HTTP 测试。
|
||||
- T-003 对账进度与指数退避持久化,覆盖取消和 SQLite 重启恢复;T-006 增加真实 ONVIF adapter、RTSP router、实验室播种/状态工具、故障代理和五路自动验收。当前仍不枚举/删除孤儿。
|
||||
- Windows 隔离测试使用 `D:\pgsql17\bin` 启动随机回环端口临时集群,`001`~`011` migration 可重放、权限断言和 29 个真实 PostgreSQL 测试通过后自动清理;现有 `D:\pgsql17\data` 和 5432 服务未被读取、停止或修改。
|
||||
- MediaMTX 固定为独立二进制 `v1.19.3`,官方 OpenAPI 已按 SHA-256 vendoring,并由固定 `oapi-codegen v2.8.0` 生成客户端;手写薄封装有 create/read/delete、幂等 ensure、探活和只返回名称的受限分页枚举测试。
|
||||
- T-003 对账进度与指数退避持久化,覆盖取消和 SQLite 重启恢复;T-006 增加真实 ONVIF adapter、RTSP router、实验室播种/状态工具、故障代理和五路自动验收。T-012 的普通调和不枚举孤儿;独立 PostgreSQL 扫描默认只报告,未知归属永不删除。
|
||||
- T-006 正式使用 1 台准入实机和 4 个独立合成 publisher 连续观察 `1806.6 s` / 180 次采样,四类恢复均通过,最大与最终 `unconverged` 均为 0;详细证据见 `docs/research/sense-5-stream-integration.md`。
|
||||
- `docs/raw/01`~`08` 已记录需求、分析、方案、客户场景、事件比对和三系统职责。
|
||||
- `docs/raw/contracts/event-v0.1.schema.json` 已冻结,并有多份示例与语义说明。
|
||||
- `docs/contracts/sense-control-v1.openapi.json` 的 7 个站点作用域/operation endpoint 已由 T-011 实现:外部静态 SHA-256 主体注册表、tenant/Site scope、HMAC cursor、ETag、PostgreSQL 24 小时幂等收据和最多 128 项 batch operation 均有代码与隔离集成测试。默认 SQLite 仍只暴露探针;Bell 管理服务、JWT/OIDC 和 Outbox relay 尚未实现。
|
||||
- `docs/contracts/sense-control-v1.openapi.json` 的 7 个站点作用域/operation endpoint 已由 T-011 实现:外部静态 SHA-256 主体注册表、tenant/Site scope、HMAC cursor、ETag、PostgreSQL 24 小时幂等收据和最多 128 项 batch operation 均有代码与隔离集成测试。默认 SQLite 只暴露运维探针与低基数 `/metrics`,不注册业务路由;Bell 管理服务、JWT/OIDC 和 Outbox relay 尚未实现。
|
||||
- T-012 把 PostgreSQL schema 提升到 v5:due row 用数据库时钟、`FOR UPDATE SKIP LOCKED`、逐项续租和 fencing token 协调;MediaMTX Path 历史归属、15 分钟孤儿快照、最多 10%/128 项安全闸、无 bypass 的本地处置命令及 `/metrics` 已实现。SQLite 明确保留单实例开发语义。
|
||||
- harness coding 文档、上下文清单、Gitea Issue/PR 模板和治理脚本已接入。
|
||||
- Gitea 已初始化 12 个协作标签;`status/waiting` 用于依赖或外部条件未满足的未领取任务,实时可领取状态必须从 Gitea 查询,不在本文复制。
|
||||
- T-002 已关闭架构影响型问题:首期 S2 民办寄宿学校、客户侧私有部署、ONVIF/RTSP、NVIDIA 主路径、自研 Bell、Web/H5 与客户侧证据存储等边界已批准。
|
||||
@@ -55,7 +56,7 @@ python scripts/validate_harness_governance.py
|
||||
./scripts/test_postgres.ps1 -PgRoot D:\pgsql17
|
||||
```
|
||||
|
||||
Sense 默认监听 `127.0.0.1:8080`,提供 `/healthz` 与 `/readyz` 运维探针;它们不代表摄像头或 M1 里程碑健康。MediaMTX 获取、校验和独立启动方法见 `Sense/README.md`。
|
||||
Sense 默认监听 `127.0.0.1:8080`,提供 `/healthz`、`/readyz` 运维探针和低基数 `/metrics`;探针不代表摄像头或 M1 里程碑健康。MediaMTX 获取、校验和独立启动方法见 `Sense/README.md`。
|
||||
|
||||
## 当前 blocker / 待确认
|
||||
|
||||
@@ -66,12 +67,12 @@ Sense 默认监听 `127.0.0.1:8080`,提供 `/healthz` 与 `/readyz` 运维探
|
||||
- 人脸方向已延后至 M5 的 S4 成人园区候选试点;必要性/PIP 影响评估、单独同意与替代方式、合法底库来源和删除流程未完成,阻塞人脸能力上线。
|
||||
- 短信/语音具体供应商未选;生产前必须选定两条独立投递路径并验证故障切换。
|
||||
- Python/Savant 的精确版本、目标硬件和 Bell 前端栈尚未冻结;Sense M1 的 Go、SQLite driver、MediaMTX、生成器及生成运行时版本已在 T-003 冻结,PostgreSQL/pgx 版本已在 T-009 冻结。
|
||||
- 本机现有 PostgreSQL 5432 实例使用 SCRAM 且当前开发进程没有管理员密码;T-009~T-011 不绕过认证,自动验收使用隔离临时集群。向共享/生产实例安装 migration 前仍需管理员私下提供专用数据库、登录角色、外部 Control API 安全文件与备份方案。
|
||||
- 本机现有 PostgreSQL 5432 实例使用 SCRAM 且当前开发进程没有管理员密码;T-009~T-012 不绕过认证,自动验收使用隔离临时集群。向共享/生产实例安装 migration 前仍需管理员私下提供专用数据库、登录角色、外部 Control API 安全文件与备份方案。
|
||||
- 代码知识图谱在无业务代码阶段可能为空;工具不可用时使用 `rg` 处理文档与配置。
|
||||
|
||||
## 下一步
|
||||
|
||||
下一项建议创建 T-012,补齐 M2 对账安全闸、孤儿检测(只报告/受控处置)与多实例可观测性,再进入 WireGuard 和 16 路批量开通基准;不要把 Control API 完成误报为 M2 容量出口。客户授权、借用或租赁条件具备后再执行 T-007 五条独立真实上游现场门禁。T-006 的合成结果不解除 T-007,也不形成容量或生产 SLA 承诺。
|
||||
下一项建议先冻结并实施 WireGuard 边缘隧道与断网恢复,再创建 16 路批量开通/停用和分维度容量基准任务;T-012 完成不等于 M2 容量出口。客户授权、借用或租赁条件具备后再执行 T-007 五条独立真实上游现场门禁。T-006 的合成结果不解除 T-007,也不形成容量或生产 SLA 承诺。
|
||||
|
||||
## 已知风险
|
||||
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
# Sense 对账与孤儿处置 Runbook
|
||||
|
||||
本文适用于 PostgreSQL v5(migration `001`~`011`)与 MediaMTX `v1.19.3`。SQLite 仅用于单进程开发,不具备本 runbook 的多实例 fencing 或孤儿处置语义。
|
||||
|
||||
## 观察入口
|
||||
|
||||
- `GET /healthz`:进程存活。
|
||||
- `GET /readyz`:数据库 schema 和权限前置检查已完成。
|
||||
- `GET /metrics`:低基数进程指标;不包含 tenant、Site、device、Path 或 URI 标签。
|
||||
- 日志中的 `background convergence error`:只输出稳定、脱敏错误,不输出 stream URI 或凭据。
|
||||
|
||||
重点指标:
|
||||
|
||||
- `sense_reconcile_devices{state="unconverged"}`:未收敛的 enabled 视频设备数。
|
||||
- `sense_reconcile_items_total{result="lease_lost"}`:worker 在执行前或提交时失去 fencing token 的次数;失去租约的 worker 不得继续产生外部变更。
|
||||
- `sense_orphan_paths{classification="owned_stale|unowned"}`:最近一次成功扫描的两类差异。
|
||||
- `sense_orphan_cleanup_blocked_total`、`sense_orphan_cleanup_items_total`:安全闸阻断和人工处置结果。
|
||||
|
||||
## 多实例调和
|
||||
|
||||
每个实例应注入唯一、稳定且不含客户信息的 `SENSE_INSTANCE_ID`。PostgreSQL 使用数据库时钟、短事务、`FOR UPDATE SKIP LOCKED` 和随机 fencing token 领取 due row;批量等待中的每一项在调用 ONVIF/MediaMTX 前续租。默认租期 30 秒、单项 deadline 20 秒;operation timeout 必须严格短于租期。
|
||||
|
||||
发现 `lease_lost` 时先检查实例 ID 是否重复、数据库时钟和外部调用延迟。不要通过延长到超过 5 分钟或关闭 fencing 规避问题;先定位超时,再在变更评审后同时调整租期和单项 deadline。
|
||||
|
||||
## 生成孤儿报告
|
||||
|
||||
在已安装 v5 migration、能访问同一 PostgreSQL 和目标 MediaMTX 的受控运维主机执行:
|
||||
|
||||
```powershell
|
||||
$env:SENSE_DB_DRIVER = 'postgres'
|
||||
$env:SENSE_DB_DSN = '由部署环境私下设置'
|
||||
$env:SENSE_MEDIAMTX_URL = '受控 MediaMTX API 地址'
|
||||
go -C Sense run ./cmd/sense-orphan -mode report
|
||||
```
|
||||
|
||||
命令只输出 `scan_id`、计数、安全闸结论和过期时间。分类含义:
|
||||
|
||||
- `owned_stale`:Sense 有历史 Path 归属记录,但当前没有同一设备继续声明该 Path;可进入受控候选。
|
||||
- `unowned`:没有 Sense 归属证据;可能属于人工或其他系统,永远只报告。
|
||||
- 当前设备仍声明的 Path 不计入 finding。
|
||||
|
||||
不要因为名称相似把 `unowned` 手工改成 owned。先调查其创建者和用途;需要接管时应走独立、可审计的迁移任务。
|
||||
|
||||
## 执行受控处置
|
||||
|
||||
仅当报告显示 `safety_allowed=true`,并在 15 分钟有效期内由授权运维人员执行:
|
||||
|
||||
```powershell
|
||||
go -C Sense run ./cmd/sense-orphan -mode apply `
|
||||
-scan-id 'scan_...' `
|
||||
-actor 'operator-id' `
|
||||
-confirm 'DELETE scan_...'
|
||||
```
|
||||
|
||||
执行前程序会重新枚举 Path、重新读取归属并重新计算:候选必须仍是报告集合的子集、数量 1~128,且满足 `候选数 × 100 <= 当前 Path 总数 × 10`。过期、比例超限、归属变化、确认不匹配或租约冲突都会整批零删除;没有 `force`/bypass。删除逐项记入 PostgreSQL,成功项重复执行会跳过,失败项在快照仍有效且重新计算仍通过时可重试。
|
||||
|
||||
## 故障与恢复
|
||||
|
||||
- 扫描失败:不产生可执行快照;修复数据库/MediaMTX 连通性后重新 `report`。
|
||||
- `ratio_exceeded`:停止自动化,核对 MediaMTX 实例/分片是否选错、台账是否缺失或发生大面积配置漂移;不得拆小批次规避 10% 闸。
|
||||
- 部分删除失败:保留输出与数据库处置事实,修复 MediaMTX 后用同一 scan ID 重试;超过 15 分钟必须重新报告和审批。
|
||||
- 误删怀疑:立即停止 apply。数据库期望态仍是来源;当前设备声明的 Path 会由调和器重建。无当前设备声明的历史 Path 不自动恢复,应根据变更记录人工确认来源。
|
||||
- 回滚 Sense 二进制:先停用 `SENSE_ORPHAN_SCAN_ENABLED` 并停止所有新实例,再回滚;v5 表和已记录事实保留,不执行破坏性 down migration。
|
||||
|
||||
本流程只处置 MediaMTX 配置 Path,不删除设备、录像、证据对象或 Bell 数据。
|
||||
+11
-1
@@ -3,7 +3,7 @@ id: T-012
|
||||
title: 补齐 Sense 对账安全闸、孤儿受控处置与多实例可观测性
|
||||
phase: 2
|
||||
deps: [T-011]
|
||||
status: DOING
|
||||
status: DONE
|
||||
created: 2026-08-07
|
||||
issue: 43
|
||||
context_ref: 25357723a09218bae58626a9934309abcda34899
|
||||
@@ -88,6 +88,16 @@ T-003 已实现幂等调和与持久化退避,T-011 已完成 Control API、
|
||||
|
||||
## 执行记录
|
||||
|
||||
### 2026-08-07 完成多实例对账安全与孤儿受控处置
|
||||
|
||||
- PostgreSQL schema 提升到 v5:`010`/`011` 为 due row 增加 owner/token/expiry fencing,新增不含 source URI 的 MediaMTX Path 历史归属、孤儿扫描租约、15 分钟报告、发现项和处置审计,并补齐 `sense_app` 最小权限与 PUBLIC 拒绝;启动会拒绝旧 schema 或越权运行角色。
|
||||
- 调和器改为 `FOR UPDATE SKIP LOCKED` 批量领取、逐项续租、短于租期的外部调用 deadline 和带 token 的完成/失败;数据库时钟判定租约,旧 worker 不能覆盖接管者结果。成功启用在同一完成事务中刷新 Path 归属;SQLite 明确保留单进程开发语义。
|
||||
- MediaMTX 薄客户端新增只返回排序去重 Path 名称的有界分页枚举,包含重复页保护且不读取/返回 source;独立孤儿扫描把差异分为 `owned_stale` 与永不删除的 `unowned`。周期任务默认只报告,人工 `sense-orphan apply` 必须使用 15 分钟内 scan ID、合法 actor 和精确确认文本,并在删除前重取库存/归属。
|
||||
- 处置安全闸固定为 1~128 项且不超过当前 Path 的 10%,使用整数交叉相乘,无 force/bypass;目标只能从原快照收窄。逐项结果可审计和幂等重试,数据库外键/约束也拒绝为 `unowned` 写入删除记录。
|
||||
- 新增无第三方运行时依赖的 `/metrics`:只使用构建/实例及固定结果枚举标签,覆盖调和 run/item/lease-lost/时长/未收敛和孤儿扫描/阻断/删除汇总;tenant、Site、device、Path、URI 与错误正文均不进入标签。
|
||||
- `./scripts/test_postgres.ps1 -PgRoot D:\pgsql17` 通过:隔离 PostgreSQL 17.10 临时集群连续重放 `001`~`011`,SQL 权限/约束断言及 29 个 `TestPostgres*` 全绿,覆盖两个 store 并发唯一领取、租约接管/旧 token fencing、归属记录、扫描租约、未知 Path 数据库拒删和处置幂等;随机端口实例已停止并清理,现有 `D:\pgsql17\data`/5432 未被读取、停止或修改。
|
||||
- `./init.ps1` 通过;Python 治理/契约测试共 52 项通过,T-012 定向静态契约 12 项通过;全部 Go 测试、生成漂移、`go vet`、`go build` 与 `git diff --check` 通过;`go test -race ./internal/metrics ./internal/mtx ./internal/orphan ./internal/reconcile ./internal/store` 通过。不需要摄像头/GPU,本结果不解除 T-007,也不形成 16/128 路容量承诺。
|
||||
|
||||
### 2026-08-07 领取任务
|
||||
|
||||
- dispatcher `ila` 将 Issue #43 分配给 `codex`;`context_ref` 为 `25357723a09218bae58626a9934309abcda34899`,claim 为 `claims/T-012`,工作分支为 `agent/codex/T-012`。
|
||||
|
||||
@@ -40,7 +40,7 @@ $port = ([System.Net.IPEndPoint]$listener.LocalEndpoint).Port
|
||||
$listener.Stop()
|
||||
|
||||
$systemTemp = [IO.Path]::GetFullPath([IO.Path]::GetTempPath())
|
||||
$clusterName = 'yovision-t011-pg-' + [guid]::NewGuid().ToString('N')
|
||||
$clusterName = 'yovision-t012-pg-' + [guid]::NewGuid().ToString('N')
|
||||
$dataDir = Join-Path $systemTemp $clusterName
|
||||
$logPath = Join-Path $systemTemp ($clusterName + '.log')
|
||||
$started = $false
|
||||
@@ -69,9 +69,9 @@ try {
|
||||
$started = $true
|
||||
|
||||
$adminRootDSN = "postgres://postgres@127.0.0.1:$port/postgres?sslmode=disable"
|
||||
$databaseName = 'yovision_t011'
|
||||
$databaseName = 'yovision_t012'
|
||||
$adminDatabaseDSN = "postgres://postgres@127.0.0.1:$port/${databaseName}?sslmode=disable"
|
||||
$senseDSN = "postgres://yovision_t011_sense@127.0.0.1:$port/${databaseName}?sslmode=disable"
|
||||
$senseDSN = "postgres://yovision_t012_sense@127.0.0.1:$port/${databaseName}?sslmode=disable"
|
||||
|
||||
Invoke-Checked $psql '-X' '-v' 'ON_ERROR_STOP=1' '-d' $adminRootDSN '-f' (Join-Path $repoRoot 'deploy\postgres\001_roles.sql')
|
||||
Invoke-Checked $createdb '-h' '127.0.0.1' '-p' ([string]$port) '-U' 'postgres' $databaseName
|
||||
@@ -85,12 +85,14 @@ try {
|
||||
'006_device_operation_outbox.sql',
|
||||
'007_privileges_area_audit.sql',
|
||||
'008_control_api.sql',
|
||||
'009_privileges_control_api.sql'
|
||||
'009_privileges_control_api.sql',
|
||||
'010_reconcile_safety.sql',
|
||||
'011_privileges_reconcile_safety.sql'
|
||||
)) {
|
||||
Invoke-Checked $psql '-X' '-v' 'ON_ERROR_STOP=1' '-d' $adminDatabaseDSN '-f' (Join-Path $repoRoot "deploy\postgres\$name")
|
||||
}
|
||||
}
|
||||
Invoke-Checked $psql '-X' '-v' 'ON_ERROR_STOP=1' '-d' $adminRootDSN '-c' 'CREATE ROLE yovision_t011_sense LOGIN IN ROLE sense_app'
|
||||
Invoke-Checked $psql '-X' '-v' 'ON_ERROR_STOP=1' '-d' $adminRootDSN '-c' 'CREATE ROLE yovision_t012_sense LOGIN IN ROLE sense_app'
|
||||
Invoke-Checked $psql '-X' '-v' 'ON_ERROR_STOP=1' '-d' $adminDatabaseDSN '-f' (Join-Path $repoRoot 'deploy\postgres\tests\assertions.sql')
|
||||
|
||||
$env:YOVISION_TEST_POSTGRES_DSN = $senseDSN
|
||||
@@ -105,7 +107,7 @@ finally {
|
||||
if (-not $started -or $stopped) {
|
||||
$resolvedData = [IO.Path]::GetFullPath($dataDir)
|
||||
if (-not $resolvedData.StartsWith($systemTemp, [StringComparison]::OrdinalIgnoreCase) -or
|
||||
[IO.Path]::GetFileName($resolvedData) -notlike 'yovision-t011-pg-*') {
|
||||
[IO.Path]::GetFileName($resolvedData) -notlike 'yovision-t012-pg-*') {
|
||||
throw "Refusing to clean unexpected temporary path."
|
||||
}
|
||||
if (Test-Path -LiteralPath $resolvedData) {
|
||||
|
||||
@@ -56,6 +56,8 @@ class PostgresContractTests(unittest.TestCase):
|
||||
"007_privileges_area_audit.sql",
|
||||
"008_control_api.sql",
|
||||
"009_privileges_control_api.sql",
|
||||
"010_reconcile_safety.sql",
|
||||
"011_privileges_reconcile_safety.sql",
|
||||
],
|
||||
names,
|
||||
)
|
||||
@@ -109,7 +111,7 @@ class PostgresContractTests(unittest.TestCase):
|
||||
"initdb.exe",
|
||||
"pg_ctl.exe",
|
||||
"127.0.0.1",
|
||||
"yovision-t011-pg-",
|
||||
"yovision-t012-pg-",
|
||||
"YOVISION_TEST_POSTGRES_DSN",
|
||||
"Get-NetTCPConnection",
|
||||
):
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Static safety checks for the T-012 reconciliation/orphan boundary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pathlib
|
||||
import re
|
||||
import unittest
|
||||
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def text(relative: str) -> str:
|
||||
return (ROOT / relative).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def normalized(value: str) -> str:
|
||||
return re.sub(r"\s+", " ", value.lower())
|
||||
|
||||
|
||||
class SenseReconcileSafetyTests(unittest.TestCase):
|
||||
def test_postgres_v5_has_fencing_ownership_and_private_reports(self) -> None:
|
||||
schema = normalized(text("deploy/postgres/010_reconcile_safety.sql"))
|
||||
privileges = normalized(
|
||||
text("deploy/postgres/011_privileges_reconcile_safety.sql")
|
||||
)
|
||||
implementation = normalized(text("Sense/internal/store/postgres.go"))
|
||||
for table in (
|
||||
"media_path_ownership",
|
||||
"operational_leases",
|
||||
"orphan_scan_runs",
|
||||
"orphan_scan_findings",
|
||||
"orphan_cleanup_actions",
|
||||
):
|
||||
self.assertIn(f"sense.{table}", schema)
|
||||
self.assertIn(f"sense.{table}", privileges)
|
||||
self.assertIn("insert into sense.schema_migrations(version) values (5)", schema)
|
||||
self.assertIn("clock_timestamp()", implementation)
|
||||
self.assertIn("for update of r skip locked", implementation)
|
||||
self.assertIn("and lease_token = $5", implementation)
|
||||
self.assertNotIn("endpoint_ref", schema)
|
||||
self.assertNotIn("credential_ref", schema)
|
||||
self.assertNotIn("source_uri", schema)
|
||||
|
||||
def test_cleanup_gate_has_no_force_or_unowned_delete_path(self) -> None:
|
||||
source = text("Sense/internal/orphan/orphan.go")
|
||||
normalized_source = normalized(source)
|
||||
self.assertIn("candidates*100 > observed*10", normalized_source)
|
||||
self.assertIn("candidates > maxcleanupitems", normalized_source)
|
||||
self.assertIn('confirmation != "delete "+scanid', normalized_source)
|
||||
self.assertIn(
|
||||
"finding.classification != store.orphanownedstale", normalized_source
|
||||
)
|
||||
self.assertNotRegex(normalized_source, r"\bforce\b|bypass")
|
||||
migration = normalized(text("deploy/postgres/010_reconcile_safety.sql"))
|
||||
self.assertIn("classification = 'owned_stale'", migration)
|
||||
|
||||
def test_mediamtx_inventory_discards_sources_and_bounds_pagination(self) -> None:
|
||||
source = text("Sense/internal/mtx/client.go")
|
||||
self.assertIn("ListPathNames", source)
|
||||
self.assertIn("maxPages", source)
|
||||
self.assertIn("list paths repeated page", source)
|
||||
inventory = source[source.index("func (c *Client) ListPathNames") :]
|
||||
inventory = inventory[: inventory.index("func (c *Client) CreatePath")]
|
||||
self.assertNotIn("item.Source", inventory)
|
||||
|
||||
def test_metrics_are_fixed_and_control_contract_is_untouched(self) -> None:
|
||||
metrics = text("Sense/internal/metrics/metrics.go")
|
||||
main = text("Sense/cmd/sense-api/main.go")
|
||||
self.assertIn('mux.Handle("GET /metrics", registry.Handler())', main)
|
||||
for forbidden in ("tenant_id", "site_id", "device_id", "path_name"):
|
||||
self.assertNotIn(forbidden, metrics)
|
||||
control = text("docs/contracts/sense-control-v1.openapi.json")
|
||||
self.assertEqual(7, len(re.findall(r'"operationId"\s*:', control)))
|
||||
|
||||
def test_operator_command_requires_fresh_scan_confirmation(self) -> None:
|
||||
command = text("Sense/cmd/sense-orphan/main.go")
|
||||
manager = text("Sense/internal/orphan/orphan.go")
|
||||
self.assertIn("exact confirmation: DELETE <scan-id>", command)
|
||||
self.assertIn("scan.ExpiresAt.After(now)", manager)
|
||||
self.assertIn("cleanupCandidates(scan, paths, ownership)", manager)
|
||||
self.assertIn("RecordOrphanCleanup", manager)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user