feat(sense): add reconciliation safety controls [T-012]

2026-08-07 23:00:03 +08:00
parent 677ed732f7
commit 12857fdf32
35 changed files with 2817 additions and 124 deletions
@@ -287,12 +287,12 @@ func TestPostgresOpenRejectsOverprivilegedRuntimeRole(t *testing.T) {
 	_, admin := openPostgresTestStore(t)
 	ctx := context.Background()
 	if _, err := admin.ExecContext(ctx,
-		`GRANT UPDATE ON bell.site_quota_v1 TO yovision_t011_sense`); err != nil {
+		`GRANT UPDATE ON bell.site_quota_v1 TO yovision_t012_sense`); err != nil {
 		t.Fatal(err)
 	}
 	defer func() {
 		_, _ = admin.ExecContext(context.Background(),
-			`REVOKE UPDATE ON bell.site_quota_v1 FROM yovision_t011_sense`)
+			`REVOKE UPDATE ON bell.site_quota_v1 FROM yovision_t012_sense`)
 	}()
 	value, err := OpenPostgres(ctx, os.Getenv(postgresTestDSNEnv))
 	if value != nil {
@@ -304,6 +304,27 @@ func TestPostgresOpenRejectsOverprivilegedRuntimeRole(t *testing.T) {
 	}
 }

+func TestPostgresOpenRejectsPublicReconciliationStatePrivilege(t *testing.T) {
+	_, admin := openPostgresTestStore(t)
+	ctx := context.Background()
+	if _, err := admin.ExecContext(ctx,
+		`GRANT SELECT ON sense.orphan_scan_runs TO PUBLIC`); err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		_, _ = admin.ExecContext(context.Background(),
+			`REVOKE SELECT ON sense.orphan_scan_runs FROM PUBLIC`)
+	}()
+	value, err := OpenPostgres(ctx, os.Getenv(postgresTestDSNEnv))
+	if value != nil {
+		_ = value.Close()
+		t.Fatal("PUBLIC reconciliation state privilege was accepted")
+	}
+	if err == nil || !strings.Contains(err.Error(), "reconciliation safety privilege boundary") {
+		t.Fatalf("expected reconciliation privilege-boundary error, got %v", err)
+	}
+}
+
 func TestPostgresAreaPolicyAllowsNonImagingAndDeniesImagingCreate(t *testing.T) {
 	store, admin := openPostgresTestStore(t)
 	ctx := context.Background()
@@ -597,12 +618,12 @@ func TestPostgresOpenRejectsAreaSourcePrivilege(t *testing.T) {
 	_, admin := openPostgresTestStore(t)
 	ctx := context.Background()
 	if _, err := admin.ExecContext(ctx,
-		`GRANT SELECT ON bell.areas TO yovision_t011_sense`); err != nil {
+		`GRANT SELECT ON bell.areas TO yovision_t012_sense`); err != nil {
 		t.Fatal(err)
 	}
 	defer func() {
 		_, _ = admin.ExecContext(context.Background(),
-			`REVOKE SELECT ON bell.areas FROM yovision_t011_sense`)
+			`REVOKE SELECT ON bell.areas FROM yovision_t012_sense`)
 	}()
 	value, err := OpenPostgres(ctx, os.Getenv(postgresTestDSNEnv))
 	if value != nil {
@@ -969,6 +990,180 @@ func TestPostgresConcurrentControlBatchesUseStableDeviceLockOrder(t *testing.T)
 	}
 }

+func TestPostgresConcurrentReconcileClaimHasOneWinner(t *testing.T) {
+	first, admin := openPostgresTestStore(t)
+	insertBellSite(t, admin, "tenant", "site", 2)
+	if err := first.CreateDevice(context.Background(), videoDevice(1, "tenant", "site")); err != nil {
+		t.Fatal(err)
+	}
+	second, err := OpenPostgres(context.Background(), os.Getenv(postgresTestDSNEnv))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = second.Close() })
+	now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC)
+	start := make(chan struct{})
+	counts := make(chan int, 2)
+	errorsFound := make(chan error, 2)
+	var wait sync.WaitGroup
+	for index, repository := range []*Postgres{first, second} {
+		wait.Add(1)
+		go func(index int, repository *Postgres) {
+			defer wait.Done()
+			<-start
+			values, err := repository.ClaimDueReconcile(context.Background(), ReconcileClaim{
+				Owner: fmt.Sprintf("ins-%d", index), Token: fmt.Sprintf("token-%d", index),
+				Now: now, LeaseDuration: 30 * time.Second, Limit: 1,
+			})
+			if err != nil {
+				errorsFound <- err
+				return
+			}
+			counts <- len(values)
+		}(index, repository)
+	}
+	close(start)
+	wait.Wait()
+	close(counts)
+	close(errorsFound)
+	for err := range errorsFound {
+		t.Fatal(err)
+	}
+	total, winners := 0, 0
+	for count := range counts {
+		total += count
+		if count == 1 {
+			winners++
+		}
+	}
+	if total != 1 || winners != 1 {
+		t.Fatalf("due row was not exclusively claimed: total=%d winners=%d", total, winners)
+	}
+}
+
+func TestPostgresExpiredReconcileLeaseFencesOldWorkerAndRecordsOwnership(t *testing.T) {
+	postgres, admin := openPostgresTestStore(t)
+	insertBellSite(t, admin, "tenant", "site", 2)
+	value := videoDevice(1, "tenant", "site")
+	if err := postgres.CreateDevice(context.Background(), value); err != nil {
+		t.Fatal(err)
+	}
+	now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC)
+	first, err := postgres.ClaimDueReconcile(context.Background(), ReconcileClaim{
+		Owner: "ins-a", Token: "token-a", Now: now, LeaseDuration: 30 * time.Second, Limit: 1,
+	})
+	if err != nil || len(first) != 1 {
+		t.Fatalf("first claim failed: %+v %v", first, err)
+	}
+	early, err := postgres.ClaimDueReconcile(context.Background(), ReconcileClaim{
+		Owner: "ins-b", Token: "token-b", Now: now.Add(10 * time.Second),
+		LeaseDuration: 30 * time.Second, Limit: 1,
+	})
+	if err != nil || len(early) != 0 {
+		t.Fatalf("live lease was stolen: %+v %v", early, err)
+	}
+	if _, err := admin.Exec(`UPDATE sense.reconcile_state
+		SET lease_until = clock_timestamp() - interval '1 second'
+		WHERE device_id = $1`, value.ID); err != nil {
+		t.Fatal(err)
+	}
+	second, err := postgres.ClaimDueReconcile(context.Background(), ReconcileClaim{
+		Owner: "ins-b", Token: "token-b", Now: now.Add(31 * time.Second),
+		LeaseDuration: 30 * time.Second, Limit: 1,
+	})
+	if err != nil || len(second) != 1 {
+		t.Fatalf("expired lease was not recoverable: %+v %v", second, err)
+	}
+	if err := postgres.CompleteReconcile(
+		context.Background(), value.ID, value.Generation, "ins-a", "token-a", now.Add(32*time.Second),
+	); !errors.Is(err, ErrReconcileLeaseLost) {
+		t.Fatalf("old worker was not fenced: %v", err)
+	}
+	if err := postgres.CompleteReconcile(
+		context.Background(), value.ID, value.Generation, "ins-b", "token-b", now.Add(32*time.Second),
+	); err != nil {
+		t.Fatal(err)
+	}
+	var ownershipDevice string
+	if err := admin.QueryRow(`SELECT device_id FROM sense.media_path_ownership WHERE path_name = $1`,
+		value.PathName).Scan(&ownershipDevice); err != nil {
+		t.Fatal(err)
+	}
+	if ownershipDevice != value.ID {
+		t.Fatalf("wrong ownership was recorded: %q", ownershipDevice)
+	}
+	var leaseToken sql.NullString
+	if err := admin.QueryRow(`SELECT lease_token FROM sense.reconcile_state WHERE device_id = $1`,
+		value.ID).Scan(&leaseToken); err != nil {
+		t.Fatal(err)
+	}
+	if leaseToken.Valid {
+		t.Fatal("completion did not release the reconcile lease")
+	}
+}
+
+func TestPostgresOrphanReportLeaseAndCleanupAuditAreFencedAndIdempotent(t *testing.T) {
+	postgres, _ := openPostgresTestStore(t)
+	ctx := context.Background()
+	now := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC)
+	acquired, err := postgres.AcquireOperationalLease(
+		ctx, OperationalLeaseOrphanScan, "ins-a", "scan-token-a", now, 30*time.Second,
+	)
+	if err != nil || !acquired {
+		t.Fatalf("scan lease failed: %v %v", acquired, err)
+	}
+	scan := OrphanScan{
+		ID: "scan_" + strings.Repeat("0", 26), InstanceID: "ins-a",
+		ObservedCount: 10, OwnedStaleCount: 1, UnownedCount: 1,
+		SafetyAllowed: true, SafetyReason: "allowed",
+		CompletedAt: now.Add(time.Second), ExpiresAt: now.Add(15 * time.Minute),
+		Findings: []OrphanFinding{
+			{PathName: "stale", Classification: OrphanOwnedStale, DeviceID: "old-device"},
+			{PathName: "unknown", Classification: OrphanUnowned},
+		},
+	}
+	if err := postgres.SaveOrphanScan(ctx, scan, "ins-a", "scan-token-a"); err != nil {
+		t.Fatal(err)
+	}
+	loaded, err := postgres.GetOrphanScan(ctx, scan.ID)
+	if err != nil || len(loaded.Findings) != 2 || !loaded.SafetyAllowed {
+		t.Fatalf("stored scan mismatch: %+v %v", loaded, err)
+	}
+	if err := postgres.RecordOrphanCleanup(
+		ctx, scan.ID, "stale", "operator", "deleted", "", now.Add(2*time.Second),
+	); err != nil {
+		t.Fatal(err)
+	}
+	if err := postgres.RecordOrphanCleanup(
+		ctx, scan.ID, "unknown", "operator", "deleted", "", now.Add(2*time.Second),
+	); err == nil {
+		t.Fatal("unowned path accepted a cleanup audit record")
+	}
+	if err := postgres.RecordOrphanCleanup(
+		ctx, scan.ID, "stale", "operator-2", "failed", "media_error", now.Add(3*time.Second),
+	); err != nil {
+		t.Fatal(err)
+	}
+	loaded, err = postgres.GetOrphanScan(ctx, scan.ID)
+	if err != nil || !loaded.Findings[0].Deleted {
+		t.Fatalf("successful cleanup was downgraded: %+v %v", loaded, err)
+	}
+
+	acquired, err = postgres.AcquireOperationalLease(
+		ctx, OperationalLeaseOrphanScan, "ins-b", "scan-token-b", now.Add(31*time.Second), 30*time.Second,
+	)
+	if err != nil || !acquired {
+		t.Fatalf("expired scan lease was not recoverable: %v %v", acquired, err)
+	}
+	staleScan := scan
+	staleScan.ID = "scan_" + strings.Repeat("1", 26)
+	staleScan.CompletedAt = now.Add(32 * time.Second)
+	staleScan.ExpiresAt = staleScan.CompletedAt.Add(15 * time.Minute)
+	if err := postgres.SaveOrphanScan(ctx, staleScan, "ins-a", "scan-token-a"); !errors.Is(err, ErrOperationalLeaseLost) {
+		t.Fatalf("stale scan worker was not fenced: %v", err)
+	}
+}
+
 func openPostgresTestStore(t *testing.T) (*Postgres, *sql.DB) {
 	t.Helper()
 	dsn := os.Getenv(postgresTestDSNEnv)
@@ -985,6 +1180,11 @@ func openPostgresTestStore(t *testing.T) (*Postgres, *sql.DB) {
 		t.Fatal("connect PostgreSQL test administrator")
 	}
 	if _, err := admin.ExecContext(context.Background(), `TRUNCATE
+		sense.orphan_cleanup_actions,
+		sense.orphan_scan_findings,
+		sense.orphan_scan_runs,
+		sense.operational_leases,
+		sense.media_path_ownership,
 		sense.control_idempotency_receipts,
 		sense.batch_operation_items,
 		sense.batch_operations,