feat(sense): add reconciliation safety controls [T-012]

2026-08-07 23:00:03 +08:00
parent 677ed732f7
commit 12857fdf32
35 changed files with 2817 additions and 124 deletions
@@ -53,8 +53,8 @@ func (s *Postgres) Close() error {
 func (s *Postgres) verifySchemaAndPrivileges(ctx context.Context) error {
 	var version sql.NullInt64
 	if err := s.db.QueryRowContext(ctx,
-		`SELECT MAX(version) FROM sense.schema_migrations`).Scan(&version); err != nil || !version.Valid || version.Int64 < 4 {
-		return errors.New("postgres sense schema migration v4 is required")
+		`SELECT MAX(version) FROM sense.schema_migrations`).Scan(&version); err != nil || !version.Valid || version.Int64 < 5 {
+		return errors.New("postgres sense schema migration v5 is required")
 	}
 	var canReadQuotaView, canWriteQuotaView, canReadSiteSource, canWriteSiteSource bool
 	var canReadAreaView, canWriteAreaView, canReadAreaSource, canWriteAreaSource bool
@@ -95,6 +95,28 @@ func (s *Postgres) verifySchemaAndPrivileges(ctx context.Context) error {
 		publicReceipts || publicOperations || publicOperationItems {
 		return errors.New("postgres role violates Control API state privilege boundary")
 	}
+	var canUseOwnership, canUseLeases, canUseScans, canUseFindings, canUseActions bool
+	var publicOwnership, publicLeases, publicScans, publicFindings, publicActions bool
+	if err := s.db.QueryRowContext(ctx, `SELECT
+		has_table_privilege(current_user, 'sense.media_path_ownership', 'SELECT,INSERT,UPDATE,DELETE'),
+		has_table_privilege(current_user, 'sense.operational_leases', 'SELECT,INSERT,UPDATE,DELETE'),
+		has_table_privilege(current_user, 'sense.orphan_scan_runs', 'SELECT,INSERT,UPDATE,DELETE'),
+		has_table_privilege(current_user, 'sense.orphan_scan_findings', 'SELECT,INSERT,UPDATE,DELETE'),
+		has_table_privilege(current_user, 'sense.orphan_cleanup_actions', 'SELECT,INSERT,UPDATE,DELETE'),
+		has_table_privilege('public', 'sense.media_path_ownership', 'SELECT,INSERT,UPDATE,DELETE'),
+		has_table_privilege('public', 'sense.operational_leases', 'SELECT,INSERT,UPDATE,DELETE'),
+		has_table_privilege('public', 'sense.orphan_scan_runs', 'SELECT,INSERT,UPDATE,DELETE'),
+		has_table_privilege('public', 'sense.orphan_scan_findings', 'SELECT,INSERT,UPDATE,DELETE'),
+		has_table_privilege('public', 'sense.orphan_cleanup_actions', 'SELECT,INSERT,UPDATE,DELETE')`).Scan(
+		&canUseOwnership, &canUseLeases, &canUseScans, &canUseFindings, &canUseActions,
+		&publicOwnership, &publicLeases, &publicScans, &publicFindings, &publicActions,
+	); err != nil {
+		return errors.New("verify postgres reconciliation safety privileges")
+	}
+	if !canUseOwnership || !canUseLeases || !canUseScans || !canUseFindings || !canUseActions ||
+		publicOwnership || publicLeases || publicScans || publicFindings || publicActions {
+		return errors.New("postgres role violates reconciliation safety privilege boundary")
+	}
 	return nil
 }

@@ -391,7 +413,8 @@ func (s *Postgres) SetDesiredState(ctx context.Context, id string, desired devic
 		return errors.New("update postgres desired state")
 	}
 	if _, err := tx.ExecContext(ctx, `UPDATE sense.reconcile_state SET
-		failure_count = 0, next_attempt_at = NULL, last_error_code = NULL, updated_at = $1
+		failure_count = 0, next_attempt_at = NULL, last_error_code = NULL,
+		lease_owner = NULL, lease_token = NULL, lease_until = NULL, updated_at = $1
 		WHERE device_id = $2`, now, id); err != nil {
 		return errors.New("reset postgres reconcile state")
 	}
@@ -486,6 +509,217 @@ func (s *Postgres) ListDueReconcile(ctx context.Context, now time.Time, limit in
 	return values, nil
 }

+func (s *Postgres) ClaimDueReconcile(
+	ctx context.Context,
+	claim ReconcileClaim,
+) ([]ReconcileCandidate, error) {
+	if claim.Limit <= 0 {
+		return nil, nil
+	}
+	if strings.TrimSpace(claim.Owner) == "" || strings.TrimSpace(claim.Token) == "" ||
+		claim.LeaseDuration <= 0 {
+		return nil, errors.New("invalid postgres reconcile claim")
+	}
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return nil, errors.New("begin postgres reconcile claim")
+	}
+	defer tx.Rollback()
+	rows, err := tx.QueryContext(ctx, `SELECT `+postgresDeviceColumns+`, r.failure_count, r.next_attempt_at
+		FROM sense.devices d
+		JOIN sense.reconcile_state r ON r.device_id = d.id
+		WHERE EXISTS (SELECT 1 FROM sense.device_capabilities c
+				WHERE c.device_id = d.id AND c.capability = 'video_capture')
+			AND (r.observed_generation < d.generation
+				OR (d.desired_state = 'enabled' AND r.failure_count > 0))
+			AND (r.next_attempt_at IS NULL OR r.next_attempt_at <= clock_timestamp())
+			AND (r.lease_until IS NULL OR r.lease_until <= clock_timestamp())
+		ORDER BY d.updated_at, d.id
+		FOR UPDATE OF r SKIP LOCKED
+		LIMIT $1`, claim.Limit)
+	if err != nil {
+		return nil, errors.New("select postgres reconcile claims")
+	}
+	values := make([]ReconcileCandidate, 0)
+	for rows.Next() {
+		var candidate ReconcileCandidate
+		var areaID sql.NullString
+		var quotaVersion, areaVersion sql.NullInt64
+		var nextAttempt sql.NullTime
+		if err := rows.Scan(
+			&candidate.Device.ID, &candidate.Device.TenantID, &candidate.Device.SiteID,
+			&areaID, &candidate.Device.SerialNumber, &candidate.Device.Name, &candidate.Device.Modality,
+			&candidate.Device.DesiredState, &candidate.Device.ActualState,
+			&candidate.Device.EndpointRef, &candidate.Device.CredentialRef,
+			&candidate.Device.ProfileToken, &candidate.Device.PathName,
+			&candidate.Device.Generation, &candidate.Device.ResourceVersion,
+			&quotaVersion, &areaVersion,
+			&candidate.Device.CreatedAt, &candidate.Device.UpdatedAt,
+			&candidate.FailureCount, &nextAttempt,
+		); err != nil {
+			rows.Close()
+			return nil, errors.New("scan postgres reconcile claim")
+		}
+		candidate.Device.AreaID = areaID.String
+		candidate.Device.QuotaSourceVersion = quotaVersion.Int64
+		candidate.Device.AreaPolicySourceVersion = areaVersion.Int64
+		if nextAttempt.Valid {
+			value := nextAttempt.Time
+			candidate.NextAttempt = &value
+		}
+		values = append(values, candidate)
+	}
+	if err := rows.Close(); err != nil {
+		return nil, errors.New("close postgres reconcile claims")
+	}
+	if err := rows.Err(); err != nil {
+		return nil, errors.New("iterate postgres reconcile claims")
+	}
+	for _, candidate := range values {
+		if _, err := tx.ExecContext(ctx, `UPDATE sense.reconcile_state SET
+			lease_owner = $1, lease_token = $2,
+			lease_until = clock_timestamp() + ($3 * interval '1 second'),
+			updated_at = clock_timestamp()
+			WHERE device_id = $4`,
+			claim.Owner, claim.Token, claim.LeaseDuration.Seconds(), candidate.Device.ID,
+		); err != nil {
+			return nil, errors.New("persist postgres reconcile claim")
+		}
+	}
+	if err := tx.Commit(); err != nil {
+		return nil, errors.New("commit postgres reconcile claim")
+	}
+	for index := range values {
+		values[index].Device.Capabilities, err = s.capabilities(ctx, values[index].Device.ID)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return values, nil
+}
+
+func (s *Postgres) RenewReconcileLease(
+	ctx context.Context,
+	id, owner, token string,
+	now time.Time,
+	duration time.Duration,
+) (bool, error) {
+	if strings.TrimSpace(owner) == "" || strings.TrimSpace(token) == "" || duration <= 0 {
+		return false, errors.New("invalid postgres reconcile lease renewal")
+	}
+	result, err := s.db.ExecContext(ctx, `UPDATE sense.reconcile_state SET
+		lease_until = clock_timestamp() + ($1 * interval '1 second'),
+		updated_at = clock_timestamp()
+		WHERE device_id = $2 AND lease_owner = $3 AND lease_token = $4
+			AND lease_until > clock_timestamp()`, duration.Seconds(), id, owner, token)
+	if err != nil {
+		return false, errors.New("renew postgres reconcile lease")
+	}
+	affected, err := result.RowsAffected()
+	if err != nil {
+		return false, errors.New("read postgres reconcile lease renewal")
+	}
+	return affected == 1, nil
+}
+
+func (s *Postgres) CompleteReconcile(
+	ctx context.Context,
+	id string,
+	generation int64,
+	owner, token string,
+	now time.Time,
+) error {
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return errors.New("begin postgres fenced reconciliation completion")
+	}
+	defer tx.Rollback()
+	result, err := tx.ExecContext(ctx, `UPDATE sense.reconcile_state SET
+		failure_count = 0, next_attempt_at = NULL, last_error_code = NULL,
+		observed_generation = $1, lease_owner = NULL, lease_token = NULL,
+		lease_until = NULL, updated_at = $2
+		WHERE device_id = $3 AND lease_owner = $4 AND lease_token = $5
+			AND lease_until > clock_timestamp()`, generation, now, id, owner, token)
+	if err != nil {
+		return errors.New("complete postgres fenced reconciliation")
+	}
+	affected, err := result.RowsAffected()
+	if err != nil {
+		return errors.New("read postgres fenced reconciliation completion")
+	}
+	if affected != 1 {
+		return ErrReconcileLeaseLost
+	}
+	if _, err := tx.ExecContext(ctx, `INSERT INTO sense.media_path_ownership(
+		path_name, device_id, tenant_id, site_id, first_claimed_at, last_confirmed_at
+	)
+	SELECT path_name, id, tenant_id, site_id, $1, $1
+	FROM sense.devices
+	WHERE id = $2 AND desired_state = 'enabled' AND btrim(path_name) <> ''
+	ON CONFLICT (path_name) DO UPDATE SET
+		device_id = EXCLUDED.device_id,
+		tenant_id = EXCLUDED.tenant_id,
+		site_id = EXCLUDED.site_id,
+		last_confirmed_at = EXCLUDED.last_confirmed_at`, now, id); err != nil {
+		return errors.New("record postgres MediaMTX path ownership")
+	}
+	result, err = tx.ExecContext(ctx, `UPDATE sense.devices
+		SET actual_state = CASE WHEN desired_state = 'disabled' THEN 'offline' ELSE 'pending' END,
+		updated_at = $1 WHERE id = $2`, now, id)
+	if err != nil {
+		return errors.New("mark postgres fenced device state")
+	}
+	if affected, _ := result.RowsAffected(); affected != 1 {
+		return ErrNotFound
+	}
+	if err := tx.Commit(); err != nil {
+		return errors.New("commit postgres fenced reconciliation completion")
+	}
+	return nil
+}
+
+func (s *Postgres) FailReconcile(
+	ctx context.Context,
+	id string,
+	failureCount int,
+	nextAttempt time.Time,
+	errorCode, owner, token string,
+	now time.Time,
+) error {
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return errors.New("begin postgres fenced reconciliation failure")
+	}
+	defer tx.Rollback()
+	result, err := tx.ExecContext(ctx, `UPDATE sense.reconcile_state SET
+		failure_count = $1, next_attempt_at = $2, last_error_code = $3,
+		lease_owner = NULL, lease_token = NULL, lease_until = NULL, updated_at = $4
+		WHERE device_id = $5 AND lease_owner = $6 AND lease_token = $7
+			AND lease_until > clock_timestamp()`, failureCount, nextAttempt, errorCode, now, id, owner, token)
+	if err != nil {
+		return errors.New("persist postgres fenced reconciliation failure")
+	}
+	affected, err := result.RowsAffected()
+	if err != nil {
+		return errors.New("read postgres fenced reconciliation failure")
+	}
+	if affected != 1 {
+		return ErrReconcileLeaseLost
+	}
+	result, err = tx.ExecContext(ctx, `UPDATE sense.devices
+		SET actual_state = 'failed', updated_at = $1 WHERE id = $2`, now, id)
+	if err != nil {
+		return errors.New("mark postgres fenced failed device")
+	}
+	if affected, _ := result.RowsAffected(); affected != 1 {
+		return ErrNotFound
+	}
+	if err := tx.Commit(); err != nil {
+		return errors.New("commit postgres fenced reconciliation failure")
+	}
+	return nil
+}
+
 func (s *Postgres) ListEnabledVideoDevices(ctx context.Context, limit int) ([]device.Device, error) {
 	if limit <= 0 {
 		return nil, nil