feat(bell): add alert acknowledgement vertical slice

2026-08-11 17:01:53 +08:00
parent 477afa6ba2
commit 8208118904
29 changed files with 1820 additions and 56 deletions
@@ -0,0 +1,355 @@
+package store
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/oklog/ulid/v2"
+
+	"yovision/bell/internal/alert"
+)
+
+func (p *Postgres) AlertReady(ctx context.Context) error {
+	var version int64
+	if err := p.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(version),0) FROM bell.schema_migrations`).Scan(&version); err != nil || version < 6 {
+		return errors.New("postgres Bell schema migration v6 is required for alerts")
+	}
+	for _, table := range []string{"rule_versions", "event_rule_sweeps", "rule_evaluations", "alerts", "alert_events", "alert_transitions", "alert_command_receipts"} {
+		var selectAllowed, insertAllowed, updateAllowed, deleteAllowed, truncateAllowed bool
+		if err := p.db.QueryRowContext(ctx, `SELECT
+			has_table_privilege(current_user,$1,'SELECT'), has_table_privilege(current_user,$1,'INSERT'),
+			has_table_privilege(current_user,$1,'UPDATE'), has_table_privilege(current_user,$1,'DELETE'),
+			has_table_privilege(current_user,$1,'TRUNCATE')`, "bell."+table).Scan(
+			&selectAllowed, &insertAllowed, &updateAllowed, &deleteAllowed, &truncateAllowed,
+		); err != nil || !selectAllowed || !insertAllowed || updateAllowed || deleteAllowed || truncateAllowed {
+			return fmt.Errorf("Bell runtime alert privileges violate append-only boundary for %s", table)
+		}
+	}
+	return nil
+}
+
+func (p *Postgres) PublishRule(ctx context.Context, rule alert.RuleSpec) (bool, error) {
+	if err := rule.Validate(); err != nil {
+		return false, err
+	}
+	digest, err := rule.Digest()
+	if err != nil {
+		return false, err
+	}
+	tx, err := p.db.BeginTx(ctx, nil)
+	if err != nil {
+		return false, errors.New("begin Bell rule publish")
+	}
+	defer tx.Rollback()
+	if _, err := tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, fmt.Sprintf("rule:%d:%s", rule.TenantID, rule.RuleKey)); err != nil {
+		return false, errors.New("lock Bell rule key")
+	}
+	var existing string
+	err = tx.QueryRowContext(ctx, `SELECT id FROM bell.rule_versions WHERE tenant_id=$1 AND rule_key=$2 AND config_hash=$3`, rule.TenantID, rule.RuleKey, digest[:]).Scan(&existing)
+	if err == nil {
+		return false, tx.Commit()
+	}
+	if !errors.Is(err, sql.ErrNoRows) {
+		return false, errors.New("read Bell rule version")
+	}
+	var version int
+	if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(version),0)+1 FROM bell.rule_versions WHERE tenant_id=$1 AND rule_key=$2`, rule.TenantID, rule.RuleKey).Scan(&version); err != nil {
+		return false, errors.New("allocate Bell rule version")
+	}
+	if _, err := tx.ExecContext(ctx, `INSERT INTO bell.rule_versions(
+		id,tenant_id,site_id,rule_key,version,display_name,event_kind,minimum_severity,enabled,effective_from,config_hash
+	) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)`, newPrefixedID("ruv_"), rule.TenantID, rule.SiteID,
+		rule.RuleKey, version, rule.DisplayName, rule.EventKind, rule.MinimumSeverity, rule.Enabled, rule.EffectiveFrom, digest[:]); err != nil {
+		return false, fmt.Errorf("insert Bell rule version: %w", err)
+	}
+	if err := tx.Commit(); err != nil {
+		return false, errors.New("commit Bell rule version")
+	}
+	return true, nil
+}
+
+func (p *Postgres) EvaluateNext(ctx context.Context) (bool, error) {
+	tx, err := p.db.BeginTx(ctx, nil)
+	if err != nil {
+		return false, errors.New("begin Bell rule evaluation")
+	}
+	defer tx.Rollback()
+	var eventID, kind, severity string
+	var tenantID, siteID, deviceID int64
+	var occurredAt time.Time
+	err = tx.QueryRowContext(ctx, `SELECT e.id,e.tenant_id,e.site_id,e.device_id,e.kind,e.severity,e.occurred_at
+		FROM bell.events e WHERE NOT EXISTS(SELECT 1 FROM bell.event_rule_sweeps s WHERE s.event_id=e.id)
+		ORDER BY e.created_at,e.id LIMIT 1`).Scan(&eventID, &tenantID, &siteID, &deviceID, &kind, &severity, &occurredAt)
+	if errors.Is(err, sql.ErrNoRows) {
+		return false, nil
+	}
+	if err != nil {
+		return false, errors.New("select Bell event for rules")
+	}
+	if _, err := tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, "event-rule:"+eventID); err != nil {
+		return false, errors.New("lock Bell event evaluation")
+	}
+	result, err := tx.ExecContext(ctx, `INSERT INTO bell.event_rule_sweeps(event_id) VALUES($1) ON CONFLICT DO NOTHING`, eventID)
+	if err != nil {
+		return false, errors.New("claim Bell event evaluation")
+	}
+	rowsAffected, _ := result.RowsAffected()
+	if rowsAffected == 0 {
+		return false, tx.Commit()
+	}
+	rows, err := tx.QueryContext(ctx, `SELECT id,rule_key,version,display_name,event_kind,minimum_severity,enabled
+		FROM (SELECT DISTINCT ON (rule_key) id,rule_key,version,display_name,event_kind,minimum_severity,enabled,effective_from
+		      FROM bell.rule_versions WHERE tenant_id=$1 AND effective_from <= $3 AND (site_id IS NULL OR site_id=$2)
+		      ORDER BY rule_key,version DESC,effective_from DESC) latest ORDER BY rule_key`, tenantID, siteID, occurredAt)
+	if err != nil {
+		return false, errors.New("read effective Bell rules")
+	}
+	defer rows.Close()
+	type effectiveRule struct {
+		id, key, name, kind, minimum string
+		version                      int
+		enabled                      bool
+	}
+	rules := make([]effectiveRule, 0)
+	for rows.Next() {
+		var rule effectiveRule
+		if err := rows.Scan(&rule.id, &rule.key, &rule.version, &rule.name, &rule.kind, &rule.minimum, &rule.enabled); err != nil {
+			return false, errors.New("scan effective Bell rule")
+		}
+		rules = append(rules, rule)
+	}
+	if err := rows.Err(); err != nil {
+		return false, errors.New("iterate effective Bell rules")
+	}
+	if err := rows.Close(); err != nil {
+		return false, errors.New("close effective Bell rules")
+	}
+	eventRank, _ := alert.SeverityRank(severity)
+	for _, rule := range rules {
+		matched, reason := true, "matched"
+		minimumRank, _ := alert.SeverityRank(rule.minimum)
+		switch {
+		case !rule.enabled:
+			matched, reason = false, "disabled"
+		case rule.kind != kind:
+			matched, reason = false, "event_kind"
+		case eventRank < minimumRank:
+			matched, reason = false, "severity"
+		}
+		value := "no_match"
+		if matched {
+			value = "matched"
+		}
+		evaluationID := newPrefixedID("eva_")
+		if _, err := tx.ExecContext(ctx, `INSERT INTO bell.rule_evaluations(id,event_id,rule_version_id,result,reason) VALUES($1,$2,$3,$4,$5)`, evaluationID, eventID, rule.id, value, reason); err != nil {
+			return false, fmt.Errorf("append Bell rule evaluation: %w", err)
+		}
+		if !matched {
+			continue
+		}
+		alertID := newPrefixedID("alt_")
+		if _, err := tx.ExecContext(ctx, `INSERT INTO bell.alerts(id,tenant_id,site_id,rule_evaluation_id,rule_version_id,severity,title) VALUES($1,$2,$3,$4,$5,$6,$7)`, alertID, tenantID, siteID, evaluationID, rule.id, severity, rule.name); err != nil {
+			return false, errors.New("create Bell alert")
+		}
+		if _, err := tx.ExecContext(ctx, `INSERT INTO bell.alert_events(alert_id,event_id) VALUES($1,$2)`, alertID, eventID); err != nil {
+			return false, errors.New("link Bell alert event")
+		}
+		if _, err := tx.ExecContext(ctx, `INSERT INTO bell.alert_transitions(id,alert_id,sequence,from_state,to_state,actor_ref) VALUES($1,$2,1,NULL,'open','system:rule-worker')`, newPrefixedID("trn_"), alertID); err != nil {
+			return false, errors.New("open Bell alert")
+		}
+		_ = deviceID
+	}
+	if err := tx.Commit(); err != nil {
+		return false, errors.New("commit Bell rule evaluation")
+	}
+	return true, nil
+}
+
+func (p *Postgres) ListAlerts(ctx context.Context, tenantID, siteID int64, state string, limit int, cursor string) (alert.Page, error) {
+	if limit < 1 || limit > alert.MaxPageSize {
+		return alert.Page{}, errors.New("invalid alert page size")
+	}
+	if state != "" && state != "open" && state != "acknowledged" && state != "closed" {
+		return alert.Page{}, errors.New("invalid alert state")
+	}
+	rows, err := p.db.QueryContext(ctx, `SELECT a.id,a.severity,a.title,t.to_state,r.rule_key,r.version,a.created_at
+		FROM bell.alerts a JOIN bell.rule_versions r ON r.id=a.rule_version_id
+		JOIN LATERAL(SELECT to_state FROM bell.alert_transitions WHERE alert_id=a.id ORDER BY sequence DESC LIMIT 1)t ON true
+		WHERE a.tenant_id=$1 AND a.site_id=$2 AND ($3='' OR t.to_state=$3)
+		  AND ($4='' OR (a.created_at,a.id) < (SELECT c.created_at,c.id FROM bell.alerts c WHERE c.id=$4 AND c.tenant_id=$1 AND c.site_id=$2))
+		ORDER BY a.created_at DESC,a.id DESC LIMIT $5`, tenantID, siteID, state, cursor, limit+1)
+	if err != nil {
+		return alert.Page{}, errors.New("list Bell alerts")
+	}
+	defer rows.Close()
+	page := alert.Page{Items: make([]alert.Summary, 0, limit)}
+	for rows.Next() {
+		var item alert.Summary
+		if err := rows.Scan(&item.ID, &item.Severity, &item.Title, &item.State, &item.RuleKey, &item.RuleVersion, &item.CreatedAt); err != nil {
+			return alert.Page{}, errors.New("scan Bell alert list")
+		}
+		page.Items = append(page.Items, item)
+	}
+	if err := rows.Err(); err != nil {
+		return alert.Page{}, errors.New("iterate Bell alert list")
+	}
+	if len(page.Items) > limit {
+		cursor := page.Items[limit-1].ID
+		page.NextCursor = &cursor
+		page.Items = page.Items[:limit]
+	}
+	return page, nil
+}
+
+func (p *Postgres) GetAlert(ctx context.Context, tenantID, siteID int64, alertID string) (alert.Detail, error) {
+	var detail alert.Detail
+	err := p.db.QueryRowContext(ctx, `SELECT a.id,a.severity,a.title,t.to_state,r.rule_key,r.version,a.created_at
+		FROM bell.alerts a JOIN bell.rule_versions r ON r.id=a.rule_version_id
+		JOIN LATERAL(SELECT to_state FROM bell.alert_transitions WHERE alert_id=a.id ORDER BY sequence DESC LIMIT 1)t ON true
+		WHERE a.tenant_id=$1 AND a.site_id=$2 AND a.id=$3`, tenantID, siteID, alertID).Scan(
+		&detail.ID, &detail.Severity, &detail.Title, &detail.State, &detail.RuleKey, &detail.RuleVersion, &detail.CreatedAt)
+	if errors.Is(err, sql.ErrNoRows) {
+		return alert.Detail{}, alert.ErrNotFound
+	}
+	if err != nil {
+		return alert.Detail{}, errors.New("read Bell alert")
+	}
+	detail.EvidenceStatus = "not_enabled"
+	detail.DeliveryStatus = "not_enabled"
+	detail.Events = []alert.EventRef{}
+	detail.Transitions = []alert.Transition{}
+	rows, err := p.db.QueryContext(ctx, `SELECT e.id,e.device_id,e.kind,e.severity,e.occurred_at FROM bell.alert_events ae JOIN bell.events e ON e.id=ae.event_id WHERE ae.alert_id=$1 ORDER BY e.occurred_at,e.id`, alertID)
+	if err != nil {
+		return alert.Detail{}, errors.New("read Bell alert events")
+	}
+	for rows.Next() {
+		var value alert.EventRef
+		if err := rows.Scan(&value.ID, &value.DeviceID, &value.Kind, &value.Severity, &value.OccurredAt); err != nil {
+			rows.Close()
+			return alert.Detail{}, errors.New("scan Bell alert event")
+		}
+		detail.Events = append(detail.Events, value)
+	}
+	if err := rows.Close(); err != nil {
+		return alert.Detail{}, errors.New("close Bell alert events")
+	}
+	rows, err = p.db.QueryContext(ctx, `SELECT sequence,from_state,to_state,actor_ref,note,occurred_at FROM bell.alert_transitions WHERE alert_id=$1 ORDER BY sequence`, alertID)
+	if err != nil {
+		return alert.Detail{}, errors.New("read Bell alert transitions")
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var value alert.Transition
+		if err := rows.Scan(&value.Sequence, &value.FromState, &value.ToState, &value.ActorRef, &value.Note, &value.OccurredAt); err != nil {
+			return alert.Detail{}, errors.New("scan Bell alert transition")
+		}
+		detail.Transitions = append(detail.Transitions, value)
+	}
+	return detail, rows.Err()
+}
+
+func (p *Postgres) Command(ctx context.Context, tenantID, siteID int64, alertID, command, key, actorRef string, note *string) (int, alert.CommandResponse, error) {
+	if (command != "ack" && command != "close") || tenantID < 1 || siteID < 1 || !alert.ValidIdempotencyKey(key) || !alert.ValidActorRef(actorRef) {
+		return 0, alert.CommandResponse{}, errors.New("invalid Bell alert command")
+	}
+	if note != nil && len([]rune(*note)) > 500 {
+		return 0, alert.CommandResponse{}, errors.New("Bell alert command note is too long")
+	}
+	digest := sha256.Sum256([]byte(strings.Join([]string{alertID, command, actorRef, valueOrEmpty(note)}, "\x00")))
+	tx, err := p.db.BeginTx(ctx, nil)
+	if err != nil {
+		return 0, alert.CommandResponse{}, errors.New("begin Bell alert command")
+	}
+	defer tx.Rollback()
+	if _, err := tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, fmt.Sprintf("alert-key:%d:%s", tenantID, key)); err != nil {
+		return 0, alert.CommandResponse{}, errors.New("lock Bell alert command key")
+	}
+	var storedHash, storedBody []byte
+	var storedStatus int
+	err = tx.QueryRowContext(ctx, `SELECT command_hash,response_status,response_body::text FROM bell.alert_command_receipts WHERE tenant_id=$1 AND idempotency_key=$2`, tenantID, key).Scan(&storedHash, &storedStatus, &storedBody)
+	if err == nil {
+		if !bytes.Equal(storedHash, digest[:]) {
+			return 0, alert.CommandResponse{}, alert.ErrIdempotencyConflict
+		}
+		var response alert.CommandResponse
+		if err := json.Unmarshal(storedBody, &response); err != nil {
+			return 0, alert.CommandResponse{}, errors.New("decode Bell alert command receipt")
+		}
+		return storedStatus, response, tx.Commit()
+	}
+	if !errors.Is(err, sql.ErrNoRows) {
+		return 0, alert.CommandResponse{}, errors.New("read Bell alert command receipt")
+	}
+	if _, err := tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, "alert:"+alertID); err != nil {
+		return 0, alert.CommandResponse{}, errors.New("lock Bell alert")
+	}
+	var current, currentActor string
+	var currentTime time.Time
+	var sequence int
+	err = tx.QueryRowContext(ctx, `SELECT t.to_state,t.actor_ref,t.occurred_at,t.sequence FROM bell.alerts a
+		JOIN LATERAL(SELECT to_state,actor_ref,occurred_at,sequence FROM bell.alert_transitions WHERE alert_id=a.id ORDER BY sequence DESC LIMIT 1)t ON true
+		WHERE a.id=$1 AND a.tenant_id=$2 AND a.site_id=$3`, alertID, tenantID, siteID).Scan(&current, &currentActor, &currentTime, &sequence)
+	if errors.Is(err, sql.ErrNoRows) {
+		return 0, alert.CommandResponse{}, alert.ErrNotFound
+	}
+	if err != nil {
+		return 0, alert.CommandResponse{}, errors.New("read Bell alert state")
+	}
+	status := 200
+	response := alert.CommandResponse{AlertID: alertID, State: current, ActorRef: currentActor, OccurredAt: currentTime}
+	allowed := (command == "ack" && current == "open") || (command == "close" && current == "acknowledged")
+	if !allowed {
+		status = 409
+		switch {
+		case command == "ack" && (current == "acknowledged" || current == "closed"):
+			response.Code = "already_acknowledged"
+			_ = tx.QueryRowContext(ctx, `SELECT actor_ref,occurred_at FROM bell.alert_transitions WHERE alert_id=$1 AND to_state='acknowledged' ORDER BY sequence LIMIT 1`, alertID).Scan(&response.ActorRef, &response.OccurredAt)
+		case command == "close" && current == "open":
+			response.Code = "acknowledgement_required"
+		case command == "close" && current == "closed":
+			response.Code = "already_closed"
+		default:
+			response.Code = "invalid_state"
+		}
+	} else {
+		next := "acknowledged"
+		if command == "close" {
+			next = "closed"
+		}
+		response.State = next
+		response.ActorRef = ""
+		response.OccurredAt = time.Time{}
+		if err := tx.QueryRowContext(ctx, `INSERT INTO bell.alert_transitions(id,alert_id,sequence,from_state,to_state,actor_ref,note)
+			VALUES($1,$2,$3,$4,$5,$6,$7) RETURNING actor_ref,occurred_at`,
+			newPrefixedID("trn_"), alertID, sequence+1, current, next, actorRef, note).Scan(&response.ActorRef, &response.OccurredAt); err != nil {
+			return 0, alert.CommandResponse{}, errors.New("append Bell alert transition")
+		}
+	}
+	body, err := json.Marshal(response)
+	if err != nil {
+		return 0, alert.CommandResponse{}, errors.New("encode Bell alert command response")
+	}
+	if _, err := tx.ExecContext(ctx, `INSERT INTO bell.alert_command_receipts(tenant_id,idempotency_key,command_hash,response_status,response_body) VALUES($1,$2,$3,$4,$5::jsonb)`, tenantID, key, digest[:], status, body); err != nil {
+		return 0, alert.CommandResponse{}, errors.New("append Bell alert command receipt")
+	}
+	if err := tx.Commit(); err != nil {
+		return 0, alert.CommandResponse{}, errors.New("commit Bell alert command")
+	}
+	return status, response, nil
+}
+
+func newPrefixedID(prefix string) string { return prefix + ulid.Make().String() }
+
+func valueOrEmpty(value *string) string {
+	if value == nil {
+		return ""
+	}
+	return *value
+}
@@ -0,0 +1,174 @@
+package store
+
+import (
+	"context"
+	"crypto/sha256"
+	"database/sql"
+	"errors"
+	"fmt"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	_ "github.com/jackc/pgx/v5/stdlib"
+
+	"yovision/bell/internal/alert"
+)
+
+func TestPostgresAlertRuleEvaluationAndFirstAckWins(t *testing.T) {
+	dsn := os.Getenv("YOVISION_TEST_BELL_POSTGRES_DSN")
+	adminDSN := os.Getenv("YOVISION_TEST_POSTGRES_ADMIN_DSN")
+	if dsn == "" || adminDSN == "" {
+		t.Skip("Bell runtime and admin PostgreSQL DSNs are not set")
+	}
+	ctx := context.Background()
+	admin, err := sql.Open("pgx", adminDSN)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer admin.Close()
+	db, err := sql.Open("pgx", dsn)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+	repository, err := OpenPostgres(ctx, db)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := repository.AlertReady(ctx); err != nil {
+		t.Fatal(err)
+	}
+	siteID := int64(902)
+	rule := alert.RuleSpec{TenantID: 901, SiteID: &siteID, RuleKey: "zone-entry", DisplayName: "区域闯入", EventKind: "zone_entry", MinimumSeverity: "medium", Enabled: true, EffectiveFrom: time.Now().Add(-time.Hour).UTC()}
+	created, err := repository.PublishRule(ctx, rule)
+	if err != nil || !created {
+		t.Fatalf("publish rule: created=%v err=%v", created, err)
+	}
+	created, err = repository.PublishRule(ctx, rule)
+	if err != nil || created {
+		t.Fatalf("idempotent rule publish: created=%v err=%v", created, err)
+	}
+	eventID := "evt_01J8XQ2K7M3P5R9T0V4W6Y8Z2Q"
+	payload := fmt.Sprintf(`{"id":%q,"tenant_id":901,"site_id":902,"device_id":903,"source_event_id":"T020-EVENT-1","kind":"zone_entry","severity":"high"}`, eventID)
+	payloadHash := sha256.Sum256([]byte(payload))
+	if _, err := admin.ExecContext(ctx, `INSERT INTO bell.events(id,tenant_id,site_id,device_id,source_event_id,kind,severity,occurred_at,detected_at,payload_hash,payload)
+		VALUES($1,901,902,903,'T020-EVENT-1','zone_entry','high',clock_timestamp(),clock_timestamp(),$3,$2::jsonb)`, eventID, payload, payloadHash[:]); err != nil {
+		t.Fatal(err)
+	}
+	noMatchID := "evt_01J8XQ2K7M3P5R9T0V4W6Y8Z2R"
+	noMatchPayload := fmt.Sprintf(`{"id":%q,"tenant_id":901,"site_id":902,"device_id":903,"source_event_id":"T020-EVENT-2","kind":"crowd","severity":"high"}`, noMatchID)
+	noMatchHash := sha256.Sum256([]byte(noMatchPayload))
+	if _, err := admin.ExecContext(ctx, `INSERT INTO bell.events(id,tenant_id,site_id,device_id,source_event_id,kind,severity,occurred_at,detected_at,payload_hash,payload)
+		VALUES($1,901,902,903,'T020-EVENT-2','crowd','high',clock_timestamp(),clock_timestamp(),$3,$2::jsonb)`, noMatchID, noMatchPayload, noMatchHash[:]); err != nil {
+		t.Fatal(err)
+	}
+
+	var evaluators sync.WaitGroup
+	errorsSeen := make(chan error, 8)
+	for index := 0; index < 8; index++ {
+		evaluators.Add(1)
+		go func() {
+			defer evaluators.Done()
+			for attempts := 0; attempts < 32; attempts++ {
+				worked, err := repository.EvaluateNext(ctx)
+				if err != nil {
+					errorsSeen <- err
+					return
+				}
+				if !worked {
+					return
+				}
+			}
+		}()
+	}
+	evaluators.Wait()
+	close(errorsSeen)
+	for err := range errorsSeen {
+		t.Error(err)
+	}
+	var alertID string
+	var alertCount, evaluationCount, sweepCount int
+	if err := db.QueryRowContext(ctx, `SELECT
+		(SELECT count(*) FROM bell.alerts WHERE tenant_id=901 AND site_id=902),
+		(SELECT count(*) FROM bell.rule_evaluations e JOIN bell.events v ON v.id=e.event_id WHERE v.tenant_id=901),
+		(SELECT count(*) FROM bell.event_rule_sweeps s JOIN bell.events v ON v.id=s.event_id WHERE v.tenant_id=901),
+		(SELECT id FROM bell.alerts WHERE tenant_id=901 AND site_id=902)`).Scan(&alertCount, &evaluationCount, &sweepCount, &alertID); err != nil {
+		t.Fatal(err)
+	}
+	if alertCount != 1 || evaluationCount != 2 || sweepCount != 2 {
+		t.Fatalf("evaluation did not converge: alerts=%d evaluations=%d sweeps=%d", alertCount, evaluationCount, sweepCount)
+	}
+	preStatus, preResponse, err := repository.Command(ctx, 901, 902, alertID, "close", "close-before-ack", "operator:closer", nil)
+	if err != nil || preStatus != 409 || preResponse.Code != "acknowledgement_required" || preResponse.State != "open" {
+		t.Fatalf("close-before-ack: status=%d response=%+v err=%v", preStatus, preResponse, err)
+	}
+
+	type outcome struct {
+		status   int
+		response alert.CommandResponse
+		err      error
+	}
+	results := make(chan outcome, 8)
+	var acknowledgers sync.WaitGroup
+	for index := 0; index < 8; index++ {
+		acknowledgers.Add(1)
+		go func(index int) {
+			defer acknowledgers.Done()
+			status, response, err := repository.Command(ctx, 901, 902, alertID, "ack", fmt.Sprintf("ack-key-%02d", index), fmt.Sprintf("operator:%d", index), nil)
+			results <- outcome{status, response, err}
+		}(index)
+	}
+	acknowledgers.Wait()
+	close(results)
+	winners := 0
+	var winner alert.CommandResponse
+	losers := make([]alert.CommandResponse, 0, 7)
+	for result := range results {
+		if result.err != nil {
+			t.Fatal(result.err)
+		}
+		if result.status == 200 {
+			winners++
+			winner = result.response
+		} else if result.status == 409 {
+			losers = append(losers, result.response)
+		}
+	}
+	if winners != 1 || len(losers) != 7 {
+		t.Fatalf("ack winners=%d losers=%d", winners, len(losers))
+	}
+	for _, loser := range losers {
+		if loser.ActorRef != winner.ActorRef || !loser.OccurredAt.Equal(winner.OccurredAt) || loser.Code != "already_acknowledged" {
+			t.Fatalf("late ack did not expose first winner: winner=%+v loser=%+v", winner, loser)
+		}
+	}
+	status, replayed, err := repository.Command(ctx, 901, 902, alertID, "ack", "ack-replay-key", "operator:replay", nil)
+	if err != nil || status != 409 || replayed.ActorRef != winner.ActorRef {
+		t.Fatalf("first replay receipt: status=%d response=%+v err=%v", status, replayed, err)
+	}
+	status2, replayed2, err := repository.Command(ctx, 901, 902, alertID, "ack", "ack-replay-key", "operator:replay", nil)
+	if err != nil || status2 != status || replayed2 != replayed {
+		t.Fatalf("stable receipt replay: status=%d response=%+v err=%v", status2, replayed2, err)
+	}
+	note := "different"
+	if _, _, err := repository.Command(ctx, 901, 902, alertID, "ack", "ack-replay-key", "operator:replay", &note); !errors.Is(err, alert.ErrIdempotencyConflict) {
+		t.Fatalf("expected idempotency conflict, got %v", err)
+	}
+	closeStatus, closeResponse, err := repository.Command(ctx, 901, 902, alertID, "close", "close-after-ack", "operator:closer", nil)
+	if err != nil || closeStatus != 200 || closeResponse.State != "closed" || closeResponse.ActorRef != "operator:closer" {
+		t.Fatalf("close-after-ack: status=%d response=%+v err=%v", closeStatus, closeResponse, err)
+	}
+	if _, err := db.ExecContext(ctx, `UPDATE bell.alert_transitions SET actor_ref=actor_ref WHERE alert_id=$1`, alertID); err == nil {
+		t.Fatal("runtime updated immutable alert transition")
+	}
+	reopened, err := OpenPostgres(ctx, db)
+	if err != nil {
+		t.Fatal(err)
+	}
+	detail, err := reopened.GetAlert(ctx, 901, 902, alertID)
+	if err != nil || detail.State != "closed" || len(detail.Transitions) != 3 || detail.EvidenceStatus != "not_enabled" {
+		t.Fatalf("restart detail: %+v %v", detail, err)
+	}
+}