package store import ( "context" "crypto/sha256" "database/sql" "errors" "fmt" "os" "sync" "testing" "time" _ "github.com/jackc/pgx/v5/stdlib" "yovision/bell/internal/alert" ) func TestPostgresAlertRuleEvaluationAndFirstAckWins(t *testing.T) { dsn := os.Getenv("YOVISION_TEST_BELL_POSTGRES_DSN") adminDSN := os.Getenv("YOVISION_TEST_POSTGRES_ADMIN_DSN") if dsn == "" || adminDSN == "" { t.Skip("Bell runtime and admin PostgreSQL DSNs are not set") } ctx := context.Background() admin, err := sql.Open("pgx", adminDSN) if err != nil { t.Fatal(err) } defer admin.Close() db, err := sql.Open("pgx", dsn) if err != nil { t.Fatal(err) } defer db.Close() repository, err := OpenPostgres(ctx, db) if err != nil { t.Fatal(err) } if err := repository.AlertReady(ctx); err != nil { t.Fatal(err) } siteID := int64(902) rule := alert.RuleSpec{TenantID: 901, SiteID: &siteID, RuleKey: "zone-entry", DisplayName: "区域闯入", EventKind: "zone_entry", MinimumSeverity: "medium", Enabled: true, EffectiveFrom: time.Now().Add(-time.Hour).UTC()} created, err := repository.PublishRule(ctx, rule) if err != nil || !created { t.Fatalf("publish rule: created=%v err=%v", created, err) } created, err = repository.PublishRule(ctx, rule) if err != nil || created { t.Fatalf("idempotent rule publish: created=%v err=%v", created, err) } eventID := "evt_01J8XQ2K7M3P5R9T0V4W6Y8Z2Q" payload := fmt.Sprintf(`{"id":%q,"tenant_id":901,"site_id":902,"device_id":903,"source_event_id":"T020-EVENT-1","kind":"zone_entry","severity":"high"}`, eventID) payloadHash := sha256.Sum256([]byte(payload)) if _, err := admin.ExecContext(ctx, `INSERT INTO bell.events(id,tenant_id,site_id,device_id,source_event_id,kind,severity,occurred_at,detected_at,payload_hash,payload) VALUES($1,901,902,903,'T020-EVENT-1','zone_entry','high',clock_timestamp(),clock_timestamp(),$3,$2::jsonb)`, eventID, payload, payloadHash[:]); err != nil { t.Fatal(err) } noMatchID := "evt_01J8XQ2K7M3P5R9T0V4W6Y8Z2R" noMatchPayload := fmt.Sprintf(`{"id":%q,"tenant_id":901,"site_id":902,"device_id":903,"source_event_id":"T020-EVENT-2","kind":"crowd","severity":"high"}`, noMatchID) noMatchHash := sha256.Sum256([]byte(noMatchPayload)) if _, err := admin.ExecContext(ctx, `INSERT INTO bell.events(id,tenant_id,site_id,device_id,source_event_id,kind,severity,occurred_at,detected_at,payload_hash,payload) VALUES($1,901,902,903,'T020-EVENT-2','crowd','high',clock_timestamp(),clock_timestamp(),$3,$2::jsonb)`, noMatchID, noMatchPayload, noMatchHash[:]); err != nil { t.Fatal(err) } var evaluators sync.WaitGroup errorsSeen := make(chan error, 8) for index := 0; index < 8; index++ { evaluators.Add(1) go func() { defer evaluators.Done() for attempts := 0; attempts < 32; attempts++ { worked, err := repository.EvaluateNext(ctx) if err != nil { errorsSeen <- err return } if !worked { return } } }() } evaluators.Wait() close(errorsSeen) for err := range errorsSeen { t.Error(err) } var alertID string var alertCount, evaluationCount, sweepCount int if err := db.QueryRowContext(ctx, `SELECT (SELECT count(*) FROM bell.alerts WHERE tenant_id=901 AND site_id=902), (SELECT count(*) FROM bell.rule_evaluations e JOIN bell.events v ON v.id=e.event_id WHERE v.tenant_id=901), (SELECT count(*) FROM bell.event_rule_sweeps s JOIN bell.events v ON v.id=s.event_id WHERE v.tenant_id=901), (SELECT id FROM bell.alerts WHERE tenant_id=901 AND site_id=902)`).Scan(&alertCount, &evaluationCount, &sweepCount, &alertID); err != nil { t.Fatal(err) } if alertCount != 1 || evaluationCount != 2 || sweepCount != 2 { t.Fatalf("evaluation did not converge: alerts=%d evaluations=%d sweeps=%d", alertCount, evaluationCount, sweepCount) } preStatus, preResponse, err := repository.Command(ctx, 901, 902, alertID, "close", "close-before-ack", "operator:closer", nil) if err != nil || preStatus != 409 || preResponse.Code != "acknowledgement_required" || preResponse.State != "open" { t.Fatalf("close-before-ack: status=%d response=%+v err=%v", preStatus, preResponse, err) } type outcome struct { status int response alert.CommandResponse err error } results := make(chan outcome, 8) var acknowledgers sync.WaitGroup for index := 0; index < 8; index++ { acknowledgers.Add(1) go func(index int) { defer acknowledgers.Done() status, response, err := repository.Command(ctx, 901, 902, alertID, "ack", fmt.Sprintf("ack-key-%02d", index), fmt.Sprintf("operator:%d", index), nil) results <- outcome{status, response, err} }(index) } acknowledgers.Wait() close(results) winners := 0 var winner alert.CommandResponse losers := make([]alert.CommandResponse, 0, 7) for result := range results { if result.err != nil { t.Fatal(result.err) } if result.status == 200 { winners++ winner = result.response } else if result.status == 409 { losers = append(losers, result.response) } } if winners != 1 || len(losers) != 7 { t.Fatalf("ack winners=%d losers=%d", winners, len(losers)) } for _, loser := range losers { if loser.ActorRef != winner.ActorRef || !loser.OccurredAt.Equal(winner.OccurredAt) || loser.Code != "already_acknowledged" { t.Fatalf("late ack did not expose first winner: winner=%+v loser=%+v", winner, loser) } } status, replayed, err := repository.Command(ctx, 901, 902, alertID, "ack", "ack-replay-key", "operator:replay", nil) if err != nil || status != 409 || replayed.ActorRef != winner.ActorRef { t.Fatalf("first replay receipt: status=%d response=%+v err=%v", status, replayed, err) } status2, replayed2, err := repository.Command(ctx, 901, 902, alertID, "ack", "ack-replay-key", "operator:replay", nil) if err != nil || status2 != status || replayed2 != replayed { t.Fatalf("stable receipt replay: status=%d response=%+v err=%v", status2, replayed2, err) } note := "different" if _, _, err := repository.Command(ctx, 901, 902, alertID, "ack", "ack-replay-key", "operator:replay", ¬e); !errors.Is(err, alert.ErrIdempotencyConflict) { t.Fatalf("expected idempotency conflict, got %v", err) } closeStatus, closeResponse, err := repository.Command(ctx, 901, 902, alertID, "close", "close-after-ack", "operator:closer", nil) if err != nil || closeStatus != 200 || closeResponse.State != "closed" || closeResponse.ActorRef != "operator:closer" { t.Fatalf("close-after-ack: status=%d response=%+v err=%v", closeStatus, closeResponse, err) } if _, err := db.ExecContext(ctx, `UPDATE bell.alert_transitions SET actor_ref=actor_ref WHERE alert_id=$1`, alertID); err == nil { t.Fatal("runtime updated immutable alert transition") } reopened, err := OpenPostgres(ctx, db) if err != nil { t.Fatal(err) } detail, err := reopened.GetAlert(ctx, 901, 902, alertID) if err != nil || detail.State != "closed" || len(detail.Transitions) != 3 || detail.EvidenceStatus != "not_enabled" { t.Fatalf("restart detail: %+v %v", detail, err) } }