feat(bell): add alert acknowledgement vertical slice
Harness governance / validate (pull_request) Has been cancelled
Harness governance / validate (pull_request) Has been cancelled
This commit is contained in:
QiuSW
@@ -0,0 +1,174 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "github.com/jackc/pgx/v5/stdlib"
|
||||
|
||||
"yovision/bell/internal/alert"
|
||||
)
|
||||
|
||||
func TestPostgresAlertRuleEvaluationAndFirstAckWins(t *testing.T) {
|
||||
dsn := os.Getenv("YOVISION_TEST_BELL_POSTGRES_DSN")
|
||||
adminDSN := os.Getenv("YOVISION_TEST_POSTGRES_ADMIN_DSN")
|
||||
if dsn == "" || adminDSN == "" {
|
||||
t.Skip("Bell runtime and admin PostgreSQL DSNs are not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
admin, err := sql.Open("pgx", adminDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer admin.Close()
|
||||
db, err := sql.Open("pgx", dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
repository, err := OpenPostgres(ctx, db)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := repository.AlertReady(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
siteID := int64(902)
|
||||
rule := alert.RuleSpec{TenantID: 901, SiteID: &siteID, RuleKey: "zone-entry", DisplayName: "区域闯入", EventKind: "zone_entry", MinimumSeverity: "medium", Enabled: true, EffectiveFrom: time.Now().Add(-time.Hour).UTC()}
|
||||
created, err := repository.PublishRule(ctx, rule)
|
||||
if err != nil || !created {
|
||||
t.Fatalf("publish rule: created=%v err=%v", created, err)
|
||||
}
|
||||
created, err = repository.PublishRule(ctx, rule)
|
||||
if err != nil || created {
|
||||
t.Fatalf("idempotent rule publish: created=%v err=%v", created, err)
|
||||
}
|
||||
eventID := "evt_01J8XQ2K7M3P5R9T0V4W6Y8Z2Q"
|
||||
payload := fmt.Sprintf(`{"id":%q,"tenant_id":901,"site_id":902,"device_id":903,"source_event_id":"T020-EVENT-1","kind":"zone_entry","severity":"high"}`, eventID)
|
||||
payloadHash := sha256.Sum256([]byte(payload))
|
||||
if _, err := admin.ExecContext(ctx, `INSERT INTO bell.events(id,tenant_id,site_id,device_id,source_event_id,kind,severity,occurred_at,detected_at,payload_hash,payload)
|
||||
VALUES($1,901,902,903,'T020-EVENT-1','zone_entry','high',clock_timestamp(),clock_timestamp(),$3,$2::jsonb)`, eventID, payload, payloadHash[:]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
noMatchID := "evt_01J8XQ2K7M3P5R9T0V4W6Y8Z2R"
|
||||
noMatchPayload := fmt.Sprintf(`{"id":%q,"tenant_id":901,"site_id":902,"device_id":903,"source_event_id":"T020-EVENT-2","kind":"crowd","severity":"high"}`, noMatchID)
|
||||
noMatchHash := sha256.Sum256([]byte(noMatchPayload))
|
||||
if _, err := admin.ExecContext(ctx, `INSERT INTO bell.events(id,tenant_id,site_id,device_id,source_event_id,kind,severity,occurred_at,detected_at,payload_hash,payload)
|
||||
VALUES($1,901,902,903,'T020-EVENT-2','crowd','high',clock_timestamp(),clock_timestamp(),$3,$2::jsonb)`, noMatchID, noMatchPayload, noMatchHash[:]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var evaluators sync.WaitGroup
|
||||
errorsSeen := make(chan error, 8)
|
||||
for index := 0; index < 8; index++ {
|
||||
evaluators.Add(1)
|
||||
go func() {
|
||||
defer evaluators.Done()
|
||||
for attempts := 0; attempts < 32; attempts++ {
|
||||
worked, err := repository.EvaluateNext(ctx)
|
||||
if err != nil {
|
||||
errorsSeen <- err
|
||||
return
|
||||
}
|
||||
if !worked {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
evaluators.Wait()
|
||||
close(errorsSeen)
|
||||
for err := range errorsSeen {
|
||||
t.Error(err)
|
||||
}
|
||||
var alertID string
|
||||
var alertCount, evaluationCount, sweepCount int
|
||||
if err := db.QueryRowContext(ctx, `SELECT
|
||||
(SELECT count(*) FROM bell.alerts WHERE tenant_id=901 AND site_id=902),
|
||||
(SELECT count(*) FROM bell.rule_evaluations e JOIN bell.events v ON v.id=e.event_id WHERE v.tenant_id=901),
|
||||
(SELECT count(*) FROM bell.event_rule_sweeps s JOIN bell.events v ON v.id=s.event_id WHERE v.tenant_id=901),
|
||||
(SELECT id FROM bell.alerts WHERE tenant_id=901 AND site_id=902)`).Scan(&alertCount, &evaluationCount, &sweepCount, &alertID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if alertCount != 1 || evaluationCount != 2 || sweepCount != 2 {
|
||||
t.Fatalf("evaluation did not converge: alerts=%d evaluations=%d sweeps=%d", alertCount, evaluationCount, sweepCount)
|
||||
}
|
||||
preStatus, preResponse, err := repository.Command(ctx, 901, 902, alertID, "close", "close-before-ack", "operator:closer", nil)
|
||||
if err != nil || preStatus != 409 || preResponse.Code != "acknowledgement_required" || preResponse.State != "open" {
|
||||
t.Fatalf("close-before-ack: status=%d response=%+v err=%v", preStatus, preResponse, err)
|
||||
}
|
||||
|
||||
type outcome struct {
|
||||
status int
|
||||
response alert.CommandResponse
|
||||
err error
|
||||
}
|
||||
results := make(chan outcome, 8)
|
||||
var acknowledgers sync.WaitGroup
|
||||
for index := 0; index < 8; index++ {
|
||||
acknowledgers.Add(1)
|
||||
go func(index int) {
|
||||
defer acknowledgers.Done()
|
||||
status, response, err := repository.Command(ctx, 901, 902, alertID, "ack", fmt.Sprintf("ack-key-%02d", index), fmt.Sprintf("operator:%d", index), nil)
|
||||
results <- outcome{status, response, err}
|
||||
}(index)
|
||||
}
|
||||
acknowledgers.Wait()
|
||||
close(results)
|
||||
winners := 0
|
||||
var winner alert.CommandResponse
|
||||
losers := make([]alert.CommandResponse, 0, 7)
|
||||
for result := range results {
|
||||
if result.err != nil {
|
||||
t.Fatal(result.err)
|
||||
}
|
||||
if result.status == 200 {
|
||||
winners++
|
||||
winner = result.response
|
||||
} else if result.status == 409 {
|
||||
losers = append(losers, result.response)
|
||||
}
|
||||
}
|
||||
if winners != 1 || len(losers) != 7 {
|
||||
t.Fatalf("ack winners=%d losers=%d", winners, len(losers))
|
||||
}
|
||||
for _, loser := range losers {
|
||||
if loser.ActorRef != winner.ActorRef || !loser.OccurredAt.Equal(winner.OccurredAt) || loser.Code != "already_acknowledged" {
|
||||
t.Fatalf("late ack did not expose first winner: winner=%+v loser=%+v", winner, loser)
|
||||
}
|
||||
}
|
||||
status, replayed, err := repository.Command(ctx, 901, 902, alertID, "ack", "ack-replay-key", "operator:replay", nil)
|
||||
if err != nil || status != 409 || replayed.ActorRef != winner.ActorRef {
|
||||
t.Fatalf("first replay receipt: status=%d response=%+v err=%v", status, replayed, err)
|
||||
}
|
||||
status2, replayed2, err := repository.Command(ctx, 901, 902, alertID, "ack", "ack-replay-key", "operator:replay", nil)
|
||||
if err != nil || status2 != status || replayed2 != replayed {
|
||||
t.Fatalf("stable receipt replay: status=%d response=%+v err=%v", status2, replayed2, err)
|
||||
}
|
||||
note := "different"
|
||||
if _, _, err := repository.Command(ctx, 901, 902, alertID, "ack", "ack-replay-key", "operator:replay", ¬e); !errors.Is(err, alert.ErrIdempotencyConflict) {
|
||||
t.Fatalf("expected idempotency conflict, got %v", err)
|
||||
}
|
||||
closeStatus, closeResponse, err := repository.Command(ctx, 901, 902, alertID, "close", "close-after-ack", "operator:closer", nil)
|
||||
if err != nil || closeStatus != 200 || closeResponse.State != "closed" || closeResponse.ActorRef != "operator:closer" {
|
||||
t.Fatalf("close-after-ack: status=%d response=%+v err=%v", closeStatus, closeResponse, err)
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, `UPDATE bell.alert_transitions SET actor_ref=actor_ref WHERE alert_id=$1`, alertID); err == nil {
|
||||
t.Fatal("runtime updated immutable alert transition")
|
||||
}
|
||||
reopened, err := OpenPostgres(ctx, db)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
detail, err := reopened.GetAlert(ctx, 901, 902, alertID)
|
||||
if err != nil || detail.State != "closed" || len(detail.Transitions) != 3 || detail.EvidenceStatus != "not_enabled" {
|
||||
t.Fatalf("restart detail: %+v %v", detail, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user