175 lines
6.8 KiB
Go
175 lines
6.8 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
_ "github.com/jackc/pgx/v5/stdlib"
|
|
|
|
"yovision/bell/internal/alert"
|
|
)
|
|
|
|
func TestPostgresAlertRuleEvaluationAndFirstAckWins(t *testing.T) {
|
|
dsn := os.Getenv("YOVISION_TEST_BELL_POSTGRES_DSN")
|
|
adminDSN := os.Getenv("YOVISION_TEST_POSTGRES_ADMIN_DSN")
|
|
if dsn == "" || adminDSN == "" {
|
|
t.Skip("Bell runtime and admin PostgreSQL DSNs are not set")
|
|
}
|
|
ctx := context.Background()
|
|
admin, err := sql.Open("pgx", adminDSN)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer admin.Close()
|
|
db, err := sql.Open("pgx", dsn)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer db.Close()
|
|
repository, err := OpenPostgres(ctx, db)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := repository.AlertReady(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
siteID := int64(902)
|
|
rule := alert.RuleSpec{TenantID: 901, SiteID: &siteID, RuleKey: "zone-entry", DisplayName: "区域闯入", EventKind: "zone_entry", MinimumSeverity: "medium", Enabled: true, EffectiveFrom: time.Now().Add(-time.Hour).UTC()}
|
|
created, err := repository.PublishRule(ctx, rule)
|
|
if err != nil || !created {
|
|
t.Fatalf("publish rule: created=%v err=%v", created, err)
|
|
}
|
|
created, err = repository.PublishRule(ctx, rule)
|
|
if err != nil || created {
|
|
t.Fatalf("idempotent rule publish: created=%v err=%v", created, err)
|
|
}
|
|
eventID := "evt_01J8XQ2K7M3P5R9T0V4W6Y8Z2Q"
|
|
payload := fmt.Sprintf(`{"id":%q,"tenant_id":901,"site_id":902,"device_id":903,"source_event_id":"T020-EVENT-1","kind":"zone_entry","severity":"high"}`, eventID)
|
|
payloadHash := sha256.Sum256([]byte(payload))
|
|
if _, err := admin.ExecContext(ctx, `INSERT INTO bell.events(id,tenant_id,site_id,device_id,source_event_id,kind,severity,occurred_at,detected_at,payload_hash,payload)
|
|
VALUES($1,901,902,903,'T020-EVENT-1','zone_entry','high',clock_timestamp(),clock_timestamp(),$3,$2::jsonb)`, eventID, payload, payloadHash[:]); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
noMatchID := "evt_01J8XQ2K7M3P5R9T0V4W6Y8Z2R"
|
|
noMatchPayload := fmt.Sprintf(`{"id":%q,"tenant_id":901,"site_id":902,"device_id":903,"source_event_id":"T020-EVENT-2","kind":"crowd","severity":"high"}`, noMatchID)
|
|
noMatchHash := sha256.Sum256([]byte(noMatchPayload))
|
|
if _, err := admin.ExecContext(ctx, `INSERT INTO bell.events(id,tenant_id,site_id,device_id,source_event_id,kind,severity,occurred_at,detected_at,payload_hash,payload)
|
|
VALUES($1,901,902,903,'T020-EVENT-2','crowd','high',clock_timestamp(),clock_timestamp(),$3,$2::jsonb)`, noMatchID, noMatchPayload, noMatchHash[:]); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
var evaluators sync.WaitGroup
|
|
errorsSeen := make(chan error, 8)
|
|
for index := 0; index < 8; index++ {
|
|
evaluators.Add(1)
|
|
go func() {
|
|
defer evaluators.Done()
|
|
for attempts := 0; attempts < 32; attempts++ {
|
|
worked, err := repository.EvaluateNext(ctx)
|
|
if err != nil {
|
|
errorsSeen <- err
|
|
return
|
|
}
|
|
if !worked {
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
evaluators.Wait()
|
|
close(errorsSeen)
|
|
for err := range errorsSeen {
|
|
t.Error(err)
|
|
}
|
|
var alertID string
|
|
var alertCount, evaluationCount, sweepCount int
|
|
if err := db.QueryRowContext(ctx, `SELECT
|
|
(SELECT count(*) FROM bell.alerts WHERE tenant_id=901 AND site_id=902),
|
|
(SELECT count(*) FROM bell.rule_evaluations e JOIN bell.events v ON v.id=e.event_id WHERE v.tenant_id=901),
|
|
(SELECT count(*) FROM bell.event_rule_sweeps s JOIN bell.events v ON v.id=s.event_id WHERE v.tenant_id=901),
|
|
(SELECT id FROM bell.alerts WHERE tenant_id=901 AND site_id=902)`).Scan(&alertCount, &evaluationCount, &sweepCount, &alertID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if alertCount != 1 || evaluationCount != 2 || sweepCount != 2 {
|
|
t.Fatalf("evaluation did not converge: alerts=%d evaluations=%d sweeps=%d", alertCount, evaluationCount, sweepCount)
|
|
}
|
|
preStatus, preResponse, err := repository.Command(ctx, 901, 902, alertID, "close", "close-before-ack", "operator:closer", nil)
|
|
if err != nil || preStatus != 409 || preResponse.Code != "acknowledgement_required" || preResponse.State != "open" {
|
|
t.Fatalf("close-before-ack: status=%d response=%+v err=%v", preStatus, preResponse, err)
|
|
}
|
|
|
|
type outcome struct {
|
|
status int
|
|
response alert.CommandResponse
|
|
err error
|
|
}
|
|
results := make(chan outcome, 8)
|
|
var acknowledgers sync.WaitGroup
|
|
for index := 0; index < 8; index++ {
|
|
acknowledgers.Add(1)
|
|
go func(index int) {
|
|
defer acknowledgers.Done()
|
|
status, response, err := repository.Command(ctx, 901, 902, alertID, "ack", fmt.Sprintf("ack-key-%02d", index), fmt.Sprintf("operator:%d", index), nil)
|
|
results <- outcome{status, response, err}
|
|
}(index)
|
|
}
|
|
acknowledgers.Wait()
|
|
close(results)
|
|
winners := 0
|
|
var winner alert.CommandResponse
|
|
losers := make([]alert.CommandResponse, 0, 7)
|
|
for result := range results {
|
|
if result.err != nil {
|
|
t.Fatal(result.err)
|
|
}
|
|
if result.status == 200 {
|
|
winners++
|
|
winner = result.response
|
|
} else if result.status == 409 {
|
|
losers = append(losers, result.response)
|
|
}
|
|
}
|
|
if winners != 1 || len(losers) != 7 {
|
|
t.Fatalf("ack winners=%d losers=%d", winners, len(losers))
|
|
}
|
|
for _, loser := range losers {
|
|
if loser.ActorRef != winner.ActorRef || !loser.OccurredAt.Equal(winner.OccurredAt) || loser.Code != "already_acknowledged" {
|
|
t.Fatalf("late ack did not expose first winner: winner=%+v loser=%+v", winner, loser)
|
|
}
|
|
}
|
|
status, replayed, err := repository.Command(ctx, 901, 902, alertID, "ack", "ack-replay-key", "operator:replay", nil)
|
|
if err != nil || status != 409 || replayed.ActorRef != winner.ActorRef {
|
|
t.Fatalf("first replay receipt: status=%d response=%+v err=%v", status, replayed, err)
|
|
}
|
|
status2, replayed2, err := repository.Command(ctx, 901, 902, alertID, "ack", "ack-replay-key", "operator:replay", nil)
|
|
if err != nil || status2 != status || replayed2 != replayed {
|
|
t.Fatalf("stable receipt replay: status=%d response=%+v err=%v", status2, replayed2, err)
|
|
}
|
|
note := "different"
|
|
if _, _, err := repository.Command(ctx, 901, 902, alertID, "ack", "ack-replay-key", "operator:replay", ¬e); !errors.Is(err, alert.ErrIdempotencyConflict) {
|
|
t.Fatalf("expected idempotency conflict, got %v", err)
|
|
}
|
|
closeStatus, closeResponse, err := repository.Command(ctx, 901, 902, alertID, "close", "close-after-ack", "operator:closer", nil)
|
|
if err != nil || closeStatus != 200 || closeResponse.State != "closed" || closeResponse.ActorRef != "operator:closer" {
|
|
t.Fatalf("close-after-ack: status=%d response=%+v err=%v", closeStatus, closeResponse, err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `UPDATE bell.alert_transitions SET actor_ref=actor_ref WHERE alert_id=$1`, alertID); err == nil {
|
|
t.Fatal("runtime updated immutable alert transition")
|
|
}
|
|
reopened, err := OpenPostgres(ctx, db)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
detail, err := reopened.GetAlert(ctx, 901, 902, alertID)
|
|
if err != nil || detail.State != "closed" || len(detail.Transitions) != 3 || detail.EvidenceStatus != "not_enabled" {
|
|
t.Fatalf("restart detail: %+v %v", detail, err)
|
|
}
|
|
}
|