feat(bell): add alert acknowledgement vertical slice
Harness governance / validate (pull_request) Has been cancelled

This commit is contained in:
QiuSW
2026-08-11 17:01:53 +08:00
parent 477afa6ba2
commit 8208118904
29 changed files with 1820 additions and 56 deletions
+221
View File
@@ -0,0 +1,221 @@
// Package alert owns Bell's small, deterministic rule and Alert domain.
package alert
import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"regexp"
"strings"
"time"
"unicode/utf8"
)
const (
DefaultPageSize = 16
MaxPageSize = 100
MaxRulesFile = 256 << 10
)
var (
ErrNotFound = errors.New("alert not found")
ErrIdempotencyConflict = errors.New("idempotency key was used for another command")
ruleKeyPattern = regexp.MustCompile(`^[a-z][a-z0-9_-]{2,63}$`)
eventKindPattern = regexp.MustCompile(`^[a-z][a-z0-9_]{2,63}$`)
actorRefPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:@/-]*$`)
idempotencyKeyPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:-]*$`)
)
type RuleSpec struct {
TenantID int64 `json:"tenant_id"`
SiteID *int64 `json:"site_id,omitempty"`
RuleKey string `json:"rule_key"`
DisplayName string `json:"display_name"`
EventKind string `json:"event_kind"`
MinimumSeverity string `json:"minimum_severity"`
Enabled bool `json:"enabled"`
EffectiveFrom time.Time `json:"effective_from"`
}
func (r RuleSpec) Validate() error {
if r.TenantID < 1 || (r.SiteID != nil && *r.SiteID < 1) {
return errors.New("rule scope must use positive identifiers")
}
if !ruleKeyPattern.MatchString(r.RuleKey) || !eventKindPattern.MatchString(r.EventKind) {
return errors.New("rule key or event kind is invalid")
}
if strings.TrimSpace(r.DisplayName) == "" || utf8.RuneCountInString(r.DisplayName) > 120 || r.EffectiveFrom.IsZero() {
return errors.New("rule name or effective time is invalid")
}
if _, ok := SeverityRank(r.MinimumSeverity); !ok {
return errors.New("rule minimum severity is invalid")
}
return nil
}
func (r RuleSpec) Digest() ([sha256.Size]byte, error) {
value, err := json.Marshal(r)
if err != nil {
return [sha256.Size]byte{}, err
}
return sha256.Sum256(value), nil
}
type rulesDocument struct {
Version int `json:"version"`
Rules []RuleSpec `json:"rules"`
}
func LoadRules(path string) ([]RuleSpec, error) {
file, err := os.Open(path)
if err != nil {
return nil, errors.New("open Bell alert rules file")
}
defer file.Close()
info, err := file.Stat()
if err != nil || info.Size() > MaxRulesFile {
return nil, errors.New("Bell alert rules file is too large")
}
decoder := json.NewDecoder(file)
decoder.DisallowUnknownFields()
var document rulesDocument
if err := decoder.Decode(&document); err != nil {
return nil, errors.New("decode Bell alert rules file")
}
var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
return nil, errors.New("Bell alert rules file contains multiple JSON values")
}
if document.Version != 1 || len(document.Rules) < 1 || len(document.Rules) > 256 {
return nil, errors.New("Bell alert rules file must contain 1 to 256 v1 rules")
}
seen := make(map[string]bool, len(document.Rules))
for _, rule := range document.Rules {
if err := rule.Validate(); err != nil {
return nil, fmt.Errorf("invalid Bell alert rule %q: %w", rule.RuleKey, err)
}
key := fmt.Sprintf("%d:%s", rule.TenantID, rule.RuleKey)
if seen[key] {
return nil, fmt.Errorf("duplicate Bell alert rule %q", rule.RuleKey)
}
seen[key] = true
}
return document.Rules, nil
}
func SeverityRank(value string) (int, bool) {
for rank, severity := range []string{"low", "medium", "high", "critical"} {
if value == severity {
return rank, true
}
}
return 0, false
}
func ValidActorRef(value string) bool {
return len(value) <= 80 && actorRefPattern.MatchString(value)
}
func ValidIdempotencyKey(value string) bool {
return len(value) >= 8 && len(value) <= 128 && idempotencyKeyPattern.MatchString(value)
}
type Summary struct {
ID string `json:"id"`
Severity string `json:"severity"`
Title string `json:"title"`
State string `json:"state"`
RuleKey string `json:"rule_key"`
RuleVersion int `json:"rule_version"`
CreatedAt time.Time `json:"created_at"`
}
type EventRef struct {
ID string `json:"id"`
DeviceID int64 `json:"device_id"`
Kind string `json:"kind"`
Severity string `json:"severity"`
OccurredAt time.Time `json:"occurred_at"`
}
type Transition struct {
Sequence int `json:"sequence"`
FromState *string `json:"from_state"`
ToState string `json:"to_state"`
ActorRef string `json:"actor_ref"`
Note *string `json:"note"`
OccurredAt time.Time `json:"occurred_at"`
}
type Detail struct {
Summary
Events []EventRef `json:"events"`
Transitions []Transition `json:"transitions"`
EvidenceStatus string `json:"evidence_status"`
DeliveryStatus string `json:"delivery_status"`
}
type Page struct {
Items []Summary `json:"items"`
NextCursor *string `json:"next_cursor"`
}
type CommandResponse struct {
AlertID string `json:"alert_id"`
State string `json:"state"`
ActorRef string `json:"actor_ref"`
OccurredAt time.Time `json:"occurred_at"`
Code string `json:"code,omitempty"`
}
type Repository interface {
AlertReady(context.Context) error
PublishRule(context.Context, RuleSpec) (bool, error)
EvaluateNext(context.Context) (bool, error)
ListAlerts(context.Context, int64, int64, string, int, string) (Page, error)
GetAlert(context.Context, int64, int64, string) (Detail, error)
Command(context.Context, int64, int64, string, string, string, string, *string) (int, CommandResponse, error)
}
func Publish(ctx context.Context, repository Repository, rules []RuleSpec) error {
for _, rule := range rules {
if _, err := repository.PublishRule(ctx, rule); err != nil {
return fmt.Errorf("publish Bell alert rule %q: %w", rule.RuleKey, err)
}
}
return nil
}
func RunWorker(ctx context.Context, repository Repository, onError func(error)) {
idle := time.NewTicker(250 * time.Millisecond)
defer idle.Stop()
for {
worked, err := repository.EvaluateNext(ctx)
if err != nil {
if ctx.Err() != nil {
return
}
if onError != nil {
onError(err)
}
select {
case <-ctx.Done():
return
case <-time.After(time.Second):
}
continue
}
if worked {
continue
}
select {
case <-ctx.Done():
return
case <-idle.C:
}
}
}
+43
View File
@@ -0,0 +1,43 @@
package alert
import (
"os"
"path/filepath"
"testing"
)
func TestLoadRulesRejectsUnknownAndDuplicateValues(t *testing.T) {
path := filepath.Join(t.TempDir(), "rules.json")
valid := `{"version":1,"rules":[{"tenant_id":1,"site_id":2,"rule_key":"zone-entry","display_name":"区域闯入","event_kind":"zone_entry","minimum_severity":"medium","enabled":true,"effective_from":"2026-08-11T00:00:00Z"}]}`
if err := os.WriteFile(path, []byte(valid), 0o600); err != nil {
t.Fatal(err)
}
rules, err := LoadRules(path)
if err != nil || len(rules) != 1 {
t.Fatalf("load valid rules: %v %#v", err, rules)
}
if err := os.WriteFile(path, []byte(`{"version":1,"unknown":true,"rules":[]}`), 0o600); err != nil {
t.Fatal(err)
}
if _, err := LoadRules(path); err == nil {
t.Fatal("unknown rule document property was accepted")
}
if err := os.WriteFile(path, []byte(valid+` trailing`), 0o600); err != nil {
t.Fatal(err)
}
if _, err := LoadRules(path); err == nil {
t.Fatal("trailing rule file data was accepted")
}
}
func TestSeverityRankIsStable(t *testing.T) {
for index, value := range []string{"low", "medium", "high", "critical"} {
rank, ok := SeverityRank(value)
if !ok || rank != index {
t.Fatalf("rank %q: %d %v", value, rank, ok)
}
}
if _, ok := SeverityRank("urgent"); ok {
t.Fatal("unknown severity was accepted")
}
}
+355
View File
@@ -0,0 +1,355 @@
package store
import (
"bytes"
"context"
"crypto/sha256"
"database/sql"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/oklog/ulid/v2"
"yovision/bell/internal/alert"
)
func (p *Postgres) AlertReady(ctx context.Context) error {
var version int64
if err := p.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(version),0) FROM bell.schema_migrations`).Scan(&version); err != nil || version < 6 {
return errors.New("postgres Bell schema migration v6 is required for alerts")
}
for _, table := range []string{"rule_versions", "event_rule_sweeps", "rule_evaluations", "alerts", "alert_events", "alert_transitions", "alert_command_receipts"} {
var selectAllowed, insertAllowed, updateAllowed, deleteAllowed, truncateAllowed bool
if err := p.db.QueryRowContext(ctx, `SELECT
has_table_privilege(current_user,$1,'SELECT'), has_table_privilege(current_user,$1,'INSERT'),
has_table_privilege(current_user,$1,'UPDATE'), has_table_privilege(current_user,$1,'DELETE'),
has_table_privilege(current_user,$1,'TRUNCATE')`, "bell."+table).Scan(
&selectAllowed, &insertAllowed, &updateAllowed, &deleteAllowed, &truncateAllowed,
); err != nil || !selectAllowed || !insertAllowed || updateAllowed || deleteAllowed || truncateAllowed {
return fmt.Errorf("Bell runtime alert privileges violate append-only boundary for %s", table)
}
}
return nil
}
func (p *Postgres) PublishRule(ctx context.Context, rule alert.RuleSpec) (bool, error) {
if err := rule.Validate(); err != nil {
return false, err
}
digest, err := rule.Digest()
if err != nil {
return false, err
}
tx, err := p.db.BeginTx(ctx, nil)
if err != nil {
return false, errors.New("begin Bell rule publish")
}
defer tx.Rollback()
if _, err := tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, fmt.Sprintf("rule:%d:%s", rule.TenantID, rule.RuleKey)); err != nil {
return false, errors.New("lock Bell rule key")
}
var existing string
err = tx.QueryRowContext(ctx, `SELECT id FROM bell.rule_versions WHERE tenant_id=$1 AND rule_key=$2 AND config_hash=$3`, rule.TenantID, rule.RuleKey, digest[:]).Scan(&existing)
if err == nil {
return false, tx.Commit()
}
if !errors.Is(err, sql.ErrNoRows) {
return false, errors.New("read Bell rule version")
}
var version int
if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(version),0)+1 FROM bell.rule_versions WHERE tenant_id=$1 AND rule_key=$2`, rule.TenantID, rule.RuleKey).Scan(&version); err != nil {
return false, errors.New("allocate Bell rule version")
}
if _, err := tx.ExecContext(ctx, `INSERT INTO bell.rule_versions(
id,tenant_id,site_id,rule_key,version,display_name,event_kind,minimum_severity,enabled,effective_from,config_hash
) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)`, newPrefixedID("ruv_"), rule.TenantID, rule.SiteID,
rule.RuleKey, version, rule.DisplayName, rule.EventKind, rule.MinimumSeverity, rule.Enabled, rule.EffectiveFrom, digest[:]); err != nil {
return false, fmt.Errorf("insert Bell rule version: %w", err)
}
if err := tx.Commit(); err != nil {
return false, errors.New("commit Bell rule version")
}
return true, nil
}
func (p *Postgres) EvaluateNext(ctx context.Context) (bool, error) {
tx, err := p.db.BeginTx(ctx, nil)
if err != nil {
return false, errors.New("begin Bell rule evaluation")
}
defer tx.Rollback()
var eventID, kind, severity string
var tenantID, siteID, deviceID int64
var occurredAt time.Time
err = tx.QueryRowContext(ctx, `SELECT e.id,e.tenant_id,e.site_id,e.device_id,e.kind,e.severity,e.occurred_at
FROM bell.events e WHERE NOT EXISTS(SELECT 1 FROM bell.event_rule_sweeps s WHERE s.event_id=e.id)
ORDER BY e.created_at,e.id LIMIT 1`).Scan(&eventID, &tenantID, &siteID, &deviceID, &kind, &severity, &occurredAt)
if errors.Is(err, sql.ErrNoRows) {
return false, nil
}
if err != nil {
return false, errors.New("select Bell event for rules")
}
if _, err := tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, "event-rule:"+eventID); err != nil {
return false, errors.New("lock Bell event evaluation")
}
result, err := tx.ExecContext(ctx, `INSERT INTO bell.event_rule_sweeps(event_id) VALUES($1) ON CONFLICT DO NOTHING`, eventID)
if err != nil {
return false, errors.New("claim Bell event evaluation")
}
rowsAffected, _ := result.RowsAffected()
if rowsAffected == 0 {
return false, tx.Commit()
}
rows, err := tx.QueryContext(ctx, `SELECT id,rule_key,version,display_name,event_kind,minimum_severity,enabled
FROM (SELECT DISTINCT ON (rule_key) id,rule_key,version,display_name,event_kind,minimum_severity,enabled,effective_from
FROM bell.rule_versions WHERE tenant_id=$1 AND effective_from <= $3 AND (site_id IS NULL OR site_id=$2)
ORDER BY rule_key,version DESC,effective_from DESC) latest ORDER BY rule_key`, tenantID, siteID, occurredAt)
if err != nil {
return false, errors.New("read effective Bell rules")
}
defer rows.Close()
type effectiveRule struct {
id, key, name, kind, minimum string
version int
enabled bool
}
rules := make([]effectiveRule, 0)
for rows.Next() {
var rule effectiveRule
if err := rows.Scan(&rule.id, &rule.key, &rule.version, &rule.name, &rule.kind, &rule.minimum, &rule.enabled); err != nil {
return false, errors.New("scan effective Bell rule")
}
rules = append(rules, rule)
}
if err := rows.Err(); err != nil {
return false, errors.New("iterate effective Bell rules")
}
if err := rows.Close(); err != nil {
return false, errors.New("close effective Bell rules")
}
eventRank, _ := alert.SeverityRank(severity)
for _, rule := range rules {
matched, reason := true, "matched"
minimumRank, _ := alert.SeverityRank(rule.minimum)
switch {
case !rule.enabled:
matched, reason = false, "disabled"
case rule.kind != kind:
matched, reason = false, "event_kind"
case eventRank < minimumRank:
matched, reason = false, "severity"
}
value := "no_match"
if matched {
value = "matched"
}
evaluationID := newPrefixedID("eva_")
if _, err := tx.ExecContext(ctx, `INSERT INTO bell.rule_evaluations(id,event_id,rule_version_id,result,reason) VALUES($1,$2,$3,$4,$5)`, evaluationID, eventID, rule.id, value, reason); err != nil {
return false, fmt.Errorf("append Bell rule evaluation: %w", err)
}
if !matched {
continue
}
alertID := newPrefixedID("alt_")
if _, err := tx.ExecContext(ctx, `INSERT INTO bell.alerts(id,tenant_id,site_id,rule_evaluation_id,rule_version_id,severity,title) VALUES($1,$2,$3,$4,$5,$6,$7)`, alertID, tenantID, siteID, evaluationID, rule.id, severity, rule.name); err != nil {
return false, errors.New("create Bell alert")
}
if _, err := tx.ExecContext(ctx, `INSERT INTO bell.alert_events(alert_id,event_id) VALUES($1,$2)`, alertID, eventID); err != nil {
return false, errors.New("link Bell alert event")
}
if _, err := tx.ExecContext(ctx, `INSERT INTO bell.alert_transitions(id,alert_id,sequence,from_state,to_state,actor_ref) VALUES($1,$2,1,NULL,'open','system:rule-worker')`, newPrefixedID("trn_"), alertID); err != nil {
return false, errors.New("open Bell alert")
}
_ = deviceID
}
if err := tx.Commit(); err != nil {
return false, errors.New("commit Bell rule evaluation")
}
return true, nil
}
func (p *Postgres) ListAlerts(ctx context.Context, tenantID, siteID int64, state string, limit int, cursor string) (alert.Page, error) {
if limit < 1 || limit > alert.MaxPageSize {
return alert.Page{}, errors.New("invalid alert page size")
}
if state != "" && state != "open" && state != "acknowledged" && state != "closed" {
return alert.Page{}, errors.New("invalid alert state")
}
rows, err := p.db.QueryContext(ctx, `SELECT a.id,a.severity,a.title,t.to_state,r.rule_key,r.version,a.created_at
FROM bell.alerts a JOIN bell.rule_versions r ON r.id=a.rule_version_id
JOIN LATERAL(SELECT to_state FROM bell.alert_transitions WHERE alert_id=a.id ORDER BY sequence DESC LIMIT 1)t ON true
WHERE a.tenant_id=$1 AND a.site_id=$2 AND ($3='' OR t.to_state=$3)
AND ($4='' OR (a.created_at,a.id) < (SELECT c.created_at,c.id FROM bell.alerts c WHERE c.id=$4 AND c.tenant_id=$1 AND c.site_id=$2))
ORDER BY a.created_at DESC,a.id DESC LIMIT $5`, tenantID, siteID, state, cursor, limit+1)
if err != nil {
return alert.Page{}, errors.New("list Bell alerts")
}
defer rows.Close()
page := alert.Page{Items: make([]alert.Summary, 0, limit)}
for rows.Next() {
var item alert.Summary
if err := rows.Scan(&item.ID, &item.Severity, &item.Title, &item.State, &item.RuleKey, &item.RuleVersion, &item.CreatedAt); err != nil {
return alert.Page{}, errors.New("scan Bell alert list")
}
page.Items = append(page.Items, item)
}
if err := rows.Err(); err != nil {
return alert.Page{}, errors.New("iterate Bell alert list")
}
if len(page.Items) > limit {
cursor := page.Items[limit-1].ID
page.NextCursor = &cursor
page.Items = page.Items[:limit]
}
return page, nil
}
func (p *Postgres) GetAlert(ctx context.Context, tenantID, siteID int64, alertID string) (alert.Detail, error) {
var detail alert.Detail
err := p.db.QueryRowContext(ctx, `SELECT a.id,a.severity,a.title,t.to_state,r.rule_key,r.version,a.created_at
FROM bell.alerts a JOIN bell.rule_versions r ON r.id=a.rule_version_id
JOIN LATERAL(SELECT to_state FROM bell.alert_transitions WHERE alert_id=a.id ORDER BY sequence DESC LIMIT 1)t ON true
WHERE a.tenant_id=$1 AND a.site_id=$2 AND a.id=$3`, tenantID, siteID, alertID).Scan(
&detail.ID, &detail.Severity, &detail.Title, &detail.State, &detail.RuleKey, &detail.RuleVersion, &detail.CreatedAt)
if errors.Is(err, sql.ErrNoRows) {
return alert.Detail{}, alert.ErrNotFound
}
if err != nil {
return alert.Detail{}, errors.New("read Bell alert")
}
detail.EvidenceStatus = "not_enabled"
detail.DeliveryStatus = "not_enabled"
detail.Events = []alert.EventRef{}
detail.Transitions = []alert.Transition{}
rows, err := p.db.QueryContext(ctx, `SELECT e.id,e.device_id,e.kind,e.severity,e.occurred_at FROM bell.alert_events ae JOIN bell.events e ON e.id=ae.event_id WHERE ae.alert_id=$1 ORDER BY e.occurred_at,e.id`, alertID)
if err != nil {
return alert.Detail{}, errors.New("read Bell alert events")
}
for rows.Next() {
var value alert.EventRef
if err := rows.Scan(&value.ID, &value.DeviceID, &value.Kind, &value.Severity, &value.OccurredAt); err != nil {
rows.Close()
return alert.Detail{}, errors.New("scan Bell alert event")
}
detail.Events = append(detail.Events, value)
}
if err := rows.Close(); err != nil {
return alert.Detail{}, errors.New("close Bell alert events")
}
rows, err = p.db.QueryContext(ctx, `SELECT sequence,from_state,to_state,actor_ref,note,occurred_at FROM bell.alert_transitions WHERE alert_id=$1 ORDER BY sequence`, alertID)
if err != nil {
return alert.Detail{}, errors.New("read Bell alert transitions")
}
defer rows.Close()
for rows.Next() {
var value alert.Transition
if err := rows.Scan(&value.Sequence, &value.FromState, &value.ToState, &value.ActorRef, &value.Note, &value.OccurredAt); err != nil {
return alert.Detail{}, errors.New("scan Bell alert transition")
}
detail.Transitions = append(detail.Transitions, value)
}
return detail, rows.Err()
}
func (p *Postgres) Command(ctx context.Context, tenantID, siteID int64, alertID, command, key, actorRef string, note *string) (int, alert.CommandResponse, error) {
if (command != "ack" && command != "close") || tenantID < 1 || siteID < 1 || !alert.ValidIdempotencyKey(key) || !alert.ValidActorRef(actorRef) {
return 0, alert.CommandResponse{}, errors.New("invalid Bell alert command")
}
if note != nil && len([]rune(*note)) > 500 {
return 0, alert.CommandResponse{}, errors.New("Bell alert command note is too long")
}
digest := sha256.Sum256([]byte(strings.Join([]string{alertID, command, actorRef, valueOrEmpty(note)}, "\x00")))
tx, err := p.db.BeginTx(ctx, nil)
if err != nil {
return 0, alert.CommandResponse{}, errors.New("begin Bell alert command")
}
defer tx.Rollback()
if _, err := tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, fmt.Sprintf("alert-key:%d:%s", tenantID, key)); err != nil {
return 0, alert.CommandResponse{}, errors.New("lock Bell alert command key")
}
var storedHash, storedBody []byte
var storedStatus int
err = tx.QueryRowContext(ctx, `SELECT command_hash,response_status,response_body::text FROM bell.alert_command_receipts WHERE tenant_id=$1 AND idempotency_key=$2`, tenantID, key).Scan(&storedHash, &storedStatus, &storedBody)
if err == nil {
if !bytes.Equal(storedHash, digest[:]) {
return 0, alert.CommandResponse{}, alert.ErrIdempotencyConflict
}
var response alert.CommandResponse
if err := json.Unmarshal(storedBody, &response); err != nil {
return 0, alert.CommandResponse{}, errors.New("decode Bell alert command receipt")
}
return storedStatus, response, tx.Commit()
}
if !errors.Is(err, sql.ErrNoRows) {
return 0, alert.CommandResponse{}, errors.New("read Bell alert command receipt")
}
if _, err := tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, "alert:"+alertID); err != nil {
return 0, alert.CommandResponse{}, errors.New("lock Bell alert")
}
var current, currentActor string
var currentTime time.Time
var sequence int
err = tx.QueryRowContext(ctx, `SELECT t.to_state,t.actor_ref,t.occurred_at,t.sequence FROM bell.alerts a
JOIN LATERAL(SELECT to_state,actor_ref,occurred_at,sequence FROM bell.alert_transitions WHERE alert_id=a.id ORDER BY sequence DESC LIMIT 1)t ON true
WHERE a.id=$1 AND a.tenant_id=$2 AND a.site_id=$3`, alertID, tenantID, siteID).Scan(&current, &currentActor, &currentTime, &sequence)
if errors.Is(err, sql.ErrNoRows) {
return 0, alert.CommandResponse{}, alert.ErrNotFound
}
if err != nil {
return 0, alert.CommandResponse{}, errors.New("read Bell alert state")
}
status := 200
response := alert.CommandResponse{AlertID: alertID, State: current, ActorRef: currentActor, OccurredAt: currentTime}
allowed := (command == "ack" && current == "open") || (command == "close" && current == "acknowledged")
if !allowed {
status = 409
switch {
case command == "ack" && (current == "acknowledged" || current == "closed"):
response.Code = "already_acknowledged"
_ = tx.QueryRowContext(ctx, `SELECT actor_ref,occurred_at FROM bell.alert_transitions WHERE alert_id=$1 AND to_state='acknowledged' ORDER BY sequence LIMIT 1`, alertID).Scan(&response.ActorRef, &response.OccurredAt)
case command == "close" && current == "open":
response.Code = "acknowledgement_required"
case command == "close" && current == "closed":
response.Code = "already_closed"
default:
response.Code = "invalid_state"
}
} else {
next := "acknowledged"
if command == "close" {
next = "closed"
}
response.State = next
response.ActorRef = ""
response.OccurredAt = time.Time{}
if err := tx.QueryRowContext(ctx, `INSERT INTO bell.alert_transitions(id,alert_id,sequence,from_state,to_state,actor_ref,note)
VALUES($1,$2,$3,$4,$5,$6,$7) RETURNING actor_ref,occurred_at`,
newPrefixedID("trn_"), alertID, sequence+1, current, next, actorRef, note).Scan(&response.ActorRef, &response.OccurredAt); err != nil {
return 0, alert.CommandResponse{}, errors.New("append Bell alert transition")
}
}
body, err := json.Marshal(response)
if err != nil {
return 0, alert.CommandResponse{}, errors.New("encode Bell alert command response")
}
if _, err := tx.ExecContext(ctx, `INSERT INTO bell.alert_command_receipts(tenant_id,idempotency_key,command_hash,response_status,response_body) VALUES($1,$2,$3,$4,$5::jsonb)`, tenantID, key, digest[:], status, body); err != nil {
return 0, alert.CommandResponse{}, errors.New("append Bell alert command receipt")
}
if err := tx.Commit(); err != nil {
return 0, alert.CommandResponse{}, errors.New("commit Bell alert command")
}
return status, response, nil
}
func newPrefixedID(prefix string) string { return prefix + ulid.Make().String() }
func valueOrEmpty(value *string) string {
if value == nil {
return ""
}
return *value
}
+174
View File
@@ -0,0 +1,174 @@
package store
import (
"context"
"crypto/sha256"
"database/sql"
"errors"
"fmt"
"os"
"sync"
"testing"
"time"
_ "github.com/jackc/pgx/v5/stdlib"
"yovision/bell/internal/alert"
)
func TestPostgresAlertRuleEvaluationAndFirstAckWins(t *testing.T) {
dsn := os.Getenv("YOVISION_TEST_BELL_POSTGRES_DSN")
adminDSN := os.Getenv("YOVISION_TEST_POSTGRES_ADMIN_DSN")
if dsn == "" || adminDSN == "" {
t.Skip("Bell runtime and admin PostgreSQL DSNs are not set")
}
ctx := context.Background()
admin, err := sql.Open("pgx", adminDSN)
if err != nil {
t.Fatal(err)
}
defer admin.Close()
db, err := sql.Open("pgx", dsn)
if err != nil {
t.Fatal(err)
}
defer db.Close()
repository, err := OpenPostgres(ctx, db)
if err != nil {
t.Fatal(err)
}
if err := repository.AlertReady(ctx); err != nil {
t.Fatal(err)
}
siteID := int64(902)
rule := alert.RuleSpec{TenantID: 901, SiteID: &siteID, RuleKey: "zone-entry", DisplayName: "区域闯入", EventKind: "zone_entry", MinimumSeverity: "medium", Enabled: true, EffectiveFrom: time.Now().Add(-time.Hour).UTC()}
created, err := repository.PublishRule(ctx, rule)
if err != nil || !created {
t.Fatalf("publish rule: created=%v err=%v", created, err)
}
created, err = repository.PublishRule(ctx, rule)
if err != nil || created {
t.Fatalf("idempotent rule publish: created=%v err=%v", created, err)
}
eventID := "evt_01J8XQ2K7M3P5R9T0V4W6Y8Z2Q"
payload := fmt.Sprintf(`{"id":%q,"tenant_id":901,"site_id":902,"device_id":903,"source_event_id":"T020-EVENT-1","kind":"zone_entry","severity":"high"}`, eventID)
payloadHash := sha256.Sum256([]byte(payload))
if _, err := admin.ExecContext(ctx, `INSERT INTO bell.events(id,tenant_id,site_id,device_id,source_event_id,kind,severity,occurred_at,detected_at,payload_hash,payload)
VALUES($1,901,902,903,'T020-EVENT-1','zone_entry','high',clock_timestamp(),clock_timestamp(),$3,$2::jsonb)`, eventID, payload, payloadHash[:]); err != nil {
t.Fatal(err)
}
noMatchID := "evt_01J8XQ2K7M3P5R9T0V4W6Y8Z2R"
noMatchPayload := fmt.Sprintf(`{"id":%q,"tenant_id":901,"site_id":902,"device_id":903,"source_event_id":"T020-EVENT-2","kind":"crowd","severity":"high"}`, noMatchID)
noMatchHash := sha256.Sum256([]byte(noMatchPayload))
if _, err := admin.ExecContext(ctx, `INSERT INTO bell.events(id,tenant_id,site_id,device_id,source_event_id,kind,severity,occurred_at,detected_at,payload_hash,payload)
VALUES($1,901,902,903,'T020-EVENT-2','crowd','high',clock_timestamp(),clock_timestamp(),$3,$2::jsonb)`, noMatchID, noMatchPayload, noMatchHash[:]); err != nil {
t.Fatal(err)
}
var evaluators sync.WaitGroup
errorsSeen := make(chan error, 8)
for index := 0; index < 8; index++ {
evaluators.Add(1)
go func() {
defer evaluators.Done()
for attempts := 0; attempts < 32; attempts++ {
worked, err := repository.EvaluateNext(ctx)
if err != nil {
errorsSeen <- err
return
}
if !worked {
return
}
}
}()
}
evaluators.Wait()
close(errorsSeen)
for err := range errorsSeen {
t.Error(err)
}
var alertID string
var alertCount, evaluationCount, sweepCount int
if err := db.QueryRowContext(ctx, `SELECT
(SELECT count(*) FROM bell.alerts WHERE tenant_id=901 AND site_id=902),
(SELECT count(*) FROM bell.rule_evaluations e JOIN bell.events v ON v.id=e.event_id WHERE v.tenant_id=901),
(SELECT count(*) FROM bell.event_rule_sweeps s JOIN bell.events v ON v.id=s.event_id WHERE v.tenant_id=901),
(SELECT id FROM bell.alerts WHERE tenant_id=901 AND site_id=902)`).Scan(&alertCount, &evaluationCount, &sweepCount, &alertID); err != nil {
t.Fatal(err)
}
if alertCount != 1 || evaluationCount != 2 || sweepCount != 2 {
t.Fatalf("evaluation did not converge: alerts=%d evaluations=%d sweeps=%d", alertCount, evaluationCount, sweepCount)
}
preStatus, preResponse, err := repository.Command(ctx, 901, 902, alertID, "close", "close-before-ack", "operator:closer", nil)
if err != nil || preStatus != 409 || preResponse.Code != "acknowledgement_required" || preResponse.State != "open" {
t.Fatalf("close-before-ack: status=%d response=%+v err=%v", preStatus, preResponse, err)
}
type outcome struct {
status int
response alert.CommandResponse
err error
}
results := make(chan outcome, 8)
var acknowledgers sync.WaitGroup
for index := 0; index < 8; index++ {
acknowledgers.Add(1)
go func(index int) {
defer acknowledgers.Done()
status, response, err := repository.Command(ctx, 901, 902, alertID, "ack", fmt.Sprintf("ack-key-%02d", index), fmt.Sprintf("operator:%d", index), nil)
results <- outcome{status, response, err}
}(index)
}
acknowledgers.Wait()
close(results)
winners := 0
var winner alert.CommandResponse
losers := make([]alert.CommandResponse, 0, 7)
for result := range results {
if result.err != nil {
t.Fatal(result.err)
}
if result.status == 200 {
winners++
winner = result.response
} else if result.status == 409 {
losers = append(losers, result.response)
}
}
if winners != 1 || len(losers) != 7 {
t.Fatalf("ack winners=%d losers=%d", winners, len(losers))
}
for _, loser := range losers {
if loser.ActorRef != winner.ActorRef || !loser.OccurredAt.Equal(winner.OccurredAt) || loser.Code != "already_acknowledged" {
t.Fatalf("late ack did not expose first winner: winner=%+v loser=%+v", winner, loser)
}
}
status, replayed, err := repository.Command(ctx, 901, 902, alertID, "ack", "ack-replay-key", "operator:replay", nil)
if err != nil || status != 409 || replayed.ActorRef != winner.ActorRef {
t.Fatalf("first replay receipt: status=%d response=%+v err=%v", status, replayed, err)
}
status2, replayed2, err := repository.Command(ctx, 901, 902, alertID, "ack", "ack-replay-key", "operator:replay", nil)
if err != nil || status2 != status || replayed2 != replayed {
t.Fatalf("stable receipt replay: status=%d response=%+v err=%v", status2, replayed2, err)
}
note := "different"
if _, _, err := repository.Command(ctx, 901, 902, alertID, "ack", "ack-replay-key", "operator:replay", &note); !errors.Is(err, alert.ErrIdempotencyConflict) {
t.Fatalf("expected idempotency conflict, got %v", err)
}
closeStatus, closeResponse, err := repository.Command(ctx, 901, 902, alertID, "close", "close-after-ack", "operator:closer", nil)
if err != nil || closeStatus != 200 || closeResponse.State != "closed" || closeResponse.ActorRef != "operator:closer" {
t.Fatalf("close-after-ack: status=%d response=%+v err=%v", closeStatus, closeResponse, err)
}
if _, err := db.ExecContext(ctx, `UPDATE bell.alert_transitions SET actor_ref=actor_ref WHERE alert_id=$1`, alertID); err == nil {
t.Fatal("runtime updated immutable alert transition")
}
reopened, err := OpenPostgres(ctx, db)
if err != nil {
t.Fatal(err)
}
detail, err := reopened.GetAlert(ctx, 901, 902, alertID)
if err != nil || detail.State != "closed" || len(detail.Transitions) != 3 || detail.EvidenceStatus != "not_enabled" {
t.Fatalf("restart detail: %+v %v", detail, err)
}
}