feat: deliver Sense audits to Bell (T-016)

Sense/internal/auditrelay/auditrelay_test.go

@@ -0,0 +1,132 @@
+package auditrelay
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func testEnvelope(id string) Envelope {
+	return Envelope{SchemaVersion: 1, Event: Event{
+		EventID: id, EventType: "device.created", TenantID: "tenant", SiteID: "site", DeviceID: "camera-1",
+		Actor: Actor{Type: "system", ID: "sense"}, AggregateGeneration: 1,
+		ProjectionVersions: ProjectionVersions{}, Data: json.RawMessage(`{"kind":"device_created"}`), OccurredAt: time.Unix(1, 0).UTC(),
+	}}
+}
+
+func TestClientSignsCanonicalRequest(t *testing.T) {
+	secret := bytesOf(32, 7)
+	server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		body := make([]byte, request.ContentLength)
+		_, _ = request.Body.Read(body)
+		canonical := CanonicalString(request.Method, request.URL.Path, request.Header.Get(HeaderTimestamp), request.Header.Get(HeaderNonce), body)
+		if request.Header.Get(HeaderSignature) != Signature(secret, canonical) {
+			t.Error("request signature did not match canonical vector")
+		}
+		_ = json.NewEncoder(writer).Encode(BatchResponse{Results: []Result{{EventID: "audit_00000000000000000000000000000001", Status: "accepted"}}})
+	}))
+	defer server.Close()
+	client, err := NewClient(server.URL+RelayPath, "sense-a", secret, server.Client())
+	if err != nil {
+		t.Fatal(err)
+	}
+	client.now = func() time.Time { return time.Unix(1_800_000_000, 0) }
+	client.nonce = func() (string, error) { return "AAAAAAAAAAAAAAAAAAAAAA", nil }
+	results, err := client.Send(context.Background(), []Envelope{testEnvelope("audit_00000000000000000000000000000001")})
+	if err != nil || len(results) != 1 || results[0].Status != "accepted" {
+		t.Fatalf("unexpected relay result: %+v, %v", results, err)
+	}
+}
+
+func TestEndpointAndKeySecurity(t *testing.T) {
+	if _, err := ValidateEndpoint("http://example.com" + RelayPath); err == nil {
+		t.Fatal("remote plaintext relay URL was accepted")
+	}
+	if _, err := ValidateEndpoint("https://example.com" + RelayPath + "?secret=x"); err == nil {
+		t.Fatal("relay URL query was accepted")
+	}
+	secret := bytesOf(32, 9)
+	path := filepath.Join(t.TempDir(), "keys.json")
+	document := map[string]any{"version": 1, "keys": []any{map[string]any{"key_id": "sense-a", "secret_base64url": base64.RawURLEncoding.EncodeToString(secret)}}}
+	raw, _ := json.Marshal(document)
+	if err := os.WriteFile(path, raw, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	loaded, err := LoadKey(path, "sense-a")
+	if err != nil || sha256.Sum256(loaded) != sha256.Sum256(secret) {
+		t.Fatalf("external key was not loaded: %v", err)
+	}
+}
+
+type fakeRepository struct {
+	queued    []QueuedEvent
+	completed []Completion
+}
+
+func (*fakeRepository) AuditRelayReady(context.Context) error { return nil }
+func (f *fakeRepository) ClaimAuditRelayBatch(context.Context, string, int, time.Duration) ([]QueuedEvent, error) {
+	return f.queued, nil
+}
+func (f *fakeRepository) CompleteAuditRelayBatch(_ context.Context, _ string, values []Completion) error {
+	f.completed = append([]Completion(nil), values...)
+	return nil
+}
+
+type fakeSender struct {
+	results []Result
+	err     error
+}
+
+func (f fakeSender) Send(context.Context, []Envelope) ([]Result, error) { return f.results, f.err }
+
+func TestWorkerDispositionAndBackoff(t *testing.T) {
+	first := "audit_00000000000000000000000000000001"
+	second := "audit_00000000000000000000000000000002"
+	repository := &fakeRepository{queued: []QueuedEvent{{Envelope: testEnvelope(first), LeaseToken: 1, AttemptCount: 1}, {Envelope: testEnvelope(second), LeaseToken: 2, AttemptCount: 10}}}
+	code := "schema_invalid"
+	worker, _ := NewWorker(repository, fakeSender{results: []Result{{EventID: first, Status: "accepted"}, {EventID: second, Status: "rejected", ErrorCode: &code}}}, "worker")
+	if err := worker.RelayOnce(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if repository.completed[0].Disposition != Delivered || repository.completed[1].Disposition != DeadLetter {
+		t.Fatalf("unexpected dispositions: %+v", repository.completed)
+	}
+
+	repository.completed = nil
+	worker.sender = fakeSender{err: errors.New("network")}
+	if err := worker.RelayOnce(context.Background()); err == nil {
+		t.Fatal("network failure was hidden")
+	}
+	if repository.completed[0].RetryAfter != time.Second || repository.completed[1].RetryAfter != 300*time.Second {
+		t.Fatalf("retry bounds drifted: %+v", repository.completed)
+	}
+}
+
+func TestWorkerRetriesWholeBatchForIncompleteResponse(t *testing.T) {
+	first := "audit_00000000000000000000000000000001"
+	second := "audit_00000000000000000000000000000002"
+	repository := &fakeRepository{queued: []QueuedEvent{{Envelope: testEnvelope(first), LeaseToken: 1, AttemptCount: 1}, {Envelope: testEnvelope(second), LeaseToken: 2, AttemptCount: 2}}}
+	worker, _ := NewWorker(repository, fakeSender{results: []Result{{EventID: first, Status: "accepted"}}}, "worker")
+	if err := worker.RelayOnce(context.Background()); err == nil {
+		t.Fatal("incomplete response was accepted")
+	}
+	if len(repository.completed) != 2 || repository.completed[0].Disposition != Retry || repository.completed[1].Disposition != Retry {
+		t.Fatalf("incomplete response partially completed the batch: %+v", repository.completed)
+	}
+}
+
+func bytesOf(size int, value byte) []byte {
+	result := make([]byte, size)
+	for index := range result {
+		result[index] = value
+	}
+	return result
+}

Sense/internal/auditrelay/client.go

@@ -0,0 +1,131 @@
+package auditrelay
+
+import (
+	"bytes"
+	"context"
+	"crypto/hmac"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	HeaderKeyID     = "X-YoVision-Key-Id"
+	HeaderTimestamp = "X-YoVision-Timestamp"
+	HeaderNonce     = "X-YoVision-Nonce"
+	HeaderSignature = "X-YoVision-Signature"
+	RelayPath       = "/internal/v1/audit-events:batch"
+)
+
+type Client struct {
+	endpoint   *url.URL
+	keyID      string
+	secret     []byte
+	httpClient *http.Client
+	now        func() time.Time
+	nonce      func() (string, error)
+}
+
+func NewClient(rawURL, keyID string, secret []byte, client *http.Client) (*Client, error) {
+	endpoint, err := ValidateEndpoint(rawURL)
+	if err != nil {
+		return nil, err
+	}
+	if keyID == "" || len(secret) < 32 {
+		return nil, errors.New("audit relay key ID and 32-byte secret are required")
+	}
+	if client == nil {
+		client = &http.Client{Timeout: 10 * time.Second}
+	}
+	return &Client{endpoint: endpoint, keyID: keyID, secret: append([]byte(nil), secret...), httpClient: client, now: time.Now, nonce: randomNonce}, nil
+}
+
+func ValidateEndpoint(rawURL string) (*url.URL, error) {
+	parsed, err := url.Parse(rawURL)
+	if err != nil || parsed.Host == "" || parsed.Path != RelayPath || parsed.RawQuery != "" || parsed.Fragment != "" || parsed.User != nil {
+		return nil, errors.New("invalid Bell audit relay URL")
+	}
+	host := parsed.Hostname()
+	ip := net.ParseIP(host)
+	loopback := strings.EqualFold(host, "localhost") || (ip != nil && ip.IsLoopback())
+	if parsed.Scheme != "https" && !(parsed.Scheme == "http" && loopback) {
+		return nil, errors.New("Bell audit relay URL requires HTTPS outside loopback")
+	}
+	return parsed, nil
+}
+
+func CanonicalString(method, path, timestamp, nonce string, body []byte) string {
+	digest := sha256.Sum256(body)
+	return strings.Join([]string{method, path, timestamp, nonce, hex.EncodeToString(digest[:])}, "\n")
+}
+
+func Signature(secret []byte, canonical string) string {
+	mac := hmac.New(sha256.New, secret)
+	_, _ = mac.Write([]byte(canonical))
+	return base64.RawURLEncoding.EncodeToString(mac.Sum(nil))
+}
+
+func randomNonce() (string, error) {
+	value := make([]byte, 16)
+	if _, err := rand.Read(value); err != nil {
+		return "", err
+	}
+	return base64.RawURLEncoding.EncodeToString(value), nil
+}
+
+func (c *Client) Send(ctx context.Context, events []Envelope) ([]Result, error) {
+	if len(events) < 1 || len(events) > MaxBatchSize {
+		return nil, errors.New("audit relay batch must contain 1 to 100 events")
+	}
+	body, err := json.Marshal(BatchRequest{Events: events})
+	if err != nil || len(body) > MaxBodyBytes {
+		return nil, errors.New("encode audit relay batch")
+	}
+	requestContext, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+	timestamp := strconv.FormatInt(c.now().UTC().Unix(), 10)
+	nonce, err := c.nonce()
+	if err != nil {
+		return nil, errors.New("generate audit relay nonce")
+	}
+	request, err := http.NewRequestWithContext(requestContext, http.MethodPost, c.endpoint.String(), bytes.NewReader(body))
+	if err != nil {
+		return nil, errors.New("create audit relay request")
+	}
+	request.Header.Set("Content-Type", "application/json")
+	request.Header.Set(HeaderKeyID, c.keyID)
+	request.Header.Set(HeaderTimestamp, timestamp)
+	request.Header.Set(HeaderNonce, nonce)
+	request.Header.Set(HeaderSignature, Signature(c.secret, CanonicalString(http.MethodPost, RelayPath, timestamp, nonce, body)))
+	response, err := c.httpClient.Do(request)
+	if err != nil {
+		return nil, errors.New("send audit relay request")
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
+		return nil, fmt.Errorf("Bell audit relay returned HTTP %d", response.StatusCode)
+	}
+	decoder := json.NewDecoder(io.LimitReader(response.Body, MaxBodyBytes+1))
+	decoder.DisallowUnknownFields()
+	var decoded BatchResponse
+	if err := decoder.Decode(&decoded); err != nil {
+		return nil, errors.New("decode audit relay response")
+	}
+	var trailing any
+	if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
+		return nil, errors.New("audit relay response contains trailing data")
+	}
+	return decoded.Results, nil
+}

Sense/internal/auditrelay/key.go

@@ -0,0 +1,53 @@
+package auditrelay
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"io"
+	"os"
+	"regexp"
+)
+
+var keyIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`)
+
+type keyFile struct {
+	Version int `json:"version"`
+	Keys    []struct {
+		KeyID  string `json:"key_id"`
+		Secret string `json:"secret_base64url"`
+	} `json:"keys"`
+}
+
+func LoadKey(path, keyID string) ([]byte, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, errors.New("read audit relay key file")
+	}
+	var document keyFile
+	decoder := json.NewDecoder(bytes.NewReader(raw))
+	decoder.DisallowUnknownFields()
+	if err := decoder.Decode(&document); err != nil || document.Version != 1 || len(document.Keys) == 0 {
+		return nil, errors.New("invalid audit relay key file")
+	}
+	var trailing any
+	if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
+		return nil, errors.New("invalid audit relay key file")
+	}
+	values := make(map[string][]byte, len(document.Keys))
+	for _, candidate := range document.Keys {
+		secret, err := base64.RawURLEncoding.DecodeString(candidate.Secret)
+		if err != nil || !keyIDPattern.MatchString(candidate.KeyID) || len(secret) < 32 {
+			return nil, errors.New("invalid audit relay secret")
+		}
+		if _, duplicate := values[candidate.KeyID]; duplicate {
+			return nil, errors.New("duplicate audit relay key ID")
+		}
+		values[candidate.KeyID] = secret
+	}
+	if secret, exists := values[keyID]; exists {
+		return secret, nil
+	}
+	return nil, errors.New("audit relay key ID not found")
+}

Sense/internal/auditrelay/model.go

@@ -0,0 +1,89 @@
+// Package auditrelay delivers Sense-owned audit facts to Bell without sharing databases.
+package auditrelay
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"time"
+)
+
+const (
+	MaxBatchSize  = 100
+	MaxBodyBytes  = 1 << 20
+	LeaseDuration = 30 * time.Second
+)
+
+type Actor struct {
+	Type string `json:"type"`
+	ID   string `json:"id"`
+}
+
+type ProjectionVersions struct {
+	QuotaSourceVersion      *int64 `json:"quota_source_version"`
+	AreaPolicySourceVersion *int64 `json:"area_policy_source_version"`
+}
+
+type Event struct {
+	EventID             string             `json:"event_id"`
+	EventType           string             `json:"event_type"`
+	TenantID            string             `json:"tenant_id"`
+	SiteID              string             `json:"site_id"`
+	DeviceID            string             `json:"device_id"`
+	Actor               Actor              `json:"actor"`
+	Reason              *string            `json:"reason"`
+	TraceID             *string            `json:"trace_id"`
+	AggregateGeneration int64              `json:"aggregate_generation"`
+	ProjectionVersions  ProjectionVersions `json:"projection_versions"`
+	Data                json.RawMessage    `json:"data"`
+	OccurredAt          time.Time          `json:"occurred_at"`
+}
+
+type Envelope struct {
+	SchemaVersion int   `json:"schema_version"`
+	Event         Event `json:"event"`
+}
+
+type BatchRequest struct {
+	Events []Envelope `json:"events"`
+}
+
+type Result struct {
+	EventID   string  `json:"event_id"`
+	Status    string  `json:"status"`
+	ErrorCode *string `json:"error_code,omitempty"`
+}
+
+type BatchResponse struct {
+	Results []Result `json:"results"`
+}
+
+type QueuedEvent struct {
+	Envelope
+	LeaseToken   int64
+	AttemptCount int
+}
+
+type Disposition string
+
+const (
+	Delivered  Disposition = "delivered"
+	DeadLetter Disposition = "dead_letter"
+	Retry      Disposition = "retry"
+)
+
+type Completion struct {
+	EventID     string
+	LeaseToken  int64
+	Disposition Disposition
+	ErrorCode   string
+	RetryAfter  time.Duration
+}
+
+var ErrLeaseLost = errors.New("audit relay lease lost")
+
+type Repository interface {
+	AuditRelayReady(context.Context) error
+	ClaimAuditRelayBatch(context.Context, string, int, time.Duration) ([]QueuedEvent, error)
+	CompleteAuditRelayBatch(context.Context, string, []Completion) error
+}

Sense/internal/auditrelay/worker.go

@@ -0,0 +1,126 @@
+package auditrelay
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"regexp"
+	"time"
+)
+
+var stableErrorCode = regexp.MustCompile(`^[a-z][a-z0-9_]{0,63}$`)
+
+type Sender interface {
+	Send(context.Context, []Envelope) ([]Result, error)
+}
+
+type Worker struct {
+	repository Repository
+	sender     Sender
+	owner      string
+}
+
+func NewWorker(repository Repository, sender Sender, owner string) (*Worker, error) {
+	if repository == nil || sender == nil || owner == "" {
+		return nil, errors.New("audit relay worker dependencies are required")
+	}
+	return &Worker{repository: repository, sender: sender, owner: owner}, nil
+}
+
+func RetryDelay(attempt int) time.Duration {
+	if attempt < 1 {
+		attempt = 1
+	}
+	if attempt > 9 {
+		return 300 * time.Second
+	}
+	delay := time.Second << (attempt - 1)
+	if delay > 300*time.Second {
+		return 300 * time.Second
+	}
+	return delay
+}
+
+func (w *Worker) RelayOnce(ctx context.Context) error {
+	queued, err := w.repository.ClaimAuditRelayBatch(ctx, w.owner, MaxBatchSize, LeaseDuration)
+	if err != nil || len(queued) == 0 {
+		return err
+	}
+	events := make([]Envelope, len(queued))
+	for index := range queued {
+		events[index] = queued[index].Envelope
+	}
+	results, sendErr := w.sender.Send(ctx, events)
+	if sendErr != nil {
+		completions := make([]Completion, len(queued))
+		for index, value := range queued {
+			completions[index] = Completion{EventID: value.Event.EventID, LeaseToken: value.LeaseToken, Disposition: Retry, ErrorCode: "delivery_failed", RetryAfter: RetryDelay(value.AttemptCount)}
+		}
+		if err := w.repository.CompleteAuditRelayBatch(ctx, w.owner, completions); err != nil {
+			return err
+		}
+		return sendErr
+	}
+	if len(results) != len(queued) {
+		return w.retryAll(ctx, queued, "invalid_response")
+	}
+	expected := make(map[string]bool, len(queued))
+	for _, value := range queued {
+		expected[value.Event.EventID] = true
+	}
+	byID := make(map[string]Result, len(results))
+	for _, result := range results {
+		validStatus := ((result.Status == "accepted" || result.Status == "duplicate") && result.ErrorCode == nil) ||
+			(result.Status == "rejected" && result.ErrorCode != nil && stableErrorCode.MatchString(*result.ErrorCode))
+		if !expected[result.EventID] || !validStatus {
+			return w.retryAll(ctx, queued, "invalid_response")
+		}
+		if _, duplicate := byID[result.EventID]; duplicate {
+			return w.retryAll(ctx, queued, "invalid_response")
+		}
+		byID[result.EventID] = result
+	}
+	completions := make([]Completion, 0, len(queued))
+	for _, value := range queued {
+		result, ok := byID[value.Event.EventID]
+		if !ok {
+			return w.retryAll(ctx, queued, "invalid_response")
+		}
+		switch result.Status {
+		case "accepted", "duplicate":
+			completions = append(completions, Completion{EventID: value.Event.EventID, LeaseToken: value.LeaseToken, Disposition: Delivered})
+		case "rejected":
+			completions = append(completions, Completion{EventID: value.Event.EventID, LeaseToken: value.LeaseToken, Disposition: DeadLetter, ErrorCode: *result.ErrorCode})
+		}
+	}
+	return w.repository.CompleteAuditRelayBatch(ctx, w.owner, completions)
+}
+
+func (w *Worker) retryAll(ctx context.Context, queued []QueuedEvent, code string) error {
+	values := make([]Completion, len(queued))
+	for index, value := range queued {
+		values[index] = Completion{EventID: value.Event.EventID, LeaseToken: value.LeaseToken, Disposition: Retry, ErrorCode: code, RetryAfter: RetryDelay(value.AttemptCount)}
+	}
+	if err := w.repository.CompleteAuditRelayBatch(ctx, w.owner, values); err != nil {
+		return err
+	}
+	return fmt.Errorf("audit relay %s", code)
+}
+
+func (w *Worker) Run(ctx context.Context, interval time.Duration, report func(error)) {
+	if interval <= 0 {
+		interval = time.Second
+	}
+	for {
+		if err := w.RelayOnce(ctx); err != nil && ctx.Err() == nil && report != nil {
+			report(err)
+		}
+		timer := time.NewTimer(interval)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return
+		case <-timer.C:
+		}
+	}
+}