feat: implement T-019 reliable event ingress
Harness governance / validate (pull_request) Has been cancelled
Harness governance / validate (pull_request) Has been cancelled
This commit is contained in:
+36
-3
@@ -1,6 +1,6 @@
|
||||
# YoVision Brain 单路工程原型
|
||||
|
||||
T-017 提供一条可见、可重复的工程链路:单路 frame source → 匿名人员检测/fixture → track → 归一化多边形区域 → `zone_entry` 候选事实。它不是生产模型、不是 NVR,也没有 Brain→Bell transport。
|
||||
T-017 提供可见、可重复的单路工程链路;T-019 为 `zone_entry` 候选增加默认关闭的可靠 Brain→Bell transport。它仍不是生产模型或 NVR:推理演示保持单路,事件先进入仓库外 SQLite Outbox,再由内部 HMAC client 投递给 Bell。
|
||||
|
||||
## 环境
|
||||
|
||||
@@ -36,6 +36,39 @@ URL 文件只允许一行、最多 4096 字节,服务不会在状态、页面
|
||||
|
||||
真实流使用 OpenCV 内置 HOG/SVM 人员检测和轻量 centroid tracker,只验证可替换端口与区域链路。它不是 M3 生产模型,不能据此承诺召回率、误报率、GPU 容量或 16/128 路能力。
|
||||
|
||||
## 可选 Brain → Bell 事件投递
|
||||
|
||||
默认不传 `--event-ingress-config`,工程原型行为与 T-017 相同。启用时,配置、key 和 SQLite 文件都必须位于仓库外绝对路径;HTTP 只允许显式回环,非回环必须 HTTPS。key 文件也供 Bell event ingress 读取,但不得与 Sense 审计 key 混用:
|
||||
|
||||
```json
|
||||
{"version":1,"keys":[{"key_id":"brain-a","producer_id":"brain-main","secret_base64url":"<至少32字节随机值的无填充base64url>"}]}
|
||||
```
|
||||
|
||||
Brain 配置示例(占位 ID 必须与 Bell 管理员创建的 `bell.event_ingress_bindings` 一致):
|
||||
|
||||
```json
|
||||
{
|
||||
"version": 1,
|
||||
"producer_id": "brain-main",
|
||||
"tenant_id": 1,
|
||||
"site_id": 1,
|
||||
"device_id": 1,
|
||||
"modality": "video",
|
||||
"severity": "medium",
|
||||
"config_version": "brain-demo-v1",
|
||||
"bell_url": "http://127.0.0.1:8081/internal/v1/event-candidates",
|
||||
"key_id": "brain-a",
|
||||
"key_file": "D:\\private\\brain-event-keys.json",
|
||||
"outbox_path": "D:\\private\\brain-event-outbox.sqlite3"
|
||||
}
|
||||
```
|
||||
|
||||
```powershell
|
||||
python -m Brain.yovision_brain --source synthetic --event-ingress-config D:\private\brain-event-ingress.json
|
||||
```
|
||||
|
||||
Outbox 使用 WAL 和 `synchronous=FULL`,首次打开即绑定 producer/tenant/site/device;更换身份必须使用新的 Outbox 路径,不能让旧队列借新 producer 发送。最多保留 10,000 条待投递;只有 Bell `accepted/duplicate` 才确认 delivered。网络、401 与 5xx 按 1~300 秒退避、最多 100 次;稳定 4xx 进入 dead letter。终态行不会自动删除。状态页只显示计数和稳定错误码,不显示 URL、key、payload 或数据库路径;client 显式忽略环境 HTTP proxy。
|
||||
|
||||
## 验证
|
||||
|
||||
```powershell
|
||||
@@ -48,6 +81,6 @@ python -m compileall -q Brain
|
||||
## 边界
|
||||
|
||||
- `source_event_id` 由 Brain 产生,平台 `evt_` ULID 由 Bell 产生。
|
||||
- 事件只保留在最多 100 项的内存环中;重启即丢失是刻意边界。
|
||||
- T-018 冻结并实现身份映射、候选契约、持久 Outbox、HMAC、幂等和 Bell ingress。
|
||||
- 页面仍只保留最多 100 项的内存环;启用 T-019 后,可靠性由独立 SQLite Outbox 承担,不能从页面事件环推断投递状态。
|
||||
- Bell 平台 ID 只由 Bell 生成;相同 producer/source candidate 重投返回原 ID,Brain 不改写 source ID 规避冲突。
|
||||
- `D:\OPC\silver_pose` 保持独立,不是本模块的源码目录、运行依赖或模型来源路径。
|
||||
|
||||
@@ -0,0 +1,291 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from datetime import datetime, timezone
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
from Brain.yovision_brain.domain import EventCandidate
|
||||
from Brain.yovision_brain.ingress import (
|
||||
BrainEventIngress,
|
||||
DeliveryResult,
|
||||
EventIngressClient,
|
||||
EventOutbox,
|
||||
EventRelayWorker,
|
||||
IngressConfig,
|
||||
KeyMaterial,
|
||||
PermanentDelivery,
|
||||
RetryableDelivery,
|
||||
load_config,
|
||||
map_candidate,
|
||||
)
|
||||
|
||||
|
||||
EVENT_ID = "evt_01J8XQ2K7M3P5R9T0V4W6Y8Z2B"
|
||||
|
||||
|
||||
def candidate(source_event_id: str = "BRN-test-0001", fixture: bool = True) -> EventCandidate:
|
||||
return EventCandidate(
|
||||
source_event_id=source_event_id,
|
||||
kind="zone_entry",
|
||||
source_ref="must-not-leave-brain",
|
||||
track_id="P-1",
|
||||
zone_id="zone-1",
|
||||
zone_version=3,
|
||||
occurred_at=datetime(2026, 8, 11, tzinfo=timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||
confidence=None,
|
||||
fixture=fixture,
|
||||
)
|
||||
|
||||
|
||||
def config(root: Path, bell_url: str = "http://127.0.0.1:8081/internal/v1/event-candidates") -> IngressConfig:
|
||||
return IngressConfig(
|
||||
producer_id="brain-main",
|
||||
tenant_id=1,
|
||||
site_id=2,
|
||||
device_id=3,
|
||||
modality="video",
|
||||
severity="medium",
|
||||
config_version="brain-demo-v1",
|
||||
bell_url=bell_url,
|
||||
key_id="brain-a",
|
||||
key_file=root / "keys.json",
|
||||
outbox_path=root / "outbox.sqlite3",
|
||||
)
|
||||
|
||||
|
||||
class ConfigAndMapperTests(unittest.TestCase):
|
||||
def test_external_config_binds_key_and_rejects_remote_plaintext(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
secret = bytes(range(32))
|
||||
key_file = root / "keys.json"
|
||||
key_file.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"version": 1,
|
||||
"keys": [
|
||||
{
|
||||
"key_id": "brain-a",
|
||||
"producer_id": "brain-main",
|
||||
"secret_base64url": base64.urlsafe_b64encode(secret).rstrip(b"=").decode("ascii"),
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
config_file = root / "config.json"
|
||||
document = {
|
||||
"version": 1,
|
||||
"producer_id": "brain-main",
|
||||
"tenant_id": 1,
|
||||
"site_id": 2,
|
||||
"device_id": 3,
|
||||
"modality": "video",
|
||||
"severity": "medium",
|
||||
"config_version": "brain-demo-v1",
|
||||
"bell_url": "http://127.0.0.1:8081/internal/v1/event-candidates",
|
||||
"key_id": "brain-a",
|
||||
"key_file": str(key_file.resolve()),
|
||||
"outbox_path": str((root / "outbox.sqlite3").resolve()),
|
||||
}
|
||||
config_file.write_text(json.dumps(document), encoding="utf-8")
|
||||
loaded, key = load_config(str(config_file.resolve()))
|
||||
self.assertEqual(("brain-main", secret), (loaded.producer_id, key.secret))
|
||||
|
||||
document["bell_url"] = "http://camera.example/internal/v1/event-candidates"
|
||||
config_file.write_text(json.dumps(document), encoding="utf-8")
|
||||
with self.assertRaisesRegex(ValueError, "HTTPS"):
|
||||
load_config(str(config_file.resolve()))
|
||||
|
||||
def test_mapper_produces_complete_candidate_without_sensitive_source(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
payload = json.loads(map_candidate(candidate(), config(Path(directory))))
|
||||
self.assertNotIn("id", payload)
|
||||
self.assertNotIn("source_ref", json.dumps(payload))
|
||||
self.assertEqual("test", payload["outcome"])
|
||||
self.assertEqual("auto", payload["outcome_source"])
|
||||
self.assertIsNone(payload["confidence"])
|
||||
self.assertEqual(payload["occurred_at"], payload["detected_at"])
|
||||
self.assertEqual(0.0, payload["latency_seconds"])
|
||||
self.assertEqual([{"device_id": 3, "modality": "video", "role": "primary"}], payload["sensors"])
|
||||
expected = {
|
||||
"schema_version", "source_event_id", "tenant_id", "site_id", "device_id", "sensors", "kind",
|
||||
"severity", "confidence", "occurred_at", "detected_at", "latency_seconds", "config_version",
|
||||
"rule", "subject", "observation", "evidence", "dedup_key", "aggregated_into", "outcome",
|
||||
"outcome_source", "outcome_reason", "diagnostics", "ext",
|
||||
}
|
||||
self.assertEqual(expected, set(payload))
|
||||
|
||||
|
||||
class OutboxTests(unittest.TestCase):
|
||||
def test_crash_recovery_retry_and_terminal_records_are_durable(self) -> None:
|
||||
clock = [1_000.0]
|
||||
now = lambda: clock[0]
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "outbox.sqlite3"
|
||||
payload = map_candidate(candidate(), config(Path(directory)))
|
||||
outbox = EventOutbox(path, now=now)
|
||||
self.assertTrue(outbox.enqueue(payload))
|
||||
self.assertFalse(outbox.enqueue(payload))
|
||||
leased = outbox.lease_one()
|
||||
self.assertIsNotNone(leased)
|
||||
outbox.close() # Simulate a process crash while the lease is held.
|
||||
|
||||
clock[0] += 31.0
|
||||
outbox = EventOutbox(path, now=now)
|
||||
recovered = outbox.lease_one()
|
||||
self.assertIsNotNone(recovered)
|
||||
self.assertEqual(2, recovered.attempt_count)
|
||||
outbox.mark_delivered(recovered, DeliveryResult("duplicate", EVENT_ID))
|
||||
self.assertEqual(1, outbox.status()["delivered"])
|
||||
outbox.close()
|
||||
|
||||
outbox = EventOutbox(path, now=now)
|
||||
status = outbox.status()
|
||||
self.assertEqual((1, 0), (status["delivered"], status["queued"]))
|
||||
outbox.close()
|
||||
|
||||
def test_source_id_conflict_is_not_silently_overwritten(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
outbox = EventOutbox(root / "outbox.sqlite3")
|
||||
outbox.enqueue(map_candidate(candidate(), config(root)))
|
||||
changed = json.loads(map_candidate(candidate(), config(root)))
|
||||
changed["config_version"] = "different"
|
||||
with self.assertRaisesRegex(ValueError, "different candidate"):
|
||||
outbox.enqueue(json.dumps(changed, sort_keys=True, separators=(",", ":")).encode())
|
||||
outbox.close()
|
||||
|
||||
def test_crash_on_final_attempt_becomes_dead_letter(self) -> None:
|
||||
clock = [1_000.0]
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
path = root / "outbox.sqlite3"
|
||||
outbox = EventOutbox(path, now=lambda: clock[0])
|
||||
outbox.enqueue(map_candidate(candidate(), config(root)))
|
||||
for _ in range(99):
|
||||
item = outbox.lease_one()
|
||||
self.assertIsNotNone(item)
|
||||
outbox.mark_retry(item, "network_error")
|
||||
clock[0] += 301.0
|
||||
final = outbox.lease_one()
|
||||
self.assertEqual(100, final.attempt_count)
|
||||
outbox.close()
|
||||
clock[0] += 31.0
|
||||
outbox = EventOutbox(path, now=lambda: clock[0])
|
||||
self.assertIsNone(outbox.lease_one())
|
||||
self.assertEqual(1, outbox.status()["dead_letter"])
|
||||
outbox.close()
|
||||
|
||||
def test_outbox_cannot_be_reused_for_another_producer_identity(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
path = root / "outbox.sqlite3"
|
||||
outbox = EventOutbox(path)
|
||||
original = config(root)
|
||||
outbox.bind_identity(original)
|
||||
outbox.close()
|
||||
outbox = EventOutbox(path)
|
||||
changed = IngressConfig(**{**original.__dict__, "producer_id": "brain-other"})
|
||||
with self.assertRaisesRegex(ValueError, "different producer"):
|
||||
outbox.bind_identity(changed)
|
||||
outbox.close()
|
||||
|
||||
|
||||
class _BellHandler(BaseHTTPRequestHandler):
|
||||
secret = bytes(range(32))
|
||||
verified = False
|
||||
failure: str | None = None
|
||||
|
||||
def log_message(self, _format: str, *_args: object) -> None:
|
||||
return
|
||||
|
||||
def do_POST(self) -> None: # noqa: N802
|
||||
try:
|
||||
length = int(self.headers["Content-Length"])
|
||||
body = self.rfile.read(length)
|
||||
timestamp = self.headers["X-YoVision-Timestamp"]
|
||||
nonce = self.headers["X-YoVision-Nonce"]
|
||||
canonical = "\n".join(("POST", self.path, timestamp, nonce, hashlib.sha256(body).hexdigest())).encode()
|
||||
encoded_signature = self.headers["X-YoVision-Signature"]
|
||||
supplied = base64.urlsafe_b64decode(encoded_signature + "=" * (-len(encoded_signature) % 4))
|
||||
type(self).verified = hmac.compare_digest(supplied, hmac.new(self.secret, canonical, hashlib.sha256).digest())
|
||||
envelope = json.loads(body)
|
||||
response = json.dumps(
|
||||
{
|
||||
"schema_version": 1,
|
||||
"producer_id": envelope["producer_id"],
|
||||
"source_event_id": envelope["candidate"]["source_event_id"],
|
||||
"event_id": EVENT_ID,
|
||||
"status": "accepted",
|
||||
}
|
||||
).encode()
|
||||
self.send_response(201)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(response)))
|
||||
self.end_headers()
|
||||
self.wfile.write(response)
|
||||
except Exception as exc: # pragma: no cover - only improves test diagnostics
|
||||
type(self).failure = repr(exc)
|
||||
self.send_response(500)
|
||||
self.end_headers()
|
||||
|
||||
|
||||
class ClientAndWorkerTests(unittest.TestCase):
|
||||
def test_client_signs_and_accepts_bell_response(self) -> None:
|
||||
server = HTTPServer(("127.0.0.1", 0), _BellHandler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
client = EventIngressClient(
|
||||
config(root, f"http://127.0.0.1:{server.server_port}/internal/v1/event-candidates"),
|
||||
KeyMaterial("brain-a", "brain-main", _BellHandler.secret),
|
||||
now=lambda: 1_800_000_000.0,
|
||||
)
|
||||
try:
|
||||
result = client.deliver(map_candidate(candidate(), config(root)))
|
||||
except RetryableDelivery as exc:
|
||||
self.fail(f"test Bell handler failed: {_BellHandler.failure}; client={exc.code}")
|
||||
self.assertEqual(("accepted", EVENT_ID), (result.status, result.event_id))
|
||||
self.assertTrue(_BellHandler.verified)
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join(timeout=2.0)
|
||||
|
||||
def test_worker_distinguishes_retryable_and_permanent_failures(self) -> None:
|
||||
class FailingClient:
|
||||
def __init__(self, failure: Exception) -> None:
|
||||
self.failure = failure
|
||||
|
||||
def deliver(self, _payload: bytes) -> DeliveryResult:
|
||||
raise self.failure
|
||||
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
payload = map_candidate(candidate(), config(root))
|
||||
outbox = EventOutbox(root / "retry.sqlite3", now=lambda: 100.0)
|
||||
outbox.enqueue(payload)
|
||||
EventRelayWorker(outbox, FailingClient(RetryableDelivery("unauthorized"))).run_once()
|
||||
self.assertEqual((1, 0), (outbox.status()["queued"], outbox.status()["dead_letter"]))
|
||||
outbox.close()
|
||||
|
||||
outbox = EventOutbox(root / "dead.sqlite3", now=lambda: 100.0)
|
||||
outbox.enqueue(payload)
|
||||
EventRelayWorker(outbox, FailingClient(PermanentDelivery("source_event_conflict"))).run_once()
|
||||
self.assertEqual(1, outbox.status()["dead_letter"])
|
||||
outbox.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -2,8 +2,50 @@ from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from Brain.yovision_brain.runtime import DemoEngine
|
||||
from Brain.yovision_brain.source import SyntheticSource, cv2
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from Brain.yovision_brain.domain import Box, Detection
|
||||
from Brain.yovision_brain.runtime import DemoEngine, EventIngressUnavailable
|
||||
from Brain.yovision_brain.source import FramePacket, SyntheticSource, cv2, np
|
||||
|
||||
|
||||
class _TwoFrameSource:
|
||||
fixture = True
|
||||
mode = "test"
|
||||
label = "test"
|
||||
source_ref = "safe-source"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.index = 0
|
||||
|
||||
def read(self) -> FramePacket:
|
||||
boxes = (Box(0.10, 0.4, 0.20, 0.8), Box(0.60, 0.4, 0.70, 0.8))
|
||||
detection = Detection("P-1", "person", boxes[min(self.index, 1)])
|
||||
self.index += 1
|
||||
return FramePacket(np.zeros((180, 320, 3), dtype=np.uint8), datetime.now(timezone.utc), (detection,))
|
||||
|
||||
def close(self) -> None:
|
||||
return
|
||||
|
||||
|
||||
class _Ingress:
|
||||
def __init__(self, fail: bool = False) -> None:
|
||||
self.items = []
|
||||
self.fail = fail
|
||||
|
||||
def submit(self, value: object) -> None:
|
||||
if self.fail:
|
||||
raise RuntimeError("disk unavailable")
|
||||
self.items.append(value)
|
||||
|
||||
def start(self) -> None:
|
||||
return
|
||||
|
||||
def stop(self) -> None:
|
||||
return
|
||||
|
||||
def status(self) -> dict[str, object]:
|
||||
return {"enabled": True, "queued": len(self.items)}
|
||||
|
||||
|
||||
@unittest.skipIf(cv2 is None, "pinned OpenCV package is not installed")
|
||||
@@ -28,6 +70,23 @@ class RuntimeTests(unittest.TestCase):
|
||||
self.assertEqual(updated.version, 2)
|
||||
self.assertEqual(engine.state()["zone"]["name"], "新区域")
|
||||
|
||||
def test_event_is_persisted_before_it_is_exposed(self) -> None:
|
||||
ingress = _Ingress()
|
||||
engine = DemoEngine(_TwoFrameSource(), event_ingress=ingress)
|
||||
engine.step()
|
||||
engine.step()
|
||||
state = engine.state()
|
||||
self.assertEqual(1, len(ingress.items))
|
||||
self.assertEqual("outbox_persisted", state["events"][0]["delivery_status"])
|
||||
self.assertEqual(1, state["event_ingress"]["queued"])
|
||||
|
||||
def test_outbox_failure_does_not_claim_delivery(self) -> None:
|
||||
engine = DemoEngine(_TwoFrameSource(), event_ingress=_Ingress(fail=True))
|
||||
engine.step()
|
||||
with self.assertRaises(EventIngressUnavailable):
|
||||
engine.step()
|
||||
self.assertEqual([], engine.state()["events"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -40,6 +40,22 @@ class UIContractTests(unittest.TestCase):
|
||||
self.assertNotRegex(visible_without_script, r"[\U0001F300-\U0001FAFF]")
|
||||
self.assertEqual(self.html.count("__BRAIN_DEMO_TOKEN__"), 1)
|
||||
|
||||
def test_reliable_delivery_status_is_visible_without_configuration_details(self) -> None:
|
||||
required = (
|
||||
'id="event-ingress"',
|
||||
'id="ingress-queued"',
|
||||
'id="ingress-dead-letter"',
|
||||
"Outbox 已持久化",
|
||||
"Bell 返回 accepted 或 duplicate 后才计为已送达",
|
||||
"event_ingress: {enabled: false}",
|
||||
)
|
||||
for marker in required:
|
||||
with self.subTest(marker=marker):
|
||||
self.assertIn(marker, self.html)
|
||||
for forbidden in ("bell_url", "key_file", "outbox_path"):
|
||||
with self.subTest(forbidden=forbidden):
|
||||
self.assertNotIn(forbidden, self.html)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
from .ingress import BrainEventIngress
|
||||
from .runtime import DemoEngine
|
||||
from .server import parse_bind, serve
|
||||
from .source import StreamSource, SyntheticSource, read_stream_url, require_opencv
|
||||
@@ -14,6 +15,7 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
parser.add_argument("--stream-url-file", help="absolute external file containing one RTSP URL")
|
||||
parser.add_argument("--bind", default="127.0.0.1:8090", help="loopback bind address")
|
||||
parser.add_argument("--fps", type=float, default=2.0, help="prototype processing FPS (0, 30]")
|
||||
parser.add_argument("--event-ingress-config", help="absolute external Brain-to-Bell ingress configuration")
|
||||
return parser
|
||||
|
||||
|
||||
@@ -30,7 +32,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
if args.stream_url_file:
|
||||
raise ValueError("--stream-url-file is only valid for stream mode")
|
||||
source = SyntheticSource()
|
||||
engine = DemoEngine(source, fps=args.fps)
|
||||
event_ingress = BrainEventIngress.from_file(args.event_ingress_config) if args.event_ingress_config else None
|
||||
engine = DemoEngine(source, fps=args.fps, event_ingress=event_ingress)
|
||||
except (RuntimeError, ValueError) as exc:
|
||||
print(f"Brain demo configuration error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
@@ -0,0 +1,645 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import hmac
|
||||
import ipaddress
|
||||
import json
|
||||
import re
|
||||
import secrets
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlsplit
|
||||
from urllib.request import HTTPRedirectHandler, ProxyHandler, Request, build_opener
|
||||
|
||||
from .domain import EventCandidate
|
||||
|
||||
|
||||
INGRESS_PATH = "/internal/v1/event-candidates"
|
||||
MAX_PAYLOAD_BYTES = 1 << 20
|
||||
MAX_QUEUED = 10_000
|
||||
MAX_ATTEMPTS = 100
|
||||
HTTP_TIMEOUT_SECONDS = 10.0
|
||||
_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$")
|
||||
_SOURCE_ID = re.compile(r"^[A-Za-z0-9_-]{1,128}$")
|
||||
_EVENT_ID = re.compile(r"^evt_[0-9A-HJKMNP-TV-Z]{26}$")
|
||||
_ERROR_CODE = re.compile(r"^[a-z][a-z0-9_]{0,63}$")
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class IngressConfig:
|
||||
producer_id: str
|
||||
tenant_id: int
|
||||
site_id: int
|
||||
device_id: int
|
||||
modality: str
|
||||
severity: str
|
||||
config_version: str
|
||||
bell_url: str
|
||||
key_id: str
|
||||
key_file: Path
|
||||
outbox_path: Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class KeyMaterial:
|
||||
key_id: str
|
||||
producer_id: str
|
||||
secret: bytes
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LeasedEvent:
|
||||
source_event_id: str
|
||||
payload: bytes
|
||||
attempt_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DeliveryResult:
|
||||
status: str
|
||||
event_id: str
|
||||
|
||||
|
||||
class RetryableDelivery(RuntimeError):
|
||||
def __init__(self, code: str) -> None:
|
||||
super().__init__(code)
|
||||
self.code = _safe_error(code)
|
||||
|
||||
|
||||
class PermanentDelivery(RuntimeError):
|
||||
def __init__(self, code: str) -> None:
|
||||
super().__init__(code)
|
||||
self.code = _safe_error(code)
|
||||
|
||||
|
||||
class _NoRedirect(HTTPRedirectHandler):
|
||||
def redirect_request(self, _request: Request, _file_pointer: object, _code: int, _message: str, _headers: object, _new_url: str) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _is_within(path: Path, parent: Path) -> bool:
|
||||
try:
|
||||
path.relative_to(parent)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _external_absolute_path(raw: object, field: str, *, must_exist: bool) -> Path:
|
||||
if not isinstance(raw, str) or not raw:
|
||||
raise ValueError(f"{field} must be an absolute external path")
|
||||
path = Path(raw)
|
||||
if not path.is_absolute():
|
||||
raise ValueError(f"{field} must be an absolute external path")
|
||||
resolved = path.resolve(strict=must_exist)
|
||||
if _is_within(resolved, _REPO_ROOT):
|
||||
raise ValueError(f"{field} must stay outside the repository")
|
||||
return resolved
|
||||
|
||||
|
||||
def _load_json_file(path: Path, maximum: int = 64 << 10) -> object:
|
||||
raw = path.read_bytes()
|
||||
if len(raw) > maximum:
|
||||
raise ValueError("external configuration file is too large")
|
||||
try:
|
||||
return json.loads(raw.decode("utf-8"))
|
||||
except (UnicodeError, json.JSONDecodeError) as exc:
|
||||
raise ValueError("external configuration is not valid JSON") from exc
|
||||
|
||||
|
||||
def _positive_int(value: object, field: str) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
|
||||
raise ValueError(f"{field} must be a positive integer")
|
||||
return value
|
||||
|
||||
|
||||
def _validate_bell_url(raw: object) -> str:
|
||||
if not isinstance(raw, str):
|
||||
raise ValueError("bell_url must be a URL")
|
||||
parsed = urlsplit(raw)
|
||||
if parsed.scheme not in {"http", "https"} or parsed.hostname is None or parsed.path != INGRESS_PATH:
|
||||
raise ValueError("bell_url must target the versioned event ingress path")
|
||||
if parsed.username is not None or parsed.password is not None or parsed.query or parsed.fragment:
|
||||
raise ValueError("bell_url cannot contain credentials, query or fragment")
|
||||
try:
|
||||
port = parsed.port
|
||||
except ValueError as exc:
|
||||
raise ValueError("bell_url has an invalid port") from exc
|
||||
if port is not None and not 1 <= port <= 65535:
|
||||
raise ValueError("bell_url has an invalid port")
|
||||
if parsed.scheme == "http":
|
||||
host = parsed.hostname
|
||||
loopback = host == "localhost"
|
||||
if not loopback:
|
||||
try:
|
||||
loopback = ipaddress.ip_address(host).is_loopback
|
||||
except ValueError:
|
||||
loopback = False
|
||||
if not loopback:
|
||||
raise ValueError("non-loopback event ingress requires HTTPS")
|
||||
return raw
|
||||
|
||||
|
||||
def load_config(path_value: str) -> tuple[IngressConfig, KeyMaterial]:
|
||||
config_path = _external_absolute_path(path_value, "event ingress config", must_exist=True)
|
||||
value = _load_json_file(config_path)
|
||||
expected = {
|
||||
"version",
|
||||
"producer_id",
|
||||
"tenant_id",
|
||||
"site_id",
|
||||
"device_id",
|
||||
"modality",
|
||||
"severity",
|
||||
"config_version",
|
||||
"bell_url",
|
||||
"key_id",
|
||||
"key_file",
|
||||
"outbox_path",
|
||||
}
|
||||
if not isinstance(value, dict) or set(value) != expected or value.get("version") != 1:
|
||||
raise ValueError("invalid event ingress configuration shape")
|
||||
producer_id = value["producer_id"]
|
||||
key_id = value["key_id"]
|
||||
config_version = value["config_version"]
|
||||
if not isinstance(producer_id, str) or not _ID.fullmatch(producer_id):
|
||||
raise ValueError("invalid producer_id")
|
||||
if not isinstance(key_id, str) or not _ID.fullmatch(key_id):
|
||||
raise ValueError("invalid key_id")
|
||||
if not isinstance(config_version, str) or not 1 <= len(config_version) <= 128:
|
||||
raise ValueError("config_version must contain 1 to 128 characters")
|
||||
if value["modality"] != "video":
|
||||
raise ValueError("T-019 Brain ingress supports only video primary sensors")
|
||||
if value["severity"] not in {"low", "medium", "high", "critical"}:
|
||||
raise ValueError("invalid severity")
|
||||
key_file = _external_absolute_path(value["key_file"], "key_file", must_exist=True)
|
||||
outbox_path = _external_absolute_path(value["outbox_path"], "outbox_path", must_exist=False)
|
||||
if not outbox_path.parent.is_dir():
|
||||
raise ValueError("outbox_path parent directory must already exist")
|
||||
config = IngressConfig(
|
||||
producer_id=producer_id,
|
||||
tenant_id=_positive_int(value["tenant_id"], "tenant_id"),
|
||||
site_id=_positive_int(value["site_id"], "site_id"),
|
||||
device_id=_positive_int(value["device_id"], "device_id"),
|
||||
modality="video",
|
||||
severity=value["severity"],
|
||||
config_version=config_version,
|
||||
bell_url=_validate_bell_url(value["bell_url"]),
|
||||
key_id=key_id,
|
||||
key_file=key_file,
|
||||
outbox_path=outbox_path,
|
||||
)
|
||||
document = _load_json_file(key_file)
|
||||
if not isinstance(document, dict) or set(document) != {"version", "keys"} or document.get("version") != 1:
|
||||
raise ValueError("invalid event ingress key document")
|
||||
keys = document.get("keys")
|
||||
if not isinstance(keys, list) or not keys:
|
||||
raise ValueError("event ingress key document has no keys")
|
||||
selected: KeyMaterial | None = None
|
||||
seen: set[str] = set()
|
||||
for item in keys:
|
||||
if not isinstance(item, dict) or set(item) != {"key_id", "producer_id", "secret_base64url"}:
|
||||
raise ValueError("invalid event ingress key entry")
|
||||
item_key = item["key_id"]
|
||||
item_producer = item["producer_id"]
|
||||
encoded = item["secret_base64url"]
|
||||
if not isinstance(item_key, str) or not _ID.fullmatch(item_key) or item_key in seen:
|
||||
raise ValueError("invalid or duplicate event ingress key ID")
|
||||
if not isinstance(item_producer, str) or not _ID.fullmatch(item_producer) or not isinstance(encoded, str):
|
||||
raise ValueError("invalid event ingress key entry")
|
||||
seen.add(item_key)
|
||||
try:
|
||||
secret = base64.b64decode(encoded + "=" * (-len(encoded) % 4), altchars=b"-_", validate=True)
|
||||
except (ValueError, binascii.Error) as exc:
|
||||
raise ValueError("invalid event ingress secret") from exc
|
||||
if len(secret) < 32:
|
||||
raise ValueError("event ingress secret must contain at least 32 bytes")
|
||||
if item_key == key_id:
|
||||
selected = KeyMaterial(item_key, item_producer, secret)
|
||||
if selected is None or selected.producer_id != producer_id:
|
||||
raise ValueError("selected key is not bound to the configured producer")
|
||||
return config, selected
|
||||
|
||||
|
||||
def map_candidate(value: EventCandidate, config: IngressConfig) -> bytes:
|
||||
if not _SOURCE_ID.fullmatch(value.source_event_id):
|
||||
raise ValueError("invalid source_event_id")
|
||||
payload = {
|
||||
"schema_version": "0.1",
|
||||
"source_event_id": value.source_event_id,
|
||||
"tenant_id": config.tenant_id,
|
||||
"site_id": config.site_id,
|
||||
"device_id": config.device_id,
|
||||
"sensors": [{"device_id": config.device_id, "modality": config.modality, "role": "primary"}],
|
||||
"kind": value.kind,
|
||||
"severity": config.severity,
|
||||
"confidence": None,
|
||||
"occurred_at": value.occurred_at,
|
||||
"detected_at": value.occurred_at,
|
||||
"latency_seconds": 0.0,
|
||||
"config_version": config.config_version,
|
||||
"rule": None,
|
||||
"subject": {
|
||||
"class": "person",
|
||||
"track_id": value.track_id,
|
||||
"attributes": {},
|
||||
"anon_id": None,
|
||||
"identity": None,
|
||||
"identity_status": "not_enabled",
|
||||
},
|
||||
"observation": {
|
||||
"zone": value.zone_id,
|
||||
"dwell_sec": 0.0,
|
||||
"bbox_seq_uri": None,
|
||||
"keypoint_seq_uri": None,
|
||||
"signal_seq_uri": None,
|
||||
},
|
||||
"evidence": {"snapshot_uris": [], "clip_uri": None, "clip_range": None},
|
||||
"dedup_key": None,
|
||||
"aggregated_into": None,
|
||||
"outcome": "test" if value.fixture else "unknown",
|
||||
"outcome_source": "auto" if value.fixture else None,
|
||||
"outcome_reason": None,
|
||||
"diagnostics": None,
|
||||
"ext": {"brain_candidate_version": "brain-demo-v1", "zone_version": value.zone_version},
|
||||
}
|
||||
encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
if len(encoded) > MAX_PAYLOAD_BYTES:
|
||||
raise ValueError("event candidate exceeds 1 MiB")
|
||||
return encoded
|
||||
|
||||
|
||||
class EventOutbox:
|
||||
def __init__(self, path: Path, *, now: Callable[[], float] = time.time) -> None:
|
||||
if not path.is_absolute() or _is_within(path.resolve(), _REPO_ROOT):
|
||||
raise ValueError("outbox path must be absolute and external")
|
||||
self._now = now
|
||||
self._lock = threading.RLock()
|
||||
self._connection = sqlite3.connect(str(path), timeout=5.0, isolation_level=None, check_same_thread=False)
|
||||
self._connection.row_factory = sqlite3.Row
|
||||
self._connection.execute("PRAGMA journal_mode=WAL")
|
||||
self._connection.execute("PRAGMA synchronous=FULL")
|
||||
self._connection.execute("PRAGMA foreign_keys=ON")
|
||||
self._connection.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS event_outbox (
|
||||
source_event_id TEXT PRIMARY KEY,
|
||||
candidate_hash BLOB NOT NULL,
|
||||
payload BLOB NOT NULL,
|
||||
state TEXT NOT NULL CHECK(state IN ('queued','delivering','delivered','dead_letter')),
|
||||
attempt_count INTEGER NOT NULL DEFAULT 0 CHECK(attempt_count BETWEEN 0 AND 100),
|
||||
available_at REAL NOT NULL,
|
||||
lease_until REAL,
|
||||
bell_event_id TEXT,
|
||||
last_error_code TEXT,
|
||||
created_at REAL NOT NULL,
|
||||
updated_at REAL NOT NULL,
|
||||
delivered_at REAL,
|
||||
dead_lettered_at REAL,
|
||||
CHECK(length(candidate_hash)=32),
|
||||
CHECK(length(payload)<=1048576),
|
||||
CHECK((state='delivering')=(lease_until IS NOT NULL)),
|
||||
CHECK(delivered_at IS NULL OR dead_lettered_at IS NULL)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS event_outbox_due_idx
|
||||
ON event_outbox(available_at, created_at, source_event_id)
|
||||
WHERE state='queued';
|
||||
CREATE TABLE IF NOT EXISTS event_outbox_identity (
|
||||
singleton INTEGER PRIMARY KEY CHECK(singleton=1),
|
||||
producer_id TEXT NOT NULL,
|
||||
tenant_id INTEGER NOT NULL CHECK(tenant_id>=1),
|
||||
site_id INTEGER NOT NULL CHECK(site_id>=1),
|
||||
device_id INTEGER NOT NULL CHECK(device_id>=1)
|
||||
);
|
||||
"""
|
||||
)
|
||||
|
||||
def bind_identity(self, config: IngressConfig) -> None:
|
||||
identity = (config.producer_id, config.tenant_id, config.site_id, config.device_id)
|
||||
with self._lock:
|
||||
self._connection.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
row = self._connection.execute(
|
||||
"SELECT producer_id,tenant_id,site_id,device_id FROM event_outbox_identity WHERE singleton=1"
|
||||
).fetchone()
|
||||
if row is None:
|
||||
self._connection.execute(
|
||||
"""INSERT INTO event_outbox_identity(
|
||||
singleton,producer_id,tenant_id,site_id,device_id
|
||||
) VALUES (1,?,?,?,?)""",
|
||||
identity,
|
||||
)
|
||||
elif tuple(row) != identity:
|
||||
raise ValueError("event outbox is bound to a different producer or device identity")
|
||||
self._connection.execute("COMMIT")
|
||||
except Exception:
|
||||
self._connection.execute("ROLLBACK")
|
||||
raise
|
||||
|
||||
def enqueue(self, payload: bytes) -> bool:
|
||||
if not 0 < len(payload) <= MAX_PAYLOAD_BYTES:
|
||||
raise ValueError("event candidate payload must contain at most 1 MiB")
|
||||
try:
|
||||
candidate = json.loads(payload)
|
||||
source_event_id = candidate["source_event_id"]
|
||||
except (UnicodeError, json.JSONDecodeError, KeyError, TypeError) as exc:
|
||||
raise ValueError("invalid event candidate payload") from exc
|
||||
if not isinstance(source_event_id, str) or not _SOURCE_ID.fullmatch(source_event_id):
|
||||
raise ValueError("invalid source_event_id")
|
||||
digest = hashlib.sha256(payload).digest()
|
||||
now = self._now()
|
||||
with self._lock:
|
||||
self._connection.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
existing = self._connection.execute(
|
||||
"SELECT candidate_hash FROM event_outbox WHERE source_event_id=?", (source_event_id,)
|
||||
).fetchone()
|
||||
if existing is not None:
|
||||
if not hmac.compare_digest(existing["candidate_hash"], digest):
|
||||
raise ValueError("source_event_id already has a different candidate")
|
||||
self._connection.execute("COMMIT")
|
||||
return False
|
||||
active = self._connection.execute(
|
||||
"SELECT count(*) FROM event_outbox WHERE state IN ('queued','delivering')"
|
||||
).fetchone()[0]
|
||||
if active >= MAX_QUEUED:
|
||||
raise RuntimeError("event outbox capacity exceeded")
|
||||
self._connection.execute(
|
||||
"""INSERT INTO event_outbox(
|
||||
source_event_id,candidate_hash,payload,state,available_at,created_at,updated_at
|
||||
) VALUES (?,?,?,'queued',?,?,?)""",
|
||||
(source_event_id, digest, payload, now, now, now),
|
||||
)
|
||||
self._connection.execute("COMMIT")
|
||||
return True
|
||||
except Exception:
|
||||
self._connection.execute("ROLLBACK")
|
||||
raise
|
||||
|
||||
def lease_one(self, lease_seconds: float = 30.0) -> LeasedEvent | None:
|
||||
now = self._now()
|
||||
with self._lock:
|
||||
self._connection.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
self._connection.execute(
|
||||
"""UPDATE event_outbox SET state='dead_letter', lease_until=NULL,
|
||||
last_error_code='retry_exhausted', dead_lettered_at=?, updated_at=?
|
||||
WHERE attempt_count>=? AND (
|
||||
state='queued' OR (state='delivering' AND lease_until<=?)
|
||||
)""",
|
||||
(now, now, MAX_ATTEMPTS, now),
|
||||
)
|
||||
self._connection.execute(
|
||||
"""UPDATE event_outbox SET state='queued', lease_until=NULL,
|
||||
available_at=?, last_error_code='lease_expired', updated_at=?
|
||||
WHERE state='delivering' AND lease_until<=? AND attempt_count<?""",
|
||||
(now, now, now, MAX_ATTEMPTS),
|
||||
)
|
||||
row = self._connection.execute(
|
||||
"""SELECT source_event_id,payload,attempt_count FROM event_outbox
|
||||
WHERE state='queued' AND available_at<=? AND attempt_count<?
|
||||
ORDER BY available_at,created_at,source_event_id LIMIT 1""",
|
||||
(now, MAX_ATTEMPTS),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
self._connection.execute("COMMIT")
|
||||
return None
|
||||
attempt = row["attempt_count"] + 1
|
||||
self._connection.execute(
|
||||
"""UPDATE event_outbox SET state='delivering',attempt_count=?,lease_until=?,updated_at=?
|
||||
WHERE source_event_id=? AND state='queued'""",
|
||||
(attempt, now + lease_seconds, now, row["source_event_id"]),
|
||||
)
|
||||
self._connection.execute("COMMIT")
|
||||
return LeasedEvent(row["source_event_id"], bytes(row["payload"]), attempt)
|
||||
except Exception:
|
||||
self._connection.execute("ROLLBACK")
|
||||
raise
|
||||
|
||||
def mark_delivered(self, item: LeasedEvent, result: DeliveryResult) -> None:
|
||||
now = self._now()
|
||||
with self._lock:
|
||||
changed = self._connection.execute(
|
||||
"""UPDATE event_outbox SET state='delivered',lease_until=NULL,bell_event_id=?,
|
||||
last_error_code=NULL,delivered_at=?,updated_at=?
|
||||
WHERE source_event_id=? AND state='delivering' AND attempt_count=?""",
|
||||
(result.event_id, now, now, item.source_event_id, item.attempt_count),
|
||||
).rowcount
|
||||
if changed != 1:
|
||||
raise RuntimeError("event outbox delivery lease was lost")
|
||||
|
||||
def mark_retry(self, item: LeasedEvent, code: str) -> None:
|
||||
now = self._now()
|
||||
if item.attempt_count >= MAX_ATTEMPTS:
|
||||
self.mark_dead(item, "retry_exhausted")
|
||||
return
|
||||
delay = min(300.0, float(2 ** min(item.attempt_count - 1, 9)))
|
||||
with self._lock:
|
||||
changed = self._connection.execute(
|
||||
"""UPDATE event_outbox SET state='queued',lease_until=NULL,available_at=?,
|
||||
last_error_code=?,updated_at=?
|
||||
WHERE source_event_id=? AND state='delivering' AND attempt_count=?""",
|
||||
(now + delay, _safe_error(code), now, item.source_event_id, item.attempt_count),
|
||||
).rowcount
|
||||
if changed != 1:
|
||||
raise RuntimeError("event outbox retry lease was lost")
|
||||
|
||||
def mark_dead(self, item: LeasedEvent, code: str) -> None:
|
||||
now = self._now()
|
||||
with self._lock:
|
||||
changed = self._connection.execute(
|
||||
"""UPDATE event_outbox SET state='dead_letter',lease_until=NULL,
|
||||
last_error_code=?,dead_lettered_at=?,updated_at=?
|
||||
WHERE source_event_id=? AND state='delivering' AND attempt_count=?""",
|
||||
(_safe_error(code), now, now, item.source_event_id, item.attempt_count),
|
||||
).rowcount
|
||||
if changed != 1:
|
||||
raise RuntimeError("event outbox dead-letter lease was lost")
|
||||
|
||||
def status(self) -> dict[str, object]:
|
||||
with self._lock:
|
||||
counts = {row["state"]: row["count"] for row in self._connection.execute(
|
||||
"SELECT state,count(*) AS count FROM event_outbox GROUP BY state"
|
||||
)}
|
||||
error = self._connection.execute(
|
||||
"""SELECT last_error_code FROM event_outbox WHERE last_error_code IS NOT NULL
|
||||
ORDER BY updated_at DESC LIMIT 1"""
|
||||
).fetchone()
|
||||
return {
|
||||
"enabled": True,
|
||||
"queued": counts.get("queued", 0),
|
||||
"delivering": counts.get("delivering", 0),
|
||||
"delivered": counts.get("delivered", 0),
|
||||
"dead_letter": counts.get("dead_letter", 0),
|
||||
"last_error_code": None if error is None else error["last_error_code"],
|
||||
}
|
||||
|
||||
def close(self) -> None:
|
||||
with self._lock:
|
||||
self._connection.close()
|
||||
|
||||
|
||||
class EventIngressClient:
|
||||
def __init__(self, config: IngressConfig, key: KeyMaterial, *, now: Callable[[], float] = time.time) -> None:
|
||||
self._config = config
|
||||
self._key = key
|
||||
self._now = now
|
||||
# Internal event payloads must never be redirected through ambient
|
||||
# HTTP(S)_PROXY settings.
|
||||
self._opener = build_opener(ProxyHandler({}), _NoRedirect())
|
||||
|
||||
def deliver(self, candidate: bytes) -> DeliveryResult:
|
||||
try:
|
||||
candidate_object = json.loads(candidate)
|
||||
except (UnicodeError, json.JSONDecodeError) as exc:
|
||||
raise PermanentDelivery("candidate_invalid") from exc
|
||||
body = json.dumps(
|
||||
{"schema_version": 1, "producer_id": self._config.producer_id, "candidate": candidate_object},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
if len(body) > MAX_PAYLOAD_BYTES:
|
||||
raise PermanentDelivery("payload_too_large")
|
||||
timestamp = str(int(self._now()))
|
||||
nonce = base64.urlsafe_b64encode(secrets.token_bytes(16)).rstrip(b"=").decode("ascii")
|
||||
digest = hashlib.sha256(body).hexdigest()
|
||||
canonical = "\n".join(("POST", INGRESS_PATH, timestamp, nonce, digest)).encode("utf-8")
|
||||
signature_value = base64.urlsafe_b64encode(hmac.new(self._key.secret, canonical, hashlib.sha256).digest()).rstrip(b"=").decode("ascii")
|
||||
request = Request(self._config.bell_url, data=body, method="POST")
|
||||
request.add_header("Content-Type", "application/json")
|
||||
request.add_header("X-YoVision-Key-Id", self._key.key_id)
|
||||
request.add_header("X-YoVision-Timestamp", timestamp)
|
||||
request.add_header("X-YoVision-Nonce", nonce)
|
||||
request.add_header("X-YoVision-Signature", signature_value)
|
||||
try:
|
||||
with self._opener.open(request, timeout=HTTP_TIMEOUT_SECONDS) as response:
|
||||
status = response.status
|
||||
response_body = response.read(64 << 10)
|
||||
except HTTPError as exc:
|
||||
status = exc.code
|
||||
response_body = exc.read(64 << 10)
|
||||
except (URLError, TimeoutError, OSError) as exc:
|
||||
raise RetryableDelivery("network_error") from exc
|
||||
if status in {200, 201}:
|
||||
try:
|
||||
value = json.loads(response_body)
|
||||
response_status = value["status"]
|
||||
event_id = value["event_id"]
|
||||
if (
|
||||
value["schema_version"] != 1
|
||||
or value["producer_id"] != self._config.producer_id
|
||||
or value["source_event_id"] != candidate_object["source_event_id"]
|
||||
or response_status not in {"accepted", "duplicate"}
|
||||
or not isinstance(event_id, str)
|
||||
or not _EVENT_ID.fullmatch(event_id)
|
||||
):
|
||||
raise ValueError
|
||||
except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
|
||||
raise RetryableDelivery("invalid_response") from exc
|
||||
return DeliveryResult(response_status, event_id)
|
||||
code = "http_error"
|
||||
try:
|
||||
error_value = json.loads(response_body)
|
||||
if isinstance(error_value, dict) and isinstance(error_value.get("error"), str):
|
||||
code = error_value["error"]
|
||||
except (UnicodeError, json.JSONDecodeError):
|
||||
pass
|
||||
if status == 401 or status >= 500:
|
||||
raise RetryableDelivery(code)
|
||||
raise PermanentDelivery(code)
|
||||
|
||||
|
||||
class EventRelayWorker:
|
||||
def __init__(self, outbox: EventOutbox, client: EventIngressClient) -> None:
|
||||
self._outbox = outbox
|
||||
self._client = client
|
||||
self._stop = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
self._last_error_code: str | None = None
|
||||
|
||||
def run_once(self) -> bool:
|
||||
item = self._outbox.lease_one()
|
||||
if item is None:
|
||||
return False
|
||||
try:
|
||||
result = self._client.deliver(item.payload)
|
||||
except PermanentDelivery as exc:
|
||||
self._outbox.mark_dead(item, exc.code)
|
||||
except RetryableDelivery as exc:
|
||||
self._outbox.mark_retry(item, exc.code)
|
||||
except Exception:
|
||||
self._outbox.mark_retry(item, "client_error")
|
||||
else:
|
||||
self._outbox.mark_delivered(item, result)
|
||||
return True
|
||||
|
||||
def start(self) -> None:
|
||||
if self._thread is not None:
|
||||
return
|
||||
self._thread = threading.Thread(target=self._run, name="brain-event-relay", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def _run(self) -> None:
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
worked = self.run_once()
|
||||
self._last_error_code = None
|
||||
except Exception:
|
||||
self._last_error_code = "event_outbox_unavailable"
|
||||
self._stop.wait(1.0)
|
||||
continue
|
||||
if not worked:
|
||||
self._stop.wait(0.25)
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
if self._thread is not None:
|
||||
self._thread.join(timeout=HTTP_TIMEOUT_SECONDS + 2.0)
|
||||
self._thread = None
|
||||
|
||||
def last_error_code(self) -> str | None:
|
||||
return self._last_error_code
|
||||
|
||||
|
||||
class BrainEventIngress:
|
||||
def __init__(self, config: IngressConfig, key: KeyMaterial) -> None:
|
||||
self._config = config
|
||||
self._outbox = EventOutbox(config.outbox_path)
|
||||
self._outbox.bind_identity(config)
|
||||
self._worker = EventRelayWorker(self._outbox, EventIngressClient(config, key))
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, path: str) -> "BrainEventIngress":
|
||||
config, key = load_config(path)
|
||||
return cls(config, key)
|
||||
|
||||
def submit(self, value: EventCandidate) -> None:
|
||||
self._outbox.enqueue(map_candidate(value, self._config))
|
||||
|
||||
def start(self) -> None:
|
||||
self._worker.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._worker.stop()
|
||||
self._outbox.close()
|
||||
|
||||
def status(self) -> dict[str, object]:
|
||||
value = self._outbox.status()
|
||||
if self._worker.last_error_code() is not None:
|
||||
value["last_error_code"] = self._worker.last_error_code()
|
||||
return value
|
||||
|
||||
|
||||
def _safe_error(value: str) -> str:
|
||||
if not isinstance(value, str) or not _ERROR_CODE.fullmatch(value):
|
||||
return "unknown_error"
|
||||
return value
|
||||
@@ -24,8 +24,12 @@ DEFAULT_ZONE = Zone(
|
||||
)
|
||||
|
||||
|
||||
class EventIngressUnavailable(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class DemoEngine:
|
||||
def __init__(self, source: Any, fps: float = 2.0, event_limit: int = 100) -> None:
|
||||
def __init__(self, source: Any, fps: float = 2.0, event_limit: int = 100, event_ingress: Any | None = None) -> None:
|
||||
if not math.isfinite(fps) or fps <= 0.0 or fps > 30.0:
|
||||
raise ValueError("fps must be within (0, 30]")
|
||||
if not 1 <= event_limit <= 100:
|
||||
@@ -36,6 +40,7 @@ class DemoEngine:
|
||||
self._detector = None if source.fixture else HOGPersonDetector()
|
||||
self._tracker = CentroidTracker()
|
||||
self._evaluator = ZoneEntryEvaluator(source.source_ref, source.fixture)
|
||||
self._event_ingress = event_ingress
|
||||
self._zone = DEFAULT_ZONE
|
||||
self._events: deque[dict[str, object]] = deque(maxlen=event_limit)
|
||||
self._lock = threading.RLock()
|
||||
@@ -54,6 +59,8 @@ class DemoEngine:
|
||||
def start(self) -> None:
|
||||
if self._thread is not None:
|
||||
return
|
||||
if self._event_ingress is not None:
|
||||
self._event_ingress.start()
|
||||
self._thread = threading.Thread(target=self._run, name="brain-demo", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
@@ -62,6 +69,8 @@ class DemoEngine:
|
||||
if self._thread is not None:
|
||||
self._thread.join(timeout=3.0)
|
||||
self._thread = None
|
||||
if self._event_ingress is not None:
|
||||
self._event_ingress.stop()
|
||||
self._source.close()
|
||||
|
||||
def _run(self) -> None:
|
||||
@@ -70,6 +79,9 @@ class DemoEngine:
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
self.step()
|
||||
except EventIngressUnavailable:
|
||||
with self._lock:
|
||||
self._last_error_code = "event_outbox_unavailable"
|
||||
except RuntimeError:
|
||||
with self._lock:
|
||||
self._connected = False
|
||||
@@ -89,6 +101,12 @@ class DemoEngine:
|
||||
with self._lock:
|
||||
zone = self._zone
|
||||
new_events, inside_by_track = self._evaluator.evaluate(self._sequence, packet.captured_at, zone, detections)
|
||||
if self._event_ingress is not None:
|
||||
try:
|
||||
for item in new_events:
|
||||
self._event_ingress.submit(item)
|
||||
except Exception as exc:
|
||||
raise EventIngressUnavailable("persist event candidate") from exc
|
||||
ok, encoded = cv2.imencode(".jpg", packet.frame, [int(cv2.IMWRITE_JPEG_QUALITY), 82])
|
||||
if not ok:
|
||||
raise RuntimeError("frame encoding failed")
|
||||
@@ -96,7 +114,12 @@ class DemoEngine:
|
||||
serialized_detections = [self._serialize_detection(item, inside_by_track.get(item.track_id, False)) for item in detections]
|
||||
with self._lock:
|
||||
for event in new_events:
|
||||
self._events.appendleft(event.as_dict())
|
||||
serialized_event = event.as_dict()
|
||||
if self._event_ingress is not None:
|
||||
# This is an immutable handoff fact, not a live delivery
|
||||
# status. Current counts live under event_ingress.
|
||||
serialized_event["delivery_status"] = "outbox_persisted"
|
||||
self._events.appendleft(serialized_event)
|
||||
self._frame_jpeg = encoded.tobytes()
|
||||
self._frame_width = int(width)
|
||||
self._frame_height = int(height)
|
||||
@@ -130,6 +153,13 @@ class DemoEngine:
|
||||
def state(self) -> dict[str, object]:
|
||||
with self._lock:
|
||||
zone = self._zone
|
||||
if self._event_ingress is None:
|
||||
ingress_status: dict[str, object] = {"enabled": False}
|
||||
else:
|
||||
try:
|
||||
ingress_status = self._event_ingress.status()
|
||||
except Exception:
|
||||
ingress_status = {"enabled": True, "last_error_code": "event_outbox_unavailable"}
|
||||
return {
|
||||
"prototype": True,
|
||||
"notice": "工程原型;合成回放不是模型输出,HOG 适配器不是生产检测模型。",
|
||||
@@ -159,6 +189,7 @@ class DemoEngine:
|
||||
},
|
||||
"detections": list(self._detections),
|
||||
"events": list(self._events),
|
||||
"event_ingress": ingress_status,
|
||||
"last_error_code": self._last_error_code,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user