feat: implement T-019 reliable event ingress
Harness governance / validate (pull_request) Has been cancelled

This commit is contained in:
QiuSW
2026-08-11 15:41:07 +08:00
parent b7fe44eeb0
commit bd964e8831
33 changed files with 2762 additions and 57 deletions
+36 -3
View File
@@ -1,6 +1,6 @@
# YoVision Brain 单路工程原型
T-017 提供一条可见、可重复的工程链路:单路 frame source → 匿名人员检测/fixture → track → 归一化多边形区域 → `zone_entry` 候选事实。它不是生产模型、不是 NVR,也没有 Brain→Bell transport。
T-017 提供可见、可重复的单路工程链路;T-019 为 `zone_entry` 候选增加默认关闭的可靠 Brain→Bell transport。它仍不是生产模型或 NVR:推理演示保持单路,事件先进入仓库外 SQLite Outbox,再由内部 HMAC client 投递给 Bell。
## 环境
@@ -36,6 +36,39 @@ URL 文件只允许一行、最多 4096 字节,服务不会在状态、页面
真实流使用 OpenCV 内置 HOG/SVM 人员检测和轻量 centroid tracker,只验证可替换端口与区域链路。它不是 M3 生产模型,不能据此承诺召回率、误报率、GPU 容量或 16/128 路能力。
## 可选 Brain → Bell 事件投递
默认不传 `--event-ingress-config`,工程原型行为与 T-017 相同。启用时,配置、key 和 SQLite 文件都必须位于仓库外绝对路径;HTTP 只允许显式回环,非回环必须 HTTPS。key 文件也供 Bell event ingress 读取,但不得与 Sense 审计 key 混用:
```json
{"version":1,"keys":[{"key_id":"brain-a","producer_id":"brain-main","secret_base64url":"<至少32字节随机值的无填充base64url>"}]}
```
Brain 配置示例(占位 ID 必须与 Bell 管理员创建的 `bell.event_ingress_bindings` 一致):
```json
{
"version": 1,
"producer_id": "brain-main",
"tenant_id": 1,
"site_id": 1,
"device_id": 1,
"modality": "video",
"severity": "medium",
"config_version": "brain-demo-v1",
"bell_url": "http://127.0.0.1:8081/internal/v1/event-candidates",
"key_id": "brain-a",
"key_file": "D:\\private\\brain-event-keys.json",
"outbox_path": "D:\\private\\brain-event-outbox.sqlite3"
}
```
```powershell
python -m Brain.yovision_brain --source synthetic --event-ingress-config D:\private\brain-event-ingress.json
```
Outbox 使用 WAL 和 `synchronous=FULL`,首次打开即绑定 producer/tenant/site/device;更换身份必须使用新的 Outbox 路径,不能让旧队列借新 producer 发送。最多保留 10,000 条待投递;只有 Bell `accepted/duplicate` 才确认 delivered。网络、401 与 5xx 按 1~300 秒退避、最多 100 次;稳定 4xx 进入 dead letter。终态行不会自动删除。状态页只显示计数和稳定错误码,不显示 URL、key、payload 或数据库路径;client 显式忽略环境 HTTP proxy。
## 验证
```powershell
@@ -48,6 +81,6 @@ python -m compileall -q Brain
## 边界
- `source_event_id` 由 Brain 产生,平台 `evt_` ULID 由 Bell 产生。
- 事件只保留在最多 100 项的内存环中;重启即丢失是刻意边界。
- T-018 冻结并实现身份映射、候选契约、持久 Outbox、HMAC、幂等和 Bell ingress。
- 页面仍只保留最多 100 项的内存环;启用 T-019 后,可靠性由独立 SQLite Outbox 承担,不能从页面事件环推断投递状态。
- Bell 平台 ID 只由 Bell 生成;相同 producer/source candidate 重投返回原 ID,Brain 不改写 source ID 规避冲突。
- `D:\OPC\silver_pose` 保持独立,不是本模块的源码目录、运行依赖或模型来源路径。
+291
View File
@@ -0,0 +1,291 @@
from __future__ import annotations
import base64
import hashlib
import hmac
import json
import tempfile
import threading
import unittest
from datetime import datetime, timezone
from http.server import BaseHTTPRequestHandler, HTTPServer
from pathlib import Path
from Brain.yovision_brain.domain import EventCandidate
from Brain.yovision_brain.ingress import (
BrainEventIngress,
DeliveryResult,
EventIngressClient,
EventOutbox,
EventRelayWorker,
IngressConfig,
KeyMaterial,
PermanentDelivery,
RetryableDelivery,
load_config,
map_candidate,
)
EVENT_ID = "evt_01J8XQ2K7M3P5R9T0V4W6Y8Z2B"
def candidate(source_event_id: str = "BRN-test-0001", fixture: bool = True) -> EventCandidate:
return EventCandidate(
source_event_id=source_event_id,
kind="zone_entry",
source_ref="must-not-leave-brain",
track_id="P-1",
zone_id="zone-1",
zone_version=3,
occurred_at=datetime(2026, 8, 11, tzinfo=timezone.utc).isoformat().replace("+00:00", "Z"),
confidence=None,
fixture=fixture,
)
def config(root: Path, bell_url: str = "http://127.0.0.1:8081/internal/v1/event-candidates") -> IngressConfig:
return IngressConfig(
producer_id="brain-main",
tenant_id=1,
site_id=2,
device_id=3,
modality="video",
severity="medium",
config_version="brain-demo-v1",
bell_url=bell_url,
key_id="brain-a",
key_file=root / "keys.json",
outbox_path=root / "outbox.sqlite3",
)
class ConfigAndMapperTests(unittest.TestCase):
def test_external_config_binds_key_and_rejects_remote_plaintext(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
secret = bytes(range(32))
key_file = root / "keys.json"
key_file.write_text(
json.dumps(
{
"version": 1,
"keys": [
{
"key_id": "brain-a",
"producer_id": "brain-main",
"secret_base64url": base64.urlsafe_b64encode(secret).rstrip(b"=").decode("ascii"),
}
],
}
),
encoding="utf-8",
)
config_file = root / "config.json"
document = {
"version": 1,
"producer_id": "brain-main",
"tenant_id": 1,
"site_id": 2,
"device_id": 3,
"modality": "video",
"severity": "medium",
"config_version": "brain-demo-v1",
"bell_url": "http://127.0.0.1:8081/internal/v1/event-candidates",
"key_id": "brain-a",
"key_file": str(key_file.resolve()),
"outbox_path": str((root / "outbox.sqlite3").resolve()),
}
config_file.write_text(json.dumps(document), encoding="utf-8")
loaded, key = load_config(str(config_file.resolve()))
self.assertEqual(("brain-main", secret), (loaded.producer_id, key.secret))
document["bell_url"] = "http://camera.example/internal/v1/event-candidates"
config_file.write_text(json.dumps(document), encoding="utf-8")
with self.assertRaisesRegex(ValueError, "HTTPS"):
load_config(str(config_file.resolve()))
def test_mapper_produces_complete_candidate_without_sensitive_source(self) -> None:
with tempfile.TemporaryDirectory() as directory:
payload = json.loads(map_candidate(candidate(), config(Path(directory))))
self.assertNotIn("id", payload)
self.assertNotIn("source_ref", json.dumps(payload))
self.assertEqual("test", payload["outcome"])
self.assertEqual("auto", payload["outcome_source"])
self.assertIsNone(payload["confidence"])
self.assertEqual(payload["occurred_at"], payload["detected_at"])
self.assertEqual(0.0, payload["latency_seconds"])
self.assertEqual([{"device_id": 3, "modality": "video", "role": "primary"}], payload["sensors"])
expected = {
"schema_version", "source_event_id", "tenant_id", "site_id", "device_id", "sensors", "kind",
"severity", "confidence", "occurred_at", "detected_at", "latency_seconds", "config_version",
"rule", "subject", "observation", "evidence", "dedup_key", "aggregated_into", "outcome",
"outcome_source", "outcome_reason", "diagnostics", "ext",
}
self.assertEqual(expected, set(payload))
class OutboxTests(unittest.TestCase):
def test_crash_recovery_retry_and_terminal_records_are_durable(self) -> None:
clock = [1_000.0]
now = lambda: clock[0]
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "outbox.sqlite3"
payload = map_candidate(candidate(), config(Path(directory)))
outbox = EventOutbox(path, now=now)
self.assertTrue(outbox.enqueue(payload))
self.assertFalse(outbox.enqueue(payload))
leased = outbox.lease_one()
self.assertIsNotNone(leased)
outbox.close() # Simulate a process crash while the lease is held.
clock[0] += 31.0
outbox = EventOutbox(path, now=now)
recovered = outbox.lease_one()
self.assertIsNotNone(recovered)
self.assertEqual(2, recovered.attempt_count)
outbox.mark_delivered(recovered, DeliveryResult("duplicate", EVENT_ID))
self.assertEqual(1, outbox.status()["delivered"])
outbox.close()
outbox = EventOutbox(path, now=now)
status = outbox.status()
self.assertEqual((1, 0), (status["delivered"], status["queued"]))
outbox.close()
def test_source_id_conflict_is_not_silently_overwritten(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
outbox = EventOutbox(root / "outbox.sqlite3")
outbox.enqueue(map_candidate(candidate(), config(root)))
changed = json.loads(map_candidate(candidate(), config(root)))
changed["config_version"] = "different"
with self.assertRaisesRegex(ValueError, "different candidate"):
outbox.enqueue(json.dumps(changed, sort_keys=True, separators=(",", ":")).encode())
outbox.close()
def test_crash_on_final_attempt_becomes_dead_letter(self) -> None:
clock = [1_000.0]
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
path = root / "outbox.sqlite3"
outbox = EventOutbox(path, now=lambda: clock[0])
outbox.enqueue(map_candidate(candidate(), config(root)))
for _ in range(99):
item = outbox.lease_one()
self.assertIsNotNone(item)
outbox.mark_retry(item, "network_error")
clock[0] += 301.0
final = outbox.lease_one()
self.assertEqual(100, final.attempt_count)
outbox.close()
clock[0] += 31.0
outbox = EventOutbox(path, now=lambda: clock[0])
self.assertIsNone(outbox.lease_one())
self.assertEqual(1, outbox.status()["dead_letter"])
outbox.close()
def test_outbox_cannot_be_reused_for_another_producer_identity(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
path = root / "outbox.sqlite3"
outbox = EventOutbox(path)
original = config(root)
outbox.bind_identity(original)
outbox.close()
outbox = EventOutbox(path)
changed = IngressConfig(**{**original.__dict__, "producer_id": "brain-other"})
with self.assertRaisesRegex(ValueError, "different producer"):
outbox.bind_identity(changed)
outbox.close()
class _BellHandler(BaseHTTPRequestHandler):
secret = bytes(range(32))
verified = False
failure: str | None = None
def log_message(self, _format: str, *_args: object) -> None:
return
def do_POST(self) -> None: # noqa: N802
try:
length = int(self.headers["Content-Length"])
body = self.rfile.read(length)
timestamp = self.headers["X-YoVision-Timestamp"]
nonce = self.headers["X-YoVision-Nonce"]
canonical = "\n".join(("POST", self.path, timestamp, nonce, hashlib.sha256(body).hexdigest())).encode()
encoded_signature = self.headers["X-YoVision-Signature"]
supplied = base64.urlsafe_b64decode(encoded_signature + "=" * (-len(encoded_signature) % 4))
type(self).verified = hmac.compare_digest(supplied, hmac.new(self.secret, canonical, hashlib.sha256).digest())
envelope = json.loads(body)
response = json.dumps(
{
"schema_version": 1,
"producer_id": envelope["producer_id"],
"source_event_id": envelope["candidate"]["source_event_id"],
"event_id": EVENT_ID,
"status": "accepted",
}
).encode()
self.send_response(201)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(response)))
self.end_headers()
self.wfile.write(response)
except Exception as exc: # pragma: no cover - only improves test diagnostics
type(self).failure = repr(exc)
self.send_response(500)
self.end_headers()
class ClientAndWorkerTests(unittest.TestCase):
def test_client_signs_and_accepts_bell_response(self) -> None:
server = HTTPServer(("127.0.0.1", 0), _BellHandler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
client = EventIngressClient(
config(root, f"http://127.0.0.1:{server.server_port}/internal/v1/event-candidates"),
KeyMaterial("brain-a", "brain-main", _BellHandler.secret),
now=lambda: 1_800_000_000.0,
)
try:
result = client.deliver(map_candidate(candidate(), config(root)))
except RetryableDelivery as exc:
self.fail(f"test Bell handler failed: {_BellHandler.failure}; client={exc.code}")
self.assertEqual(("accepted", EVENT_ID), (result.status, result.event_id))
self.assertTrue(_BellHandler.verified)
finally:
server.shutdown()
server.server_close()
thread.join(timeout=2.0)
def test_worker_distinguishes_retryable_and_permanent_failures(self) -> None:
class FailingClient:
def __init__(self, failure: Exception) -> None:
self.failure = failure
def deliver(self, _payload: bytes) -> DeliveryResult:
raise self.failure
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
payload = map_candidate(candidate(), config(root))
outbox = EventOutbox(root / "retry.sqlite3", now=lambda: 100.0)
outbox.enqueue(payload)
EventRelayWorker(outbox, FailingClient(RetryableDelivery("unauthorized"))).run_once()
self.assertEqual((1, 0), (outbox.status()["queued"], outbox.status()["dead_letter"]))
outbox.close()
outbox = EventOutbox(root / "dead.sqlite3", now=lambda: 100.0)
outbox.enqueue(payload)
EventRelayWorker(outbox, FailingClient(PermanentDelivery("source_event_conflict"))).run_once()
self.assertEqual(1, outbox.status()["dead_letter"])
outbox.close()
if __name__ == "__main__":
unittest.main()
+61 -2
View File
@@ -2,8 +2,50 @@ from __future__ import annotations
import unittest
from Brain.yovision_brain.runtime import DemoEngine
from Brain.yovision_brain.source import SyntheticSource, cv2
from datetime import datetime, timezone
from Brain.yovision_brain.domain import Box, Detection
from Brain.yovision_brain.runtime import DemoEngine, EventIngressUnavailable
from Brain.yovision_brain.source import FramePacket, SyntheticSource, cv2, np
class _TwoFrameSource:
fixture = True
mode = "test"
label = "test"
source_ref = "safe-source"
def __init__(self) -> None:
self.index = 0
def read(self) -> FramePacket:
boxes = (Box(0.10, 0.4, 0.20, 0.8), Box(0.60, 0.4, 0.70, 0.8))
detection = Detection("P-1", "person", boxes[min(self.index, 1)])
self.index += 1
return FramePacket(np.zeros((180, 320, 3), dtype=np.uint8), datetime.now(timezone.utc), (detection,))
def close(self) -> None:
return
class _Ingress:
def __init__(self, fail: bool = False) -> None:
self.items = []
self.fail = fail
def submit(self, value: object) -> None:
if self.fail:
raise RuntimeError("disk unavailable")
self.items.append(value)
def start(self) -> None:
return
def stop(self) -> None:
return
def status(self) -> dict[str, object]:
return {"enabled": True, "queued": len(self.items)}
@unittest.skipIf(cv2 is None, "pinned OpenCV package is not installed")
@@ -28,6 +70,23 @@ class RuntimeTests(unittest.TestCase):
self.assertEqual(updated.version, 2)
self.assertEqual(engine.state()["zone"]["name"], "新区域")
def test_event_is_persisted_before_it_is_exposed(self) -> None:
ingress = _Ingress()
engine = DemoEngine(_TwoFrameSource(), event_ingress=ingress)
engine.step()
engine.step()
state = engine.state()
self.assertEqual(1, len(ingress.items))
self.assertEqual("outbox_persisted", state["events"][0]["delivery_status"])
self.assertEqual(1, state["event_ingress"]["queued"])
def test_outbox_failure_does_not_claim_delivery(self) -> None:
engine = DemoEngine(_TwoFrameSource(), event_ingress=_Ingress(fail=True))
engine.step()
with self.assertRaises(EventIngressUnavailable):
engine.step()
self.assertEqual([], engine.state()["events"])
if __name__ == "__main__":
unittest.main()
+16
View File
@@ -40,6 +40,22 @@ class UIContractTests(unittest.TestCase):
self.assertNotRegex(visible_without_script, r"[\U0001F300-\U0001FAFF]")
self.assertEqual(self.html.count("__BRAIN_DEMO_TOKEN__"), 1)
def test_reliable_delivery_status_is_visible_without_configuration_details(self) -> None:
required = (
'id="event-ingress"',
'id="ingress-queued"',
'id="ingress-dead-letter"',
"Outbox 已持久化",
"Bell 返回 accepted 或 duplicate 后才计为已送达",
"event_ingress: {enabled: false}",
)
for marker in required:
with self.subTest(marker=marker):
self.assertIn(marker, self.html)
for forbidden in ("bell_url", "key_file", "outbox_path"):
with self.subTest(forbidden=forbidden):
self.assertNotIn(forbidden, self.html)
if __name__ == "__main__":
unittest.main()
+4 -1
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import argparse
import sys
from .ingress import BrainEventIngress
from .runtime import DemoEngine
from .server import parse_bind, serve
from .source import StreamSource, SyntheticSource, read_stream_url, require_opencv
@@ -14,6 +15,7 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument("--stream-url-file", help="absolute external file containing one RTSP URL")
parser.add_argument("--bind", default="127.0.0.1:8090", help="loopback bind address")
parser.add_argument("--fps", type=float, default=2.0, help="prototype processing FPS (0, 30]")
parser.add_argument("--event-ingress-config", help="absolute external Brain-to-Bell ingress configuration")
return parser
@@ -30,7 +32,8 @@ def main(argv: list[str] | None = None) -> int:
if args.stream_url_file:
raise ValueError("--stream-url-file is only valid for stream mode")
source = SyntheticSource()
engine = DemoEngine(source, fps=args.fps)
event_ingress = BrainEventIngress.from_file(args.event_ingress_config) if args.event_ingress_config else None
engine = DemoEngine(source, fps=args.fps, event_ingress=event_ingress)
except (RuntimeError, ValueError) as exc:
print(f"Brain demo configuration error: {exc}", file=sys.stderr)
return 2
+645
View File
@@ -0,0 +1,645 @@
from __future__ import annotations
import base64
import binascii
import hashlib
import hmac
import ipaddress
import json
import re
import secrets
import sqlite3
import threading
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Callable
from urllib.error import HTTPError, URLError
from urllib.parse import urlsplit
from urllib.request import HTTPRedirectHandler, ProxyHandler, Request, build_opener
from .domain import EventCandidate
INGRESS_PATH = "/internal/v1/event-candidates"
MAX_PAYLOAD_BYTES = 1 << 20
MAX_QUEUED = 10_000
MAX_ATTEMPTS = 100
HTTP_TIMEOUT_SECONDS = 10.0
_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$")
_SOURCE_ID = re.compile(r"^[A-Za-z0-9_-]{1,128}$")
_EVENT_ID = re.compile(r"^evt_[0-9A-HJKMNP-TV-Z]{26}$")
_ERROR_CODE = re.compile(r"^[a-z][a-z0-9_]{0,63}$")
_REPO_ROOT = Path(__file__).resolve().parents[2]
@dataclass(frozen=True)
class IngressConfig:
producer_id: str
tenant_id: int
site_id: int
device_id: int
modality: str
severity: str
config_version: str
bell_url: str
key_id: str
key_file: Path
outbox_path: Path
@dataclass(frozen=True)
class KeyMaterial:
key_id: str
producer_id: str
secret: bytes
@dataclass(frozen=True)
class LeasedEvent:
source_event_id: str
payload: bytes
attempt_count: int
@dataclass(frozen=True)
class DeliveryResult:
status: str
event_id: str
class RetryableDelivery(RuntimeError):
def __init__(self, code: str) -> None:
super().__init__(code)
self.code = _safe_error(code)
class PermanentDelivery(RuntimeError):
def __init__(self, code: str) -> None:
super().__init__(code)
self.code = _safe_error(code)
class _NoRedirect(HTTPRedirectHandler):
def redirect_request(self, _request: Request, _file_pointer: object, _code: int, _message: str, _headers: object, _new_url: str) -> None:
return None
def _is_within(path: Path, parent: Path) -> bool:
try:
path.relative_to(parent)
return True
except ValueError:
return False
def _external_absolute_path(raw: object, field: str, *, must_exist: bool) -> Path:
if not isinstance(raw, str) or not raw:
raise ValueError(f"{field} must be an absolute external path")
path = Path(raw)
if not path.is_absolute():
raise ValueError(f"{field} must be an absolute external path")
resolved = path.resolve(strict=must_exist)
if _is_within(resolved, _REPO_ROOT):
raise ValueError(f"{field} must stay outside the repository")
return resolved
def _load_json_file(path: Path, maximum: int = 64 << 10) -> object:
raw = path.read_bytes()
if len(raw) > maximum:
raise ValueError("external configuration file is too large")
try:
return json.loads(raw.decode("utf-8"))
except (UnicodeError, json.JSONDecodeError) as exc:
raise ValueError("external configuration is not valid JSON") from exc
def _positive_int(value: object, field: str) -> int:
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
raise ValueError(f"{field} must be a positive integer")
return value
def _validate_bell_url(raw: object) -> str:
if not isinstance(raw, str):
raise ValueError("bell_url must be a URL")
parsed = urlsplit(raw)
if parsed.scheme not in {"http", "https"} or parsed.hostname is None or parsed.path != INGRESS_PATH:
raise ValueError("bell_url must target the versioned event ingress path")
if parsed.username is not None or parsed.password is not None or parsed.query or parsed.fragment:
raise ValueError("bell_url cannot contain credentials, query or fragment")
try:
port = parsed.port
except ValueError as exc:
raise ValueError("bell_url has an invalid port") from exc
if port is not None and not 1 <= port <= 65535:
raise ValueError("bell_url has an invalid port")
if parsed.scheme == "http":
host = parsed.hostname
loopback = host == "localhost"
if not loopback:
try:
loopback = ipaddress.ip_address(host).is_loopback
except ValueError:
loopback = False
if not loopback:
raise ValueError("non-loopback event ingress requires HTTPS")
return raw
def load_config(path_value: str) -> tuple[IngressConfig, KeyMaterial]:
config_path = _external_absolute_path(path_value, "event ingress config", must_exist=True)
value = _load_json_file(config_path)
expected = {
"version",
"producer_id",
"tenant_id",
"site_id",
"device_id",
"modality",
"severity",
"config_version",
"bell_url",
"key_id",
"key_file",
"outbox_path",
}
if not isinstance(value, dict) or set(value) != expected or value.get("version") != 1:
raise ValueError("invalid event ingress configuration shape")
producer_id = value["producer_id"]
key_id = value["key_id"]
config_version = value["config_version"]
if not isinstance(producer_id, str) or not _ID.fullmatch(producer_id):
raise ValueError("invalid producer_id")
if not isinstance(key_id, str) or not _ID.fullmatch(key_id):
raise ValueError("invalid key_id")
if not isinstance(config_version, str) or not 1 <= len(config_version) <= 128:
raise ValueError("config_version must contain 1 to 128 characters")
if value["modality"] != "video":
raise ValueError("T-019 Brain ingress supports only video primary sensors")
if value["severity"] not in {"low", "medium", "high", "critical"}:
raise ValueError("invalid severity")
key_file = _external_absolute_path(value["key_file"], "key_file", must_exist=True)
outbox_path = _external_absolute_path(value["outbox_path"], "outbox_path", must_exist=False)
if not outbox_path.parent.is_dir():
raise ValueError("outbox_path parent directory must already exist")
config = IngressConfig(
producer_id=producer_id,
tenant_id=_positive_int(value["tenant_id"], "tenant_id"),
site_id=_positive_int(value["site_id"], "site_id"),
device_id=_positive_int(value["device_id"], "device_id"),
modality="video",
severity=value["severity"],
config_version=config_version,
bell_url=_validate_bell_url(value["bell_url"]),
key_id=key_id,
key_file=key_file,
outbox_path=outbox_path,
)
document = _load_json_file(key_file)
if not isinstance(document, dict) or set(document) != {"version", "keys"} or document.get("version") != 1:
raise ValueError("invalid event ingress key document")
keys = document.get("keys")
if not isinstance(keys, list) or not keys:
raise ValueError("event ingress key document has no keys")
selected: KeyMaterial | None = None
seen: set[str] = set()
for item in keys:
if not isinstance(item, dict) or set(item) != {"key_id", "producer_id", "secret_base64url"}:
raise ValueError("invalid event ingress key entry")
item_key = item["key_id"]
item_producer = item["producer_id"]
encoded = item["secret_base64url"]
if not isinstance(item_key, str) or not _ID.fullmatch(item_key) or item_key in seen:
raise ValueError("invalid or duplicate event ingress key ID")
if not isinstance(item_producer, str) or not _ID.fullmatch(item_producer) or not isinstance(encoded, str):
raise ValueError("invalid event ingress key entry")
seen.add(item_key)
try:
secret = base64.b64decode(encoded + "=" * (-len(encoded) % 4), altchars=b"-_", validate=True)
except (ValueError, binascii.Error) as exc:
raise ValueError("invalid event ingress secret") from exc
if len(secret) < 32:
raise ValueError("event ingress secret must contain at least 32 bytes")
if item_key == key_id:
selected = KeyMaterial(item_key, item_producer, secret)
if selected is None or selected.producer_id != producer_id:
raise ValueError("selected key is not bound to the configured producer")
return config, selected
def map_candidate(value: EventCandidate, config: IngressConfig) -> bytes:
if not _SOURCE_ID.fullmatch(value.source_event_id):
raise ValueError("invalid source_event_id")
payload = {
"schema_version": "0.1",
"source_event_id": value.source_event_id,
"tenant_id": config.tenant_id,
"site_id": config.site_id,
"device_id": config.device_id,
"sensors": [{"device_id": config.device_id, "modality": config.modality, "role": "primary"}],
"kind": value.kind,
"severity": config.severity,
"confidence": None,
"occurred_at": value.occurred_at,
"detected_at": value.occurred_at,
"latency_seconds": 0.0,
"config_version": config.config_version,
"rule": None,
"subject": {
"class": "person",
"track_id": value.track_id,
"attributes": {},
"anon_id": None,
"identity": None,
"identity_status": "not_enabled",
},
"observation": {
"zone": value.zone_id,
"dwell_sec": 0.0,
"bbox_seq_uri": None,
"keypoint_seq_uri": None,
"signal_seq_uri": None,
},
"evidence": {"snapshot_uris": [], "clip_uri": None, "clip_range": None},
"dedup_key": None,
"aggregated_into": None,
"outcome": "test" if value.fixture else "unknown",
"outcome_source": "auto" if value.fixture else None,
"outcome_reason": None,
"diagnostics": None,
"ext": {"brain_candidate_version": "brain-demo-v1", "zone_version": value.zone_version},
}
encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
if len(encoded) > MAX_PAYLOAD_BYTES:
raise ValueError("event candidate exceeds 1 MiB")
return encoded
class EventOutbox:
def __init__(self, path: Path, *, now: Callable[[], float] = time.time) -> None:
if not path.is_absolute() or _is_within(path.resolve(), _REPO_ROOT):
raise ValueError("outbox path must be absolute and external")
self._now = now
self._lock = threading.RLock()
self._connection = sqlite3.connect(str(path), timeout=5.0, isolation_level=None, check_same_thread=False)
self._connection.row_factory = sqlite3.Row
self._connection.execute("PRAGMA journal_mode=WAL")
self._connection.execute("PRAGMA synchronous=FULL")
self._connection.execute("PRAGMA foreign_keys=ON")
self._connection.executescript(
"""
CREATE TABLE IF NOT EXISTS event_outbox (
source_event_id TEXT PRIMARY KEY,
candidate_hash BLOB NOT NULL,
payload BLOB NOT NULL,
state TEXT NOT NULL CHECK(state IN ('queued','delivering','delivered','dead_letter')),
attempt_count INTEGER NOT NULL DEFAULT 0 CHECK(attempt_count BETWEEN 0 AND 100),
available_at REAL NOT NULL,
lease_until REAL,
bell_event_id TEXT,
last_error_code TEXT,
created_at REAL NOT NULL,
updated_at REAL NOT NULL,
delivered_at REAL,
dead_lettered_at REAL,
CHECK(length(candidate_hash)=32),
CHECK(length(payload)<=1048576),
CHECK((state='delivering')=(lease_until IS NOT NULL)),
CHECK(delivered_at IS NULL OR dead_lettered_at IS NULL)
);
CREATE INDEX IF NOT EXISTS event_outbox_due_idx
ON event_outbox(available_at, created_at, source_event_id)
WHERE state='queued';
CREATE TABLE IF NOT EXISTS event_outbox_identity (
singleton INTEGER PRIMARY KEY CHECK(singleton=1),
producer_id TEXT NOT NULL,
tenant_id INTEGER NOT NULL CHECK(tenant_id>=1),
site_id INTEGER NOT NULL CHECK(site_id>=1),
device_id INTEGER NOT NULL CHECK(device_id>=1)
);
"""
)
def bind_identity(self, config: IngressConfig) -> None:
identity = (config.producer_id, config.tenant_id, config.site_id, config.device_id)
with self._lock:
self._connection.execute("BEGIN IMMEDIATE")
try:
row = self._connection.execute(
"SELECT producer_id,tenant_id,site_id,device_id FROM event_outbox_identity WHERE singleton=1"
).fetchone()
if row is None:
self._connection.execute(
"""INSERT INTO event_outbox_identity(
singleton,producer_id,tenant_id,site_id,device_id
) VALUES (1,?,?,?,?)""",
identity,
)
elif tuple(row) != identity:
raise ValueError("event outbox is bound to a different producer or device identity")
self._connection.execute("COMMIT")
except Exception:
self._connection.execute("ROLLBACK")
raise
def enqueue(self, payload: bytes) -> bool:
if not 0 < len(payload) <= MAX_PAYLOAD_BYTES:
raise ValueError("event candidate payload must contain at most 1 MiB")
try:
candidate = json.loads(payload)
source_event_id = candidate["source_event_id"]
except (UnicodeError, json.JSONDecodeError, KeyError, TypeError) as exc:
raise ValueError("invalid event candidate payload") from exc
if not isinstance(source_event_id, str) or not _SOURCE_ID.fullmatch(source_event_id):
raise ValueError("invalid source_event_id")
digest = hashlib.sha256(payload).digest()
now = self._now()
with self._lock:
self._connection.execute("BEGIN IMMEDIATE")
try:
existing = self._connection.execute(
"SELECT candidate_hash FROM event_outbox WHERE source_event_id=?", (source_event_id,)
).fetchone()
if existing is not None:
if not hmac.compare_digest(existing["candidate_hash"], digest):
raise ValueError("source_event_id already has a different candidate")
self._connection.execute("COMMIT")
return False
active = self._connection.execute(
"SELECT count(*) FROM event_outbox WHERE state IN ('queued','delivering')"
).fetchone()[0]
if active >= MAX_QUEUED:
raise RuntimeError("event outbox capacity exceeded")
self._connection.execute(
"""INSERT INTO event_outbox(
source_event_id,candidate_hash,payload,state,available_at,created_at,updated_at
) VALUES (?,?,?,'queued',?,?,?)""",
(source_event_id, digest, payload, now, now, now),
)
self._connection.execute("COMMIT")
return True
except Exception:
self._connection.execute("ROLLBACK")
raise
def lease_one(self, lease_seconds: float = 30.0) -> LeasedEvent | None:
now = self._now()
with self._lock:
self._connection.execute("BEGIN IMMEDIATE")
try:
self._connection.execute(
"""UPDATE event_outbox SET state='dead_letter', lease_until=NULL,
last_error_code='retry_exhausted', dead_lettered_at=?, updated_at=?
WHERE attempt_count>=? AND (
state='queued' OR (state='delivering' AND lease_until<=?)
)""",
(now, now, MAX_ATTEMPTS, now),
)
self._connection.execute(
"""UPDATE event_outbox SET state='queued', lease_until=NULL,
available_at=?, last_error_code='lease_expired', updated_at=?
WHERE state='delivering' AND lease_until<=? AND attempt_count<?""",
(now, now, now, MAX_ATTEMPTS),
)
row = self._connection.execute(
"""SELECT source_event_id,payload,attempt_count FROM event_outbox
WHERE state='queued' AND available_at<=? AND attempt_count<?
ORDER BY available_at,created_at,source_event_id LIMIT 1""",
(now, MAX_ATTEMPTS),
).fetchone()
if row is None:
self._connection.execute("COMMIT")
return None
attempt = row["attempt_count"] + 1
self._connection.execute(
"""UPDATE event_outbox SET state='delivering',attempt_count=?,lease_until=?,updated_at=?
WHERE source_event_id=? AND state='queued'""",
(attempt, now + lease_seconds, now, row["source_event_id"]),
)
self._connection.execute("COMMIT")
return LeasedEvent(row["source_event_id"], bytes(row["payload"]), attempt)
except Exception:
self._connection.execute("ROLLBACK")
raise
def mark_delivered(self, item: LeasedEvent, result: DeliveryResult) -> None:
now = self._now()
with self._lock:
changed = self._connection.execute(
"""UPDATE event_outbox SET state='delivered',lease_until=NULL,bell_event_id=?,
last_error_code=NULL,delivered_at=?,updated_at=?
WHERE source_event_id=? AND state='delivering' AND attempt_count=?""",
(result.event_id, now, now, item.source_event_id, item.attempt_count),
).rowcount
if changed != 1:
raise RuntimeError("event outbox delivery lease was lost")
def mark_retry(self, item: LeasedEvent, code: str) -> None:
now = self._now()
if item.attempt_count >= MAX_ATTEMPTS:
self.mark_dead(item, "retry_exhausted")
return
delay = min(300.0, float(2 ** min(item.attempt_count - 1, 9)))
with self._lock:
changed = self._connection.execute(
"""UPDATE event_outbox SET state='queued',lease_until=NULL,available_at=?,
last_error_code=?,updated_at=?
WHERE source_event_id=? AND state='delivering' AND attempt_count=?""",
(now + delay, _safe_error(code), now, item.source_event_id, item.attempt_count),
).rowcount
if changed != 1:
raise RuntimeError("event outbox retry lease was lost")
def mark_dead(self, item: LeasedEvent, code: str) -> None:
now = self._now()
with self._lock:
changed = self._connection.execute(
"""UPDATE event_outbox SET state='dead_letter',lease_until=NULL,
last_error_code=?,dead_lettered_at=?,updated_at=?
WHERE source_event_id=? AND state='delivering' AND attempt_count=?""",
(_safe_error(code), now, now, item.source_event_id, item.attempt_count),
).rowcount
if changed != 1:
raise RuntimeError("event outbox dead-letter lease was lost")
def status(self) -> dict[str, object]:
with self._lock:
counts = {row["state"]: row["count"] for row in self._connection.execute(
"SELECT state,count(*) AS count FROM event_outbox GROUP BY state"
)}
error = self._connection.execute(
"""SELECT last_error_code FROM event_outbox WHERE last_error_code IS NOT NULL
ORDER BY updated_at DESC LIMIT 1"""
).fetchone()
return {
"enabled": True,
"queued": counts.get("queued", 0),
"delivering": counts.get("delivering", 0),
"delivered": counts.get("delivered", 0),
"dead_letter": counts.get("dead_letter", 0),
"last_error_code": None if error is None else error["last_error_code"],
}
def close(self) -> None:
with self._lock:
self._connection.close()
class EventIngressClient:
def __init__(self, config: IngressConfig, key: KeyMaterial, *, now: Callable[[], float] = time.time) -> None:
self._config = config
self._key = key
self._now = now
# Internal event payloads must never be redirected through ambient
# HTTP(S)_PROXY settings.
self._opener = build_opener(ProxyHandler({}), _NoRedirect())
def deliver(self, candidate: bytes) -> DeliveryResult:
try:
candidate_object = json.loads(candidate)
except (UnicodeError, json.JSONDecodeError) as exc:
raise PermanentDelivery("candidate_invalid") from exc
body = json.dumps(
{"schema_version": 1, "producer_id": self._config.producer_id, "candidate": candidate_object},
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
if len(body) > MAX_PAYLOAD_BYTES:
raise PermanentDelivery("payload_too_large")
timestamp = str(int(self._now()))
nonce = base64.urlsafe_b64encode(secrets.token_bytes(16)).rstrip(b"=").decode("ascii")
digest = hashlib.sha256(body).hexdigest()
canonical = "\n".join(("POST", INGRESS_PATH, timestamp, nonce, digest)).encode("utf-8")
signature_value = base64.urlsafe_b64encode(hmac.new(self._key.secret, canonical, hashlib.sha256).digest()).rstrip(b"=").decode("ascii")
request = Request(self._config.bell_url, data=body, method="POST")
request.add_header("Content-Type", "application/json")
request.add_header("X-YoVision-Key-Id", self._key.key_id)
request.add_header("X-YoVision-Timestamp", timestamp)
request.add_header("X-YoVision-Nonce", nonce)
request.add_header("X-YoVision-Signature", signature_value)
try:
with self._opener.open(request, timeout=HTTP_TIMEOUT_SECONDS) as response:
status = response.status
response_body = response.read(64 << 10)
except HTTPError as exc:
status = exc.code
response_body = exc.read(64 << 10)
except (URLError, TimeoutError, OSError) as exc:
raise RetryableDelivery("network_error") from exc
if status in {200, 201}:
try:
value = json.loads(response_body)
response_status = value["status"]
event_id = value["event_id"]
if (
value["schema_version"] != 1
or value["producer_id"] != self._config.producer_id
or value["source_event_id"] != candidate_object["source_event_id"]
or response_status not in {"accepted", "duplicate"}
or not isinstance(event_id, str)
or not _EVENT_ID.fullmatch(event_id)
):
raise ValueError
except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
raise RetryableDelivery("invalid_response") from exc
return DeliveryResult(response_status, event_id)
code = "http_error"
try:
error_value = json.loads(response_body)
if isinstance(error_value, dict) and isinstance(error_value.get("error"), str):
code = error_value["error"]
except (UnicodeError, json.JSONDecodeError):
pass
if status == 401 or status >= 500:
raise RetryableDelivery(code)
raise PermanentDelivery(code)
class EventRelayWorker:
def __init__(self, outbox: EventOutbox, client: EventIngressClient) -> None:
self._outbox = outbox
self._client = client
self._stop = threading.Event()
self._thread: threading.Thread | None = None
self._last_error_code: str | None = None
def run_once(self) -> bool:
item = self._outbox.lease_one()
if item is None:
return False
try:
result = self._client.deliver(item.payload)
except PermanentDelivery as exc:
self._outbox.mark_dead(item, exc.code)
except RetryableDelivery as exc:
self._outbox.mark_retry(item, exc.code)
except Exception:
self._outbox.mark_retry(item, "client_error")
else:
self._outbox.mark_delivered(item, result)
return True
def start(self) -> None:
if self._thread is not None:
return
self._thread = threading.Thread(target=self._run, name="brain-event-relay", daemon=True)
self._thread.start()
def _run(self) -> None:
while not self._stop.is_set():
try:
worked = self.run_once()
self._last_error_code = None
except Exception:
self._last_error_code = "event_outbox_unavailable"
self._stop.wait(1.0)
continue
if not worked:
self._stop.wait(0.25)
def stop(self) -> None:
self._stop.set()
if self._thread is not None:
self._thread.join(timeout=HTTP_TIMEOUT_SECONDS + 2.0)
self._thread = None
def last_error_code(self) -> str | None:
return self._last_error_code
class BrainEventIngress:
def __init__(self, config: IngressConfig, key: KeyMaterial) -> None:
self._config = config
self._outbox = EventOutbox(config.outbox_path)
self._outbox.bind_identity(config)
self._worker = EventRelayWorker(self._outbox, EventIngressClient(config, key))
@classmethod
def from_file(cls, path: str) -> "BrainEventIngress":
config, key = load_config(path)
return cls(config, key)
def submit(self, value: EventCandidate) -> None:
self._outbox.enqueue(map_candidate(value, self._config))
def start(self) -> None:
self._worker.start()
def stop(self) -> None:
self._worker.stop()
self._outbox.close()
def status(self) -> dict[str, object]:
value = self._outbox.status()
if self._worker.last_error_code() is not None:
value["last_error_code"] = self._worker.last_error_code()
return value
def _safe_error(value: str) -> str:
if not isinstance(value, str) or not _ERROR_CODE.fullmatch(value):
return "unknown_error"
return value
+33 -2
View File
@@ -24,8 +24,12 @@ DEFAULT_ZONE = Zone(
)
class EventIngressUnavailable(RuntimeError):
pass
class DemoEngine:
def __init__(self, source: Any, fps: float = 2.0, event_limit: int = 100) -> None:
def __init__(self, source: Any, fps: float = 2.0, event_limit: int = 100, event_ingress: Any | None = None) -> None:
if not math.isfinite(fps) or fps <= 0.0 or fps > 30.0:
raise ValueError("fps must be within (0, 30]")
if not 1 <= event_limit <= 100:
@@ -36,6 +40,7 @@ class DemoEngine:
self._detector = None if source.fixture else HOGPersonDetector()
self._tracker = CentroidTracker()
self._evaluator = ZoneEntryEvaluator(source.source_ref, source.fixture)
self._event_ingress = event_ingress
self._zone = DEFAULT_ZONE
self._events: deque[dict[str, object]] = deque(maxlen=event_limit)
self._lock = threading.RLock()
@@ -54,6 +59,8 @@ class DemoEngine:
def start(self) -> None:
if self._thread is not None:
return
if self._event_ingress is not None:
self._event_ingress.start()
self._thread = threading.Thread(target=self._run, name="brain-demo", daemon=True)
self._thread.start()
@@ -62,6 +69,8 @@ class DemoEngine:
if self._thread is not None:
self._thread.join(timeout=3.0)
self._thread = None
if self._event_ingress is not None:
self._event_ingress.stop()
self._source.close()
def _run(self) -> None:
@@ -70,6 +79,9 @@ class DemoEngine:
started = time.perf_counter()
try:
self.step()
except EventIngressUnavailable:
with self._lock:
self._last_error_code = "event_outbox_unavailable"
except RuntimeError:
with self._lock:
self._connected = False
@@ -89,6 +101,12 @@ class DemoEngine:
with self._lock:
zone = self._zone
new_events, inside_by_track = self._evaluator.evaluate(self._sequence, packet.captured_at, zone, detections)
if self._event_ingress is not None:
try:
for item in new_events:
self._event_ingress.submit(item)
except Exception as exc:
raise EventIngressUnavailable("persist event candidate") from exc
ok, encoded = cv2.imencode(".jpg", packet.frame, [int(cv2.IMWRITE_JPEG_QUALITY), 82])
if not ok:
raise RuntimeError("frame encoding failed")
@@ -96,7 +114,12 @@ class DemoEngine:
serialized_detections = [self._serialize_detection(item, inside_by_track.get(item.track_id, False)) for item in detections]
with self._lock:
for event in new_events:
self._events.appendleft(event.as_dict())
serialized_event = event.as_dict()
if self._event_ingress is not None:
# This is an immutable handoff fact, not a live delivery
# status. Current counts live under event_ingress.
serialized_event["delivery_status"] = "outbox_persisted"
self._events.appendleft(serialized_event)
self._frame_jpeg = encoded.tobytes()
self._frame_width = int(width)
self._frame_height = int(height)
@@ -130,6 +153,13 @@ class DemoEngine:
def state(self) -> dict[str, object]:
with self._lock:
zone = self._zone
if self._event_ingress is None:
ingress_status: dict[str, object] = {"enabled": False}
else:
try:
ingress_status = self._event_ingress.status()
except Exception:
ingress_status = {"enabled": True, "last_error_code": "event_outbox_unavailable"}
return {
"prototype": True,
"notice": "工程原型;合成回放不是模型输出,HOG 适配器不是生产检测模型。",
@@ -159,6 +189,7 @@ class DemoEngine:
},
"detections": list(self._detections),
"events": list(self._events),
"event_ingress": ingress_status,
"last_error_code": self._last_error_code,
"generated_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
}