feat(v1): latest-frame-only reader thread for low-latency RTSP

Adds _FrameGrabber which decodes in a background thread and keeps only the
newest frame; VideoSource drop_stale (STREAM only) returns the latest frame
so a slow consumer never builds a backlog. app enables it for streams.
REPLAY is unchanged. 82 tests pass.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

v1/app.py

@@ -38,7 +38,9 @@ class FrameWorker(QtCore.QThread):
         # OpenCV reads this env var when it opens the stream, so set it first.
         os.environ["OPENCV_FFMPEG_CAPTURE_OPTIONS"] = self._config.ffmpeg_capture_options
         mode = SourceMode.STREAM if self._config.source_mode == "stream" else SourceMode.REPLAY
-        source = VideoSource(self._config.source_url, mode=mode)
+        source = VideoSource(
+            self._config.source_url, mode=mode, drop_stale=mode is SourceMode.STREAM
+        )
         pipeline = FallPipeline.from_config(self._config, self._pose_adapter)
         try:
             while not self._stop:

v1/tests/test_video_source.py

@@ -1,8 +1,10 @@
+import time
+
 import cv2
 import numpy as np
 import pytest
 
-from v1.video_source import SourceMode, SourceStatus, VideoSource
+from v1.video_source import SourceMode, SourceStatus, VideoSource, _FrameGrabber
 
 
 def _write_sample_video(path):
@@ -135,6 +137,72 @@ def test_stream_recovers_after_mid_stream_drop():
     assert recovered.image is not None
 
 
+class _ScriptedCapture:
+    def __init__(self, frames, fail_after=False):
+        self._frames = list(frames)
+        self._fail_after = fail_after
+
+    def isOpened(self):
+        return True
+
+    def read(self):
+        if self._frames:
+            return True, self._frames.pop(0)
+        return False, None
+
+    def get(self, _property_id):
+        return 0.0
+
+    def release(self):
+        pass
+
+
+def test_frame_grabber_keeps_only_the_latest_frame():
+    frame_a = np.full((4, 4, 3), 1, dtype=np.uint8)
+    frame_b = np.full((4, 4, 3), 2, dtype=np.uint8)
+    frame_c = np.full((4, 4, 3), 3, dtype=np.uint8)
+    grabber = _FrameGrabber(_ScriptedCapture([frame_a, frame_b, frame_c]))
+
+    assert grabber._pump() is True
+    assert grabber._pump() is True
+    assert grabber._pump() is True
+    image, failed = grabber.take_latest()
+
+    assert failed is False
+    assert image is frame_c  # older frames dropped
+
+
+def test_frame_grabber_marks_failed_on_read_failure():
+    grabber = _FrameGrabber(_ScriptedCapture([np.zeros((4, 4, 3), dtype=np.uint8)]))
+
+    assert grabber._pump() is True
+    assert grabber._pump() is False
+    _, failed = grabber.take_latest()
+
+    assert failed is True
+
+
+def test_stream_with_drop_stale_returns_a_connected_frame():
+    source = VideoSource(
+        "rtsp://demo",
+        mode=SourceMode.STREAM,
+        drop_stale=True,
+        capture_factory=lambda _s: _OpenCapture(),
+    )
+    try:
+        packet = None
+        for _ in range(40):
+            packet = source.read()
+            if packet.status is SourceStatus.CONNECTED and packet.image is not None:
+                break
+            time.sleep(0.02)
+        assert packet is not None
+        assert packet.status is SourceStatus.CONNECTED
+        assert packet.image is not None
+    finally:
+        source.close()
+
+
 class _NegativeFirstTimestampCapture:
     def __init__(self):
         self._read_count = 0

v1/video_source.py

@@ -1,5 +1,6 @@
 """OpenCV frame acquisition with explicit replay and reconnect states."""
 
+import threading
 import time
 from dataclasses import dataclass
 from enum import Enum
@@ -10,6 +11,48 @@ import cv2
 import numpy as np
 
 
+class _FrameGrabber(threading.Thread):
+    """Continuously read a capture, keeping only the newest frame.
+
+    Decoding runs here so a slow consumer never builds a backlog: it always gets
+    the latest decoded frame and older frames are dropped. This bounds live RTSP
+    latency to roughly one inference regardless of stream vs processing rate.
+    """
+
+    def __init__(self, capture: Any) -> None:
+        super().__init__(daemon=True)
+        self._capture = capture
+        self._lock = threading.Lock()
+        self._latest: Optional[np.ndarray] = None
+        self._failed = False
+        self._stopped = threading.Event()
+
+    def _pump(self) -> bool:
+        try:
+            success, image = self._capture.read()
+        except Exception:  # noqa: BLE001 - a released capture must not crash the thread
+            success, image = False, None
+        if not success or image is None:
+            with self._lock:
+                self._failed = True
+            return False
+        with self._lock:
+            self._latest = image
+        return True
+
+    def run(self) -> None:
+        while not self._stopped.is_set():
+            if not self._pump():
+                break
+
+    def take_latest(self):
+        with self._lock:
+            return self._latest, self._failed
+
+    def stop(self) -> None:
+        self._stopped.set()
+
+
 class SourceStatus(str, Enum):
     CONNECTED = "connected"
     RETRYING = "retrying"
@@ -44,6 +87,7 @@ class VideoSource:
         retry_initial_seconds: float = 1.0,
         retry_max_seconds: float = 16.0,
         capture_factory: Optional[CaptureFactory] = None,
+        drop_stale: bool = False,
     ) -> None:
         if retry_initial_seconds <= 0:
             raise ValueError("retry_initial_seconds must be positive")
@@ -53,6 +97,8 @@ class VideoSource:
             raise ValueError("mode must be a SourceMode")
         self._source = str(source)
         self._mode = mode
+        self._drop_stale = bool(drop_stale) and mode is SourceMode.STREAM
+        self._grabber: Optional[_FrameGrabber] = None
         self._retry_initial_seconds = retry_initial_seconds
         self._retry_max_seconds = retry_max_seconds
         self._capture_factory = capture_factory or cv2.VideoCapture
@@ -69,7 +115,7 @@ class VideoSource:
         return self._status
 
     def close(self) -> None:
-        self._release_capture()
+        self._stop_grabber()
         self._closed = True
         self._status = SourceStatus.CLOSED
 
@@ -84,6 +130,9 @@ class VideoSource:
         if self._closed:
             return self._packet(timestamp, SourceStatus.CLOSED, "source is closed")
 
+        if self._drop_stale:
+            return self._read_latest(timestamp)
+
         if self._mode is SourceMode.REPLAY and self._status is SourceStatus.EOF:
             return self._packet(timestamp, SourceStatus.EOF, self._last_error)
 
@@ -109,6 +158,37 @@ class VideoSource:
         frame_timestamp = self._frame_timestamp(timestamp)
         return FramePacket(image=image, timestamp_monotonic=frame_timestamp, status=SourceStatus.CONNECTED)
 
+    def _read_latest(self, timestamp: float) -> FramePacket:
+        """STREAM path: return the newest grabbed frame, dropping any backlog."""
+
+        if self._capture is None:
+            if self._status is SourceStatus.RETRYING and timestamp < self._next_retry_at:
+                return self._packet(timestamp, SourceStatus.RETRYING, self._last_error)
+            if not self._open_capture(timestamp):
+                return self._packet(timestamp, self._status, self._last_error)
+            self._grabber = _FrameGrabber(self._capture)
+            self._grabber.start()
+
+        image, failed = self._grabber.take_latest()
+        if failed:
+            self._stop_grabber()
+            self._schedule_retry(timestamp, "stream read failed; retry scheduled")
+            return self._packet(timestamp, SourceStatus.RETRYING, self._last_error)
+        if image is None:
+            return self._packet(timestamp, SourceStatus.RETRYING, "connecting")
+
+        self._status = SourceStatus.CONNECTED
+        self._last_error = None
+        self._retry_delay_seconds = self._retry_initial_seconds
+        self._last_timestamp = timestamp
+        return FramePacket(image=image, timestamp_monotonic=timestamp, status=SourceStatus.CONNECTED)
+
+    def _stop_grabber(self) -> None:
+        if self._grabber is not None:
+            self._grabber.stop()
+            self._grabber = None
+        self._release_capture()
+
     def _open_capture(self, now: float) -> bool:
         capture = self._capture_factory(self._source)
         if capture is None or not capture.isOpened():