chengma/cmbuyer
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(client): add durable HTTP task state

Browse Source
This commit is contained in:
QiuSW
2026-08-05 00:59:32 +08:00
parent 488005ac93
commit 31f07ac245
37 changed files with 4654 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/tests/remote/__init__.py
+1
View File
@@ -0,0 +1 @@
"""remote tests。"""
client/tests/remote/test_evidence_sink.py
+167
View File
@@ -0,0 +1,167 @@
from __future__ import annotations
import hashlib
import json
import base64
import re
import unittest
from cmbuyer_client.core.models import DeviceCredentials, EvidenceUpload, SecretToken
from cmbuyer_client.core.errors import AmbiguousRemoteError, ValidationError
from cmbuyer_client.remote.evidence_sink import HttpEvidenceSink
from cmbuyer_client.remote.http_transport import HttpResponse
from tests.core.test_models import ATTEMPT_ID, TASK_ID, TOKEN
from tests.remote.test_task_source import DEVICE_ID, FakeTransport
UPLOAD_ID = "43c9f507-7473-4fa6-8d71-8786c34c6301"
ASSET_ID = "63c9f507-7473-4fa6-8d71-8786c34c6301"
PNG = base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
)
class EvidenceSinkTests(unittest.TestCase):
def test_upload_has_fixed_fields_and_never_contains_local_path(self) -> None:
digest = hashlib.sha256(PNG).hexdigest()
upload = EvidenceUpload(TASK_ID, UPLOAD_ID, ATTEMPT_ID, digest, "2026-08-04T09:01:00Z", PNG)
payload = {
"asset_id": ASSET_ID,
"task_id": TASK_ID,
"attempt_id": ATTEMPT_ID,
"kind": "SKU_PANEL_GATE_1",
"privacy_tier": "INTERNAL_RAW",
"sha256": digest,
"byte_size": len(PNG),
"content_type": "image/png",
"width_px": 1,
"height_px": 1,
"captured_at": "2026-08-04T09:01:00Z",
}
raw = json.dumps(payload, separators=(",", ":")).encode()
transport = FakeTransport(HttpResponse(201, (("Content-Type", "application/json"),), raw))
receipt = HttpEvidenceSink(transport).upload(DeviceCredentials(DEVICE_ID, SecretToken(TOKEN)), upload)
self.assertEqual(receipt.asset_id, ASSET_ID)
self.assertEqual(len(transport.calls), 1)
body = transport.calls[0][3]
self.assertIn(b'filename="evidence.png"', body)
self.assertNotIn(b"C:\\", body)
self.assertNotIn(b"manifest", body)
self.assertNotIn(b".xml", body)
names = re.findall(br'Content-Disposition: form-data; name="([^"]+)"', body)
self.assertEqual(
names,
[b"upload_key", b"attempt_id", b"kind", b"privacy_tier", b"sha256", b"captured_at", b"file"],
)
self.assertEqual(body.count(b'filename="evidence.png"'), 1)
self.assertNotIn(b"claim_token", body)
self.assertNotIn(b"session_id", body)
boundary = dict(transport.calls[0][2])["Content-Type"].split("boundary=", 1)[1]
self.assertEqual(boundary, "cmbuyer-" + UPLOAD_ID.replace("-", ""))
def test_captured_at_equivalent_trailing_zeros_are_accepted(self) -> None:
digest = hashlib.sha256(PNG).hexdigest()
upload = EvidenceUpload(TASK_ID, UPLOAD_ID, ATTEMPT_ID, digest, "2026-08-04T09:01:00.120000Z", PNG)
payload = {
"asset_id": ASSET_ID,
"task_id": TASK_ID,
"attempt_id": ATTEMPT_ID,
"kind": "SKU_PANEL_GATE_1",
"privacy_tier": "INTERNAL_RAW",
"sha256": digest,
"byte_size": len(PNG),
"content_type": "image/png",
"width_px": 1,
"height_px": 1,
"captured_at": "2026-08-04T09:01:00.12Z",
}
raw = json.dumps(payload, separators=(",", ":")).encode()
for status in (200, 201):
with self.subTest(status=status):
transport = FakeTransport(HttpResponse(status, (("Content-Type", "application/json"),), raw))
receipt = HttpEvidenceSink(transport).upload(DeviceCredentials(DEVICE_ID, SecretToken(TOKEN)), upload)
self.assertEqual(receipt.captured_at, "2026-08-04T09:01:00.12Z")
def test_wrong_content_hash_fails_before_http_object_can_be_built(self) -> None:
transport = FakeTransport(HttpResponse(500, (), b""))
with self.assertRaises(ValidationError):
upload = EvidenceUpload(TASK_ID, UPLOAD_ID, ATTEMPT_ID, "0" * 64, "2026-08-04T09:01:00Z", PNG)
HttpEvidenceSink(transport).upload(DeviceCredentials(DEVICE_ID, SecretToken(TOKEN)), upload)
self.assertEqual(transport.calls, [])
def test_unknown_2xx_is_ambiguous(self) -> None:
digest = hashlib.sha256(PNG).hexdigest()
upload = EvidenceUpload(TASK_ID, UPLOAD_ID, ATTEMPT_ID, digest, "2026-08-04T09:01:00Z", PNG)
for status in (202, 204, 206):
with self.subTest(status=status):
transport = FakeTransport(HttpResponse(status, (), b""))
with self.assertRaises(AmbiguousRemoteError):
HttpEvidenceSink(transport).upload(DeviceCredentials(DEVICE_ID, SecretToken(TOKEN)), upload)
self.assertEqual(len(transport.calls), 1)
def test_boundary_collision_and_receipt_mismatch_fail_closed(self) -> None:
marker = ("--cmbuyer-" + UPLOAD_ID.replace("-", "")).encode()
collision_content = PNG + marker
collision = EvidenceUpload(
TASK_ID,
UPLOAD_ID,
ATTEMPT_ID,
hashlib.sha256(collision_content).hexdigest(),
"2026-08-04T09:01:00Z",
collision_content,
)
transport = FakeTransport(HttpResponse(500, (), b""))
from cmbuyer_client.core.errors import ProtocolRemoteError
with self.assertRaises(ProtocolRemoteError):
HttpEvidenceSink(transport).upload(DeviceCredentials(DEVICE_ID, SecretToken(TOKEN)), collision)
self.assertEqual(transport.calls, [])
digest = hashlib.sha256(PNG).hexdigest()
upload = EvidenceUpload(TASK_ID, UPLOAD_ID, ATTEMPT_ID, digest, "2026-08-04T09:01:00Z", PNG)
mismatch = {
"asset_id": ASSET_ID,
"task_id": TASK_ID,
"attempt_id": ATTEMPT_ID,
"kind": "SKU_PANEL_GATE_1",
"privacy_tier": "INTERNAL_RAW",
"sha256": "f" * 64,
"byte_size": len(PNG),
"content_type": "image/png",
"width_px": 1,
"height_px": 1,
"captured_at": "2026-08-04T09:01:00Z",
}
raw = json.dumps(mismatch, separators=(",", ":")).encode()
with self.assertRaises(AmbiguousRemoteError):
HttpEvidenceSink(FakeTransport(HttpResponse(201, (("Content-Type", "application/json"),), raw))).upload(
DeviceCredentials(DEVICE_ID, SecretToken(TOKEN)), upload
)
dimension_mismatch = dict(mismatch)
dimension_mismatch["sha256"] = digest
dimension_mismatch["width_px"] = 2
raw = json.dumps(dimension_mismatch, separators=(",", ":")).encode()
with self.assertRaises(AmbiguousRemoteError):
HttpEvidenceSink(FakeTransport(HttpResponse(201, (("Content-Type", "application/json"),), raw))).upload(
DeviceCredentials(DEVICE_ID, SecretToken(TOKEN)), upload
)
def test_evidence_error_status_matrix(self) -> None:
digest = hashlib.sha256(PNG).hexdigest()
upload = EvidenceUpload(TASK_ID, UPLOAD_ID, ATTEMPT_ID, digest, "2026-08-04T09:01:00Z", PNG)
credentials = DeviceCredentials(DEVICE_ID, SecretToken(TOKEN))
from cmbuyer_client.core.errors import CredentialRemoteError, ManualRemoteError, ProtocolRemoteError
cases = (
(401, CredentialRemoteError),
(400, ProtocolRemoteError),
(403, ProtocolRemoteError),
(409, ManualRemoteError),
(413, ProtocolRemoteError),
(415, ProtocolRemoteError),
(500, AmbiguousRemoteError),
(503, AmbiguousRemoteError),
)
for status, expected in cases:
with self.subTest(status=status), self.assertRaises(expected):
HttpEvidenceSink(FakeTransport(HttpResponse(status, (), b""))).upload(credentials, upload)
client/tests/remote/test_http_transport.py
+184
View File
@@ -0,0 +1,184 @@
from __future__ import annotations
import os
import http.client
import unittest
from unittest import mock
from cmbuyer_client.core.errors import AmbiguousRemoteError, ProtocolRemoteError
from cmbuyer_client.remote.http_transport import HttpTransport
class FakeResponse:
status = 200
def __init__(self, body: bytes = b"{}", headers: list[tuple[str, str]] | None = None) -> None:
self.body = body
self.headers = headers or [("Content-Type", "application/json")]
def getheader(self, name: str) -> str | None:
return str(len(self.body)) if name == "Content-Length" else None
def getheaders(self) -> list[tuple[str, str]]:
return self.headers
def read(self, maximum: int) -> bytes:
return self.body[:maximum]
class FakeConnection:
def __init__(self, host: str, port: int, timeout: int) -> None:
self.created = (host, port, timeout)
self.calls = 0
self.closed = False
self.sent_headers: dict[str, str] = {}
self.response = FakeResponse()
def request(self, method: str, path: str, body: bytes, headers: dict[str, str]) -> None:
self.calls += 1
self.sent_headers = headers
def getresponse(self) -> FakeResponse:
return self.response
def close(self) -> None:
self.closed = True
class HttpTransportTests(unittest.TestCase):
def test_exact_loopback_and_proxy_environment_is_irrelevant(self) -> None:
made: list[FakeConnection] = []
def factory(*args: object, **kwargs: object) -> FakeConnection:
connection = FakeConnection(*args, **kwargs)
made.append(connection)
return connection
with mock.patch.dict(os.environ, {"HTTP_PROXY": "http://example.invalid:9999"}):
result = HttpTransport(connection_factory=factory).request(
"POST", "/api/v1/tasks/claim-next", (("Content-Type", "application/json"),), b"{}", response_limit=10
)
self.assertEqual(result.status, 200)
self.assertEqual(made[0].created, ("127.0.0.1", 8080, 10))
self.assertEqual(made[0].calls, 1)
self.assertTrue(made[0].closed)
for url in ("http://localhost:8080", "http://127.0.0.1:8081", "http://127.0.0.1:8080/", "https://127.0.0.1:8080"):
with self.subTest(url=url), self.assertRaises(ProtocolRemoteError):
HttpTransport(url)
def test_network_failure_is_ambiguous_without_retry(self) -> None:
class Broken(FakeConnection):
def getresponse(self) -> FakeResponse:
raise OSError("offline")
made: list[Broken] = []
def factory(*args: object, **kwargs: object) -> Broken:
connection = Broken(*args, **kwargs)
made.append(connection)
return connection
with self.assertRaises(AmbiguousRemoteError):
HttpTransport(connection_factory=factory).request(
"POST", "/api/v1/tasks/claim-next", (), b"{}", response_limit=10
)
self.assertEqual(made[0].calls, 1)
def test_generator_headers_and_content_length_framing(self) -> None:
made: list[FakeConnection] = []
def factory(*args: object, **kwargs: object) -> FakeConnection:
connection = FakeConnection(*args, **kwargs)
made.append(connection)
return connection
headers = ((name, value) for name, value in (("Content-Type", "application/json"), ("Accept", "application/json")))
HttpTransport(connection_factory=factory).request(
"POST", "/api/v1/tasks/claim-next", headers, b"{}", response_limit=8
)
self.assertEqual(made[0].sent_headers["Content-Type"], "application/json")
self.assertEqual(made[0].sent_headers["Accept"], "application/json")
legal = FakeConnection("127.0.0.1", 8080, 10)
legal.response = FakeResponse(b"{}", [("Content-Length", "2")])
accepted = HttpTransport(connection_factory=lambda *args, **kwargs: legal).request(
"POST", "/api/v1/tasks/claim-next", (), b"{}", response_limit=8
)
self.assertEqual(accepted.body, b"{}")
chunked = FakeConnection("127.0.0.1", 8080, 10)
chunked.response = FakeResponse(b"{}", [("Transfer-Encoding", "Chunked")])
accepted_chunked = HttpTransport(connection_factory=lambda *args, **kwargs: chunked).request(
"POST", "/api/v1/tasks/claim-next", (), b"{}", response_limit=8
)
self.assertEqual(accepted_chunked.body, b"{}")
cases = (
([('Transfer-Encoding', 'chunked'), ('Content-Length', '2')], b'{}'),
([('Transfer-Encoding', 'gzip')], b'{}'),
([('Transfer-Encoding', ' chunked ')], b'{}'),
([('Transfer-Encoding', 'chunked,gzip')], b'{}'),
([('Transfer-Encoding', 'chunked'), ('Transfer-Encoding', 'chunked')], b'{}'),
([("Content-Length", "2"), ("Content-Length", "2")], b"{}"),
([("Content-Length", "+2")], b"{}"),
([("Content-Length", "-0")], b""),
([("Content-Length", "2x")], b"{}"),
([("Content-Length", "3")], b"{}"),
([("Content-Length", "1")], b"{}"),
([("Content-Length", "999")], b"{}"),
([], b"0123456789"),
)
for response_headers, body in cases:
with self.subTest(headers=response_headers, body=body):
connection = FakeConnection("127.0.0.1", 8080, 10)
connection.response = FakeResponse(body, response_headers)
with self.assertRaises(AmbiguousRemoteError):
HttpTransport(connection_factory=lambda *args, value=connection, **kwargs: value).request(
"POST", "/api/v1/tasks/claim-next", (), b"{}", response_limit=8
)
def test_timeout_incomplete_read_and_close_do_not_expose_partial_body(self) -> None:
token = ("a" * 64).encode()
class Incomplete(FakeResponse):
def read(self, maximum: int) -> bytes:
raise http.client.IncompleteRead(token, 1)
class Connection(FakeConnection):
def getresponse(self) -> FakeResponse:
return Incomplete()
def close(self) -> None:
self.closed = True
raise OSError("close failed")
with self.assertRaises(AmbiguousRemoteError) as captured:
HttpTransport(connection_factory=Connection).request(
"POST", "/api/v1/tasks/claim-next", (), b"{}", response_limit=128
)
self.assertNotIn(token.decode(), _exception_graph(captured.exception))
class Timeout(FakeConnection):
def getresponse(self) -> FakeResponse:
raise TimeoutError("timed out")
with self.assertRaises(AmbiguousRemoteError):
HttpTransport(connection_factory=Timeout).request(
"POST", "/api/v1/tasks/claim-next", (), b"{}", response_limit=8
)
def _exception_graph(error: BaseException) -> str:
seen: set[int] = set()
values: list[str] = []
pending: list[object] = [error]
while pending:
value = pending.pop()
if id(value) in seen:
continue
seen.add(id(value))
values.append(repr(value))
if isinstance(value, BaseException):
pending.extend(item for item in (value.__cause__, value.__context__) if item is not None)
pending.extend(value.__dict__.values())
return "\n".join(values)
client/tests/remote/test_task_source.py
+136
View File
@@ -0,0 +1,136 @@
from __future__ import annotations
import json
import unittest
from cmbuyer_client.core.errors import AmbiguousRemoteError, CredentialRemoteError, ManualRemoteError, ProtocolRemoteError
from cmbuyer_client.core.models import ClaimRequest, DeviceCredentials, RenewRequest, SecretToken
from cmbuyer_client.remote.http_transport import HttpResponse
from cmbuyer_client.remote.task_source import HttpTaskSource
from tests.core.test_models import ATTEMPT_ID, TASK_ID, TOKEN, claim_wire
from tests.remote.test_http_transport import _exception_graph
DEVICE_ID = "e3c9f507-7473-4fa6-8d71-8786c34c6301"
SESSION_ID = "23c9f507-7473-4fa6-8d71-8786c34c6301"
REQUEST_ID = "33c9f507-7473-4fa6-8d71-8786c34c6301"
RENEW_ID = "43c9f507-7473-4fa6-8d71-8786c34c6301"
class FakeTransport:
def __init__(self, response: HttpResponse) -> None:
self.response = response
self.calls: list[tuple[object, ...]] = []
def request(self, *args: object, **kwargs: object) -> HttpResponse:
self.calls.append(args + (kwargs,))
return self.response
def response(status: int, value: object | None = None) -> HttpResponse:
body = b"" if value is None else json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode()
headers = () if value is None else (("Content-Type", "application/json; charset=utf-8"),)
return HttpResponse(status, headers, body)
class TaskSourceTests(unittest.TestCase):
def setUp(self) -> None:
self.credentials = DeviceCredentials(DEVICE_ID, SecretToken(TOKEN))
def test_claim_success_and_empty_each_send_once_with_exact_headers(self) -> None:
transport = FakeTransport(response(200, claim_wire()))
claimed = HttpTaskSource(transport).claim_next(self.credentials, ClaimRequest(SESSION_ID, REQUEST_ID))
self.assertEqual(claimed.task.id, TASK_ID)
self.assertEqual(len(transport.calls), 1)
args = transport.calls[0]
self.assertEqual(args[1], "/api/v1/tasks/claim-next")
headers = dict(args[2])
self.assertEqual(headers["Authorization"], "Bearer " + TOKEN)
self.assertEqual(headers["X-CMBuyer-Device-ID"], DEVICE_ID)
empty = FakeTransport(response(204))
self.assertIsNone(HttpTaskSource(empty).claim_next(self.credentials, ClaimRequest(SESSION_ID, REQUEST_ID)))
self.assertEqual(len(empty.calls), 1)
def test_invalid_2xx_is_unknown_and_redirect_is_not_followed(self) -> None:
malformed = FakeTransport(HttpResponse(200, (("Content-Type", "application/json"),), b'{"task":'))
with self.assertRaises(AmbiguousRemoteError):
HttpTaskSource(malformed).claim_next(self.credentials, ClaimRequest(SESSION_ID, REQUEST_ID))
self.assertEqual(len(malformed.calls), 1)
secret_body = b'{"claim_token":"' + TOKEN.encode()
leaking = FakeTransport(HttpResponse(200, (("Content-Type", "application/json"),), secret_body))
with self.assertRaises(AmbiguousRemoteError) as captured:
HttpTaskSource(leaking).claim_next(self.credentials, ClaimRequest(SESSION_ID, REQUEST_ID))
self.assertNotIn(TOKEN, _exception_graph(captured.exception))
for status in (201, 202, 206):
with self.subTest(status=status), self.assertRaises(AmbiguousRemoteError):
HttpTaskSource(FakeTransport(response(status, claim_wire()))).claim_next(
self.credentials, ClaimRequest(SESSION_ID, REQUEST_ID)
)
redirect = FakeTransport(HttpResponse(302, (("Location", "http://example.invalid"),), b""))
with self.assertRaises(ProtocolRemoteError):
HttpTaskSource(redirect).claim_next(self.credentials, ClaimRequest(SESSION_ID, REQUEST_ID))
self.assertEqual(len(redirect.calls), 1)
def test_fixed_conflict_and_renew_cas(self) -> None:
conflict = FakeTransport(response(409, {"error": "claim_requires_manual"}))
with self.assertRaises(ManualRemoteError):
HttpTaskSource(conflict).claim_next(self.credentials, ClaimRequest(SESSION_ID, REQUEST_ID))
request = RenewRequest(TASK_ID, RENEW_ID, SESSION_ID, ATTEMPT_ID, 1, SecretToken(TOKEN), "2026-08-04T09:05:00Z", "2026-08-04T10:00:00Z")
renewed = response(200, {"task_id": TASK_ID, "attempt_id": ATTEMPT_ID, "claim_generation": 1, "lease_expires_at": "2026-08-04T09:06:00Z"})
result = HttpTaskSource(FakeTransport(renewed)).renew(self.credentials, request)
self.assertEqual(result.claim_generation, 1)
capped = RenewRequest(TASK_ID, RENEW_ID, SESSION_ID, ATTEMPT_ID, 1, SecretToken(TOKEN), "2026-08-04T10:00:00.000000000Z", "2026-08-04T10:00:00Z")
capped_result = response(200, {"task_id": TASK_ID, "attempt_id": ATTEMPT_ID, "claim_generation": 1, "lease_expires_at": "2026-08-04T10:00:00Z"})
self.assertEqual(HttpTaskSource(FakeTransport(capped_result)).renew(self.credentials, capped).lease_expires_at, "2026-08-04T10:00:00Z")
beyond_cap = response(200, {"task_id": TASK_ID, "attempt_id": ATTEMPT_ID, "claim_generation": 1, "lease_expires_at": "2026-08-04T10:00:00.000000001Z"})
with self.assertRaises(AmbiguousRemoteError):
HttpTaskSource(FakeTransport(beyond_cap)).renew(self.credentials, request)
stale = response(200, {"task_id": TASK_ID, "attempt_id": ATTEMPT_ID, "claim_generation": 1, "lease_expires_at": "2026-08-04T09:04:00Z"})
with self.assertRaises(AmbiguousRemoteError):
HttpTaskSource(FakeTransport(stale)).renew(self.credentials, request)
for status in (201, 204):
with self.subTest(status=status), self.assertRaises(AmbiguousRemoteError):
HttpTaskSource(FakeTransport(response(status, None if status == 204 else {
"task_id": TASK_ID,
"attempt_id": ATTEMPT_ID,
"claim_generation": 1,
"lease_expires_at": "2026-08-04T09:06:00Z",
}))).renew(self.credentials, request)
def test_claim_and_renew_error_status_matrix(self) -> None:
claim_request = ClaimRequest(SESSION_ID, REQUEST_ID)
claim_cases = (
(HttpResponse(401, (), b""), CredentialRemoteError),
(response(400, {"error": "invalid_request"}), ProtocolRemoteError),
(HttpResponse(403, (), b""), ProtocolRemoteError),
(response(413, {"error": "request_too_large"}), ProtocolRemoteError),
(response(415, {"error": "unsupported_media_type"}), ProtocolRemoteError),
(HttpResponse(500, (), b""), AmbiguousRemoteError),
(HttpResponse(503, (), b""), AmbiguousRemoteError),
(HttpResponse(418, (), b""), ProtocolRemoteError),
)
for wire_response, expected in claim_cases:
with self.subTest(status=wire_response.status), self.assertRaises(expected):
HttpTaskSource(FakeTransport(wire_response)).claim_next(self.credentials, claim_request)
renew_request = RenewRequest(
TASK_ID,
RENEW_ID,
SESSION_ID,
ATTEMPT_ID,
1,
SecretToken(TOKEN),
"2026-08-04T09:05:00Z",
"2026-08-04T10:00:00Z",
)
for code in ("idempotency_conflict", "claim_not_current"):
with self.subTest(code=code), self.assertRaises(ManualRemoteError):
HttpTaskSource(FakeTransport(response(409, {"error": code}))).renew(self.credentials, renew_request)