feat: 执行并提交 PDD 采集任务 (#32)

This commit is contained in:
chengma
2026-08-07 17:38:29 +08:00
parent 34b4b54b63
commit 2d1fdbea3f
17 changed files with 2156 additions and 108 deletions
+229
View File
@@ -0,0 +1,229 @@
"""一条采集任务的应用层流程:领取、采集、落库、提交。"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Callable, Optional
from .admin_gateway import (
AdminGateway,
AdminGatewayError,
AndroidDeviceInfo,
ClaimCapabilities,
ClientInfo,
)
from .pdd_collect_service import PddCollectError, PddCollectService
from .pdd_device_service import PddDeviceService
from .db import data_dir
from .task_models import OutboxEventRecord, OutboxEventType, TaskStatus, TaskType
from .task_models import NewClaimedTask
from .task_repository import DuplicateTaskError, TaskRepository
@dataclass(frozen=True)
class CollectTaskOutcome:
"""工作线程返回给界面的简短结果。"""
kind: str
message: str
task_id: str = ""
CollectServiceFactory = Callable[
[str, str, Callable[[], bool]], PddCollectService
]
class CollectTaskService:
"""一次调用只处理一条本地工作或一个待提交事件。"""
def __init__(
self,
gateway: AdminGateway,
repository: TaskRepository,
client: ClientInfo,
device_address: str,
*,
cancelled: Callable[[], bool] = lambda: False,
collect_service_factory: Optional[CollectServiceFactory] = None,
) -> None:
self._gateway = gateway
self._repository = repository
self._client = client
self._device_address = device_address
self._cancelled = cancelled
self._factory = collect_service_factory or self._default_factory
def execute_one(self) -> CollectTaskOutcome:
"""先补交已有结果,再执行最早的本地任务,最后才领取新任务。"""
pending = self._repository.next_pending_outbox()
if pending is not None:
return self._submit(pending)
if not self._device_address.strip():
raise ValueError("请先在设置页选择并保存 Android 设备")
task = self._repository.next_collect_task()
if task is None:
remote = self._gateway.claim_next(
self._client,
ClaimCapabilities(
device=AndroidDeviceInfo(self._device_address),
supported_types=(TaskType.COLLECT,),
purchase_mode="dry_run",
schema_versions=(1,),
),
)
if remote is None:
return CollectTaskOutcome("no_task", "暂无可领取的采集任务")
try:
self._repository.add_claimed_task(
NewClaimedTask(
remote_task_id=remote.task_id,
task_type=remote.task_type,
goods_url=str(remote.payload.get("goods_url") or ""),
goods_id=(
str(remote.payload["goods_id"])
if remote.payload.get("goods_id") is not None
else None
),
priority=remote.priority,
version=remote.version,
admin_payload={
"id": remote.task_id,
"type": remote.task_type.value,
"version": remote.version,
"priority": remote.priority,
"payload": dict(remote.payload),
"created_at": remote.created_at,
"updated_at": remote.updated_at,
},
)
)
except DuplicateTaskError:
pass
except Exception as exc:
raise RuntimeError(
f"任务 {remote.task_id} 已领取,但本地保存失败:{exc}"
) from exc
task = self._repository.get_task(remote.task_id)
if task is None:
raise RuntimeError(f"任务 {remote.task_id} 未能保存到本地")
if self._cancelled():
return CollectTaskOutcome("cancelled", "本次采集已取消", task.remote_task_id)
started = self._repository.start_collect_run(
task.remote_task_id, self._device_address
)
collector = self._factory(
self._device_address, self._client.client_id, self._cancelled
)
try:
result = collector.collect(started.task)
event = self._repository.save_collect_result(
task.remote_task_id, started.attempt_id, result.to_pdd_data()
)
except PddCollectError as exc:
status, retryable = self._classify_error(exc.code)
report_code = self._report_error_code(exc.code)
event = self._repository.save_collect_failure(
task.remote_task_id,
started.attempt_id,
status,
report_code,
exc.message,
retryable,
exc.diagnostics,
)
return self._submit(event)
def _submit(self, event: OutboxEventRecord) -> CollectTaskOutcome:
task_id = self._repository.outbox_task_id(event.id)
self._repository.mark_outbox_sending(event.id)
try:
if event.event_type is OutboxEventType.TASK_FAILURE:
receipt = self._gateway.submit_failure(
task_id, event.idempotency_key, event.payload_json
)
else:
receipt = self._gateway.submit_result(
task_id, event.idempotency_key, event.payload_json
)
if not receipt.accepted:
raise AdminGatewayError(
"ADMIN_RESULT_NOT_ACCEPTED", "Admin 未确认接收结果", False
)
except AdminGatewayError as exc:
message = str(exc)
if exc.retryable:
self._repository.mark_outbox_retry(event.id, message)
return CollectTaskOutcome(
"result_pending",
f"任务 {task_id} 数据已保存在本地,等待重新提交 Admin:{message}",
task_id,
)
self._repository.mark_outbox_failed(event.id, message)
return CollectTaskOutcome(
"manual_review",
f"任务 {task_id} 提交被 Admin 拒绝:{message}",
task_id,
)
self._repository.mark_outbox_sent(event.id)
if event.event_type is OutboxEventType.TASK_FAILURE:
return CollectTaskOutcome(
"failed",
f"任务 {task_id} 采集未完成,失败信息已提交 Admin",
task_id,
)
return CollectTaskOutcome(
"succeeded", f"任务 {task_id} 采集完成并已提交 Admin", task_id
)
@staticmethod
def _classify_error(code: str) -> tuple[TaskStatus, bool]:
if code == "PDD_CANCELLED":
return TaskStatus.CANCELLED, False
if code in {
"PDD_PAGE_LOGIN_REQUIRED",
"PDD_PAGE_CAPTCHA",
"PDD_DATA_SPEC_INCOMPLETE",
"PDD_DATA_TITLE_MISSING",
"PDD_DATA_PRICE_MISSING",
"PDD_DATA_SKU_NAME_TRUNCATED",
}:
return TaskStatus.MANUAL_REVIEW, False
if code.startswith("PDD_DATA_GOODS_"):
return TaskStatus.FAILED, False
return TaskStatus.RETRY_WAIT, True
@staticmethod
def _report_error_code(code: str) -> str:
"""把内部解析代码映射为 Admin 可枚举的稳定代码。"""
return {
"PDD_DATA_SKU_NAME_TRUNCATED": "SKU_NAME_TRUNCATED",
"PDD_PAGE_SPEC_ENTRY_MISSING": "SKU_PANEL_NOT_FOUND",
"PDD_DATA_TITLE_MISSING": "TITLE_TOO_SHORT",
"PDD_PAGE_OVERALL_TIMEOUT": "COLLECT_TIMEOUT",
"DEVICE_OFFLINE": "DEVICE_DISCONNECTED",
"DEVICE_CONNECT_FAILED": "DEVICE_DISCONNECTED",
"DEVICE_DISCONNECTED": "DEVICE_DISCONNECTED",
}.get(code, code)
@staticmethod
def _default_factory(
device_address: str,
client_id: str,
cancelled: Callable[[], bool],
) -> PddCollectService:
return PddCollectService(
PddDeviceService(),
device_address,
client_id,
cancelled=cancelled,
overall_timeout=600.0,
artifact_directory=data_dir() / "artifacts",
)
+107 -5
View File
@@ -3,20 +3,20 @@
import json
import socket
from http.client import RemoteDisconnected
from typing import Callable, Mapping, Optional
from typing import Any, Callable, Mapping, Optional
from urllib.error import HTTPError, URLError
from urllib.parse import urlparse
from urllib.request import ProxyHandler, Request, build_opener
from uuid import uuid4
from .admin_gateway import (
AdminGateway,
AdminTask,
AdminGatewayError,
ClaimCapabilities,
ClientInfo,
ClientRegistrationGateway,
RegistrationReceipt,
TaskClaimGateway,
SubmissionReceipt,
)
from .task_models import TaskType
@@ -24,8 +24,8 @@ from .task_models import TaskType
DEFAULT_ADMIN_BASE_URL = "http://127.0.0.1:8080"
class HttpAdminGateway(ClientRegistrationGateway, TaskClaimGateway):
"""通过 HTTP 登记 Client 和领取采集任务;令牌只保存在内存。"""
class HttpAdminGateway(AdminGateway):
"""通过 HTTP 登记、领取和提交任务;令牌只保存在内存。"""
def __init__(
self,
@@ -33,6 +33,7 @@ class HttpAdminGateway(ClientRegistrationGateway, TaskClaimGateway):
token: str = "",
timeout_seconds: float = 3.0,
opener: Optional[Callable] = None,
client_id: str = "",
):
normalized_url = base_url.strip().rstrip("/")
parsed = urlparse(normalized_url)
@@ -44,6 +45,7 @@ class HttpAdminGateway(ClientRegistrationGateway, TaskClaimGateway):
self._base_url = normalized_url
self._token = token.strip()
self._timeout_seconds = timeout_seconds
self._client_id = client_id.strip()
# Admin 通常运行在本机或局域网。明确禁用环境代理,避免
# HTTP_PROXY 把 127.0.0.1 请求错误转发到代理服务器。
self._opener = opener or build_opener(ProxyHandler({})).open
@@ -54,6 +56,7 @@ class HttpAdminGateway(ClientRegistrationGateway, TaskClaimGateway):
"""调用独立登记接口,不领取或修改任务。"""
request_id = str(uuid4())
self._client_id = client.client_id.strip()
payload = {
"client": {"name": client.name.strip()},
"supported_types": [
@@ -146,6 +149,7 @@ class HttpAdminGateway(ClientRegistrationGateway, TaskClaimGateway):
"""领取一个采集任务;Admin 返回 204 时返回 ``None``。"""
request_id = str(uuid4())
self._client_id = client.client_id.strip()
payload = {
"client": {"name": client.name.strip()},
# #30 只允许领取采集任务。采购能力必须由安全门禁工单开启。
@@ -219,6 +223,104 @@ class HttpAdminGateway(ClientRegistrationGateway, TaskClaimGateway):
)
return self._parse_claim_response(body, request_id)
def submit_result(
self,
task_id: str,
idempotency_key: str,
result: Mapping[str, Any],
) -> SubmissionReceipt:
"""幂等提交成功结果。"""
return self._submit(task_id, idempotency_key, result, "result")
def submit_failure(
self,
task_id: str,
idempotency_key: str,
failure: Mapping[str, Any],
) -> SubmissionReceipt:
"""幂等提交失败或人工处理结果。"""
return self._submit(task_id, idempotency_key, failure, "failure")
def _submit(
self,
task_id: str,
idempotency_key: str,
payload: Mapping[str, Any],
endpoint: str,
) -> SubmissionReceipt:
if not task_id.strip():
raise ValueError("task_id 不能为空")
if not idempotency_key.strip():
raise ValueError("Idempotency-Key 不能为空")
if not self._client_id:
raise AdminGatewayError(
"CLIENT_ID_MISSING",
"提交结果前必须先配置 Client 设备号",
False,
)
request_id = str(uuid4())
headers = {
"Content-Type": "application/json; charset=utf-8",
"Accept": "application/json",
"Idempotency-Key": idempotency_key.strip(),
"X-Request-Id": request_id,
"X-Client-Id": self._client_id,
}
if self._token:
headers["Authorization"] = f"Bearer {self._token}"
request = Request(
f"{self._base_url}/api/v1/client/tasks/{task_id.strip()}/{endpoint}",
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
headers=headers,
method="POST",
)
try:
with self._opener(request, timeout=self._timeout_seconds) as response:
status = getattr(response, "status", None)
if status is None:
status = response.getcode()
body = response.read()
except HTTPError as exc:
self._raise_http_error(exc, request_id, "提交任务")
except (
URLError,
RemoteDisconnected,
ConnectionError,
socket.timeout,
TimeoutError,
) as exc:
reason = getattr(exc, "reason", exc)
code = "ADMIN_TIMEOUT" if isinstance(reason, (socket.timeout, TimeoutError)) else "ADMIN_UNAVAILABLE"
message = "Admin 提交请求超时,请稍后重试" if code == "ADMIN_TIMEOUT" else "无法连接 Admin,结果已保存在本地"
raise AdminGatewayError(code, message, True, request_id) from exc
if status not in (200, 201):
raise AdminGatewayError(
"ADMIN_UNEXPECTED_RESPONSE",
f"Admin 返回了未预期的状态码 {status}",
status >= 500,
request_id,
)
data = self._decode_json(body, request_id)
accepted = data.get("accepted")
result_id = data.get("result_id")
accepted_at = data.get("accepted_at")
if (
accepted is not True
or not isinstance(result_id, str)
or not result_id.strip()
or not isinstance(accepted_at, str)
or not accepted_at.strip()
):
raise AdminGatewayError(
"ADMIN_INVALID_RESPONSE",
"Admin 提交响应字段不完整",
False,
request_id,
)
return SubmissionReceipt(True, result_id, accepted_at)
@classmethod
def _parse_claim_response(
cls,
+197 -35
View File
@@ -14,6 +14,7 @@ import xml.etree.ElementTree as ET
from dataclasses import dataclass
from datetime import datetime, timezone
from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
from pathlib import Path
from typing import Any, Callable, Iterable, Mapping, Optional, Sequence
from urllib.parse import parse_qs, urlparse
@@ -31,13 +32,32 @@ _CAPTCHA_MARKERS = ("请完成验证", "拖动滑块", "安全验证", "点击
_READY_MARKERS = ("发起拼单", "立即购买", "单独购买", "免拼购买", "快要抢光")
def _is_device_disconnect(error: BaseException) -> bool:
details = str(error).lower()
return any(
marker in details
for marker in (
"device not found",
"device offline",
"disconnected",
"closed transport",
)
)
class PddCollectError(RuntimeError):
"""采集失败,并携带稳定错误码。"""
def __init__(self, code: str, message: str) -> None:
def __init__(
self,
code: str,
message: str,
diagnostics: Optional[Mapping[str, Any]] = None,
) -> None:
super().__init__(message)
self.code = code
self.message = message
self.diagnostics = dict(diagnostics or {})
@dataclass(frozen=True)
@@ -85,11 +105,15 @@ class SkuResult:
price_cent: Optional[int]
available: bool
raw_price: Optional[str]
price_observed_at: Mapping[str, str]
list_price_cent: Optional[int] = None
def to_dict(self) -> dict[str, Any]:
return {
"options": dict(self.options),
"price_cent": self.price_cent,
"list_price_cent": self.list_price_cent,
"price_observed_at": dict(self.price_observed_at),
"currency": "CNY",
"available": self.available,
"raw_price": self.raw_price,
@@ -110,6 +134,7 @@ class SpecSnapshot:
selected_text: Optional[str]
price_cent: Optional[int]
raw_price: Optional[str]
list_price_cent: Optional[int]
@dataclass(frozen=True)
@@ -127,16 +152,16 @@ class CollectResult:
captured_at: str
client_id: str
device_address: str
artifacts: tuple[Mapping[str, Any], ...] = ()
def to_pdd_data(self) -> dict[str, Any]:
return {
"schema_version": 1,
"goods": {
"goods_id": self.goods_id,
"url": self.goods_url,
"title": self.title,
},
"shop": {"name": self.shop_name},
"goods_id": self.goods_id,
"goods_url": self.goods_url,
"title": self.title,
"shop_name": self.shop_name,
"price_granularity": "color",
"metrics": {
"sales": self.sales.to_dict(),
"reviews": self.reviews.to_dict(),
@@ -150,7 +175,7 @@ class CollectResult:
"device_address": self.device_address,
"pdd_package": PDD_PACKAGE_NAME,
},
"artifacts": [],
"artifacts": [dict(item) for item in self.artifacts],
}
@@ -182,6 +207,14 @@ def _node_label(node: ET.Element) -> str:
)
def _preferred_node_label(node: ET.Element) -> str:
"""规格名被 text 截断时,优先采用完整的无障碍描述。"""
text = (node.get("text") or "").strip()
description = (node.get("content-desc") or "").strip()
return description or text
def _own_or_descendant_label(node: ET.Element) -> str:
own = _node_label(node)
if own:
@@ -193,6 +226,17 @@ def _own_or_descendant_label(node: ET.Element) -> str:
return ""
def _preferred_or_descendant_label(node: ET.Element) -> str:
own = _preferred_node_label(node)
if own:
return own
for child in node.iter("node"):
label = _preferred_node_label(child)
if label and label != "打开大图":
return label
return ""
def _all_labels(root: ET.Element) -> list[str]:
return [label for node in root.iter("node") if (label := _node_label(node))]
@@ -250,9 +294,21 @@ def parse_goods_page(xml_data: str | bytes) -> GoodsSnapshot:
title = candidate
break
if title is None:
line_parts: dict[int, list[tuple[int, str]]] = {}
for node in root.iter("node"):
label = _preferred_node_label(node)
bounds = _parse_bounds(node.get("bounds", ""))
if not label or bounds is None:
continue
line_key = bounds[1] // 24
line_parts.setdefault(line_key, []).append((bounds[0], label))
joined_lines = [
"".join(text for _, text in sorted(parts))
for parts in line_parts.values()
]
candidates = [
label
for label in labels
for label in labels + joined_lines
if len(label) >= 12
and not any(word in label for word in ("通知", "支付", "已拼", "评价"))
]
@@ -347,7 +403,7 @@ def _top_level_clickable_options(
def _price_from_nodes(
nodes: Iterable[ET.Element],
spec_top: int,
) -> tuple[Optional[int], Optional[str]]:
) -> tuple[Optional[int], Optional[str], Optional[int]]:
candidates: list[tuple[int, int, str]] = []
for node in nodes:
label = _node_label(node)
@@ -366,9 +422,12 @@ def _price_from_nodes(
continue
candidates.append((bounds[1], cents, match.group(0).replace(" ", "")))
if not candidates:
return None, None
_, cents, raw = min(candidates, key=lambda item: (item[0], item[1]))
return cents, raw
return None, None, None
first_top = min(item[0] for item in candidates)
first_row = [item for item in candidates if abs(item[0] - first_top) <= 12]
_, cents, raw = min(first_row, key=lambda item: item[1])
list_prices = [item[1] for item in first_row if item[1] > cents]
return cents, raw, max(list_prices, default=None)
def parse_spec_panel(xml_data: str | bytes) -> SpecSnapshot:
@@ -446,7 +505,12 @@ def parse_spec_panel(xml_data: str | bytes) -> SpecSnapshot:
values: list[DimensionValue] = []
seen: set[str] = set()
for node in option_nodes:
text = _own_or_descendant_label(node).strip()
text = _preferred_or_descendant_label(node).strip()
if text.endswith(("…", "...")):
raise PddCollectError(
"PDD_DATA_SKU_NAME_TRUNCATED",
f"规格名称被截断,无法安全采集:{text}",
)
if not text or text in seen:
continue
seen.add(text)
@@ -458,8 +522,12 @@ def parse_spec_panel(xml_data: str | bytes) -> SpecSnapshot:
dimensions.append(SpecDimension(key, name, tuple(values)))
selected_text = next((label for label in labels if label.startswith("已选")), None)
price_cent, raw_price = _price_from_nodes(root.iter("node"), outer_bounds[1])
return SpecSnapshot(tuple(dimensions), selected_text, price_cent, raw_price)
price_cent, raw_price, list_price_cent = _price_from_nodes(
root.iter("node"), outer_bounds[1]
)
return SpecSnapshot(
tuple(dimensions), selected_text, price_cent, raw_price, list_price_cent
)
def _ancestors(
@@ -561,9 +629,11 @@ class PddCollectService:
now: Callable[[], datetime] = lambda: datetime.now(timezone.utc),
cancelled: Callable[[], bool] = lambda: False,
page_timeout: float = 30.0,
overall_timeout: float = 600.0,
max_page_swipes: int = 12,
max_spec_swipes: int = 12,
max_sku_count: int = 200,
artifact_directory: Optional[Path] = None,
) -> None:
self._device_service = device_service
self._device_address = device_address
@@ -573,9 +643,15 @@ class PddCollectService:
self._now = now
self._cancelled = cancelled
self._page_timeout = page_timeout
self._overall_timeout = overall_timeout
self._overall_deadline: Optional[float] = None
self._max_page_swipes = max_page_swipes
self._max_spec_swipes = max_spec_swipes
self._max_sku_count = max_sku_count
self._artifact_directory = artifact_directory
self._last_goods_xml: Optional[str] = None
self._goods_screens_checked = 0
self._artifacts: list[Mapping[str, Any]] = []
def collect(self, task: Any) -> CollectResult:
"""执行采集;``task`` 至少提供 ``goods_url`` 和可选 ``goods_id``。"""
@@ -585,6 +661,7 @@ class PddCollectService:
raise PddCollectError("PDD_DATA_GOODS_URL_MISSING", "采集任务缺少商品链接")
goods_id = str(getattr(task, "goods_id", "") or "").strip()
goods_id = _validate_goods_url(goods_url, goods_id)
self._overall_deadline = self._monotonic() + self._overall_timeout
self._check_cancelled()
try:
@@ -593,12 +670,14 @@ class PddCollectService:
goods = self._collect_goods_details(device)
if not goods.title:
raise PddCollectError("PDD_DATA_TITLE_MISSING", "商品页没有可识别的标题")
if not goods.shop_name:
raise PddCollectError("PDD_DATA_SHOP_MISSING", "商品页没有采集到店铺名称")
if not goods.sales.raw:
raise PddCollectError("PDD_DATA_SALES_MISSING", "商品页没有采集到已拼数量")
if not goods.reviews.raw:
raise PddCollectError("PDD_DATA_REVIEWS_MISSING", "商品页没有采集到评价数量")
if not goods.shop_name and self._last_goods_xml:
artifact = self._save_xml("shop-not-found", self._last_goods_xml)
if artifact:
self._artifacts.append(artifact)
home_xml = device.dump_hierarchy()
coordinate = get_size_panel_coord(home_xml)
@@ -623,11 +702,23 @@ class PddCollectService:
)
if not skus or not has_available_price:
raise PddCollectError("PDD_DATA_PRICE_MISSING", "没有采集到可用 SKU 的价格")
except PddCollectError:
except PddCollectError as exc:
if not exc.diagnostics and self._last_goods_xml:
artifact = self._save_xml("collect-failed", self._last_goods_xml)
if artifact:
exc.diagnostics = {
"artifacts": [artifact],
"goods_screens_checked": self._goods_screens_checked,
}
raise
except PddDeviceError as exc:
raise PddCollectError(exc.code, exc.message) from exc
except Exception as exc:
if _is_device_disconnect(exc):
raise PddCollectError(
"DEVICE_DISCONNECTED",
f"Android 设备在采集过程中断开:{exc}",
) from exc
raise PddCollectError("PDD_PAGE_UNKNOWN", f"PDD 采集过程中发生未知错误:{exc}") from exc
return CollectResult(
@@ -642,11 +733,17 @@ class PddCollectService:
captured_at=self._now().astimezone(timezone.utc).isoformat().replace("+00:00", "Z"),
client_id=self._client_id,
device_address=self._device_address,
artifacts=tuple(self._artifacts),
)
def _check_cancelled(self) -> None:
if self._cancelled():
raise PddCollectError("PDD_CANCELLED", "采集任务已安全取消")
if (
self._overall_deadline is not None
and self._monotonic() >= self._overall_deadline
):
raise PddCollectError("PDD_PAGE_OVERALL_TIMEOUT", "PDD 采集超过 10 分钟")
def _open_goods(self, device: Any, goods_url: str) -> None:
try:
@@ -659,6 +756,11 @@ class PddCollectService:
except PddCollectError:
raise
except Exception as exc:
if _is_device_disconnect(exc):
raise PddCollectError(
"DEVICE_DISCONNECTED",
f"Android 设备在打开商品页时断开:{exc}",
) from exc
raise PddCollectError("DEVICE_APP_START_FAILED", f"无法打开 PDD 商品链接:{exc}") from exc
deadline = self._monotonic() + self._page_timeout
@@ -686,13 +788,14 @@ class PddCollectService:
for _ in range(self._max_page_swipes + 1):
self._check_cancelled()
xml_data = device.dump_hierarchy()
self._last_goods_xml = str(xml_data)
self._goods_screens_checked += 1
root = _parse_xml(xml_data)
labels = tuple(_all_labels(root))
snapshots.append(parse_goods_page(xml_data))
combined = _combine_goods_snapshots(snapshots)
is_complete = (
combined.title
and combined.shop_name
and combined.sales.raw
and combined.reviews.raw
)
@@ -720,6 +823,27 @@ class PddCollectService:
self._sleep(0.35)
return _combine_goods_snapshots(snapshots)
def _save_xml(self, label: str, xml_data: str) -> Optional[Mapping[str, Any]]:
"""保存本地诊断 XML;测试未提供目录时不写文件。"""
if self._artifact_directory is None:
return None
try:
digest = hashlib.sha256(xml_data.encode("utf-8")).hexdigest()
directory = self._artifact_directory / self._client_id
directory.mkdir(parents=True, exist_ok=True)
path = directory / f"{label}-{digest[:12]}.xml"
if not path.exists():
path.write_text(xml_data, encoding="utf-8")
return {
"kind": "accessibility_xml",
"path": str(path.resolve()),
"sha256": digest,
"screens_checked": self._goods_screens_checked,
}
except OSError:
return None
def _discover_dimensions(self, device: Any) -> list[SpecSnapshot]:
snapshots: list[SpecSnapshot] = []
@@ -768,28 +892,66 @@ class PddCollectService:
f"规格组合共 {len(combinations)} 个,超过安全上限 {self._max_sku_count}",
)
results: list[SkuResult] = []
for values in combinations:
color_index = next(
(index for index, item in enumerate(dimensions) if item.key == "color"),
None,
)
if color_index is None:
raise PddCollectError(
"PDD_DATA_SPEC_INCOMPLETE", "规格面板没有可识别的颜色分类"
)
samples: dict[
str,
tuple[Optional[int], Optional[str], Optional[int], dict[str, str]],
] = {}
color_dimension = dimensions[color_index]
for color in color_dimension.values:
self._check_cancelled()
options = {dimension.key: value.text for dimension, value in zip(dimensions, values)}
pairs = list(zip(dimensions, values))
selected = self._select_combination(device, pairs)
if not selected and len(pairs) > 1:
# 某个选项可能只是在当前搭配下禁用。反向选择一次,可以先改变
# 依赖维度,再重新判断目标组合是否真的缺货。
selected = self._select_combination(device, list(reversed(pairs)))
if not color.available:
samples[color.text] = (None, None, None, {"color": color.text})
continue
selected = self._select_option(device, color.text, color_dimension.key)
if not selected:
results.append(SkuResult(options, None, False, None))
samples[color.text] = (None, None, None, {"color": color.text})
continue
snapshot = parse_spec_panel(device.dump_hierarchy())
summary = snapshot.selected_text or ""
confirmed = all(value.text in summary for value in values)
observed: dict[str, str] = {}
for dimension in dimensions:
match = next(
(value.text for value in dimension.values if value.text in summary),
None,
)
if match:
observed[dimension.key] = match
confirmed = (
observed.get(color_dimension.key) == color.text
and len(observed) == len(dimensions)
)
samples[color.text] = (
snapshot.price_cent if confirmed else None,
snapshot.raw_price if confirmed else None,
snapshot.list_price_cent if confirmed else None,
observed,
)
results: list[SkuResult] = []
for values in combinations:
options = {
dimension.key: value.text
for dimension, value in zip(dimensions, values)
}
price, raw_price, list_price, observed = samples[values[color_index].text]
available = all(value.available for value in values) and price is not None
results.append(
SkuResult(
options,
snapshot.price_cent if confirmed else None,
confirmed and snapshot.price_cent is not None,
snapshot.raw_price if confirmed else None,
price if available else None,
available,
raw_price if available else None,
observed,
list_price if available else None,
)
)
return tuple(results)
@@ -864,7 +1026,7 @@ class PddCollectService:
]
screen_right = max(right_edges, default=0)
for node in root.iter("node"):
if _own_or_descendant_label(node).strip() != target:
if _preferred_or_descendant_label(node).strip() != target:
continue
bounds = _parse_bounds(node.get("bounds", ""))
if node.get("clickable") != "true" or bounds is None:
+46 -46
View File
@@ -13,7 +13,7 @@
- 窗口关闭时要断开信号并置标志位,否则迟到的后台结果会访问
已经销毁的控件、直接崩溃。做法见同文档 §5.2。
- 数据库读写走 Repository,**不要在这里拼业务 SQL**。
- “获取任务”会真的去操作手机、可能下单;“搜索”只读本地数据库。
- “获取任务”会真的去操作手机采集商品;“搜索”只读本地数据库。
两者必须分开,不得共用入口。
- 普通成功不弹窗,更新界面即可;可恢复错误用 `InfoBar`
(模板见 `docs/client/05-ui-specification.md` §9.1);
@@ -28,11 +28,10 @@ from qfluentwidgets import InfoBar, InfoBarPosition
from .admin_gateway import (
AdminGatewayError,
AdminTask,
AndroidDeviceInfo,
ClaimCapabilities,
ClientInfo,
TaskClaimGateway,
AdminGateway,
)
from .collect_task_service import CollectServiceFactory, CollectTaskService
from .current_client_service import CurrentClientService
from .http_admin_gateway import DEFAULT_ADMIN_BASE_URL, HttpAdminGateway
from .pdd_ui import PDDTaskPage, TaskRow
@@ -45,7 +44,7 @@ from .task_models import (
TaskSummary,
TaskType,
)
from .task_repository import DuplicateTaskError, TaskRepository
from .task_repository import TaskRepository
TASK_TYPE_BY_TEXT = {
@@ -116,21 +115,23 @@ def admin_task_to_new_claimed_task(task: AdminTask) -> NewClaimedTask:
class ClaimTaskWorker(QObject):
"""在后台领取至多一个采集任务,并先写入本地数据库。"""
"""在后台补交或执行至多一条采集任务。"""
noTask = pyqtSignal()
taskSaved = pyqtSignal(str)
duplicateTask = pyqtSignal(str)
localSaveFailed = pyqtSignal(str, str)
failed = pyqtSignal(str)
outcome = pyqtSignal(str, str, str)
completed = pyqtSignal()
def __init__(
self,
gateway: TaskClaimGateway,
gateway: AdminGateway,
task_repository: TaskRepository,
client_service: CurrentClientService,
android_device_service: SelectedAndroidDeviceService,
collect_service_factory: Optional[CollectServiceFactory] = None,
) -> None:
super().__init__()
self._gateway = gateway
@@ -138,6 +139,7 @@ class ClaimTaskWorker(QObject):
self._client_service = client_service
self._android_device_service = android_device_service
self._cancelled = False
self._collect_service_factory = collect_service_factory
def cancel(self) -> None:
"""阻止尚未开始的领取;已领取的任务仍必须保存到本地。"""
@@ -146,7 +148,6 @@ class ClaimTaskWorker(QObject):
@pyqtSlot()
def run(self) -> None:
claimed_task: Optional[AdminTask] = None
try:
if self._cancelled:
return
@@ -155,42 +156,20 @@ class ClaimTaskWorker(QObject):
self.failed.emit("请先在设置页保存当前设备号和设备名")
return
android_serial = self._android_device_service.load()
if not android_serial:
self.failed.emit("请先在设置页选择并保存 Android 设备")
return
capabilities = ClaimCapabilities(
device=AndroidDeviceInfo(android_serial),
supported_types=(TaskType.COLLECT,),
purchase_mode="dry_run",
schema_versions=(1,),
)
claimed_task = self._gateway.claim_next(
result = CollectTaskService(
self._gateway,
self._task_repository,
ClientInfo(
client_settings.client_id,
client_settings.client_name,
),
capabilities,
)
if claimed_task is None:
if not self._cancelled:
self.noTask.emit()
return
local_task = admin_task_to_new_claimed_task(claimed_task)
try:
self._task_repository.add_claimed_task(local_task)
except DuplicateTaskError:
if not self._cancelled:
self.duplicateTask.emit(claimed_task.task_id)
return
except Exception as exc:
if not self._cancelled:
self.localSaveFailed.emit(claimed_task.task_id, str(exc))
return
if not self._cancelled:
self.taskSaved.emit(claimed_task.task_id)
android_serial or "",
cancelled=lambda: self._cancelled,
collect_service_factory=self._collect_service_factory,
).execute_one()
if not self._cancelled or result.kind == "cancelled":
self.outcome.emit(result.kind, result.message, result.task_id)
except AdminGatewayError as exc:
if not self._cancelled:
request_hint = (
@@ -199,10 +178,7 @@ class ClaimTaskWorker(QObject):
self.failed.emit(f"{exc}{request_hint}")
except Exception as exc:
if not self._cancelled:
if claimed_task is not None:
self.localSaveFailed.emit(claimed_task.task_id, str(exc))
else:
self.failed.emit(f"领取任务失败:{exc}")
self.failed.emit(f"执行采集任务失败:{exc}")
finally:
self.completed.emit()
@@ -215,8 +191,9 @@ class PDDTaskPageEvent(QObject):
page: PDDTaskPage,
repository: Optional[TaskRepository] = None,
parent=None,
claim_gateway: Optional[TaskClaimGateway] = None,
claim_gateway: Optional[AdminGateway] = None,
settings_repository: Optional[SettingsRepository] = None,
collect_service_factory: Optional[CollectServiceFactory] = None,
):
super().__init__(parent or page)
self._page = page
@@ -226,6 +203,13 @@ class PDDTaskPageEvent(QObject):
self._claim_busy = False
self._claim_thread: Optional[QThread] = None
self._claim_worker: Optional[ClaimTaskWorker] = None
self._collect_service_factory = collect_service_factory
try:
self._repository.recover_interrupted_work()
except AttributeError:
# 测试用的只读 Repository 可以不实现恢复接口。
pass
settings = settings_repository or SettingsRepository()
self._client_service = CurrentClientService(settings)
@@ -242,6 +226,7 @@ class PDDTaskPageEvent(QObject):
self._claim_gateway = HttpAdminGateway(
base_url if isinstance(base_url, str) else "",
timeout_seconds=timeout_seconds,
client_id=self._client_service.load().client_id,
)
except (TypeError, ValueError) as exc:
self._claim_gateway_error = str(exc)
@@ -290,7 +275,7 @@ class PDDTaskPageEvent(QObject):
self._claim_busy = True
self._page.autoFetchButton.setEnabled(False)
self._page.set_engine_status("正在领取一个采集任务,请稍候…")
self._page.set_engine_status("正在处理一条采集任务,请稍候…")
thread = QThread(self)
worker = ClaimTaskWorker(
@@ -298,6 +283,7 @@ class PDDTaskPageEvent(QObject):
self._repository,
self._client_service,
self._selected_android_device_service,
self._collect_service_factory,
)
worker.moveToThread(thread)
thread.started.connect(worker.run)
@@ -306,6 +292,7 @@ class PDDTaskPageEvent(QObject):
worker.duplicateTask.connect(self._on_duplicate_claimed_task)
worker.localSaveFailed.connect(self._on_claimed_task_save_failed)
worker.failed.connect(self._on_claim_failed)
worker.outcome.connect(self._on_collect_outcome)
worker.completed.connect(thread.quit)
worker.completed.connect(worker.deleteLater)
thread.finished.connect(thread.deleteLater)
@@ -352,6 +339,15 @@ class PDDTaskPageEvent(QObject):
self._page.set_engine_status(content)
self._show_claim_error("领取任务失败", content)
@pyqtSlot(str, str, str)
def _on_collect_outcome(self, kind: str, message: str, _task_id: str) -> None:
if self._closing:
return
self._page.set_engine_status(message)
self._reload()
if kind in {"result_pending", "manual_review", "failed"}:
self._show_claim_error("采集任务需要处理", message)
def _show_claim_error(self, title: str, content: str) -> None:
"""显示不会自动消失的可恢复错误,同时保留底部状态文字。"""
@@ -413,6 +409,7 @@ class PDDTaskPageEvent(QObject):
self._on_claimed_task_save_failed,
),
(worker.failed, self._on_claim_failed),
(worker.outcome, self._on_collect_outcome),
)
except RuntimeError:
signal_slots = ()
@@ -424,7 +421,10 @@ class PDDTaskPageEvent(QObject):
if thread is not None and thread.isRunning():
thread.quit()
thread.wait(10_000)
# uiautomator2/ADB 的单次调用可能需要数秒才返回。先通过
# cancelled 标志让采集在下一个安全点退出,再等待工作线程收尾,
# 避免窗口销毁时出现 "QThread destroyed while running"。
thread.wait(60_000)
def summary_to_row(summary: TaskSummary) -> TaskRow:
+9
View File
@@ -184,6 +184,15 @@ class OutboxEventRecord:
sent_at: Optional[str]
@dataclass(frozen=True)
class StartedTaskRun:
"""已进入执行状态的一次采集尝试。"""
task: TaskDetail
attempt_id: str
attempt_no: int
@dataclass(frozen=True)
class AppSettingRecord:
"""一条非敏感应用设置。"""
+353
View File
@@ -8,10 +8,16 @@ import sqlite3
from datetime import datetime, timezone
from pathlib import Path
from typing import Dict, List, Optional, Tuple, Union
from uuid import uuid4
from .db import initialize_database, open_database
from .task_models import (
NewClaimedTask,
OutboxEventRecord,
OutboxEventType,
OutboxStatus,
RunStatus,
StartedTaskRun,
TaskDetail,
TaskFilters,
TaskStatus,
@@ -145,6 +151,353 @@ class TaskRepository:
connection.close()
return self._to_detail(row) if row is not None else None
def recover_interrupted_work(self) -> None:
"""恢复上次异常退出留下的可重试状态。"""
now = utc_now_iso()
connection = open_database(self._db_path)
try:
with connection:
connection.execute(
"UPDATE outbox_events SET status = 'pending', updated_at = ?"
" WHERE status = 'sending'",
(now,),
)
connection.execute(
"UPDATE pdd_tasks SET status = 'retry_wait',"
" current_step = 'interrupted', retry_count = retry_count + 1,"
" last_error_code = 'CLIENT_INTERRUPTED',"
" last_error_message = '客户端上次执行期间退出', updated_at = ?"
" WHERE status = 'running' AND task_type = 'collect'",
(now,),
)
connection.execute(
"UPDATE task_runs SET run_status = 'failed',"
" error_code = 'CLIENT_INTERRUPTED',"
" error_message = '客户端上次执行期间退出',"
" finished_at = ?, updated_at = ?"
" WHERE run_status = 'running' AND irreversible_action_at IS NULL"
" AND task_id IN (SELECT id FROM pdd_tasks WHERE task_type = 'collect')",
(now, now),
)
finally:
connection.close()
def next_collect_task(self) -> Optional[TaskDetail]:
"""返回最早的本地待执行采集任务。"""
connection = open_database(self._db_path)
try:
row = connection.execute(
"SELECT * FROM pdd_tasks"
" WHERE task_type = 'collect' AND status IN ('claimed', 'retry_wait')"
" ORDER BY received_at ASC, id ASC LIMIT 1"
).fetchone()
finally:
connection.close()
return self._to_detail(row) if row is not None else None
def start_collect_run(
self, remote_task_id: str, device_address: str
) -> StartedTaskRun:
"""原子地把待执行任务改为执行中,并创建一次运行记录。"""
now = utc_now_iso()
attempt_id = str(uuid4())
connection = open_database(self._db_path)
try:
with connection:
row = connection.execute(
"SELECT * FROM pdd_tasks WHERE remote_task_id = ?",
(remote_task_id,),
).fetchone()
if row is None:
raise ValueError(f"任务 {remote_task_id} 不存在")
if row["task_type"] != TaskType.COLLECT.value:
raise ValueError("当前只能执行采集任务")
if row["status"] not in {
TaskStatus.CLAIMED.value,
TaskStatus.RETRY_WAIT.value,
}:
raise ValueError(f"任务状态 {row['status']} 不能开始采集")
attempt_no = int(
connection.execute(
"SELECT COALESCE(MAX(attempt_no), 0) + 1"
" FROM task_runs WHERE task_id = ?",
(row["id"],),
).fetchone()[0]
)
connection.execute(
"UPDATE pdd_tasks SET status = 'running',"
" current_step = 'collecting', started_at = COALESCE(started_at, ?),"
" last_error_code = NULL, last_error_message = NULL, updated_at = ?"
" WHERE id = ?",
(now, now, row["id"]),
)
connection.execute(
"INSERT INTO task_runs (task_id, attempt_id, attempt_no,"
" device_address, run_status, current_step, started_at,"
" created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
(
row["id"], attempt_id, attempt_no, device_address,
RunStatus.RUNNING.value, "collecting", now, now, now,
),
)
task = self.get_task(remote_task_id)
assert task is not None
return StartedTaskRun(task, attempt_id, attempt_no)
finally:
connection.close()
def save_collect_result(
self,
remote_task_id: str,
attempt_id: str,
pdd_data: Dict[str, object],
) -> OutboxEventRecord:
"""在一个事务中保存采集结果并创建待提交事件。"""
now = utc_now_iso()
connection = open_database(self._db_path)
try:
with connection:
task = connection.execute(
"SELECT id, version FROM pdd_tasks WHERE remote_task_id = ?",
(remote_task_id,),
).fetchone()
if task is None:
raise ValueError(f"任务 {remote_task_id} 不存在")
payload = {
"task_version": task["version"],
"attempt_id": attempt_id,
"result_type": "collect",
"completed_at": now,
"pdd_data": pdd_data,
}
idempotency_key = f"{remote_task_id}:{attempt_id}:result-v1"
connection.execute(
"UPDATE pdd_tasks SET status = 'result_pending',"
" current_step = 'submit_result', pdd_data = ?, goods_id = ?,"
" title = ?, price_cent = ?, finished_at = ?, updated_at = ?"
" WHERE id = ?",
(
json.dumps(pdd_data, ensure_ascii=False),
pdd_data.get("goods_id"), pdd_data.get("title"),
self._summary_price(pdd_data), now, now, task["id"],
),
)
connection.execute(
"UPDATE task_runs SET run_status = 'succeeded',"
" current_step = 'submit_result', finished_at = ?, updated_at = ?"
" WHERE attempt_id = ?",
(now, now, attempt_id),
)
cursor = connection.execute(
"INSERT INTO outbox_events (task_id, event_type, idempotency_key,"
" payload_json, status, created_at, updated_at)"
" VALUES (?, 'collect_result', ?, ?, 'pending', ?, ?)",
(
task["id"], idempotency_key,
json.dumps(payload, ensure_ascii=False), now, now,
),
)
event_id = int(cursor.lastrowid)
event = self.get_outbox_event(event_id)
assert event is not None
return event
finally:
connection.close()
def save_collect_failure(
self,
remote_task_id: str,
attempt_id: str,
status: TaskStatus,
error_code: str,
error_message: str,
retryable: bool,
diagnostics: Optional[Dict[str, object]] = None,
) -> OutboxEventRecord:
"""保存结构化失败,并可靠排队提交 Admin。"""
if status not in {
TaskStatus.RETRY_WAIT, TaskStatus.MANUAL_REVIEW,
TaskStatus.FAILED, TaskStatus.CANCELLED,
}:
raise ValueError("失败状态无效")
now = utc_now_iso()
connection = open_database(self._db_path)
try:
with connection:
task = connection.execute(
"SELECT id, version FROM pdd_tasks WHERE remote_task_id = ?",
(remote_task_id,),
).fetchone()
if task is None:
raise ValueError(f"任务 {remote_task_id} 不存在")
payload = {
"task_version": task["version"],
"attempt_id": attempt_id,
"status": status.value,
"error": {
"code": error_code,
"message": error_message,
"retryable": retryable,
"step": "collecting",
},
"diagnostics": diagnostics or {"artifacts": []},
"reported_at": now,
}
idempotency_key = f"{remote_task_id}:{attempt_id}:failure-v1"
connection.execute(
"UPDATE pdd_tasks SET status = ?, current_step = 'failed',"
" retry_count = retry_count + ?, last_error_code = ?,"
" last_error_message = ?, finished_at = ?, updated_at = ?"
" WHERE id = ?",
(
status.value, 1 if status is TaskStatus.RETRY_WAIT else 0,
error_code, error_message, now, now, task["id"],
),
)
run_status = {
TaskStatus.CANCELLED: RunStatus.CANCELLED,
TaskStatus.MANUAL_REVIEW: RunStatus.MANUAL_REVIEW,
}.get(status, RunStatus.FAILED)
connection.execute(
"UPDATE task_runs SET run_status = ?, current_step = 'failed',"
" error_code = ?, error_message = ?, finished_at = ?, updated_at = ?"
" WHERE attempt_id = ?",
(run_status.value, error_code, error_message, now, now, attempt_id),
)
cursor = connection.execute(
"INSERT INTO outbox_events (task_id, event_type, idempotency_key,"
" payload_json, status, created_at, updated_at)"
" VALUES (?, 'task_failure', ?, ?, 'pending', ?, ?)",
(
task["id"], idempotency_key,
json.dumps(payload, ensure_ascii=False), now, now,
),
)
event_id = int(cursor.lastrowid)
event = self.get_outbox_event(event_id)
assert event is not None
return event
finally:
connection.close()
def next_pending_outbox(self) -> Optional[OutboxEventRecord]:
connection = open_database(self._db_path)
try:
row = connection.execute(
"SELECT o.* FROM outbox_events o JOIN pdd_tasks t ON t.id = o.task_id"
" WHERE o.status = 'pending'"
" AND (o.next_retry_at IS NULL OR o.next_retry_at <= ?)"
" ORDER BY o.id ASC LIMIT 1",
(utc_now_iso(),),
).fetchone()
finally:
connection.close()
return self._to_outbox(row) if row is not None else None
def get_outbox_event(self, event_id: int) -> Optional[OutboxEventRecord]:
connection = open_database(self._db_path)
try:
row = connection.execute(
"SELECT * FROM outbox_events WHERE id = ?", (event_id,)
).fetchone()
finally:
connection.close()
return self._to_outbox(row) if row is not None else None
def outbox_task_id(self, event_id: int) -> str:
connection = open_database(self._db_path)
try:
row = connection.execute(
"SELECT t.remote_task_id FROM outbox_events o"
" JOIN pdd_tasks t ON t.id = o.task_id WHERE o.id = ?",
(event_id,),
).fetchone()
finally:
connection.close()
if row is None:
raise ValueError(f"Outbox {event_id} 不存在")
return str(row[0])
def mark_outbox_sending(self, event_id: int) -> None:
self._update_outbox(event_id, "sending", None)
def mark_outbox_retry(self, event_id: int, message: str) -> None:
self._update_outbox(event_id, "pending", message, increment=True)
def mark_outbox_failed(self, event_id: int, message: str) -> None:
self._update_outbox(event_id, "failed", message, increment=True)
def mark_outbox_sent(self, event_id: int) -> None:
now = utc_now_iso()
connection = open_database(self._db_path)
try:
with connection:
row = connection.execute(
"SELECT task_id, event_type FROM outbox_events WHERE id = ?",
(event_id,),
).fetchone()
if row is None:
raise ValueError(f"Outbox {event_id} 不存在")
connection.execute(
"UPDATE outbox_events SET status = 'sent', sent_at = ?,"
" updated_at = ?, attempt_count = attempt_count + 1"
" WHERE id = ?",
(now, now, event_id),
)
if row["event_type"] == OutboxEventType.COLLECT_RESULT.value:
connection.execute(
"UPDATE pdd_tasks SET status = 'succeeded',"
" current_step = 'completed', updated_at = ? WHERE id = ?",
(now, row["task_id"]),
)
finally:
connection.close()
def _update_outbox(
self, event_id: int, status: str, message: Optional[str], increment: bool = False
) -> None:
connection = open_database(self._db_path)
try:
with connection:
cursor = connection.execute(
"UPDATE outbox_events SET status = ?, last_error = ?, updated_at = ?,"
f" attempt_count = attempt_count + {1 if increment else 0} WHERE id = ?",
(status, message, utc_now_iso(), event_id),
)
if cursor.rowcount != 1:
raise ValueError(f"Outbox {event_id} 不存在")
finally:
connection.close()
@staticmethod
def _summary_price(pdd_data: Dict[str, object]) -> Optional[int]:
skus = pdd_data.get("skus")
if not isinstance(skus, list):
return None
prices = [
item.get("price_cent") for item in skus
if isinstance(item, dict) and isinstance(item.get("price_cent"), int)
]
return min(prices) if prices else None
@staticmethod
def _to_outbox(row: sqlite3.Row) -> OutboxEventRecord:
return OutboxEventRecord(
id=row["id"], task_id=row["task_id"],
event_type=OutboxEventType(row["event_type"]),
idempotency_key=row["idempotency_key"],
payload_json=TaskRepository._load_json_object(row["payload_json"]),
status=OutboxStatus(row["status"]), attempt_count=row["attempt_count"],
next_retry_at=row["next_retry_at"], last_error=row["last_error"],
created_at=row["created_at"], updated_at=row["updated_at"],
sent_at=row["sent_at"],
)
@staticmethod
def _validate_page(limit: int, offset: int) -> None:
if not 1 <= limit <= MAX_PAGE_SIZE: