chengma/cmautobuy
2
0
Fork 0
Code Issues 89 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 执行并提交 PDD 采集任务 (#32)

Browse Source
This commit is contained in:
chengma
2026-08-07 17:38:29 +08:00
parent 34b4b54b63
commit 2d1fdbea3f
17 changed files with 2156 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/src/pdd_collect_service.py
+197 -35
View File
@@ -14,6 +14,7 @@ import xml.etree.ElementTree as ET
from dataclasses import dataclass
from datetime import datetime, timezone
from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
from pathlib import Path
from typing import Any, Callable, Iterable, Mapping, Optional, Sequence
from urllib.parse import parse_qs, urlparse
@@ -31,13 +32,32 @@ _CAPTCHA_MARKERS = ("请完成验证", "拖动滑块", "安全验证", "点击
_READY_MARKERS = ("发起拼单", "立即购买", "单独购买", "免拼购买", "快要抢光")
def _is_device_disconnect(error: BaseException) -> bool:
details = str(error).lower()
return any(
marker in details
for marker in (
"device not found",
"device offline",
"disconnected",
"closed transport",
)
)
class PddCollectError(RuntimeError):
"""采集失败，并携带稳定错误码。"""
def __init__(self, code: str, message: str) -> None:
def __init__(
self,
code: str,
message: str,
diagnostics: Optional[Mapping[str, Any]] = None,
) -> None:
super().__init__(message)
self.code = code
self.message = message
self.diagnostics = dict(diagnostics or {})
@dataclass(frozen=True)
@@ -85,11 +105,15 @@ class SkuResult:
price_cent: Optional[int]
available: bool
raw_price: Optional[str]
price_observed_at: Mapping[str, str]
list_price_cent: Optional[int] = None
def to_dict(self) -> dict[str, Any]:
return {
"options": dict(self.options),
"price_cent": self.price_cent,
"list_price_cent": self.list_price_cent,
"price_observed_at": dict(self.price_observed_at),
"currency": "CNY",
"available": self.available,
"raw_price": self.raw_price,
@@ -110,6 +134,7 @@ class SpecSnapshot:
selected_text: Optional[str]
price_cent: Optional[int]
raw_price: Optional[str]
list_price_cent: Optional[int]
@dataclass(frozen=True)
@@ -127,16 +152,16 @@ class CollectResult:
captured_at: str
client_id: str
device_address: str
artifacts: tuple[Mapping[str, Any], ...] = ()
def to_pdd_data(self) -> dict[str, Any]:
return {
"schema_version": 1,
"goods": {
"goods_id": self.goods_id,
"url": self.goods_url,
"title": self.title,
},
"shop": {"name": self.shop_name},
"goods_id": self.goods_id,
"goods_url": self.goods_url,
"title": self.title,
"shop_name": self.shop_name,
"price_granularity": "color",
"metrics": {
"sales": self.sales.to_dict(),
"reviews": self.reviews.to_dict(),
@@ -150,7 +175,7 @@ class CollectResult:
"device_address": self.device_address,
"pdd_package": PDD_PACKAGE_NAME,
},
"artifacts": [],
"artifacts": [dict(item) for item in self.artifacts],
}
@@ -182,6 +207,14 @@ def _node_label(node: ET.Element) -> str:
)
def _preferred_node_label(node: ET.Element) -> str:
"""规格名被 text 截断时，优先采用完整的无障碍描述。"""
text = (node.get("text") or "").strip()
description = (node.get("content-desc") or "").strip()
return description or text
def _own_or_descendant_label(node: ET.Element) -> str:
own = _node_label(node)
if own:
@@ -193,6 +226,17 @@ def _own_or_descendant_label(node: ET.Element) -> str:
return ""
def _preferred_or_descendant_label(node: ET.Element) -> str:
own = _preferred_node_label(node)
if own:
return own
for child in node.iter("node"):
label = _preferred_node_label(child)
if label and label != "打开大图":
return label
return ""
def _all_labels(root: ET.Element) -> list[str]:
return [label for node in root.iter("node") if (label := _node_label(node))]
@@ -250,9 +294,21 @@ def parse_goods_page(xml_data: str | bytes) -> GoodsSnapshot:
title = candidate
break
if title is None:
line_parts: dict[int, list[tuple[int, str]]] = {}
for node in root.iter("node"):
label = _preferred_node_label(node)
bounds = _parse_bounds(node.get("bounds", ""))
if not label or bounds is None:
continue
line_key = bounds[1] // 24
line_parts.setdefault(line_key, []).append((bounds[0], label))
joined_lines = [
"".join(text for _, text in sorted(parts))
for parts in line_parts.values()
]
candidates = [
label
for label in labels
for label in labels + joined_lines
if len(label) >= 12
and not any(word in label for word in ("通知", "支付", "已拼", "评价"))
]
@@ -347,7 +403,7 @@ def _top_level_clickable_options(
def _price_from_nodes(
nodes: Iterable[ET.Element],
spec_top: int,
) -> tuple[Optional[int], Optional[str]]:
) -> tuple[Optional[int], Optional[str], Optional[int]]:
candidates: list[tuple[int, int, str]] = []
for node in nodes:
label = _node_label(node)
@@ -366,9 +422,12 @@ def _price_from_nodes(
continue
candidates.append((bounds[1], cents, match.group(0).replace(" ", "")))
if not candidates:
return None, None
_, cents, raw = min(candidates, key=lambda item: (item[0], item[1]))
return cents, raw
return None, None, None
first_top = min(item[0] for item in candidates)
first_row = [item for item in candidates if abs(item[0] - first_top) <= 12]
_, cents, raw = min(first_row, key=lambda item: item[1])
list_prices = [item[1] for item in first_row if item[1] > cents]
return cents, raw, max(list_prices, default=None)
def parse_spec_panel(xml_data: str | bytes) -> SpecSnapshot:
@@ -446,7 +505,12 @@ def parse_spec_panel(xml_data: str | bytes) -> SpecSnapshot:
values: list[DimensionValue] = []
seen: set[str] = set()
for node in option_nodes:
text = _own_or_descendant_label(node).strip()
text = _preferred_or_descendant_label(node).strip()
if text.endswith(("…", "...")):
raise PddCollectError(
"PDD_DATA_SKU_NAME_TRUNCATED",
f"规格名称被截断，无法安全采集：{text}",
)
if not text or text in seen:
continue
seen.add(text)
@@ -458,8 +522,12 @@ def parse_spec_panel(xml_data: str | bytes) -> SpecSnapshot:
dimensions.append(SpecDimension(key, name, tuple(values)))
selected_text = next((label for label in labels if label.startswith("已选")), None)
price_cent, raw_price = _price_from_nodes(root.iter("node"), outer_bounds[1])
return SpecSnapshot(tuple(dimensions), selected_text, price_cent, raw_price)
price_cent, raw_price, list_price_cent = _price_from_nodes(
root.iter("node"), outer_bounds[1]
)
return SpecSnapshot(
tuple(dimensions), selected_text, price_cent, raw_price, list_price_cent
)
def _ancestors(
@@ -561,9 +629,11 @@ class PddCollectService:
now: Callable[[], datetime] = lambda: datetime.now(timezone.utc),
cancelled: Callable[[], bool] = lambda: False,
page_timeout: float = 30.0,
overall_timeout: float = 600.0,
max_page_swipes: int = 12,
max_spec_swipes: int = 12,
max_sku_count: int = 200,
artifact_directory: Optional[Path] = None,
) -> None:
self._device_service = device_service
self._device_address = device_address
@@ -573,9 +643,15 @@ class PddCollectService:
self._now = now
self._cancelled = cancelled
self._page_timeout = page_timeout
self._overall_timeout = overall_timeout
self._overall_deadline: Optional[float] = None
self._max_page_swipes = max_page_swipes
self._max_spec_swipes = max_spec_swipes
self._max_sku_count = max_sku_count
self._artifact_directory = artifact_directory
self._last_goods_xml: Optional[str] = None
self._goods_screens_checked = 0
self._artifacts: list[Mapping[str, Any]] = []
def collect(self, task: Any) -> CollectResult:
"""执行采集；``task`` 至少提供 ``goods_url`` 和可选 ``goods_id``。"""
@@ -585,6 +661,7 @@ class PddCollectService:
raise PddCollectError("PDD_DATA_GOODS_URL_MISSING", "采集任务缺少商品链接")
goods_id = str(getattr(task, "goods_id", "") or "").strip()
goods_id = _validate_goods_url(goods_url, goods_id)
self._overall_deadline = self._monotonic() + self._overall_timeout
self._check_cancelled()
try:
@@ -593,12 +670,14 @@ class PddCollectService:
goods = self._collect_goods_details(device)
if not goods.title:
raise PddCollectError("PDD_DATA_TITLE_MISSING", "商品页没有可识别的标题")
if not goods.shop_name:
raise PddCollectError("PDD_DATA_SHOP_MISSING", "商品页没有采集到店铺名称")
if not goods.sales.raw:
raise PddCollectError("PDD_DATA_SALES_MISSING", "商品页没有采集到已拼数量")
if not goods.reviews.raw:
raise PddCollectError("PDD_DATA_REVIEWS_MISSING", "商品页没有采集到评价数量")
if not goods.shop_name and self._last_goods_xml:
artifact = self._save_xml("shop-not-found", self._last_goods_xml)
if artifact:
self._artifacts.append(artifact)
home_xml = device.dump_hierarchy()
coordinate = get_size_panel_coord(home_xml)
@@ -623,11 +702,23 @@ class PddCollectService:
)
if not skus or not has_available_price:
raise PddCollectError("PDD_DATA_PRICE_MISSING", "没有采集到可用 SKU 的价格")
except PddCollectError:
except PddCollectError as exc:
if not exc.diagnostics and self._last_goods_xml:
artifact = self._save_xml("collect-failed", self._last_goods_xml)
if artifact:
exc.diagnostics = {
"artifacts": [artifact],
"goods_screens_checked": self._goods_screens_checked,
}
raise
except PddDeviceError as exc:
raise PddCollectError(exc.code, exc.message) from exc
except Exception as exc:
if _is_device_disconnect(exc):
raise PddCollectError(
"DEVICE_DISCONNECTED",
f"Android 设备在采集过程中断开：{exc}",
) from exc
raise PddCollectError("PDD_PAGE_UNKNOWN", f"PDD 采集过程中发生未知错误：{exc}") from exc
return CollectResult(
@@ -642,11 +733,17 @@ class PddCollectService:
captured_at=self._now().astimezone(timezone.utc).isoformat().replace("+00:00", "Z"),
client_id=self._client_id,
device_address=self._device_address,
artifacts=tuple(self._artifacts),
)
def _check_cancelled(self) -> None:
if self._cancelled():
raise PddCollectError("PDD_CANCELLED", "采集任务已安全取消")
if (
self._overall_deadline is not None
and self._monotonic() >= self._overall_deadline
):
raise PddCollectError("PDD_PAGE_OVERALL_TIMEOUT", "PDD 采集超过 10 分钟")
def _open_goods(self, device: Any, goods_url: str) -> None:
try:
@@ -659,6 +756,11 @@ class PddCollectService:
except PddCollectError:
raise
except Exception as exc:
if _is_device_disconnect(exc):
raise PddCollectError(
"DEVICE_DISCONNECTED",
f"Android 设备在打开商品页时断开：{exc}",
) from exc
raise PddCollectError("DEVICE_APP_START_FAILED", f"无法打开 PDD 商品链接：{exc}") from exc
deadline = self._monotonic() + self._page_timeout
@@ -686,13 +788,14 @@ class PddCollectService:
for _ in range(self._max_page_swipes + 1):
self._check_cancelled()
xml_data = device.dump_hierarchy()
self._last_goods_xml = str(xml_data)
self._goods_screens_checked += 1
root = _parse_xml(xml_data)
labels = tuple(_all_labels(root))
snapshots.append(parse_goods_page(xml_data))
combined = _combine_goods_snapshots(snapshots)
is_complete = (
combined.title
and combined.shop_name
and combined.sales.raw
and combined.reviews.raw
)
@@ -720,6 +823,27 @@ class PddCollectService:
self._sleep(0.35)
return _combine_goods_snapshots(snapshots)
def _save_xml(self, label: str, xml_data: str) -> Optional[Mapping[str, Any]]:
"""保存本地诊断 XML；测试未提供目录时不写文件。"""
if self._artifact_directory is None:
return None
try:
digest = hashlib.sha256(xml_data.encode("utf-8")).hexdigest()
directory = self._artifact_directory / self._client_id
directory.mkdir(parents=True, exist_ok=True)
path = directory / f"{label}-{digest[:12]}.xml"
if not path.exists():
path.write_text(xml_data, encoding="utf-8")
return {
"kind": "accessibility_xml",
"path": str(path.resolve()),
"sha256": digest,
"screens_checked": self._goods_screens_checked,
}
except OSError:
return None
def _discover_dimensions(self, device: Any) -> list[SpecSnapshot]:
snapshots: list[SpecSnapshot] = []
@@ -768,28 +892,66 @@ class PddCollectService:
f"规格组合共 {len(combinations)} 个，超过安全上限 {self._max_sku_count}",
)
results: list[SkuResult] = []
for values in combinations:
color_index = next(
(index for index, item in enumerate(dimensions) if item.key == "color"),
None,
)
if color_index is None:
raise PddCollectError(
"PDD_DATA_SPEC_INCOMPLETE", "规格面板没有可识别的颜色分类"
)
samples: dict[
str,
tuple[Optional[int], Optional[str], Optional[int], dict[str, str]],
] = {}
color_dimension = dimensions[color_index]
for color in color_dimension.values:
self._check_cancelled()
options = {dimension.key: value.text for dimension, value in zip(dimensions, values)}
pairs = list(zip(dimensions, values))
selected = self._select_combination(device, pairs)
if not selected and len(pairs) > 1:
# 某个选项可能只是在当前搭配下禁用。反向选择一次，可以先改变
# 依赖维度，再重新判断目标组合是否真的缺货。
selected = self._select_combination(device, list(reversed(pairs)))
if not color.available:
samples[color.text] = (None, None, None, {"color": color.text})
continue
selected = self._select_option(device, color.text, color_dimension.key)
if not selected:
results.append(SkuResult(options, None, False, None))
samples[color.text] = (None, None, None, {"color": color.text})
continue
snapshot = parse_spec_panel(device.dump_hierarchy())
summary = snapshot.selected_text or ""
confirmed = all(value.text in summary for value in values)
observed: dict[str, str] = {}
for dimension in dimensions:
match = next(
(value.text for value in dimension.values if value.text in summary),
None,
)
if match:
observed[dimension.key] = match
confirmed = (
observed.get(color_dimension.key) == color.text
and len(observed) == len(dimensions)
)
samples[color.text] = (
snapshot.price_cent if confirmed else None,
snapshot.raw_price if confirmed else None,
snapshot.list_price_cent if confirmed else None,
observed,
)
results: list[SkuResult] = []
for values in combinations:
options = {
dimension.key: value.text
for dimension, value in zip(dimensions, values)
}
price, raw_price, list_price, observed = samples[values[color_index].text]
available = all(value.available for value in values) and price is not None
results.append(
SkuResult(
options,
snapshot.price_cent if confirmed else None,
confirmed and snapshot.price_cent is not None,
snapshot.raw_price if confirmed else None,
price if available else None,
available,
raw_price if available else None,
observed,
list_price if available else None,
)
)
return tuple(results)
@@ -864,7 +1026,7 @@ class PddCollectService:
]
screen_right = max(right_edges, default=0)
for node in root.iter("node"):
if _own_or_descendant_label(node).strip() != target:
if _preferred_or_descendant_label(node).strip() != target:
continue
bounds = _parse_bounds(node.get("bounds", ""))
if node.get("clickable") != "true" or bounds is None: