fix: 商品页就绪后立即进入规格面板 (#36)
This commit is contained in:
chengma
@@ -241,22 +241,6 @@ def _all_labels(root: ET.Element) -> list[str]:
|
||||
return [label for node in root.iter("node") if (label := _node_label(node))]
|
||||
|
||||
|
||||
def _screen_bounds(root: ET.Element) -> Optional[Bounds]:
|
||||
bounds_list = [
|
||||
bounds
|
||||
for node in root.iter("node")
|
||||
if (bounds := _parse_bounds(node.get("bounds", ""))) is not None
|
||||
]
|
||||
if not bounds_list:
|
||||
return None
|
||||
return (
|
||||
min(item[0] for item in bounds_list),
|
||||
min(item[1] for item in bounds_list),
|
||||
max(item[2] for item in bounds_list),
|
||||
max(item[3] for item in bounds_list),
|
||||
)
|
||||
|
||||
|
||||
def parse_quantity(raw: Optional[str]) -> QuantityMetric:
|
||||
"""把“已拼1.2万+”等文字转成整数，同时保留原文。"""
|
||||
|
||||
@@ -600,21 +584,6 @@ def _validate_goods_url(goods_url: str, task_goods_id: str) -> str:
|
||||
return value
|
||||
|
||||
|
||||
def _combine_goods_snapshots(snapshots: Sequence[GoodsSnapshot]) -> GoodsSnapshot:
|
||||
return GoodsSnapshot(
|
||||
next((item.title for item in snapshots if item.title), None),
|
||||
next((item.shop_name for item in snapshots if item.shop_name), None),
|
||||
next(
|
||||
(item.sales for item in snapshots if item.sales.raw),
|
||||
QuantityMetric(None, None, False),
|
||||
),
|
||||
next(
|
||||
(item.reviews for item in snapshots if item.reviews.raw),
|
||||
QuantityMetric(None, None, False),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class PddCollectService:
|
||||
"""打开商品页并采集结构化商品与 SKU 数据。"""
|
||||
|
||||
@@ -629,8 +598,8 @@ class PddCollectService:
|
||||
now: Callable[[], datetime] = lambda: datetime.now(timezone.utc),
|
||||
cancelled: Callable[[], bool] = lambda: False,
|
||||
page_timeout: float = 30.0,
|
||||
spec_panel_timeout: float = 10.0,
|
||||
overall_timeout: float = 600.0,
|
||||
max_page_swipes: int = 12,
|
||||
max_spec_swipes: int = 12,
|
||||
max_sku_count: int = 200,
|
||||
artifact_directory: Optional[Path] = None,
|
||||
@@ -643,9 +612,9 @@ class PddCollectService:
|
||||
self._now = now
|
||||
self._cancelled = cancelled
|
||||
self._page_timeout = page_timeout
|
||||
self._spec_panel_timeout = spec_panel_timeout
|
||||
self._overall_timeout = overall_timeout
|
||||
self._overall_deadline: Optional[float] = None
|
||||
self._max_page_swipes = max_page_swipes
|
||||
self._max_spec_swipes = max_spec_swipes
|
||||
self._max_sku_count = max_sku_count
|
||||
self._artifact_directory = artifact_directory
|
||||
@@ -672,10 +641,13 @@ class PddCollectService:
|
||||
raise PddCollectError("PDD_DATA_TITLE_MISSING", "商品页没有可识别的标题")
|
||||
if not goods.sales.raw:
|
||||
raise PddCollectError("PDD_DATA_SALES_MISSING", "商品页没有采集到已拼数量")
|
||||
if not goods.reviews.raw:
|
||||
raise PddCollectError("PDD_DATA_REVIEWS_MISSING", "商品页没有采集到评价数量")
|
||||
if not goods.shop_name and self._last_goods_xml:
|
||||
artifact = self._save_xml("shop-not-found", self._last_goods_xml)
|
||||
if (
|
||||
(not goods.shop_name or not goods.reviews.raw)
|
||||
and self._last_goods_xml
|
||||
):
|
||||
artifact = self._save_xml(
|
||||
"goods-metadata-incomplete", self._last_goods_xml
|
||||
)
|
||||
if artifact:
|
||||
self._artifacts.append(artifact)
|
||||
|
||||
@@ -687,7 +659,7 @@ class PddCollectService:
|
||||
"商品页没有找到可靠的规格入口",
|
||||
)
|
||||
device.click(*coordinate)
|
||||
self._sleep(0.5)
|
||||
self._wait_spec_panel(device)
|
||||
|
||||
snapshots = self._discover_dimensions(device)
|
||||
dimensions = merge_dimensions(snapshots)
|
||||
@@ -791,46 +763,36 @@ class PddCollectService:
|
||||
raise PddCollectError("PDD_PAGE_TIMEOUT", "等待 PDD 商品详情页加载超时")
|
||||
|
||||
def _collect_goods_details(self, device: Any) -> GoodsSnapshot:
|
||||
snapshots: list[GoodsSnapshot] = []
|
||||
previous_signature: Optional[tuple[str, ...]] = None
|
||||
unchanged = 0
|
||||
for _ in range(self._max_page_swipes + 1):
|
||||
"""只读商品首页当前视口，不在打开规格面板前滚动详情页。"""
|
||||
|
||||
self._check_cancelled()
|
||||
xml_data = device.dump_hierarchy()
|
||||
self._last_goods_xml = str(xml_data)
|
||||
self._goods_screens_checked += 1
|
||||
return parse_goods_page(xml_data)
|
||||
|
||||
def _wait_spec_panel(self, device: Any) -> SpecSnapshot:
|
||||
"""等待点击后的规格面板真正出现，不能只依赖固定延时。"""
|
||||
|
||||
deadline = self._monotonic() + self._spec_panel_timeout
|
||||
while self._monotonic() < deadline:
|
||||
self._check_cancelled()
|
||||
xml_data = device.dump_hierarchy()
|
||||
self._last_goods_xml = str(xml_data)
|
||||
self._goods_screens_checked += 1
|
||||
root = _parse_xml(xml_data)
|
||||
labels = tuple(_all_labels(root))
|
||||
snapshots.append(parse_goods_page(xml_data))
|
||||
combined = _combine_goods_snapshots(snapshots)
|
||||
is_complete = (
|
||||
combined.title
|
||||
and combined.sales.raw
|
||||
and combined.reviews.raw
|
||||
)
|
||||
if is_complete:
|
||||
break
|
||||
if labels == previous_signature:
|
||||
unchanged += 1
|
||||
try:
|
||||
snapshot = parse_spec_panel(xml_data)
|
||||
except PddCollectError as exc:
|
||||
if exc.code != "PDD_DATA_SPEC_INCOMPLETE":
|
||||
raise
|
||||
else:
|
||||
unchanged = 0
|
||||
if unchanged >= 1:
|
||||
break
|
||||
previous_signature = labels
|
||||
screen = _screen_bounds(root)
|
||||
if screen is None:
|
||||
break
|
||||
left, top, right, bottom = screen
|
||||
x = (left + right) // 2
|
||||
device.swipe(
|
||||
x,
|
||||
top + int((bottom - top) * 0.75),
|
||||
x,
|
||||
top + int((bottom - top) * 0.30),
|
||||
duration=0.35,
|
||||
)
|
||||
self._sleep(0.35)
|
||||
return _combine_goods_snapshots(snapshots)
|
||||
if snapshot.dimensions:
|
||||
return snapshot
|
||||
self._sleep(0.25)
|
||||
raise PddCollectError(
|
||||
"PDD_PAGE_SPEC_PANEL_TIMEOUT",
|
||||
"点击规格入口后，等待规格面板加载超时",
|
||||
)
|
||||
|
||||
def _save_xml(self, label: str, xml_data: str) -> Optional[Mapping[str, Any]]:
|
||||
"""保存本地诊断 XML；测试未提供目录时不写文件。"""
|
||||
|
||||
Reference in New Issue
Block a user