From 9dadac2bff1ff8b2dd833b617d74cabd46527420 Mon Sep 17 00:00:00 2001 From: chengma Date: Sat, 8 Aug 2026 10:30:04 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=BB=9A=E5=8A=A8=E9=87=87=E9=9B=86?= =?UTF-8?q?=E8=AF=84=E4=BB=B7=E6=95=B0=E9=87=8F=E5=92=8C=E5=BA=97=E9=93=BA?= =?UTF-8?q?=E5=90=8D=20(#40)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/src/pdd_collect_service.py | 117 +++++++++++++++++++- client/test/test_pdd_collect_service.py | 137 +++++++++++++++++++++++- docs/client/01-requirements.md | 2 +- docs/client/02-architecture.md | 12 ++- 4 files changed, 257 insertions(+), 11 deletions(-) diff --git a/client/src/pdd_collect_service.py b/client/src/pdd_collect_service.py index 67d567c..ff51756 100644 --- a/client/src/pdd_collect_service.py +++ b/client/src/pdd_collect_service.py @@ -29,6 +29,8 @@ _DIMENSION_NAMES = ("颜色分类", "颜色", "尺码", "尺寸", "规格", "型 _LOGIN_MARKERS = ("手机号登录", "登录后继续", "验证码登录", "账号登录") _CAPTCHA_MARKERS = ("请完成验证", "拖动滑块", "安全验证", "点击图中") _READY_MARKERS = ("发起拼单", "立即购买", "单独购买", "免拼购买", "快要抢光") +_SHOP_NAME_EXCLUDES = frozenset({"进店", "关注", "店铺", "收藏", "客服"}) +_SHOP_ROW_TOLERANCE = 40 def _is_device_disconnect(error: BaseException) -> bool: @@ -280,6 +282,44 @@ def _find_metric(labels: Sequence[str], pattern: re.Pattern[str]) -> QuantityMet return QuantityMetric(None, None, False) +def _shop_name_by_enter_anchor(root: ET.Element) -> Optional[str]: + """以“进店”为锚点，读取同一行左侧的店铺名。""" + + anchors = [] + for node in root.iter("node"): + if _preferred_node_label(node) != "进店": + continue + bounds = _parse_bounds(node.get("bounds", "")) + if bounds is not None: + anchors.append(bounds) + if not anchors: + return None + + anchor_left, anchor_top, _, _ = min( + anchors, key=lambda bounds: (bounds[1], bounds[0]) + ) + candidates = [] + for node in root.iter("node"): + if node.get("class") != "android.widget.TextView": + continue + text = _preferred_node_label(node) + if not text or text in _SHOP_NAME_EXCLUDES: + continue + if "已拼" in text or "评价" in text or not 2 <= len(text) <= 30: + continue + bounds = _parse_bounds(node.get("bounds", "")) + if bounds is None: + continue + left, top, _, _ = bounds + if left >= anchor_left or abs(top - anchor_top) > _SHOP_ROW_TOLERANCE: + continue + candidates.append((abs(top - anchor_top), -left, text)) + + if not candidates: + return None + return min(candidates)[2] + + def parse_goods_page(xml_data: str | bytes) -> GoodsSnapshot: """解析一棵商品页控件树中的标题、店铺、销量和评价。""" @@ -317,7 +357,8 @@ def parse_goods_page(xml_data: str | bytes) -> GoodsSnapshot: sales = _find_metric(labels, re.compile(r"已拼\s*\d+(?:\.\d+)?\s*(?:万|亿)?\s*\+?\s*(?:件|人)?")) reviews_pattern = re.compile( - r"(?:\d+(?:\.\d+)?\s*(?:万|亿)?\s*\+?\s*条?评价" + r"(?:商品评价\s*[（(]\s*\d+(?:\.\d+)?\s*(?:万|亿)?\s*\+?\s*[)）]" + r"|\d+(?:\.\d+)?\s*(?:万|亿)?\s*\+?\s*条?评价" r"|评价\s*\d+(?:\.\d+)?\s*(?:万|亿)?\s*\+?)" ) reviews = _find_metric(labels, reviews_pattern) @@ -338,6 +379,8 @@ def parse_goods_page(xml_data: str | bytes) -> GoodsSnapshot: if len(label) > 2 and label.endswith(("旗舰店", "专卖店", "专营店")): shop_name = label break + if shop_name is None: + shop_name = _shop_name_by_enter_anchor(root) return GoodsSnapshot(title, shop_name, sales, reviews) @@ -592,6 +635,7 @@ class PddCollectService: page_timeout: float = 30.0, spec_panel_timeout: float = 10.0, overall_timeout: float = 600.0, + max_goods_page_swipes: int = 12, max_spec_swipes: int = 12, max_sku_count: int = 200, artifact_directory: Optional[Path] = None, @@ -607,6 +651,7 @@ class PddCollectService: self._spec_panel_timeout = spec_panel_timeout self._overall_timeout = overall_timeout self._overall_deadline: Optional[float] = None + self._max_goods_page_swipes = max_goods_page_swipes self._max_spec_swipes = max_spec_swipes self._max_sku_count = max_sku_count self._artifact_directory = artifact_directory @@ -770,11 +815,77 @@ class PddCollectService: raise PddCollectError("PDD_PAGE_TIMEOUT", "等待 PDD 商品详情页加载超时") def _collect_goods_details(self, device: Any) -> GoodsSnapshot: - """只读商品首页当前视口，不在打开规格面板前滚动详情页。""" + """保留首屏结果，并向下浏览补采评价数量和店铺名。""" self._check_cancelled() xml_data = self._dump_hierarchy(device) - return parse_goods_page(xml_data) + goods = parse_goods_page(xml_data) + if goods.shop_name and goods.reviews.raw is not None: + return goods + + unchanged_reads = 0 + for _ in range(self._max_goods_page_swipes): + self._check_cancelled() + self._swipe_goods_page(device, xml_data) + self._sleep(0.8) + self._check_cancelled() + + next_xml = self._dump_hierarchy(device) + snapshot = parse_goods_page(next_xml) + goods = GoodsSnapshot( + title=goods.title, + shop_name=goods.shop_name or snapshot.shop_name, + sales=goods.sales, + reviews=( + goods.reviews + if goods.reviews.raw is not None + else snapshot.reviews + ), + ) + if goods.shop_name and goods.reviews.raw is not None: + break + + unchanged_reads = unchanged_reads + 1 if next_xml == xml_data else 0 + xml_data = next_xml + if unchanged_reads >= 2: + break + return goods + + @staticmethod + def _swipe_goods_page(device: Any, xml_data: str | bytes) -> None: + """在商品详情区域慢速向上滑动约 30%，浏览页面下方内容。""" + + root = _parse_xml(xml_data) + scrollable_bounds = [ + bounds + for node in root.iter("node") + if node.get("scrollable") == "true" + and (bounds := _parse_bounds(node.get("bounds", ""))) is not None + ] + all_bounds = [ + bounds + for node in root.iter("node") + if (bounds := _parse_bounds(node.get("bounds", ""))) is not None + ] + candidates = scrollable_bounds or all_bounds + if not candidates: + raise PddCollectError( + "PDD_DATA_PAGE_BOUNDS_MISSING", + "商品详情页没有可用于滚动的有效区域", + ) + left, top, right, bottom = max( + candidates, + key=lambda bounds: (bounds[2] - bounds[0]) * (bounds[3] - bounds[1]), + ) + x = (left + right) // 2 + height = bottom - top + device.swipe( + x, + top + int(height * 0.65), + x, + top + int(height * 0.35), + duration=0.6, + ) def _wait_spec_panel(self, device: Any) -> SpecSnapshot: """等待点击后的规格面板真正出现，不能只依赖固定延时。""" diff --git a/client/test/test_pdd_collect_service.py b/client/test/test_pdd_collect_service.py index 369ac3a..cb0837d 100644 --- a/client/test/test_pdd_collect_service.py +++ b/client/test/test_pdd_collect_service.py @@ -60,6 +60,25 @@ class FakeCollectDevice: self.swipe_panel_states.append(self.panel_open) +class GoodsMetadataScrollDevice(FakeCollectDevice): + """模拟评价和店铺分别出现在商品页后续屏幕。""" + + def __init__(self, pages, spec_xml): + super().__init__(pages[0], spec_xml) + self.pages = pages + self.page_index = 0 + + def dump_hierarchy(self): + if self.panel_open: + return self.spec_xml + return self.pages[self.page_index] + + def swipe(self, *args, **kwargs): + super().swipe(*args, **kwargs) + if not self.panel_open: + self.page_index = min(self.page_index + 1, len(self.pages) - 1) + + class LoadingDevice(FakeCollectDevice): def dump_hierarchy(self): return '' @@ -251,6 +270,25 @@ class PddCollectParserTest(unittest.TestCase): self.assertTrue(result.sales.approximate) self.assertEqual(result.reviews.value, 2356) + def test_parse_review_title_and_shop_name_next_to_enter_shop(self): + xml_data = """ + + + + + + + """ + + result = parse_goods_page(xml_data) + + self.assertEqual(result.reviews.value, 191) + self.assertEqual(result.shop_name, "小熊服饰店") + def test_parse_spec_panel_uses_generic_dimensions_and_cents(self): result = parse_spec_panel(self.spec_xml) self.assertEqual(result.price_cent, 1000) @@ -328,6 +366,70 @@ class PddCollectParserTest(unittest.TestCase): self.assertEqual(data["source"]["device_address"], "USB-001") self.assertIsNone(data["purchase"]) + def test_collect_scrolls_before_spec_panel_and_accumulates_metadata(self): + initial_page = self.home_xml.replace( + '', + "", + ).replace( + '', + "", + ) + buy_button = """ + + + + + """ + review_page = f""" + + + {buy_button} + + """ + shop_page = f""" + + + + {buy_button} + + """ + device = GoodsMetadataScrollDevice( + [initial_page, review_page, shop_page], + keep_only_one_sku(self.spec_xml), + ) + service = PddCollectService( + PddDeviceService(lambda _serial: device), + "USB-001", + "client-001", + sleeper=lambda _seconds: None, + max_spec_swipes=0, + ) + + result = service.collect( + FakeTask("https://mobile.yangkeduo.com/goods.html?goods_id=123") + ) + + self.assertEqual(result.title, "测试纯棉短袖商品") + self.assertEqual(result.sales.value, 12000) + self.assertEqual(result.reviews.value, 191) + self.assertEqual(result.shop_name, "小熊服饰店") + self.assertEqual(device.swipe_panel_states[:2], [False, False]) + self.assertEqual(device.page_index, 2) + self.assertTrue(device.clicks) + def test_pdd_hierarchy_is_ready_even_when_focused_package_is_settings(self): pdd_home_xml = self.home_xml.replace( "', + "", + ).replace( + '', + "", + ) + cancelled = {"value": False} + device = FakeCollectDevice(incomplete_home, self.spec_xml) + service = PddCollectService( + PddDeviceService(lambda _serial: device), + "USB-001", + "client-001", + sleeper=lambda _seconds: cancelled.update(value=True), + cancelled=lambda: cancelled["value"], + ) + + with self.assertRaises(PddCollectError) as raised: + service.collect( + FakeTask("https://mobile.yangkeduo.com/goods.html?goods_id=123") + ) + + self.assertEqual(raised.exception.code, "PDD_CANCELLED") + self.assertEqual(device.swipe_panel_states, [False]) + self.assertEqual(device.clicks, []) def test_spec_panel_timeout_has_stable_code_and_xml_evidence(self): with tempfile.TemporaryDirectory() as directory: diff --git a/docs/client/01-requirements.md b/docs/client/01-requirements.md index 93d7b7f..ce55179 100644 --- a/docs/client/01-requirements.md +++ b/docs/client/01-requirements.md @@ -48,7 +48,7 @@ Client 顶级导航仅包含： - 商品编号、商品链接和商品标题； - 店铺名称（无障碍树未提供时允许留空并保存诊断证据）； - 已拼数量及原始显示文字； -- 评价数量及原始显示文字（当前首页未提供时允许留空，不能为此反复滚动并阻塞规格采集）； +- 评价数量及原始显示文字（首屏没有时，在进入规格面板前有限次缓慢向下浏览补采；到达页面底部或次数上限后仍没有时允许留空）； - 所有规格维度及其可见值； - 每个颜色对应的价格、币种，以及当前控件树显示的规格可用状态； - 采集时间、设备及必要诊断产物引用。 diff --git a/docs/client/02-architecture.md b/docs/client/02-architecture.md index 8c23be5..1caf7f5 100644 --- a/docs/client/02-architecture.md +++ b/docs/client/02-architecture.md @@ -354,15 +354,17 @@ reconcile_purchase(task, run) -> PurchaseResult | ManualReview PDD 页面可能出现登录失效、验证码、控件树不完整、A/B 页面、库存变化和价格变化。适配层必须返回结构化错误，不得把这些情况统一返回 `False`。 -采集商品时按“首页就绪 → 读取当前首页摘要 → 点击规格入口 → 确认规格面板 -出现 → 颜色列表归左 → 按行蛇形逐色点击并采价 → 向下滚动并只读尺码 → -组装结果”的顺序执行。颜色点击可能改变列表位置，因此每次点击后必须重新读取 +采集商品时按“首页就绪 → 读取当前首页摘要 → 有限滚动补采评价和店铺 → +点击规格入口 → 确认规格面板出现 → 颜色列表归左 → 按行蛇形逐色点击并采价 → +向下滚动并只读尺码 → 组装结果”的顺序执行。颜色点击可能改变列表位置，因此每次点击后必须重新读取 控件树，不能复用旧坐标；左右边缘以连续两次没有新颜色且视口稳定为准。 当前业务价格粒度固定为颜色。第一行从左到右、下一行从右到左交替遍历，操作 顺序采用蛇形以减少滑动，输出维度仍恢复为页面每行从左到右的自然顺序。尺码 -只读取文字和当前可用状态，不点击、不参与采价。打开规格面板前不得为了寻找 -评价或店铺连续滚动商品详情；这两个可选字段缺失时保存证据，但不阻塞核心采集。 +只读取文字和当前可用状态，不点击、不参与采价。商品首屏的标题和已拼数量必须 +保留；首屏缺少评价或店铺时，打开规格面板前小幅、慢速向下浏览并分别累积这两个 +字段，不要求它们同时出现在一屏。获取完整、页面不再变化或达到次数上限后停止， +字段仍缺失时保存证据，但不阻塞核心规格采集。 ## 10. 关键架构决策