feat: 滚动采集评价数量和店铺名 (#40)

client/src/pdd_collect_service.py

@@ -29,6 +29,8 @@ _DIMENSION_NAMES = ("颜色分类", "颜色", "尺码", "尺寸", "规格", "型
 _LOGIN_MARKERS = ("手机号登录", "登录后继续", "验证码登录", "账号登录")
 _CAPTCHA_MARKERS = ("请完成验证", "拖动滑块", "安全验证", "点击图中")
 _READY_MARKERS = ("发起拼单", "立即购买", "单独购买", "免拼购买", "快要抢光")
+_SHOP_NAME_EXCLUDES = frozenset({"进店", "关注", "店铺", "收藏", "客服"})
+_SHOP_ROW_TOLERANCE = 40
 
 
 def _is_device_disconnect(error: BaseException) -> bool:
@@ -280,6 +282,44 @@ def _find_metric(labels: Sequence[str], pattern: re.Pattern[str]) -> QuantityMet
     return QuantityMetric(None, None, False)
 
 
+def _shop_name_by_enter_anchor(root: ET.Element) -> Optional[str]:
+    """以“进店”为锚点，读取同一行左侧的店铺名。"""
+
+    anchors = []
+    for node in root.iter("node"):
+        if _preferred_node_label(node) != "进店":
+            continue
+        bounds = _parse_bounds(node.get("bounds", ""))
+        if bounds is not None:
+            anchors.append(bounds)
+    if not anchors:
+        return None
+
+    anchor_left, anchor_top, _, _ = min(
+        anchors, key=lambda bounds: (bounds[1], bounds[0])
+    )
+    candidates = []
+    for node in root.iter("node"):
+        if node.get("class") != "android.widget.TextView":
+            continue
+        text = _preferred_node_label(node)
+        if not text or text in _SHOP_NAME_EXCLUDES:
+            continue
+        if "已拼" in text or "评价" in text or not 2 <= len(text) <= 30:
+            continue
+        bounds = _parse_bounds(node.get("bounds", ""))
+        if bounds is None:
+            continue
+        left, top, _, _ = bounds
+        if left >= anchor_left or abs(top - anchor_top) > _SHOP_ROW_TOLERANCE:
+            continue
+        candidates.append((abs(top - anchor_top), -left, text))
+
+    if not candidates:
+        return None
+    return min(candidates)[2]
+
+
 def parse_goods_page(xml_data: str | bytes) -> GoodsSnapshot:
     """解析一棵商品页控件树中的标题、店铺、销量和评价。"""
 
@@ -317,7 +357,8 @@ def parse_goods_page(xml_data: str | bytes) -> GoodsSnapshot:
 
     sales = _find_metric(labels, re.compile(r"已拼\s*\d+(?:\.\d+)?\s*(?:万|亿)?\s*\+?\s*(?:件|人)?"))
     reviews_pattern = re.compile(
-        r"(?:\d+(?:\.\d+)?\s*(?:万|亿)?\s*\+?\s*条?评价"
+        r"(?:商品评价\s*[（(]\s*\d+(?:\.\d+)?\s*(?:万|亿)?\s*\+?\s*[)）]"
+        r"|\d+(?:\.\d+)?\s*(?:万|亿)?\s*\+?\s*条?评价"
         r"|评价\s*\d+(?:\.\d+)?\s*(?:万|亿)?\s*\+?)"
     )
     reviews = _find_metric(labels, reviews_pattern)
@@ -338,6 +379,8 @@ def parse_goods_page(xml_data: str | bytes) -> GoodsSnapshot:
         if len(label) > 2 and label.endswith(("旗舰店", "专卖店", "专营店")):
             shop_name = label
             break
+    if shop_name is None:
+        shop_name = _shop_name_by_enter_anchor(root)
     return GoodsSnapshot(title, shop_name, sales, reviews)
 
 
@@ -592,6 +635,7 @@ class PddCollectService:
         page_timeout: float = 30.0,
         spec_panel_timeout: float = 10.0,
         overall_timeout: float = 600.0,
+        max_goods_page_swipes: int = 12,
         max_spec_swipes: int = 12,
         max_sku_count: int = 200,
         artifact_directory: Optional[Path] = None,
@@ -607,6 +651,7 @@ class PddCollectService:
         self._spec_panel_timeout = spec_panel_timeout
         self._overall_timeout = overall_timeout
         self._overall_deadline: Optional[float] = None
+        self._max_goods_page_swipes = max_goods_page_swipes
         self._max_spec_swipes = max_spec_swipes
         self._max_sku_count = max_sku_count
         self._artifact_directory = artifact_directory
@@ -770,11 +815,77 @@ class PddCollectService:
         raise PddCollectError("PDD_PAGE_TIMEOUT", "等待 PDD 商品详情页加载超时")
 
     def _collect_goods_details(self, device: Any) -> GoodsSnapshot:
-        """只读商品首页当前视口，不在打开规格面板前滚动详情页。"""
+        """保留首屏结果，并向下浏览补采评价数量和店铺名。"""
 
         self._check_cancelled()
         xml_data = self._dump_hierarchy(device)
-        return parse_goods_page(xml_data)
+        goods = parse_goods_page(xml_data)
+        if goods.shop_name and goods.reviews.raw is not None:
+            return goods
+
+        unchanged_reads = 0
+        for _ in range(self._max_goods_page_swipes):
+            self._check_cancelled()
+            self._swipe_goods_page(device, xml_data)
+            self._sleep(0.8)
+            self._check_cancelled()
+
+            next_xml = self._dump_hierarchy(device)
+            snapshot = parse_goods_page(next_xml)
+            goods = GoodsSnapshot(
+                title=goods.title,
+                shop_name=goods.shop_name or snapshot.shop_name,
+                sales=goods.sales,
+                reviews=(
+                    goods.reviews
+                    if goods.reviews.raw is not None
+                    else snapshot.reviews
+                ),
+            )
+            if goods.shop_name and goods.reviews.raw is not None:
+                break
+
+            unchanged_reads = unchanged_reads + 1 if next_xml == xml_data else 0
+            xml_data = next_xml
+            if unchanged_reads >= 2:
+                break
+        return goods
+
+    @staticmethod
+    def _swipe_goods_page(device: Any, xml_data: str | bytes) -> None:
+        """在商品详情区域慢速向上滑动约 30%，浏览页面下方内容。"""
+
+        root = _parse_xml(xml_data)
+        scrollable_bounds = [
+            bounds
+            for node in root.iter("node")
+            if node.get("scrollable") == "true"
+            and (bounds := _parse_bounds(node.get("bounds", ""))) is not None
+        ]
+        all_bounds = [
+            bounds
+            for node in root.iter("node")
+            if (bounds := _parse_bounds(node.get("bounds", ""))) is not None
+        ]
+        candidates = scrollable_bounds or all_bounds
+        if not candidates:
+            raise PddCollectError(
+                "PDD_DATA_PAGE_BOUNDS_MISSING",
+                "商品详情页没有可用于滚动的有效区域",
+            )
+        left, top, right, bottom = max(
+            candidates,
+            key=lambda bounds: (bounds[2] - bounds[0]) * (bounds[3] - bounds[1]),
+        )
+        x = (left + right) // 2
+        height = bottom - top
+        device.swipe(
+            x,
+            top + int(height * 0.65),
+            x,
+            top + int(height * 0.35),
+            duration=0.6,
+        )
 
     def _wait_spec_panel(self, device: Any) -> SpecSnapshot:
         """等待点击后的规格面板真正出现，不能只依赖固定延时。"""

client/test/test_pdd_collect_service.py

@@ -60,6 +60,25 @@ class FakeCollectDevice:
         self.swipe_panel_states.append(self.panel_open)
 
 
+class GoodsMetadataScrollDevice(FakeCollectDevice):
+    """模拟评价和店铺分别出现在商品页后续屏幕。"""
+
+    def __init__(self, pages, spec_xml):
+        super().__init__(pages[0], spec_xml)
+        self.pages = pages
+        self.page_index = 0
+
+    def dump_hierarchy(self):
+        if self.panel_open:
+            return self.spec_xml
+        return self.pages[self.page_index]
+
+    def swipe(self, *args, **kwargs):
+        super().swipe(*args, **kwargs)
+        if not self.panel_open:
+            self.page_index = min(self.page_index + 1, len(self.pages) - 1)
+
+
 class LoadingDevice(FakeCollectDevice):
     def dump_hierarchy(self):
         return '<hierarchy><node text="正在加载" /></hierarchy>'
@@ -251,6 +270,25 @@ class PddCollectParserTest(unittest.TestCase):
         self.assertTrue(result.sales.approximate)
         self.assertEqual(result.reviews.value, 2356)
 
+    def test_parse_review_title_and_shop_name_next_to_enter_shop(self):
+        xml_data = """<hierarchy>
+          <node class="android.widget.FrameLayout" bounds="[0,0][1080,2000]">
+            <node class="android.widget.TextView" text="商品评价(191)"
+                  bounds="[30,700][300,760]" />
+            <node class="android.widget.TextView" text="承诺假一赔十"
+                  bounds="[30,1050][300,1110]" />
+            <node class="android.widget.TextView" text="小熊服饰店"
+                  bounds="[300,1200][650,1260]" />
+            <node class="android.widget.TextView" text="进店"
+                  bounds="[850,1200][1030,1260]" />
+          </node>
+        </hierarchy>"""
+
+        result = parse_goods_page(xml_data)
+
+        self.assertEqual(result.reviews.value, 191)
+        self.assertEqual(result.shop_name, "小熊服饰店")
+
     def test_parse_spec_panel_uses_generic_dimensions_and_cents(self):
         result = parse_spec_panel(self.spec_xml)
         self.assertEqual(result.price_cent, 1000)
@@ -328,6 +366,70 @@ class PddCollectParserTest(unittest.TestCase):
         self.assertEqual(data["source"]["device_address"], "USB-001")
         self.assertIsNone(data["purchase"])
 
+    def test_collect_scrolls_before_spec_panel_and_accumulates_metadata(self):
+        initial_page = self.home_xml.replace(
+            '<node class="android.widget.TextView" text="2356条评价" '
+            'bounds="[330,950][600,1010]" visible-to-user="true" '
+            'enabled="true" />',
+            "",
+        ).replace(
+            '<node class="android.widget.TextView" text="店铺：测试服饰旗舰店" '
+            'bounds="[30,1050][600,1110]" visible-to-user="true" '
+            'enabled="true" />',
+            "",
+        )
+        buy_button = """
+            <node class="android.view.ViewGroup" content-desc="¥10.00立即购买"
+                  clickable="true" bounds="[500,1700][1080,1950]"
+                  visible-to-user="true" enabled="true">
+              <node class="android.widget.TextView" text="¥10.00"
+                    bounds="[650,1730][850,1810]"
+                    visible-to-user="true" enabled="true" />
+              <node class="android.widget.TextView" text="立即购买"
+                    bounds="[650,1820][900,1900]"
+                    visible-to-user="true" enabled="true" />
+            </node>
+        """
+        review_page = f"""<hierarchy>
+          <node class="android.widget.FrameLayout" bounds="[0,0][1080,2000]">
+            <node class="android.widget.TextView" text="商品评价(191)"
+                  bounds="[30,700][300,760]" />
+            {buy_button}
+          </node>
+        </hierarchy>"""
+        shop_page = f"""<hierarchy>
+          <node class="android.widget.FrameLayout" bounds="[0,0][1080,2000]">
+            <node class="android.widget.TextView" text="小熊服饰店"
+                  bounds="[300,1200][650,1260]" />
+            <node class="android.widget.TextView" text="进店"
+                  bounds="[850,1200][1030,1260]" />
+            {buy_button}
+          </node>
+        </hierarchy>"""
+        device = GoodsMetadataScrollDevice(
+            [initial_page, review_page, shop_page],
+            keep_only_one_sku(self.spec_xml),
+        )
+        service = PddCollectService(
+            PddDeviceService(lambda _serial: device),
+            "USB-001",
+            "client-001",
+            sleeper=lambda _seconds: None,
+            max_spec_swipes=0,
+        )
+
+        result = service.collect(
+            FakeTask("https://mobile.yangkeduo.com/goods.html?goods_id=123")
+        )
+
+        self.assertEqual(result.title, "测试纯棉短袖商品")
+        self.assertEqual(result.sales.value, 12000)
+        self.assertEqual(result.reviews.value, 191)
+        self.assertEqual(result.shop_name, "小熊服饰店")
+        self.assertEqual(device.swipe_panel_states[:2], [False, False])
+        self.assertEqual(device.page_index, 2)
+        self.assertTrue(device.clicks)
+
     def test_pdd_hierarchy_is_ready_even_when_focused_package_is_settings(self):
         pdd_home_xml = self.home_xml.replace(
             "<node ", '<node package="com.xunmeng.pinduoduo" '
@@ -593,7 +695,7 @@ class PddCollectParserTest(unittest.TestCase):
         )
         self.assertIsNone(result.shop_name)
 
-    def test_missing_reviews_does_not_scroll_before_opening_spec_panel(self):
+    def test_missing_reviews_stops_when_goods_page_no_longer_changes(self):
         home_without_reviews = self.home_xml.replace(
             '<node class="android.widget.TextView" text="2356条评价" '
             'bounds="[330,950][600,1010]" visible-to-user="true" '
@@ -617,7 +719,38 @@ class PddCollectParserTest(unittest.TestCase):
 
         self.assertIsNone(result.reviews.raw)
         self.assertTrue(device.clicks)
-        self.assertTrue(all(device.swipe_panel_states))
+        self.assertEqual(device.swipe_panel_states[:2], [False, False])
+
+    def test_cancel_during_goods_page_scroll_stops_before_spec_panel(self):
+        incomplete_home = self.home_xml.replace(
+            '<node class="android.widget.TextView" text="2356条评价" '
+            'bounds="[330,950][600,1010]" visible-to-user="true" '
+            'enabled="true" />',
+            "",
+        ).replace(
+            '<node class="android.widget.TextView" text="店铺：测试服饰旗舰店" '
+            'bounds="[30,1050][600,1110]" visible-to-user="true" '
+            'enabled="true" />',
+            "",
+        )
+        cancelled = {"value": False}
+        device = FakeCollectDevice(incomplete_home, self.spec_xml)
+        service = PddCollectService(
+            PddDeviceService(lambda _serial: device),
+            "USB-001",
+            "client-001",
+            sleeper=lambda _seconds: cancelled.update(value=True),
+            cancelled=lambda: cancelled["value"],
+        )
+
+        with self.assertRaises(PddCollectError) as raised:
+            service.collect(
+                FakeTask("https://mobile.yangkeduo.com/goods.html?goods_id=123")
+            )
+
+        self.assertEqual(raised.exception.code, "PDD_CANCELLED")
+        self.assertEqual(device.swipe_panel_states, [False])
+        self.assertEqual(device.clicks, [])
 
     def test_spec_panel_timeout_has_stable_code_and_xml_evidence(self):
         with tempfile.TemporaryDirectory() as directory:

docs/client/01-requirements.md

@@ -48,7 +48,7 @@ Client 顶级导航仅包含：
 - 商品编号、商品链接和商品标题；
 - 店铺名称（无障碍树未提供时允许留空并保存诊断证据）；
 - 已拼数量及原始显示文字；
-- 评价数量及原始显示文字（当前首页未提供时允许留空，不能为此反复滚动并阻塞规格采集）；
+- 评价数量及原始显示文字（首屏没有时，在进入规格面板前有限次缓慢向下浏览补采；到达页面底部或次数上限后仍没有时允许留空）；
 - 所有规格维度及其可见值；
 - 每个颜色对应的价格、币种，以及当前控件树显示的规格可用状态；
 - 采集时间、设备及必要诊断产物引用。

docs/client/02-architecture.md

@@ -354,15 +354,17 @@ reconcile_purchase(task, run) -> PurchaseResult | ManualReview
 
 PDD 页面可能出现登录失效、验证码、控件树不完整、A/B 页面、库存变化和价格变化。适配层必须返回结构化错误，不得把这些情况统一返回 `False`。
 
-采集商品时按“首页就绪 → 读取当前首页摘要 → 点击规格入口 → 确认规格面板
+采集商品时按“首页就绪 → 读取当前首页摘要 → 有限滚动补采评价和店铺 →
-出现 → 颜色列表归左 → 按行蛇形逐色点击并采价 → 向下滚动并只读尺码 →
+点击规格入口 → 确认规格面板出现 → 颜色列表归左 → 按行蛇形逐色点击并采价 →
-组装结果”的顺序执行。颜色点击可能改变列表位置，因此每次点击后必须重新读取
+向下滚动并只读尺码 → 组装结果”的顺序执行。颜色点击可能改变列表位置，因此每次点击后必须重新读取
 控件树，不能复用旧坐标；左右边缘以连续两次没有新颜色且视口稳定为准。
 
 当前业务价格粒度固定为颜色。第一行从左到右、下一行从右到左交替遍历，操作
 顺序采用蛇形以减少滑动，输出维度仍恢复为页面每行从左到右的自然顺序。尺码
-只读取文字和当前可用状态，不点击、不参与采价。打开规格面板前不得为了寻找
+只读取文字和当前可用状态，不点击、不参与采价。商品首屏的标题和已拼数量必须
-评价或店铺连续滚动商品详情；这两个可选字段缺失时保存证据，但不阻塞核心采集。
+保留；首屏缺少评价或店铺时，打开规格面板前小幅、慢速向下浏览并分别累积这两个
+字段，不要求它们同时出现在一屏。获取完整、页面不再变化或达到次数上限后停止，
+字段仍缺失时保存证据，但不阻塞核心规格采集。
 
 ## 10. 关键架构决策