Files
cmautobuy/client/test/test_pdd_collect_service.py
T

655 lines
24 KiB
Python

"""PDD 商品采集与 XML 解析测试,不连接真机。"""
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
import tempfile
import unittest
import xml.etree.ElementTree as ET
from src.pdd_collect_service import (
PddCollectError,
PddCollectService,
parse_goods_page,
parse_quantity,
parse_spec_panel,
)
from src.pdd_device_service import PddDeviceService
FIXTURES = Path(__file__).parent / "fixtures"
@dataclass(frozen=True)
class FakeTask:
goods_url: str
goods_id: str | None = None
class FakeCollectDevice:
def __init__(self, home_xml: str, spec_xml: str):
self.home_xml = home_xml
self.spec_xml = spec_xml
self.panel_open = False
self.clicks = []
self.swipes = []
self.swipe_panel_states = []
def app_current(self):
return {"package": "com.xunmeng.pinduoduo"}
def app_start(self, _package):
raise AssertionError("PDD 已经在前台,不应重复启动")
def app_wait(self, _package, timeout=10):
return 123
def open_url(self, url):
self.opened_url = url
def dump_hierarchy(self):
return self.spec_xml if self.panel_open else self.home_xml
def click(self, x, y):
self.clicks.append((x, y))
if y > 1600:
self.panel_open = True
def swipe(self, *args, **kwargs):
self.swipes.append((args, kwargs))
self.swipe_panel_states.append(self.panel_open)
class LoadingDevice(FakeCollectDevice):
def dump_hierarchy(self):
return '<hierarchy><node text="正在加载" /></hierarchy>'
class DisconnectingDevice(FakeCollectDevice):
def __init__(self, home_xml, spec_xml):
super().__init__(home_xml, spec_xml)
self.current_calls = 0
def app_current(self):
self.current_calls += 1
if self.current_calls > 1:
raise RuntimeError("device not found")
return super().app_current()
class FocusMismatchDevice(FakeCollectDevice):
"""系统焦点报告为设置页,但无障碍树实际属于 PDD。"""
def app_current(self):
return {"package": "com.oplus.wirelesssettings"}
def app_start(self, _package):
self.app_started = True
class PanelDoesNotOpenDevice(FakeCollectDevice):
def click(self, x, y):
self.clicks.append((x, y))
class SnakeColorDevice(FakeCollectDevice):
"""模拟两行横向颜色网格和滚动后才出现的尺码区域。"""
prices = {
"A色": 1000,
"B色": 1000,
"C色": 1200,
"D色": 1300,
"E色": 1400,
"F色": 1500,
}
def __init__(self, home_xml):
super().__init__(home_xml, "")
self.page = 1
self.selected_color = "A色"
self.sizes_visible = False
self.clicked_colors = []
def dump_hierarchy(self):
if not self.panel_open:
return self.home_xml
return self._spec_xml()
def click(self, x, y):
self.clicks.append((x, y))
if not self.panel_open:
self.panel_open = True
return
for name, bounds in self._current_color_bounds().items():
left, top, right, bottom = bounds
if left <= x <= right and top <= y <= bottom:
self.selected_color = name
self.clicked_colors.append(name)
return
def swipe(self, x1, y1, x2, y2, duration=0.35):
self.swipes.append(((x1, y1, x2, y2), {"duration": duration}))
self.swipe_panel_states.append(self.panel_open)
if abs(x2 - x1) > abs(y2 - y1):
self.page = 1 if x1 > x2 else 0
else:
self.sizes_visible = True
def _current_color_bounds(self):
if self.page == 0:
return {
"A色": (36, 740, 340, 850),
"B色": (370, 740, 674, 850),
"C色": (1044, 740, 1080, 850),
"D色": (36, 900, 340, 1010),
"E色": (370, 900, 674, 1010),
"F色": (1044, 900, 1080, 1010),
}
return {
"A色": (0, 740, 20, 850),
"B色": (36, 740, 340, 850),
"C色": (370, 740, 674, 850),
"D色": (0, 900, 20, 1010),
"E色": (36, 900, 340, 1010),
"F色": (370, 900, 674, 1010),
}
def _spec_xml(self):
color_nodes = []
for name, bounds in self._current_color_bounds().items():
left, top, right, bottom = bounds
selected = "true" if name == self.selected_color else "false"
color_nodes.append(
f'<node class="android.view.ViewGroup" content-desc="{name}" '
f'clickable="true" selected="{selected}" '
f'bounds="[{left},{top}][{right},{bottom}]" '
'visible-to-user="true" enabled="true" />'
)
size_nodes = ""
if self.sizes_visible:
size_nodes = """
<node class="android.widget.TextView" text="尺码"
bounds="[36,1120][130,1180]" visible-to-user="true"
enabled="true" />
<node class="android.view.ViewGroup" content-desc="M"
clickable="true" bounds="[36,1210][340,1300]"
visible-to-user="true" enabled="true" />
<node class="android.view.ViewGroup" content-desc="L"
clickable="true" bounds="[370,1210][674,1300]"
visible-to-user="true" enabled="true" />
"""
price = self.prices[self.selected_color] / 100
return f"""<hierarchy>
<node class="android.widget.FrameLayout" bounds="[0,0][1080,2000]"
visible-to-user="true" enabled="true">
<node class="android.widget.TextView" text="¥{price:.2f}"
bounds="[390,300][650,380]" visible-to-user="true"
enabled="true" />
<node class="androidx.recyclerview.widget.RecyclerView"
scrollable="true" bounds="[0,600][1080,1700]"
visible-to-user="true" enabled="true">
<node class="android.widget.TextView" text="颜色分类"
bounds="[36,650][200,710]" visible-to-user="true"
enabled="true" />
<node class="androidx.recyclerview.widget.RecyclerView"
scrollable="true" bounds="[36,720][1044,1060]"
visible-to-user="true" enabled="true">
{''.join(color_nodes)}
</node>
{size_nodes}
</node>
</node>
</hierarchy>"""
class IgnoredColorClickDevice(SnakeColorDevice):
"""页面暴露选中状态,但模拟一次没有生效的颜色点击。"""
def click(self, x, y):
if not self.panel_open:
super().click(x, y)
return
self.clicks.append((x, y))
def keep_only_one_sku(xml_data: str) -> str:
"""从脱敏固件中删除蓝色和 L,只保留一个组合。"""
root = ET.fromstring(xml_data)
for parent in root.iter():
for child in list(parent):
own_label = (
child.get("text") or child.get("content-desc") or ""
).strip()
labels = " ".join(
(node.get("text") or node.get("content-desc") or "").strip()
for node in child.iter("node")
)
if own_label == "蓝色" or labels.strip() == "L":
parent.remove(child)
return ET.tostring(root, encoding="unicode")
class PddCollectParserTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.home_xml = (FIXTURES / "pdd_goods_page.xml").read_text(encoding="utf-8")
cls.spec_xml = (FIXTURES / "pdd_spec_panel.xml").read_text(encoding="utf-8")
def test_quantity_keeps_raw_value_and_approximate_flag(self):
result = parse_quantity("已拼1.2万+件")
self.assertEqual(result.value, 12000)
self.assertEqual(result.raw, "已拼1.2万+件")
self.assertTrue(result.approximate)
def test_parse_goods_page(self):
result = parse_goods_page(self.home_xml)
self.assertEqual(result.title, "测试纯棉短袖商品")
self.assertEqual(result.shop_name, "测试服饰旗舰店")
self.assertEqual(result.sales.value, 12000)
self.assertTrue(result.sales.approximate)
self.assertEqual(result.reviews.value, 2356)
def test_parse_spec_panel_uses_generic_dimensions_and_cents(self):
result = parse_spec_panel(self.spec_xml)
self.assertEqual(result.price_cent, 1000)
self.assertEqual([item.key for item in result.dimensions], ["color", "size"])
self.assertEqual(
[value.text for value in result.dimensions[0].values],
["红色", "蓝色"],
)
self.assertFalse(result.dimensions[0].values[1].available)
self.assertEqual(
[value.text for value in result.dimensions[1].values],
["M", "L"],
)
def test_truncated_spec_name_prefers_full_content_description(self):
xml_data = self.spec_xml.replace(
'content-desc="红色" clickable="true"',
'text="红…" content-desc="红色" clickable="true"',
)
result = parse_spec_panel(xml_data)
self.assertEqual(result.dimensions[0].values[0].text, "红色")
without_description = xml_data.replace(' content-desc="红色"', "")
with self.assertRaises(PddCollectError) as raised:
parse_spec_panel(without_description)
self.assertEqual(raised.exception.code, "PDD_DATA_SKU_NAME_TRUNCATED")
def test_login_and_invalid_xml_have_different_error_codes(self):
with self.assertRaises(PddCollectError) as login:
parse_goods_page('<hierarchy><node text="手机号登录" /></hierarchy>')
with self.assertRaises(PddCollectError) as invalid:
parse_goods_page("<hierarchy>")
self.assertEqual(login.exception.code, "PDD_PAGE_LOGIN_REQUIRED")
self.assertEqual(invalid.exception.code, "PDD_DATA_XML_INVALID")
def test_captcha_and_incomplete_spec_have_specific_codes(self):
with self.assertRaises(PddCollectError) as captcha:
parse_goods_page('<hierarchy><node text="请完成验证" /></hierarchy>')
with self.assertRaises(PddCollectError) as incomplete:
parse_spec_panel('<hierarchy><node text="普通页面" /></hierarchy>')
self.assertEqual(captcha.exception.code, "PDD_PAGE_CAPTCHA")
self.assertEqual(incomplete.exception.code, "PDD_DATA_SPEC_INCOMPLETE")
def test_collect_returns_versioned_pdd_data_without_real_device(self):
# 收窄到一个可用组合,便于验证完整流程而不模拟真实页面切换。
one_sku_xml = keep_only_one_sku(self.spec_xml)
device = FakeCollectDevice(self.home_xml, one_sku_xml)
service = PddCollectService(
PddDeviceService(lambda _serial: device),
"USB-001",
"client-001",
sleeper=lambda _seconds: None,
now=lambda: datetime(2026, 8, 7, 8, 0, tzinfo=timezone.utc),
max_spec_swipes=1,
)
result = service.collect(
FakeTask("https://mobile.yangkeduo.com/goods.html?goods_id=123")
)
data = result.to_pdd_data()
self.assertEqual(data["schema_version"], 1)
self.assertEqual(data["goods_id"], "123")
self.assertEqual(data["title"], "测试纯棉短袖商品")
self.assertEqual(data["shop_name"], "测试服饰旗舰店")
self.assertEqual(data["price_granularity"], "color")
self.assertEqual(data["metrics"]["sales"]["value"], 12000)
self.assertEqual(data["dimensions"][0]["key"], "color")
self.assertEqual(data["skus"][0]["options"], {"color": "红色", "size": "M"})
self.assertEqual(data["skus"][0]["price_cent"], 1000)
self.assertEqual(
data["skus"][0]["price_observed_at"],
{"color": "红色"},
)
self.assertEqual(data["source"]["device_address"], "USB-001")
self.assertIsNone(data["purchase"])
def test_pdd_hierarchy_is_ready_even_when_focused_package_is_settings(self):
pdd_home_xml = self.home_xml.replace(
"<node ", '<node package="com.xunmeng.pinduoduo" '
)
device = FocusMismatchDevice(
pdd_home_xml, keep_only_one_sku(self.spec_xml)
)
service = PddCollectService(
PddDeviceService(lambda _serial: device),
"USB-001",
"client-001",
sleeper=lambda _seconds: None,
max_spec_swipes=0,
)
result = service.collect(
FakeTask("https://mobile.yangkeduo.com/goods.html?goods_id=123")
)
self.assertEqual(result.goods_id, "123")
self.assertTrue(device.app_started)
def test_price_is_sampled_once_per_available_color(self):
device = FakeCollectDevice(self.home_xml, self.spec_xml)
service = PddCollectService(
PddDeviceService(lambda _serial: device),
"USB-001",
"client-001",
sleeper=lambda _seconds: None,
max_spec_swipes=0,
)
data = service.collect(
FakeTask("https://mobile.yangkeduo.com/goods.html?goods_id=123")
).to_pdd_data()
self.assertEqual(len(data["skus"]), 4)
red_skus = [
item for item in data["skus"] if item["options"]["color"] == "红色"
]
self.assertEqual(
{item["price_observed_at"]["color"] for item in red_skus},
{"红色"},
)
self.assertTrue(
all(
set(item["price_observed_at"]) == {"color"}
for item in red_skus
)
)
# 1 次打开规格面板,另有红色点 1 次;尺码只读不点击。
self.assertEqual(len(device.clicks), 2)
def test_colors_are_sampled_in_snake_order_before_sizes_are_read(self):
device = SnakeColorDevice(self.home_xml)
service = PddCollectService(
PddDeviceService(lambda _serial: device),
"USB-001",
"client-001",
sleeper=lambda _seconds: None,
)
data = service.collect(
FakeTask("https://mobile.yangkeduo.com/goods.html?goods_id=123")
).to_pdd_data()
self.assertEqual(
device.clicked_colors,
["A色", "B色", "C色", "F色", "E色", "D色"],
)
self.assertEqual(
[item["text"] for item in data["dimensions"][0]["values"]],
["A色", "B色", "C色", "D色", "E色", "F色"],
)
self.assertEqual(
[item["text"] for item in data["dimensions"][1]["values"]],
["M", "L"],
)
color_prices = {
item["options"]["color"]: item["price_cent"]
for item in data["skus"]
if item["options"]["size"] == "M"
}
self.assertEqual(color_prices["A色"], 1000)
self.assertEqual(color_prices["B色"], 1000)
self.assertEqual(color_prices["F色"], 1500)
self.assertTrue(all(
item["price_observed_at"] == {"color": item["options"]["color"]}
for item in data["skus"]
))
def test_visible_click_is_fallback_when_page_exposes_no_selection_state(self):
xml_data = keep_only_one_sku(self.spec_xml)
root = ET.fromstring(xml_data)
for parent in root.iter():
for child in list(parent):
if (child.get("text") or "").startswith("已选"):
parent.remove(child)
xml_without_selection = ET.tostring(root, encoding="unicode")
device = FakeCollectDevice(self.home_xml, xml_without_selection)
service = PddCollectService(
PddDeviceService(lambda _serial: device),
"USB-001",
"client-001",
sleeper=lambda _seconds: None,
max_spec_swipes=0,
)
data = service.collect(
FakeTask("https://mobile.yangkeduo.com/goods.html?goods_id=123")
).to_pdd_data()
self.assertEqual(data["skus"][0]["price_cent"], 1000)
self.assertEqual(
data["skus"][0]["price_observed_at"], {"color": "红色"}
)
def test_visible_text_does_not_confirm_an_ignored_click(self):
device = IgnoredColorClickDevice(self.home_xml)
device.panel_open = True
device.page = 0
service = PddCollectService(
PddDeviceService(lambda _serial: device),
"USB-001",
"client-001",
sleeper=lambda _seconds: None,
)
sample = service._click_and_sample_color(device, "B色")
self.assertIsNone(sample.price_cent)
def test_missing_goods_id_is_structured_error(self):
service = PddCollectService(
PddDeviceService(lambda _serial: object()),
"USB-001",
"client-001",
)
with self.assertRaises(PddCollectError) as raised:
service.collect(FakeTask("https://mobile.yangkeduo.com/goods.html"))
self.assertEqual(raised.exception.code, "PDD_DATA_GOODS_ID_MISSING")
def test_non_pdd_goods_url_is_rejected_before_opening_device(self):
service = PddCollectService(
PddDeviceService(lambda _serial: object()),
"USB-001",
"client-001",
)
with self.assertRaises(PddCollectError) as raised:
service.collect(FakeTask("https://example.com/goods.html?goods_id=123"))
self.assertEqual(raised.exception.code, "PDD_DATA_GOODS_URL_INVALID")
def test_task_goods_id_must_match_url(self):
service = PddCollectService(
PddDeviceService(lambda _serial: object()),
"USB-001",
"client-001",
)
with self.assertRaises(PddCollectError) as raised:
service.collect(
FakeTask(
"https://mobile.yangkeduo.com/goods.html?goods_id=123",
goods_id="456",
)
)
self.assertEqual(raised.exception.code, "PDD_DATA_GOODS_ID_MISMATCH")
def test_page_timeout_has_specific_error_code(self):
device = LoadingDevice(self.home_xml, self.spec_xml)
ticks = iter((0.0, 0.0, 0.0, 2.0))
service = PddCollectService(
PddDeviceService(lambda _serial: device),
"USB-001",
"client-001",
sleeper=lambda _seconds: None,
monotonic=lambda: next(ticks),
page_timeout=1.0,
)
with self.assertRaises(PddCollectError) as raised:
service.collect(
FakeTask("https://mobile.yangkeduo.com/goods.html?goods_id=123")
)
self.assertEqual(raised.exception.code, "PDD_PAGE_TIMEOUT")
def test_non_pdd_tree_with_purchase_words_is_not_ready(self):
device = FocusMismatchDevice(
'<hierarchy><node package="com.android.settings" text="免拼购买" '
'bounds="[0,0][100,100]" /></hierarchy>',
self.spec_xml,
)
ticks = iter((0.0, 0.0, 0.0, 2.0))
service = PddCollectService(
PddDeviceService(lambda _serial: device),
"USB-001",
"client-001",
sleeper=lambda _seconds: None,
monotonic=lambda: next(ticks),
page_timeout=1.0,
)
with self.assertRaises(PddCollectError) as raised:
service.collect(
FakeTask("https://mobile.yangkeduo.com/goods.html?goods_id=123")
)
self.assertEqual(raised.exception.code, "PDD_PAGE_TIMEOUT")
def test_page_timeout_saves_last_xml_diagnostic(self):
with tempfile.TemporaryDirectory() as directory:
device = LoadingDevice(self.home_xml, self.spec_xml)
ticks = iter((0.0, 0.0, 0.0, 0.0, 0.0, 2.0))
service = PddCollectService(
PddDeviceService(lambda _serial: device),
"USB-001",
"client-001",
sleeper=lambda _seconds: None,
monotonic=lambda: next(ticks),
page_timeout=1.0,
artifact_directory=Path(directory),
)
with self.assertRaises(PddCollectError) as raised:
service.collect(
FakeTask("https://mobile.yangkeduo.com/goods.html?goods_id=123")
)
artifacts = raised.exception.diagnostics["artifacts"]
self.assertEqual(len(artifacts), 1)
self.assertTrue(Path(artifacts[0]["path"]).is_file())
self.assertEqual(len(artifacts[0]["sha256"]), 64)
def test_runtime_device_disconnect_has_stable_error_code(self):
service = PddCollectService(
PddDeviceService(
lambda _serial: DisconnectingDevice(self.home_xml, self.spec_xml)
),
"USB-001",
"client-001",
)
with self.assertRaises(PddCollectError) as raised:
service.collect(
FakeTask("https://mobile.yangkeduo.com/goods.html?goods_id=123")
)
self.assertEqual(raised.exception.code, "DEVICE_DISCONNECTED")
def test_missing_shop_name_is_allowed(self):
incomplete_home = self.home_xml.replace(
'<node class="android.widget.TextView" text="店铺:测试服饰旗舰店" '
'bounds="[30,1050][600,1110]" visible-to-user="true" '
'enabled="true" />',
"",
)
device = FakeCollectDevice(incomplete_home, self.spec_xml)
service = PddCollectService(
PddDeviceService(lambda _serial: device),
"USB-001",
"client-001",
sleeper=lambda _seconds: None,
)
result = service.collect(
FakeTask("https://mobile.yangkeduo.com/goods.html?goods_id=123")
)
self.assertIsNone(result.shop_name)
def test_missing_reviews_does_not_scroll_before_opening_spec_panel(self):
home_without_reviews = self.home_xml.replace(
'<node class="android.widget.TextView" text="2356条评价" '
'bounds="[330,950][600,1010]" visible-to-user="true" '
'enabled="true" />',
"",
)
device = FakeCollectDevice(
home_without_reviews, keep_only_one_sku(self.spec_xml)
)
service = PddCollectService(
PddDeviceService(lambda _serial: device),
"USB-001",
"client-001",
sleeper=lambda _seconds: None,
max_spec_swipes=0,
)
result = service.collect(
FakeTask("https://mobile.yangkeduo.com/goods.html?goods_id=123")
)
self.assertIsNone(result.reviews.raw)
self.assertTrue(device.clicks)
self.assertTrue(all(device.swipe_panel_states))
def test_spec_panel_timeout_has_stable_code_and_xml_evidence(self):
with tempfile.TemporaryDirectory() as directory:
device = PanelDoesNotOpenDevice(self.home_xml, self.spec_xml)
tick = {"value": -0.5}
def clock():
tick["value"] += 0.5
return tick["value"]
service = PddCollectService(
PddDeviceService(lambda _serial: device),
"USB-001",
"client-001",
sleeper=lambda _seconds: None,
monotonic=clock,
spec_panel_timeout=1.0,
artifact_directory=Path(directory),
)
with self.assertRaises(PddCollectError) as raised:
service.collect(
FakeTask("https://mobile.yangkeduo.com/goods.html?goods_id=123")
)
self.assertEqual(
raised.exception.code, "PDD_PAGE_SPEC_PANEL_TIMEOUT"
)
artifacts = raised.exception.diagnostics["artifacts"]
self.assertTrue(Path(artifacts[0]["path"]).is_file())
if __name__ == "__main__":
unittest.main()