From 4658a1ca61a4c07474eed7e2448018bb5a02d4f9 Mon Sep 17 00:00:00 2001 From: chengma Date: Sat, 18 Jul 2026 16:46:59 +0800 Subject: [PATCH] feat(collect): choose product status scope --- app/editor.py | 55 ++++++++----- app/gui/tabs/collect.py | 63 ++++++++++++++- app/gui/workers.py | 64 +++++++++++++-- app/product_status.py | 21 +++++ docs/04-architecture.md | 3 +- docs/api.md | 8 +- docs/routes.md | 2 +- docs/tasks/T-662b.md | 8 +- tests/test_editor_login.py | 27 +++++++ tests/test_gui.py | 49 ++++++++++- tests/test_workers.py | 161 +++++++++++++++++++++++++++++++++++++ 11 files changed, 420 insertions(+), 41 deletions(-) diff --git a/app/editor.py b/app/editor.py index 3094666..ec8854e 100644 --- a/app/editor.py +++ b/app/editor.py @@ -1166,26 +1166,41 @@ def collect(account, task, on_step=None) -> dict: try: _notify_collect_step(on_step, "read_product_status") status_snapshot = read_product_status(cdp) - _notify_collect_step(on_step, "read_title") - old_title = read_title(cdp) - _notify_collect_step(on_step, "read_cover") - old_cover_src = read_cover_src(cdp) - out_path = _get(task, "old_cover_path") - if not out_path: - out_path = image_paths.task_image_path( - appconfig.image_dir(), - task, - account, - "old", - ) - _notify_collect_step(on_step, "download_cover") - old_cover_path = download_cover(old_cover_src, out_path) - result = { - "old_title": old_title, - "old_cover_src": old_cover_src, - "old_cover_path": old_cover_path, - **status_snapshot, - } + collect_scope = product_status.normalize_collect_scope( + _get(task, "collection_scope") + ) + if not product_status.should_collect_content( + status_snapshot.get("product_status"), + collect_scope, + ): + result = { + **status_snapshot, + "collection_skipped": True, + "collection_skip_reason": product_status.collect_skip_reason( + status_snapshot.get("product_status") + ), + } + else: + _notify_collect_step(on_step, "read_title") + old_title = read_title(cdp) + _notify_collect_step(on_step, "read_cover") + old_cover_src = read_cover_src(cdp) + out_path = _get(task, "old_cover_path") + if not out_path: + out_path = image_paths.task_image_path( + appconfig.image_dir(), + task, + account, + "old", + ) + _notify_collect_step(on_step, "download_cover") + old_cover_path = download_cover(old_cover_src, out_path) + result = { + "old_title": old_title, + "old_cover_src": old_cover_src, + "old_cover_path": old_cover_path, + **status_snapshot, + } finally: close_target_confirmed = _close_collected_product(cdp) result["close_target_confirmed"] = close_target_confirmed diff --git a/app/gui/tabs/collect.py b/app/gui/tabs/collect.py index 38d8dff..0d32036 100644 --- a/app/gui/tabs/collect.py +++ b/app/gui/tabs/collect.py @@ -8,6 +8,7 @@ from ...collect_skip import ( format_skip_reason_summary, normalize_skip_reason_counts, ) +from ... import product_status from ..models import TaskTableModel from ..widgets import * from ..workers import CollectWorker as _RealCollectWorker, WriteBackWorker as _RealWriteBackWorker @@ -28,6 +29,7 @@ COLLECT_ACTIVITY_STEP_LABELS = { "prepare_task": "准备采集", "open_product": "打开商品页", "wait_ready": "等待商品页加载", + "read_product_status": "读取商品状态", "read_title": "读取标题", "read_cover": "读取封面", "download_cover": "下载封面", @@ -718,11 +720,16 @@ class CollectTab(QWidget): if not tasks: self._set_status("没有可采集任务") return + collect_scope = self._choose_collect_scope() + if collect_scope is None: + self._set_status("已取消采集") + return worker = CollectWorker( tasks, db_path=self.db_path, config=self.config, diagnostic_log_dir=diagnostics.DEFAULT_LOG_DIR, + collect_scope=collect_scope, ) activity_signal = getattr(worker, "activity", None) if activity_signal is not None: @@ -742,6 +749,32 @@ class CollectTab(QWidget): self._start_collect_activity() thread.start() + def _choose_collect_scope(self): + box = QMessageBox(self) + box.setIcon(QMessageBox.Question) + box.setWindowTitle("选择采集范围") + box.setText("请选择本轮要采集的商品范围。") + box.setInformativeText( + "程序会逐个打开商品详情页检测状态并保存结果。" + "采集所有状态商品包含未上架、审核中和状态未知商品，" + "可能增加采集时间，但本步骤不消耗 AI 点数。" + ) + normal_button = box.addButton("只采集状态正常的商品", QMessageBox.AcceptRole) + normal_button.setObjectName("collectNormalOnlyButton") + all_button = box.addButton("采集所有状态的商品", QMessageBox.DestructiveRole) + all_button.setObjectName("collectAllStatusesButton") + all_button.setStyleSheet("color: #cf222e; font-weight: 600;") + cancel_button = box.addButton("取消", QMessageBox.RejectRole) + cancel_button.setObjectName("collectScopeCancelButton") + box.setDefaultButton(normal_button) + box.setEscapeButton(cancel_button) + box.exec() + if box.clickedButton() is normal_button: + return product_status.COLLECT_SCOPE_NORMAL_ONLY + if box.clickedButton() is all_button: + return product_status.COLLECT_SCOPE_ALL + return None + def stop_collect(self, checked=False): if self.collect_worker is not None: self.collect_worker.cancel() @@ -870,6 +903,17 @@ class CollectTab(QWidget): skipped=payload.get("skipped", 0), failed=payload.get("failed", 0), ) + status_counts = payload.get("product_status_counts") or {} + status_skipped = int(payload.get("status_scope_skipped", 0) or 0) + if status_counts: + message += "；状态正常{normal}，未上架{unlisted}，审核中{reviewing}，状态未知{unknown}".format( + normal=status_counts.get("normal", 0), + unlisted=status_counts.get("unlisted", 0), + reviewing=status_counts.get("reviewing", 0), + unknown=status_counts.get("unknown", 0), + ) + if status_skipped: + message += f"；按范围略过{status_skipped}" self._show_collect_account_summary(payload, message) if payload.get("collected", 0) > 0: batch_id = self._active_batch_id() @@ -917,14 +961,25 @@ class CollectTab(QWidget): reused = payload.get("reused_accounts") or [] login_required = payload.get("login_required_accounts") or [] skipped = max(0, int(payload.get("skipped", 0) or 0)) + status_scope_skipped = int(payload.get("status_scope_skipped", 0) or 0) + account_skipped = max(0, skipped - status_scope_skipped) skip_counts = normalize_skip_reason_counts( payload.get("skip_reason_counts"), - skipped_total=skipped, + skipped_total=account_skipped, ) - if not launched and not reused and not login_required and skipped == 0: + if ( + not launched + and not reused + and not login_required + and account_skipped == 0 + and status_scope_skipped == 0 + ): return lines = [message] - skip_summary = format_skip_reason_summary(skip_counts, skipped_total=skipped) + skip_summary = format_skip_reason_summary( + skip_counts, + skipped_total=account_skipped, + ) if skip_summary: lines.append(skip_summary) if skip_counts[ALIAS_UNMATCHED] > 0: @@ -941,6 +996,8 @@ class CollectTab(QWidget): ) if skip_counts[LOGIN_REQUIRED] > 0: lines.append("请到账号管理完成对应账号登录后，再重新采集略过任务。") + if status_scope_skipped: + lines.append(f"本轮按范围略过{status_scope_skipped}个非正常状态商品，未下载标题和封面。") if launched or reused or login_required: lines.append("采集结束后不会自动关闭账号 Chrome，请按需自行关闭。") text = "\n".join(lines) diff --git a/app/gui/workers.py b/app/gui/workers.py index 52c1611..ba13db0 100644 --- a/app/gui/workers.py +++ b/app/gui/workers.py @@ -20,6 +20,7 @@ from .. import ( image_studio_export, image_studio_generation, image_studio_images, + product_status, ) from ..collect_skip import ALIAS_UNMATCHED, LOGIN_REQUIRED, empty_skip_reason_counts from .widgets import * @@ -1878,6 +1879,7 @@ class CollectWorker(BaseWorker): config=None, preflight=True, diagnostic_log_dir=None, + collect_scope="all", ): super().__init__() self.tasks = list(tasks) @@ -1885,6 +1887,7 @@ class CollectWorker(BaseWorker): self.config = config self.preflight = preflight self.diagnostic_log_dir = diagnostic_log_dir + self.collect_scope = product_status.normalize_collect_scope(collect_scope) self._run_id = None def execute(self): @@ -1908,6 +1911,10 @@ class CollectWorker(BaseWorker): login_required_accounts = {} preflight_info = {} skip_reason_counts = empty_skip_reason_counts() + product_status_counts = { + status: 0 for status in product_status.VALID_PRODUCT_STATUSES + } + status_scope_skipped = 0 self._run_id = self._create_run_log(eligible, batch_ids) self._emit_activity( @@ -1935,6 +1942,9 @@ class CollectWorker(BaseWorker): extra={ **blocked, "skip_reason_counts": dict(skip_reason_counts), + "collect_scope": self.collect_scope, + "product_status_counts": dict(product_status_counts), + "status_scope_skipped": status_scope_skipped, }, ) self._finish_run_log("blocked", summary) @@ -2111,9 +2121,22 @@ class CollectWorker(BaseWorker): { "item_id": task.item_id, "old_cover_path": self._old_cover_path(account, task), + "collection_scope": self.collect_scope, }, on_step=on_step, ) + detected_status = product_status.normalize_status( + result.get("product_status") + ) + product_status_counts[detected_status] += 1 + if result.get("product_status_error"): + self._write_diagnostic_log( + "商品状态检测失败，已按状态未知保存", + level="WARNING", + step="read_product_status", + task=task, + payload={"error": result.get("product_status_error")}, + ) if result.get("close_target_confirmed") is False: self._log_run_event( "step=close_product result=uncertain detail=任务 {task_id} 商品 {item_id} 商品页已请求关闭，但未在短时间内确认关闭；采集结果已保留，继续处理后续任务".format( @@ -2133,6 +2156,35 @@ class CollectWorker(BaseWorker): "close_target_confirmed": False, }, ) + if result.get("collection_skipped"): + activity_result = "skipped" + current_step = "read_product_status" + reason = result.get("collection_skip_reason") or product_status.collect_skip_reason( + detected_status + ) + db.set_product_status( + task.id, + detected_status, + result.get("product_status_note"), + path=self.db_path, + ) + db.mark_skipped(task.id, reason, path=self.db_path) + skipped += 1 + status_scope_skipped += 1 + self.row_updated.emit( + task.id, + {"status": "skipped", "last_error": reason}, + ) + self._log_run_event( + "step=read_product_status result=skipped detail=任务 {task_id} 商品 {item_id} {reason}".format( + task_id=task.id, + item_id=task.item_id, + reason=reason, + ), + task=task, + level="warning", + ) + continue current_step = "db_write" self._emit_activity( "task_step", @@ -2156,14 +2208,6 @@ class CollectWorker(BaseWorker): product_status_note=result.get("product_status_note"), path=self.db_path, ) - if result.get("product_status_error"): - self._write_diagnostic_log( - "商品状态检测失败，已按状态未知保存", - level="WARNING", - step="read_product_status", - task=task, - payload={"error": result.get("product_status_error")}, - ) collected += 1 elapsed_ms = self._elapsed_ms(started) self.row_updated.emit( @@ -2235,6 +2279,9 @@ class CollectWorker(BaseWorker): **preflight_info, "login_required_accounts": list(login_required_accounts.values()), "skip_reason_counts": dict(skip_reason_counts), + "collect_scope": self.collect_scope, + "product_status_counts": dict(product_status_counts), + "status_scope_skipped": status_scope_skipped, }, ) self._finish_run_log("cancelled" if self.should_cancel() else "done", summary) @@ -2540,6 +2587,7 @@ class CollectWorker(BaseWorker): options={ "batch_ids": batch_ids, "preflight": self.preflight, + "collect_scope": self.collect_scope, }, path=self.db_path, ) diff --git a/app/product_status.py b/app/product_status.py index 71f6a92..08a589d 100644 --- a/app/product_status.py +++ b/app/product_status.py @@ -26,6 +26,14 @@ PRODUCT_STATUS_LABELS = { STATUS_UNKNOWN: "状态未知", } +SCOPE_NORMAL_ONLY = "normal_only" +SCOPE_ALL = "all" + +# Kept as explicit collection aliases while generation and apply adopt the +# shared scope values in their own tasks. +COLLECT_SCOPE_NORMAL_ONLY = SCOPE_NORMAL_ONLY +COLLECT_SCOPE_ALL = SCOPE_ALL + _WHITESPACE_RE = re.compile(r"\s+") _NOTE_LIMIT = 2000 @@ -93,6 +101,19 @@ def partition_tasks(tasks) -> dict: return grouped +def normalize_collect_scope(value) -> str: + value = str(value or "").strip().lower() + return SCOPE_ALL if value == SCOPE_ALL else SCOPE_NORMAL_ONLY + + +def should_collect_content(status, scope) -> bool: + return normalize_collect_scope(scope) == COLLECT_SCOPE_ALL or is_normal(status) + + +def collect_skip_reason(status) -> str: + return f"{status_label(status)}，按本轮范围略过" + + def _alert_note(title: str, description: str) -> str: parts = [] if title: diff --git a/docs/04-architecture.md b/docs/04-architecture.md index 06c5e71..212da15 100644 --- a/docs/04-architecture.md +++ b/docs/04-architecture.md @@ -342,7 +342,7 @@ CREATE TABLE run_log_events ( 优先级： -1. **T-207 已接入 ① 采集**：记录批次开始/结束、账号预检、每个商品开始/成功/失败/略过；关键步骤覆盖 `preflight`、`open_product`、`wait_ready`、`read_title`、`read_cover`、`download_cover`、`db_write`、`excel_write_back`。失败时 DB 事件保存最后步骤和简短错误，本地 `data/logs/cmshopee.log` 保存完整脱敏 traceback。 +1. **T-207/T-662b 已接入 ① 采集**：记录批次开始/结束、账号预检、每个商品开始/成功/失败/略过；关键步骤覆盖 `preflight`、`open_product`、`wait_ready`、`read_product_status`、`read_title`、`read_cover`、`download_cover`、`db_write`、`excel_write_back`。运行日志选项记录本轮范围，汇总记录四类商品状态与按范围略过数。失败时 DB 事件保存最后步骤和简短错误，本地 `data/logs/cmshopee.log` 保存完整脱敏 traceback。 2. **② AI生成已先补诊断**：`GenerateWorker` 创建 `run_type=generate`，按任务记录标题/封面阶段事件；关键步骤覆盖 `title_submit`、`load_text_model`、`title_build_request`、`title_request`、`title_parse_response`、`cover_prompt_render`、`cover_submit`、`cover_validate_input`、`load_image_model`、`cover_build_request`、`cover_request`、`cover_parse_response`、`cover_save`、`db_write`。失败时任务 `last_error`、DB 运行日志和本地 `data/logs/cmshopee.log` 都写脱敏错误。 3. **T-505 已扩展全流程**：Excel 导入创建 `run_type=import`，记录选中文件、缺列、脏行、入库统计；Excel 回写创建 `run_type=write_back`，记录文件写入、文件锁/保存异常和行数；③ 更新蝦皮 的 `run_type=apply` 覆盖安全/账号预检、Chrome/CDP 检查、登录检测、打开商品页、改标题、换封面、点更新、写库；账号管理中的「启动登录」创建 `run_type=chrome_launch`，账号管理中的「检测登录」创建 `run_type=login_check`；设置中的 AI 模型测试连接创建 `run_type=ai_model_test`。失败时本地 `data/logs/cmshopee.log` 写脱敏 traceback，业务日志和状态栏只写脱敏短错误。 4. **T-404b 已接入商品页失败 toast 捕获**：①采集和③更新在 `open_product`/等待详情页就绪失败时，不应只报等待超时。进入/刷新商品编辑页后应捕获 `.eds-toasts .eds-toast__content` 的文本和 `outerHTML`，记录当前 URL、时间、可见状态，并在关键元素超时时把最近错误 toast 作为用户可读失败原因写入 `last_error`、`run_log_events` 与本地诊断日志；例如商品 ID 失效时提示 `please input correct product id`。若 toast 明确属于商品失效/商品不存在/无权限类错误，底层仍写 `stage=imported/status=failed/last_error=商品失效：<原始toast>`，只在①导入采集列表“阶段”显示“商品失效”；其他打开失败仍显示“失败”。若失败发生在 `open_product()` 内部且 `cdp` 尚未返回上层，`open_product()` 必须自行清理本轮自动新建 tab；复用用户已有 tab 只断开 CDP，不关闭页面。失败现场可保存 HTML/toast JSON 片段，但不得记录 Cookie、密码、token。 @@ -421,6 +421,7 @@ data/images///__new. # AI 生成的新 - 采集前和采集中途的登录检测必须区分“明确未登录”和“暂时不确定”。明确 `LOGIN_PAGE` / 登录页 URL 才整组略过该账号后续任务；`NO_SESSION_COOKIE`、检测超时或 CDP 短暂异常只记录为不确定并继续尝试采集当前商品，不得级联跳过同账号剩余任务。Cookie API 调用失败不能伪装成空 Cookie；登录检测若连到正在关闭的旧商品 target，应在本次检测预算内重新枚举并改连其他有效 Shopee 页面。 - 若商品 ID 失效、无权限或店铺不匹配导致商品编辑页无法就绪，`open_product` 必须读取/捕获 Shopee toast，把最近错误文案写入采集失败原因和诊断日志，不能只返回泛化超时。①列表只在明确捕获商品失效类 toast 时把“阶段”显示为“商品失效”；底层 `stage` 不新增中文值。若这个失败发生在后台只读、程序自动新建的商品 tab 内，`open_product` 要关闭并执行同样的有界 target 消失确认；复用用户已有 tab 不关闭。③前台更新打开失败仍沿用既有清理路径，不引入额外等待。 - 商品状态只读取 `.eds-alert.eds-alert--warning` 内的 `.eds-alert-title/.eds-alert-desc`，不得依赖 Vue `data-v-*`。`審核中/审核中` 归为 `reviewing`，`您的商品未上架` 归为 `unlisted`，没有 warning 为 `normal`，未识别横幅、DOM 异常或非法返回为 `unknown`。保存归一化摘要而非页面 HTML；状态探测异常只写脱敏诊断，不伪造技术采集失败。 +- 每次点击采集先在主线程选择本轮范围，策略值为 `normal_only/all`，默认 `normal_only`，不跨轮记忆。两种范围都会逐条重新打开页面检测并独立保存状态，不能用旧数据库状态预过滤；`normal_only` 仅正常商品读取标题和封面，其他三类状态记为业务略过并保留旧内容，`all` 则四类状态均走现有采集流程。范围略过不是技术失败，汇总分别统计状态和略过数。 - 旧封面：取第一张 itembox 的 `img.src`（CDN 链接），下载到 `data/images///__old.jpg`。 - 写 `old_title/old_cover_path`、stage=collected；批量回写 Excel 旧字段。 diff --git a/docs/api.md b/docs/api.md index 04c3c68..2715027 100644 --- a/docs/api.md +++ b/docs/api.md @@ -269,7 +269,7 @@ read_product_status(cdp) -> dict # {product_status, product_status_ read_title(cdp) -> str read_cover_src(cdp) -> str # 第一张 itembox 的 img.src download_cover(src, out_path) -> str # 下载旧封面到本地 -collect(account, task) -> dict # -> {old_title, old_cover_path, product_status, product_status_note, product_status_error, close_target_confirmed} +collect(account, task) -> dict # task.collection_scope=normal_only/all；范围略过时返回 collection_skipped/reason，不读取标题/封面 # 应用 change_title(cdp, new_title) -> dict # {ok, value, modelvalue}，要求三者相等 @@ -292,7 +292,7 @@ apply_task(account, task, close_success_tab=False) -> dict - `read_product_status()` 在读取标题/封面前读取 `.eds-alert.eds-alert--warning` 下的 `.eds-alert-title/.eds-alert-desc`。`審核中/审核中` → `reviewing`，`您的商品未上架` → `unlisted`，无 warning → `normal`，未识别 warning、DOM 异常或非法响应 → `unknown`；多横幅按 DOM 顺序取第一个白名单命中，不依赖 `data-v-*`，只返回归一化 note，不保存 HTML。状态读取异常不阻断 `collect()`，由调用方写脱敏诊断日志和 `unknown` 快照。 -- `collect()` 结束时只关闭本轮自动新建的商品编辑页 tab，并通过 `close_tab_and_wait()` 在最多 2 秒内确认 target 从 `/json` 消失，结果写入 `close_target_confirmed`。确认超时只记警告，不覆盖已成功读取的标题/封面；如果 `open_product()` 尚未返回就失败，也由 `open_product()` 关闭本轮自动新建 tab；用户原本打开的商品 tab 不关闭、不等待。 +- `collect()` 先读取状态再决定是否读取标题/封面。`task.collection_scope=normal_only` 时，未上架、审核中、状态未知返回 `collection_skipped=True` 和中文原因，不下载封面、不覆盖旧内容；`all` 时四类状态都继续采集。两种策略都重新读取页面状态，不能用历史状态预过滤。结束时只关闭本轮自动新建的商品编辑页 tab，并通过 `close_tab_and_wait()` 在最多 2 秒内确认 target 从 `/json` 消失，结果写入 `close_target_confirmed`。确认超时只记警告，不覆盖已成功读取的标题/封面；如果 `open_product()` 尚未返回就失败，也由 `open_product()` 关闭本轮自动新建 tab；用户原本打开的商品 tab 不关闭、不等待。 - ③ 更新流程中程序自动新建的商品编辑页成功/失败都关闭，复用用户原本打开的 tab 只断开 CDP、不关闭页面；`open_product()` 内部打开失败的新建 tab 仍由 `open_product()` 自行关闭。Shopee 确认成功后可能把当前 tab 跳回 `/portal/product/list/all?operationSortBy=modified_time`，`click_update()` 会把该 URL 记录到 `post_update.url` 并标记 `redirected_to_list=true`；自动新建页成功关闭前等待 2 秒。 - `click_update()` 的提交成功定义：页面主「更新」按钮已点击，且 Shopee 站点侧确认框未出现或已在可见 `.eds-modal__content` / `.eds-modal__box` 内点击主按钮「更新」。如果确认框仍停留、只点到页面主按钮、或误入「立即優化」，必须返回失败；若 tab 是本轮自动新建，失败后由 `apply_task()` 关闭该 tab。 - T-404/T-502 封面更新删除前，`apply_task()` 应把任务的 `old_cover_path` 传给 `replace_cover()`；`replace_cover()` 只有在本地旧封面备份存在时才允许进入删第一张流程。更新封面统一先删当前第一张，不再只在满 9 张时删除；8 张商品图也按替换语义先删再上传。 @@ -459,7 +459,7 @@ class ApplyTab(QWidget) # ③ 更新蝦皮：筛选已生成 class SettingsTab(QWidget) # 设置：cmhub 网关配置 + 响应式三列布局 + 角色/生成参数/路径端口 + 蝦皮更新安全 + 未保存状态追踪 class ProductSuiteTab(QWidget) # 商品套图：多任务、原图、结构配置、AI帮写、cmhub生成、历史结果 class ImageStudioTab(QWidget) # 旧AI工场兼容实现；主窗口不再创建 -class CollectWorker(BaseWorker) # ① 后台采集：账号就绪预检 -> editor.collect -> db.set_collected/mark_skipped/mark_failed +class CollectWorker(BaseWorker) # ① 后台采集：范围 normal_only/all + 账号预检 -> editor.collect -> 状态独立落库，采集或略过 class GenerateWorker(BaseWorker) # ② 后台生成：ai.generate_batch -> db.set_generated/set_generated_cover/mark_failed + 进度 class ApplyWorker(BaseWorker) # ③ 后台更新：账号就绪预检 -> 检查或按批调用 editor.apply_task(...) -> db.set_applied/mark_skipped class WriteBackWorker(BaseWorker) # ①/③ 后台回写：旧字段或更新结果写回原 Excel @@ -520,7 +520,7 @@ T-523 后 GUI 已从旧 `app/gui.py` 拆为 `app/gui/` 包：`__init__.py` 负 - 任务列表使用 `QTableView + TaskTableModel`，列为：账号、别名、商品ID、阶段。 - 账号列优先显示匹配到的 `accounts.account_name`；未匹配账号时保留 Excel 输入账号名。 - 别名未匹配 `accounts.alias` 时列表阶段列显示“略过”；点击「采集旧标题/旧封面」后由 `CollectWorker` 逐条写库为 `skipped`，原因 `别名未匹配账号`。 -- 「采集旧标题/旧封面」通过 `CollectWorker` 后台执行，只处理 `stage=imported` 的任务；采集前先做账号就绪预检。无账号、当前批次匹配账号未启动 CDP 端口或未登录时，返回 `blocked=True`，GUI 弹窗汇总并跳转/引导去账号管理，不进入逐条采集、不写 skipped/failed。预检通过后，已匹配任务调用 `editor.collect()` 下载旧封面到 `image_dir///__old.jpg` 并 `db.set_collected()`；别名未匹配任务仍逐条 `mark_skipped`；单条失败 `mark_failed(..., "collect", error)` 后继续。采集完成且本轮成功采集数量大于 0 时，自动触发当前批次旧字段回写；锁文件失败时只提示，不回滚 SQLite。 +- 「采集旧标题/旧封面」先弹范围确认框，默认安全策略 `normal_only`，红色危险选项为 `all`，取消不创建 Worker。`CollectWorker` 只处理 `stage=imported` 的任务；采集前先做账号就绪预检。无账号、当前批次匹配账号未启动 CDP 端口或未登录时，返回 `blocked=True`，GUI 弹窗汇总并跳转/引导去账号管理，不进入逐条采集、不写 skipped/failed。预检通过后，已匹配任务调用 `editor.collect()` 先检测和保存页面状态；默认范围仅正常商品下载旧封面到 `image_dir///__old.jpg` 并 `db.set_collected()`，其他状态 `set_product_status()` 后按范围 `mark_skipped` 且保留原旧内容；选择全部范围时四类状态均采集。别名未匹配任务仍逐条 `mark_skipped`；单条技术失败 `mark_failed(..., "collect", error)` 后继续。采集完成且本轮成功采集数量大于 0 时，自动触发当前批次旧字段回写；锁文件失败时只提示，不回滚 SQLite。 - 采集打开商品页时，若本轮自动新建 tab，采集完成后会关闭该 tab，并在最多 2 秒内确认 target 已从 `/json` 消失；确认超时只写 warning/诊断，不把采集成功改成失败。若失败发生在 `open_product()` 内部且尚未返回 `cdp`，也要关闭本轮自动新建 tab；若复用用户已打开的商品页，只断开 CDP 连接不关闭页面。 - 采集中途登录检测必须快速跳过正在销毁的旧商品 target，改连其他有效 Shopee 页面。Cookie API 调用失败返回 `LOGIN_CHECK_TARGET_UNAVAILABLE`，只有 Cookie API 成功返回空会话时才返回 `NO_SESSION_COOKIE`；两者都不按明确掉登录批量略过，显式 `LOGIN_PAGE` 仍按账号需登录处理。retry/recovered 运行日志包含当前任务 ID 和商品 ID，避免与上一条采集成功日志混淆。 - 采集打开商品页失败时，`CollectWorker` 应把 `open_product()` 捕获到的 Shopee toast 文案写入 `run_log_events` 和 `tasks.last_error`；商品 ID 失效、无权限、店铺不匹配等场景不得只显示泛化超时。① `TaskTableModel` 的“阶段”列只在 `last_error` 明确为商品失效类错误时显示“商品失效”，否则仍按 `status=failed` 显示“失败”；底层不新增 `stage` 枚举。 diff --git a/docs/routes.md b/docs/routes.md index d8f24ec..2afb7b3 100644 --- a/docs/routes.md +++ b/docs/routes.md @@ -72,7 +72,7 @@ - 导入：openpyxl 解析**输入列**（账号名/别名/商品id）入 SQLite。 - **导入汇总栏**（导入后即时刷新，跑采集前的校验关口）：显示 文件数、解析行数（原始数据量）、有效/无效行、匹配账号行数（按账号细分）、未匹配行数。未匹配/无效数字标红可点，点击在列表筛出便于定位纠错。 -- 采集：点击后先为本轮匹配账号确保 Chrome 就绪（已开复用、未开启动），再检测登录；明确未登录账号的任务整组略过并汇总提示。`NO_SESSION_COOKIE`、登录检测超时或 CDP 短暂异常会重试，连续不确定时不批量略过，继续打开商品页由真实页面结果决定成功/失败。登录账号用对应 Chrome 只读打开商品页，读旧标题、下载旧封面到 `data/images///__old.jpg`，写 `old_title/old_cover_path`，stage=collected。采集不主动把商品页切到前台；程序自动新建商品页 tab 时尽量后台创建，采集结束后自动关闭；若复用用户原本打开的 tab，则不关闭。采集结束不关闭账号 Chrome，用户可自行关闭。 +- 采集：点击后先选择范围，默认「只采集状态正常的商品」，红色危险选项为「采集所有状态的商品」。随后为本轮匹配账号确保 Chrome 就绪（已开复用、未开启动），再检测登录；明确未登录账号的任务整组略过并汇总提示。`NO_SESSION_COOKIE`、登录检测超时或 CDP 短暂异常会重试，连续不确定时不批量略过，继续打开商品页由真实页面结果决定成功/失败。登录账号用对应 Chrome 只读打开商品页，先检测并保存商品状态；默认范围下只有正常商品才读旧标题、下载旧封面到 `data/images///__old.jpg`，未上架、审核中和状态未知商品按范围略过且不覆盖已有内容。选择全部范围时四类商品均继续采集。采集不主动把商品页切到前台；程序自动新建商品页 tab 时尽量后台创建，采集结束后自动关闭；若复用用户原本打开的 tab，则不关闭。采集结束不关闭账号 Chrome，用户可自行关闭。 - 若商品 ID 已失效、无权限或店铺不匹配，Shopee 可能只弹出短暂错误 toast；采集失败时界面日志应显示捕获到的 toast 文案，并把 toast HTML/URL 写入本地诊断日志，避免用户手动抢复制。只有明确捕获商品失效/商品不存在/无权限类 toast 时，①列表“阶段”列显示“商品失效”；其他商品页打开失败仍显示“失败”。如果失败发生在 `open_product()` 内部，本轮自动新建的商品 tab 必须关闭，复用用户已有 tab 不关闭。 - 回写：采集完成后自动把旧标题/旧封面路径批量回写原 Excel；保留「回写旧数据到 Excel」作为手动重试入口（原文件被锁→提示关闭后重试/另存）。 diff --git a/docs/tasks/T-662b.md b/docs/tasks/T-662b.md index c26f7e2..c4dd037 100644 --- a/docs/tasks/T-662b.md +++ b/docs/tasks/T-662b.md @@ -1,7 +1,7 @@ --- id: T-662b title: 导入采集范围确认与商品状态汇总 -status: TODO +status: DONE phase: 7 deps: [T-662a] created: 2026-07-18 @@ -61,4 +61,8 @@ git diff --check ## 执行记录 -- 待实现。 +- 已实现采集范围确认框：默认“只采集状态正常的商品”，红色“采集所有状态的商品”，取消不启动 Worker。 +- `CollectWorker` 将范围策略冻结在本轮运行日志中；逐条先读取并保存商品状态。默认范围仅采集正常商品，异常/未知商品保留旧标题和封面并按范围略过；全部范围继续采集四类状态。 +- 汇总、行结果和运行日志分别统计四类状态与按范围略过数，账号预检略过统计不再把业务范围略过误报为“其他”。 +- 已更新 `docs/routes.md`、`docs/04-architecture.md`、`docs/api.md`。 +- 验证通过：`py -3.10 -m unittest tests.test_editor_login tests.test_workers tests.test_gui`（267 项）、`py -3.10 -m unittest discover -s tests`（605 项）、`py -3.10 -m ruff check app tests main.py`、`py -3.10 -m compileall app main.py`、`git diff --check`。 diff --git a/tests/test_editor_login.py b/tests/test_editor_login.py index a1800d5..ae7c75b 100644 --- a/tests/test_editor_login.py +++ b/tests/test_editor_login.py @@ -971,6 +971,33 @@ class EditorLoginTests(unittest.TestCase): steps, ) + def test_collect_normal_only_skips_non_normal_before_reading_content(self): + cdp = FakeProductCDP( + "ws-status", + alerts=[{"title": "您的商品未上架", "description": "已下架"}], + ) + + with mock.patch("app.editor.open_product", return_value=cdp), mock.patch( + "app.editor.read_title" + ) as read_title, mock.patch("app.editor.read_cover_src") as read_cover_src, mock.patch( + "app.editor.download_cover" + ) as download_cover: + result = editor.collect( + {"debug_port": 9222}, + { + "item_id": "51100639510", + "old_cover_path": "old.jpg", + "collection_scope": "normal_only", + }, + ) + + self.assertTrue(result["collection_skipped"]) + self.assertEqual("unlisted", result["product_status"]) + self.assertEqual("未上架，按本轮范围略过", result["collection_skip_reason"]) + read_title.assert_not_called() + read_cover_src.assert_not_called() + download_cover.assert_not_called() + def test_collect_keeps_reused_product_tab_open(self): cdp = FakeProductCDP("ws-existing") cdp.target_id = "target-existing" diff --git a/tests/test_gui.py b/tests/test_gui.py index a226b5e..f988fba 100644 --- a/tests/test_gui.py +++ b/tests/test_gui.py @@ -28,7 +28,7 @@ from app import ( if gui.QT_IMPORT_ERROR is not None: raise unittest.SkipTest("PySide6 未安装") -from PySide6.QtCore import QItemSelectionModel, QModelIndex, QRect, QSize, Qt +from PySide6.QtCore import QItemSelectionModel, QModelIndex, QRect, QSize, QTimer, Qt from PySide6.QtGui import QImage, QKeyEvent, QTextCursor from PySide6.QtWidgets import ( QAbstractItemView, @@ -38,6 +38,7 @@ from PySide6.QtWidgets import ( QListView, QPlainTextEdit, QProgressBar, + QPushButton, QTableView, ) @@ -7804,7 +7805,11 @@ class GuiTests(TempDirMixin, unittest.TestCase): captured["started"] = True captured = {} - with mock.patch("app.gui.CollectWorker", FakeWorker), mock.patch( + with mock.patch.object( + tab, + "_choose_collect_scope", + return_value="normal_only", + ), mock.patch("app.gui.CollectWorker", FakeWorker), mock.patch( "app.gui.run_worker", return_value=FakeThread(), ): @@ -7813,12 +7818,52 @@ class GuiTests(TempDirMixin, unittest.TestCase): self.assertTrue(captured["started"]) self.assertEqual(["1001"], [task.item_id for task in captured["tasks"]]) self.assertEqual(cfg["db_path"], captured["kwargs"]["db_path"]) + self.assertEqual("normal_only", captured["kwargs"]["collect_scope"]) self.assertFalse(tab.shop_filter.isEnabled()) self.assertFalse(tab.item_filter.isEnabled()) self.assertFalse(tab.status_filter.isEnabled()) self.assert_removed(temp_dir) + def test_collect_scope_dialog_uses_safe_default_and_red_all_status_choice(self): + with self.make_temp_dir() as temp_dir: + tab = CollectTab(config=self.make_config(temp_dir)) + self.addCleanup(tab.close) + captured = {} + + def click_all_statuses(): + box = QApplication.activeModalWidget() + normal_button = box.findChild(QPushButton, "collectNormalOnlyButton") + all_button = box.findChild(QPushButton, "collectAllStatusesButton") + captured["default"] = box.defaultButton().objectName() + captured["all_style"] = all_button.styleSheet() + self.assertIsNotNone(normal_button) + self.assertIsNotNone(all_button) + all_button.click() + + QTimer.singleShot(0, click_all_statuses) + scope = tab._choose_collect_scope() + + self.assertEqual("all", scope) + self.assertEqual("collectNormalOnlyButton", captured["default"]) + self.assertIn("#cf222e", captured["all_style"]) + + self.assert_removed(temp_dir) + + def test_collect_without_candidates_does_not_open_scope_dialog(self): + with self.make_temp_dir() as temp_dir: + statuses = [] + tab = CollectTab(config=self.make_config(temp_dir), status_callback=statuses.append) + self.addCleanup(tab.close) + + with mock.patch.object(tab, "_choose_collect_scope") as choose_scope: + tab.collect_old_data() + + choose_scope.assert_not_called() + self.assertEqual("没有可采集任务", statuses[-1]) + + self.assert_removed(temp_dir) + def test_collect_activity_tracks_step_resets_each_task_and_freezes_on_stop(self): with self.make_temp_dir() as temp_dir: tab = CollectTab(config=self.make_config(temp_dir)) diff --git a/tests/test_workers.py b/tests/test_workers.py index 5d1712f..1115f37 100644 --- a/tests/test_workers.py +++ b/tests/test_workers.py @@ -24,6 +24,7 @@ from app.gui.workers import ( ProductSuiteAiWriteWorker, ProductSuiteGenerateWorker, ProductSuiteHistoryExportWorker, + CollectWorker, ) @@ -119,6 +120,166 @@ class WorkerTests(unittest.TestCase): self.assertEqual([(-1, "模拟失败")], failed) self.assertEqual([{"ok": False, "error": "模拟失败"}], finished) + def test_collect_worker_scope_skips_non_normal_without_overwriting_old_content(self): + with tempfile.TemporaryDirectory() as temp_dir: + db_path = os.path.join(temp_dir, "cmshopee.db") + db.init_db(db_path) + batch_id = db.create_batch(["input.xlsx"], path=db_path) + db.insert_tasks( + batch_id, + [ + { + "source_file_abs": os.path.join(temp_dir, "input.xlsx"), + "source_sheet": "Sheet1", + "source_row": index, + "account_name": "店铺", + "alias": "alias", + "item_id": str(51100639510 + index), + } + for index in range(2, 6) + ], + path=db_path, + ) + tasks = db.list_tasks(batch_id=batch_id, path=db_path) + connection = db.connect(db_path) + try: + with connection: + connection.execute( + "UPDATE tasks SET old_title = ?, old_cover_path = ? WHERE id = ?", + ("历史标题", "history.jpg", tasks[1].id), + ) + finally: + connection.close() + + status_by_item = { + tasks[0].item_id: "normal", + tasks[1].item_id: "unlisted", + tasks[2].item_id: "reviewing", + tasks[3].item_id: "unknown", + } + account = SimpleNamespace(alias="alias", account_name="店铺", debug_port=9222) + + def fake_collect(_account, task, on_step=None): + status = status_by_item[task["item_id"]] + self.assertEqual("normal_only", task["collection_scope"]) + on_step("read_product_status") + if status != "normal": + labels = { + "unlisted": "未上架", + "reviewing": "审核中", + "unknown": "状态未知", + } + return { + "product_status": status, + "product_status_note": f"{labels[status]}提示", + "collection_skipped": True, + "collection_skip_reason": f"{labels[status]}，按本轮范围略过", + } + on_step("download_cover") + return { + "product_status": status, + "product_status_note": None, + "old_title": "新采集标题", + "old_cover_path": "new.jpg", + } + + with mock.patch( + "app.gui.workers.accounts.list_accounts", + return_value=[account], + ), mock.patch( + "app.gui.workers.accounts.detect_login", + return_value={"logged_in": True, "reason": None}, + ), mock.patch( + "app.gui.workers.editor.collect", + side_effect=fake_collect, + ): + summary = CollectWorker( + tasks, + db_path=db_path, + preflight=False, + collect_scope="normal_only", + ).execute() + + self.assertEqual(1, summary["collected"]) + self.assertEqual(3, summary["skipped"]) + self.assertEqual(0, summary["failed"]) + self.assertEqual(3, summary["status_scope_skipped"]) + self.assertEqual( + {"normal": 1, "unlisted": 1, "reviewing": 1, "unknown": 1}, + summary["product_status_counts"], + ) + refreshed = db.list_tasks(batch_id=batch_id, path=db_path) + by_status = {task.product_status: task for task in refreshed} + self.assertEqual("collected", by_status["normal"].stage) + self.assertEqual("success", by_status["normal"].status) + self.assertEqual("skipped", by_status["unlisted"].status) + self.assertEqual("imported", by_status["unlisted"].stage) + self.assertEqual("历史标题", by_status["unlisted"].old_title) + self.assertEqual("history.jpg", by_status["unlisted"].old_cover_path) + + def test_collect_worker_all_scope_collects_every_detected_status(self): + with tempfile.TemporaryDirectory() as temp_dir: + db_path = os.path.join(temp_dir, "cmshopee.db") + db.init_db(db_path) + batch_id = db.create_batch(["input.xlsx"], path=db_path) + db.insert_tasks( + batch_id, + [ + { + "source_file_abs": os.path.join(temp_dir, "input.xlsx"), + "source_sheet": "Sheet1", + "source_row": index, + "account_name": "店铺", + "alias": "alias", + "item_id": str(51100639600 + index), + } + for index in range(2, 6) + ], + path=db_path, + ) + tasks = db.list_tasks(batch_id=batch_id, path=db_path) + statuses = ["normal", "unlisted", "reviewing", "unknown"] + account = SimpleNamespace(alias="alias", account_name="店铺", debug_port=9222) + + def fake_collect(_account, task, on_step=None): + self.assertEqual("all", task["collection_scope"]) + on_step("read_product_status") + status = statuses.pop(0) + return { + "product_status": status, + "product_status_note": None, + "old_title": f"标题{status}", + "old_cover_path": f"{status}.jpg", + } + + with mock.patch( + "app.gui.workers.accounts.list_accounts", + return_value=[account], + ), mock.patch( + "app.gui.workers.accounts.detect_login", + return_value={"logged_in": True, "reason": None}, + ), mock.patch( + "app.gui.workers.editor.collect", + side_effect=fake_collect, + ): + summary = CollectWorker( + tasks, + db_path=db_path, + preflight=False, + collect_scope="all", + ).execute() + + self.assertEqual(4, summary["collected"]) + self.assertEqual(0, summary["skipped"]) + self.assertEqual(0, summary["status_scope_skipped"]) + self.assertEqual( + {"normal": 1, "unlisted": 1, "reviewing": 1, "unknown": 1}, + summary["product_status_counts"], + ) + self.assertTrue( + all(task.stage == "collected" for task in db.list_tasks(batch_id=batch_id, path=db_path)) + ) + def test_run_worker_rejects_plain_object(self): with self.assertRaises(TypeError): run_worker(object(), start=False)