chengma/cmshoppe
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(collect): choose product status scope

Browse Source
This commit is contained in:
chengma
2026-07-18 16:46:59 +08:00
parent 0cf23db403
commit 4658a1ca61
11 changed files with 420 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/editor.py
+35 -20
View File
@@ -1166,26 +1166,41 @@ def collect(account, task, on_step=None) -> dict:
try:
_notify_collect_step(on_step, "read_product_status")
status_snapshot = read_product_status(cdp)
_notify_collect_step(on_step, "read_title")
old_title = read_title(cdp)
_notify_collect_step(on_step, "read_cover")
old_cover_src = read_cover_src(cdp)
out_path = _get(task, "old_cover_path")
if not out_path:
out_path = image_paths.task_image_path(
appconfig.image_dir(),
task,
account,
"old",
)
_notify_collect_step(on_step, "download_cover")
old_cover_path = download_cover(old_cover_src, out_path)
result = {
"old_title": old_title,
"old_cover_src": old_cover_src,
"old_cover_path": old_cover_path,
**status_snapshot,
}
collect_scope = product_status.normalize_collect_scope(
_get(task, "collection_scope")
)
if not product_status.should_collect_content(
status_snapshot.get("product_status"),
collect_scope,
):
result = {
**status_snapshot,
"collection_skipped": True,
"collection_skip_reason": product_status.collect_skip_reason(
status_snapshot.get("product_status")
),
}
else:
_notify_collect_step(on_step, "read_title")
old_title = read_title(cdp)
_notify_collect_step(on_step, "read_cover")
old_cover_src = read_cover_src(cdp)
out_path = _get(task, "old_cover_path")
if not out_path:
out_path = image_paths.task_image_path(
appconfig.image_dir(),
task,
account,
"old",
)
_notify_collect_step(on_step, "download_cover")
old_cover_path = download_cover(old_cover_src, out_path)
result = {
"old_title": old_title,
"old_cover_src": old_cover_src,
"old_cover_path": old_cover_path,
**status_snapshot,
}
finally:
close_target_confirmed = _close_collected_product(cdp)
result["close_target_confirmed"] = close_target_confirmed
app/gui/tabs/collect.py
+60 -3
View File
@@ -8,6 +8,7 @@ from ...collect_skip import (
format_skip_reason_summary,
normalize_skip_reason_counts,
)
from ... import product_status
from ..models import TaskTableModel
from ..widgets import *
from ..workers import CollectWorker as _RealCollectWorker, WriteBackWorker as _RealWriteBackWorker
@@ -28,6 +29,7 @@ COLLECT_ACTIVITY_STEP_LABELS = {
"prepare_task": "准备采集",
"open_product": "打开商品页",
"wait_ready": "等待商品页加载",
"read_product_status": "读取商品状态",
"read_title": "读取标题",
"read_cover": "读取封面",
"download_cover": "下载封面",
@@ -718,11 +720,16 @@ class CollectTab(QWidget):
if not tasks:
self._set_status("没有可采集任务")
return
collect_scope = self._choose_collect_scope()
if collect_scope is None:
self._set_status("已取消采集")
return
worker = CollectWorker(
tasks,
db_path=self.db_path,
config=self.config,
diagnostic_log_dir=diagnostics.DEFAULT_LOG_DIR,
collect_scope=collect_scope,
)
activity_signal = getattr(worker, "activity", None)
if activity_signal is not None:
@@ -742,6 +749,32 @@ class CollectTab(QWidget):
self._start_collect_activity()
thread.start()
def _choose_collect_scope(self):
box = QMessageBox(self)
box.setIcon(QMessageBox.Question)
box.setWindowTitle("选择采集范围")
box.setText("请选择本轮要采集的商品范围。")
box.setInformativeText(
"程序会逐个打开商品详情页检测状态并保存结果。"
"采集所有状态商品包含未上架、审核中和状态未知商品，"
"可能增加采集时间，但本步骤不消耗 AI 点数。"
)
normal_button = box.addButton("只采集状态正常的商品", QMessageBox.AcceptRole)
normal_button.setObjectName("collectNormalOnlyButton")
all_button = box.addButton("采集所有状态的商品", QMessageBox.DestructiveRole)
all_button.setObjectName("collectAllStatusesButton")
all_button.setStyleSheet("color: #cf222e; font-weight: 600;")
cancel_button = box.addButton("取消", QMessageBox.RejectRole)
cancel_button.setObjectName("collectScopeCancelButton")
box.setDefaultButton(normal_button)
box.setEscapeButton(cancel_button)
box.exec()
if box.clickedButton() is normal_button:
return product_status.COLLECT_SCOPE_NORMAL_ONLY
if box.clickedButton() is all_button:
return product_status.COLLECT_SCOPE_ALL
return None
def stop_collect(self, checked=False):
if self.collect_worker is not None:
self.collect_worker.cancel()
@@ -870,6 +903,17 @@ class CollectTab(QWidget):
skipped=payload.get("skipped", 0),
failed=payload.get("failed", 0),
)
status_counts = payload.get("product_status_counts") or {}
status_skipped = int(payload.get("status_scope_skipped", 0) or 0)
if status_counts:
message += "；状态正常{normal}，未上架{unlisted}，审核中{reviewing}，状态未知{unknown}".format(
normal=status_counts.get("normal", 0),
unlisted=status_counts.get("unlisted", 0),
reviewing=status_counts.get("reviewing", 0),
unknown=status_counts.get("unknown", 0),
)
if status_skipped:
message += f"；按范围略过{status_skipped}"
self._show_collect_account_summary(payload, message)
if payload.get("collected", 0) > 0:
batch_id = self._active_batch_id()
@@ -917,14 +961,25 @@ class CollectTab(QWidget):
reused = payload.get("reused_accounts") or []
login_required = payload.get("login_required_accounts") or []
skipped = max(0, int(payload.get("skipped", 0) or 0))
status_scope_skipped = int(payload.get("status_scope_skipped", 0) or 0)
account_skipped = max(0, skipped - status_scope_skipped)
skip_counts = normalize_skip_reason_counts(
payload.get("skip_reason_counts"),
skipped_total=skipped,
skipped_total=account_skipped,
)
if not launched and not reused and not login_required and skipped == 0:
if (
not launched
and not reused
and not login_required
and account_skipped == 0
and status_scope_skipped == 0
):
return
lines = [message]
skip_summary = format_skip_reason_summary(skip_counts, skipped_total=skipped)
skip_summary = format_skip_reason_summary(
skip_counts,
skipped_total=account_skipped,
)
if skip_summary:
lines.append(skip_summary)
if skip_counts[ALIAS_UNMATCHED] > 0:
@@ -941,6 +996,8 @@ class CollectTab(QWidget):
)
if skip_counts[LOGIN_REQUIRED] > 0:
lines.append("请到账号管理完成对应账号登录后，再重新采集略过任务。")
if status_scope_skipped:
lines.append(f"本轮按范围略过{status_scope_skipped}个非正常状态商品，未下载标题和封面。")
if launched or reused or login_required:
lines.append("采集结束后不会自动关闭账号 Chrome，请按需自行关闭。")
text = "\n".join(lines)
app/gui/workers.py
+56 -8
View File
@@ -20,6 +20,7 @@ from .. import (
image_studio_export,
image_studio_generation,
image_studio_images,
product_status,
)
from ..collect_skip import ALIAS_UNMATCHED, LOGIN_REQUIRED, empty_skip_reason_counts
from .widgets import *
@@ -1878,6 +1879,7 @@ class CollectWorker(BaseWorker):
config=None,
preflight=True,
diagnostic_log_dir=None,
collect_scope="all",
):
super().__init__()
self.tasks = list(tasks)
@@ -1885,6 +1887,7 @@ class CollectWorker(BaseWorker):
self.config = config
self.preflight = preflight
self.diagnostic_log_dir = diagnostic_log_dir
self.collect_scope = product_status.normalize_collect_scope(collect_scope)
self._run_id = None
def execute(self):
@@ -1908,6 +1911,10 @@ class CollectWorker(BaseWorker):
login_required_accounts = {}
preflight_info = {}
skip_reason_counts = empty_skip_reason_counts()
product_status_counts = {
status: 0 for status in product_status.VALID_PRODUCT_STATUSES
}
status_scope_skipped = 0
self._run_id = self._create_run_log(eligible, batch_ids)
self._emit_activity(
@@ -1935,6 +1942,9 @@ class CollectWorker(BaseWorker):
extra={
**blocked,
"skip_reason_counts": dict(skip_reason_counts),
"collect_scope": self.collect_scope,
"product_status_counts": dict(product_status_counts),
"status_scope_skipped": status_scope_skipped,
},
)
self._finish_run_log("blocked", summary)
@@ -2111,9 +2121,22 @@ class CollectWorker(BaseWorker):
{
"item_id": task.item_id,
"old_cover_path": self._old_cover_path(account, task),
"collection_scope": self.collect_scope,
},
on_step=on_step,
)
detected_status = product_status.normalize_status(
result.get("product_status")
)
product_status_counts[detected_status] += 1
if result.get("product_status_error"):
self._write_diagnostic_log(
"商品状态检测失败，已按状态未知保存",
level="WARNING",
step="read_product_status",
task=task,
payload={"error": result.get("product_status_error")},
)
if result.get("close_target_confirmed") is False:
self._log_run_event(
"step=close_product result=uncertain detail=任务 {task_id} 商品 {item_id} 商品页已请求关闭，但未在短时间内确认关闭；采集结果已保留，继续处理后续任务".format(
@@ -2133,6 +2156,35 @@ class CollectWorker(BaseWorker):
"close_target_confirmed": False,
},
)
if result.get("collection_skipped"):
activity_result = "skipped"
current_step = "read_product_status"
reason = result.get("collection_skip_reason") or product_status.collect_skip_reason(
detected_status
)
db.set_product_status(
task.id,
detected_status,
result.get("product_status_note"),
path=self.db_path,
)
db.mark_skipped(task.id, reason, path=self.db_path)
skipped += 1
status_scope_skipped += 1
self.row_updated.emit(
task.id,
{"status": "skipped", "last_error": reason},
)
self._log_run_event(
"step=read_product_status result=skipped detail=任务 {task_id} 商品 {item_id} {reason}".format(
task_id=task.id,
item_id=task.item_id,
reason=reason,
),
task=task,
level="warning",
)
continue
current_step = "db_write"
self._emit_activity(
"task_step",
@@ -2156,14 +2208,6 @@ class CollectWorker(BaseWorker):
product_status_note=result.get("product_status_note"),
path=self.db_path,
)
if result.get("product_status_error"):
self._write_diagnostic_log(
"商品状态检测失败，已按状态未知保存",
level="WARNING",
step="read_product_status",
task=task,
payload={"error": result.get("product_status_error")},
)
collected += 1
elapsed_ms = self._elapsed_ms(started)
self.row_updated.emit(
@@ -2235,6 +2279,9 @@ class CollectWorker(BaseWorker):
**preflight_info,
"login_required_accounts": list(login_required_accounts.values()),
"skip_reason_counts": dict(skip_reason_counts),
"collect_scope": self.collect_scope,
"product_status_counts": dict(product_status_counts),
"status_scope_skipped": status_scope_skipped,
},
)
self._finish_run_log("cancelled" if self.should_cancel() else "done", summary)
@@ -2540,6 +2587,7 @@ class CollectWorker(BaseWorker):
options={
"batch_ids": batch_ids,
"preflight": self.preflight,
"collect_scope": self.collect_scope,
},
path=self.db_path,
)
app/product_status.py
+21
View File
@@ -26,6 +26,14 @@ PRODUCT_STATUS_LABELS = {
STATUS_UNKNOWN: "状态未知",
}
SCOPE_NORMAL_ONLY = "normal_only"
SCOPE_ALL = "all"
# Kept as explicit collection aliases while generation and apply adopt the
# shared scope values in their own tasks.
COLLECT_SCOPE_NORMAL_ONLY = SCOPE_NORMAL_ONLY
COLLECT_SCOPE_ALL = SCOPE_ALL
_WHITESPACE_RE = re.compile(r"\s+")
_NOTE_LIMIT = 2000
@@ -93,6 +101,19 @@ def partition_tasks(tasks) -> dict:
return grouped
def normalize_collect_scope(value) -> str:
value = str(value or "").strip().lower()
return SCOPE_ALL if value == SCOPE_ALL else SCOPE_NORMAL_ONLY
def should_collect_content(status, scope) -> bool:
return normalize_collect_scope(scope) == COLLECT_SCOPE_ALL or is_normal(status)
def collect_skip_reason(status) -> str:
return f"{status_label(status)}，按本轮范围略过"
def _alert_note(title: str, description: str) -> str:
parts = []
if title:
docs/04-architecture.md
+2 -1
View File
@@ -342,7 +342,7 @@ CREATE TABLE run_log_events (
优先级：
1. **T-207 已接入 ① 采集**：记录批次开始/结束、账号预检、每个商品开始/成功/失败/略过；关键步骤覆盖 `preflight`、`open_product`、`wait_ready`、`read_title`、`read_cover`、`download_cover`、`db_write`、`excel_write_back`。失败时 DB 事件保存最后步骤和简短错误，本地 `data/logs/cmshopee.log` 保存完整脱敏 traceback。
1. **T-207/T-662b 已接入 ① 采集**：记录批次开始/结束、账号预检、每个商品开始/成功/失败/略过；关键步骤覆盖 `preflight`、`open_product`、`wait_ready`、`read_product_status`、`read_title`、`read_cover`、`download_cover`、`db_write`、`excel_write_back`。运行日志选项记录本轮范围，汇总记录四类商品状态与按范围略过数。失败时 DB 事件保存最后步骤和简短错误，本地 `data/logs/cmshopee.log` 保存完整脱敏 traceback。
2. **② AI生成已先补诊断**：`GenerateWorker` 创建 `run_type=generate`，按任务记录标题/封面阶段事件；关键步骤覆盖 `title_submit`、`load_text_model`、`title_build_request`、`title_request`、`title_parse_response`、`cover_prompt_render`、`cover_submit`、`cover_validate_input`、`load_image_model`、`cover_build_request`、`cover_request`、`cover_parse_response`、`cover_save`、`db_write`。失败时任务 `last_error`、DB 运行日志和本地 `data/logs/cmshopee.log` 都写脱敏错误。
3. **T-505 已扩展全流程**：Excel 导入创建 `run_type=import`，记录选中文件、缺列、脏行、入库统计；Excel 回写创建 `run_type=write_back`，记录文件写入、文件锁/保存异常和行数；③ 更新蝦皮 的 `run_type=apply` 覆盖安全/账号预检、Chrome/CDP 检查、登录检测、打开商品页、改标题、换封面、点更新、写库；账号管理中的「启动登录」创建 `run_type=chrome_launch`，账号管理中的「检测登录」创建 `run_type=login_check`；设置中的 AI 模型测试连接创建 `run_type=ai_model_test`。失败时本地 `data/logs/cmshopee.log` 写脱敏 traceback，业务日志和状态栏只写脱敏短错误。
4. **T-404b 已接入商品页失败 toast 捕获**：①采集和③更新在 `open_product`/等待详情页就绪失败时，不应只报等待超时。进入/刷新商品编辑页后应捕获 `.eds-toasts .eds-toast__content` 的文本和 `outerHTML`，记录当前 URL、时间、可见状态，并在关键元素超时时把最近错误 toast 作为用户可读失败原因写入 `last_error`、`run_log_events` 与本地诊断日志；例如商品 ID 失效时提示 `please input correct product id`。若 toast 明确属于商品失效/商品不存在/无权限类错误，底层仍写 `stage=imported/status=failed/last_error=商品失效：<原始toast>`，只在①导入采集列表“阶段”显示“商品失效”；其他打开失败仍显示“失败”。若失败发生在 `open_product()` 内部且 `cdp` 尚未返回上层，`open_product()` 必须自行清理本轮自动新建 tab；复用用户已有 tab 只断开 CDP，不关闭页面。失败现场可保存 HTML/toast JSON 片段，但不得记录 Cookie、密码、token。
@@ -421,6 +421,7 @@ data/images/<batch_id>/<slug>/<task_id>_<item_id>_new.<ext> # AI 生成的新
- 采集前和采集中途的登录检测必须区分“明确未登录”和“暂时不确定”。明确 `LOGIN_PAGE` / 登录页 URL 才整组略过该账号后续任务；`NO_SESSION_COOKIE`、检测超时或 CDP 短暂异常只记录为不确定并继续尝试采集当前商品，不得级联跳过同账号剩余任务。Cookie API 调用失败不能伪装成空 Cookie；登录检测若连到正在关闭的旧商品 target，应在本次检测预算内重新枚举并改连其他有效 Shopee 页面。
- 若商品 ID 失效、无权限或店铺不匹配导致商品编辑页无法就绪，`open_product` 必须读取/捕获 Shopee toast，把最近错误文案写入采集失败原因和诊断日志，不能只返回泛化超时。①列表只在明确捕获商品失效类 toast 时把“阶段”显示为“商品失效”；底层 `stage` 不新增中文值。若这个失败发生在后台只读、程序自动新建的商品 tab 内，`open_product` 要关闭并执行同样的有界 target 消失确认；复用用户已有 tab 不关闭。③前台更新打开失败仍沿用既有清理路径，不引入额外等待。
- 商品状态只读取 `.eds-alert.eds-alert--warning` 内的 `.eds-alert-title/.eds-alert-desc`，不得依赖 Vue `data-v-*`。`審核中/审核中` 归为 `reviewing`，`您的商品未上架` 归为 `unlisted`，没有 warning 为 `normal`，未识别横幅、DOM 异常或非法返回为 `unknown`。保存归一化摘要而非页面 HTML；状态探测异常只写脱敏诊断，不伪造技术采集失败。
- 每次点击采集先在主线程选择本轮范围，策略值为 `normal_only/all`，默认 `normal_only`，不跨轮记忆。两种范围都会逐条重新打开页面检测并独立保存状态，不能用旧数据库状态预过滤；`normal_only` 仅正常商品读取标题和封面，其他三类状态记为业务略过并保留旧内容，`all` 则四类状态均走现有采集流程。范围略过不是技术失败，汇总分别统计状态和略过数。
- 旧封面：取第一张 itembox 的 `img.src`（CDN 链接），下载到 `data/images/<batch_id>/<slug>/<task_id>_<item_id>_old.jpg`。
- 写 `old_title/old_cover_path`、stage=collected；批量回写 Excel 旧字段。
docs/api.md
+4 -4
View File
@@ -269,7 +269,7 @@ read_product_status(cdp) -> dict # {product_status, product_status_
read_title(cdp) -> str
read_cover_src(cdp) -> str # 第一张 itembox 的 img.src
download_cover(src, out_path) -> str # 下载旧封面到本地
collect(account, task) -> dict # -> {old_title, old_cover_path, product_status, product_status_note, product_status_error, close_target_confirmed}
collect(account, task) -> dict # task.collection_scope=normal_only/all；范围略过时返回 collection_skipped/reason，不读取标题/封面
# 应用
change_title(cdp, new_title) -> dict # {ok, value, modelvalue}，要求三者相等
@@ -292,7 +292,7 @@ apply_task(account, task, close_success_tab=False) -> dict
- `read_product_status()` 在读取标题/封面前读取 `.eds-alert.eds-alert--warning` 下的 `.eds-alert-title/.eds-alert-desc`。`審核中/审核中` → `reviewing`，`您的商品未上架` → `unlisted`，无 warning → `normal`，未识别 warning、DOM 异常或非法响应 → `unknown`；多横幅按 DOM 顺序取第一个白名单命中，不依赖 `data-v-*`，只返回归一化 note，不保存 HTML。状态读取异常不阻断 `collect()`，由调用方写脱敏诊断日志和 `unknown` 快照。
- `collect()` 结束时只关闭本轮自动新建的商品编辑页 tab，并通过 `close_tab_and_wait()` 在最多 2 秒内确认 target 从 `/json` 消失，结果写入 `close_target_confirmed`。确认超时只记警告，不覆盖已成功读取的标题/封面；如果 `open_product()` 尚未返回就失败，也由 `open_product()` 关闭本轮自动新建 tab；用户原本打开的商品 tab 不关闭、不等待。
- `collect()` 先读取状态再决定是否读取标题/封面。`task.collection_scope=normal_only` 时，未上架、审核中、状态未知返回 `collection_skipped=True` 和中文原因，不下载封面、不覆盖旧内容；`all` 时四类状态都继续采集。两种策略都重新读取页面状态，不能用历史状态预过滤。结束时只关闭本轮自动新建的商品编辑页 tab，并通过 `close_tab_and_wait()` 在最多 2 秒内确认 target 从 `/json` 消失，结果写入 `close_target_confirmed`。确认超时只记警告，不覆盖已成功读取的标题/封面；如果 `open_product()` 尚未返回就失败，也由 `open_product()` 关闭本轮自动新建 tab；用户原本打开的商品 tab 不关闭、不等待。
- ③ 更新流程中程序自动新建的商品编辑页成功/失败都关闭，复用用户原本打开的 tab 只断开 CDP、不关闭页面；`open_product()` 内部打开失败的新建 tab 仍由 `open_product()` 自行关闭。Shopee 确认成功后可能把当前 tab 跳回 `/portal/product/list/all?operationSortBy=modified_time`，`click_update()` 会把该 URL 记录到 `post_update.url` 并标记 `redirected_to_list=true`；自动新建页成功关闭前等待 2 秒。
- `click_update()` 的提交成功定义：页面主「更新」按钮已点击，且 Shopee 站点侧确认框未出现或已在可见 `.eds-modal__content` / `.eds-modal__box` 内点击主按钮「更新」。如果确认框仍停留、只点到页面主按钮、或误入「立即優化」，必须返回失败；若 tab 是本轮自动新建，失败后由 `apply_task()` 关闭该 tab。
- T-404/T-502 封面更新删除前，`apply_task()` 应把任务的 `old_cover_path` 传给 `replace_cover()`；`replace_cover()` 只有在本地旧封面备份存在时才允许进入删第一张流程。更新封面统一先删当前第一张，不再只在满 9 张时删除；8 张商品图也按替换语义先删再上传。
@@ -459,7 +459,7 @@ class ApplyTab(QWidget) # ③ 更新蝦皮：筛选已生成
class SettingsTab(QWidget) # 设置：cmhub 网关配置 + 响应式三列布局 + 角色/生成参数/路径端口 + 蝦皮更新安全 + 未保存状态追踪
class ProductSuiteTab(QWidget) # 商品套图：多任务、原图、结构配置、AI帮写、cmhub生成、历史结果
class ImageStudioTab(QWidget) # 旧AI工场兼容实现；主窗口不再创建
class CollectWorker(BaseWorker) # ① 后台采集：账号就绪预检 -> editor.collect -> db.set_collected/mark_skipped/mark_failed
class CollectWorker(BaseWorker) # ① 后台采集：范围 normal_only/all + 账号预检 -> editor.collect -> 状态独立落库，采集或略过
class GenerateWorker(BaseWorker) # ② 后台生成：ai.generate_batch -> db.set_generated/set_generated_cover/mark_failed + 进度
class ApplyWorker(BaseWorker) # ③ 后台更新：账号就绪预检 -> 检查或按批调用 editor.apply_task(...) -> db.set_applied/mark_skipped
class WriteBackWorker(BaseWorker) # ①/③ 后台回写：旧字段或更新结果写回原 Excel
@@ -520,7 +520,7 @@ T-523 后 GUI 已从旧 `app/gui.py` 拆为 `app/gui/` 包：`__init__.py` 负
- 任务列表使用 `QTableView + TaskTableModel`，列为：账号、别名、商品ID、阶段。
- 账号列优先显示匹配到的 `accounts.account_name`；未匹配账号时保留 Excel 输入账号名。
- 别名未匹配 `accounts.alias` 时列表阶段列显示“略过”；点击「采集旧标题/旧封面」后由 `CollectWorker` 逐条写库为 `skipped`，原因 `别名未匹配账号`。
- 「采集旧标题/旧封面」通过 `CollectWorker` 后台执行，只处理 `stage=imported` 的任务；采集前先做账号就绪预检。无账号、当前批次匹配账号未启动 CDP 端口或未登录时，返回 `blocked=True`，GUI 弹窗汇总并跳转/引导去账号管理，不进入逐条采集、不写 skipped/failed。预检通过后，已匹配任务调用 `editor.collect()` 下载旧封面到 `image_dir/<batch_id>/<slug>/<task_id>_<item_id>_old.jpg` 并 `db.set_collected()`；别名未匹配任务仍逐条 `mark_skipped`；单条失败 `mark_failed(..., "collect", error)` 后继续。采集完成且本轮成功采集数量大于 0 时，自动触发当前批次旧字段回写；锁文件失败时只提示，不回滚 SQLite。
- 「采集旧标题/旧封面」先弹范围确认框，默认安全策略 `normal_only`，红色危险选项为 `all`，取消不创建 Worker。`CollectWorker` 只处理 `stage=imported` 的任务；采集前先做账号就绪预检。无账号、当前批次匹配账号未启动 CDP 端口或未登录时，返回 `blocked=True`，GUI 弹窗汇总并跳转/引导去账号管理，不进入逐条采集、不写 skipped/failed。预检通过后，已匹配任务调用 `editor.collect()` 先检测和保存页面状态；默认范围仅正常商品下载旧封面到 `image_dir/<batch_id>/<slug>/<task_id>_<item_id>_old.jpg` 并 `db.set_collected()`，其他状态 `set_product_status()` 后按范围 `mark_skipped` 且保留原旧内容；选择全部范围时四类状态均采集。别名未匹配任务仍逐条 `mark_skipped`；单条技术失败 `mark_failed(..., "collect", error)` 后继续。采集完成且本轮成功采集数量大于 0 时，自动触发当前批次旧字段回写；锁文件失败时只提示，不回滚 SQLite。
- 采集打开商品页时，若本轮自动新建 tab，采集完成后会关闭该 tab，并在最多 2 秒内确认 target 已从 `/json` 消失；确认超时只写 warning/诊断，不把采集成功改成失败。若失败发生在 `open_product()` 内部且尚未返回 `cdp`，也要关闭本轮自动新建 tab；若复用用户已打开的商品页，只断开 CDP 连接不关闭页面。
- 采集中途登录检测必须快速跳过正在销毁的旧商品 target，改连其他有效 Shopee 页面。Cookie API 调用失败返回 `LOGIN_CHECK_TARGET_UNAVAILABLE`，只有 Cookie API 成功返回空会话时才返回 `NO_SESSION_COOKIE`；两者都不按明确掉登录批量略过，显式 `LOGIN_PAGE` 仍按账号需登录处理。retry/recovered 运行日志包含当前任务 ID 和商品 ID，避免与上一条采集成功日志混淆。
- 采集打开商品页失败时，`CollectWorker` 应把 `open_product()` 捕获到的 Shopee toast 文案写入 `run_log_events` 和 `tasks.last_error`；商品 ID 失效、无权限、店铺不匹配等场景不得只显示泛化超时。① `TaskTableModel` 的“阶段”列只在 `last_error` 明确为商品失效类错误时显示“商品失效”，否则仍按 `status=failed` 显示“失败”；底层不新增 `stage` 枚举。
docs/routes.md
+1 -1
View File
@@ -72,7 +72,7 @@
- 导入：openpyxl 解析**输入列**（账号名/别名/商品id）入 SQLite。
- **导入汇总栏**（导入后即时刷新，跑采集前的校验关口）：显示 文件数、解析行数（原始数据量）、有效/无效行、匹配账号行数（按账号细分）、未匹配行数。未匹配/无效数字标红可点，点击在列表筛出便于定位纠错。
- 采集：点击后先为本轮匹配账号确保 Chrome 就绪（已开复用、未开启动），再检测登录；明确未登录账号的任务整组略过并汇总提示。`NO_SESSION_COOKIE`、登录检测超时或 CDP 短暂异常会重试，连续不确定时不批量略过，继续打开商品页由真实页面结果决定成功/失败。登录账号用对应 Chrome 只读打开商品页，读旧标题、下载旧封面到 `data/images/<batch_id>/<slug>/<task_id>_<item_id>_old.jpg`，写 `old_title/old_cover_path`，stage=collected。采集不主动把商品页切到前台；程序自动新建商品页 tab 时尽量后台创建，采集结束后自动关闭；若复用用户原本打开的 tab，则不关闭。采集结束不关闭账号 Chrome，用户可自行关闭。
- 采集：点击后先选择范围，默认「只采集状态正常的商品」，红色危险选项为「采集所有状态的商品」。随后为本轮匹配账号确保 Chrome 就绪（已开复用、未开启动），再检测登录；明确未登录账号的任务整组略过并汇总提示。`NO_SESSION_COOKIE`、登录检测超时或 CDP 短暂异常会重试，连续不确定时不批量略过，继续打开商品页由真实页面结果决定成功/失败。登录账号用对应 Chrome 只读打开商品页，先检测并保存商品状态；默认范围下只有正常商品才读旧标题、下载旧封面到 `data/images/<batch_id>/<slug>/<task_id>_<item_id>_old.jpg`，未上架、审核中和状态未知商品按范围略过且不覆盖已有内容。选择全部范围时四类商品均继续采集。采集不主动把商品页切到前台；程序自动新建商品页 tab 时尽量后台创建，采集结束后自动关闭；若复用用户原本打开的 tab，则不关闭。采集结束不关闭账号 Chrome，用户可自行关闭。
- 若商品 ID 已失效、无权限或店铺不匹配，Shopee 可能只弹出短暂错误 toast；采集失败时界面日志应显示捕获到的 toast 文案，并把 toast HTML/URL 写入本地诊断日志，避免用户手动抢复制。只有明确捕获商品失效/商品不存在/无权限类 toast 时，①列表“阶段”列显示“商品失效”；其他商品页打开失败仍显示“失败”。如果失败发生在 `open_product()` 内部，本轮自动新建的商品 tab 必须关闭，复用用户已有 tab 不关闭。
- 回写：采集完成后自动把旧标题/旧封面路径批量回写原 Excel；保留「回写旧数据到 Excel」作为手动重试入口（原文件被锁→提示关闭后重试/另存）。
docs/tasks/T-662b.md
+6 -2
View File
@@ -1,7 +1,7 @@
---
id: T-662b
title: 导入采集范围确认与商品状态汇总
status: TODO
status: DONE
phase: 7
deps: [T-662a]
created: 2026-07-18
@@ -61,4 +61,8 @@ git diff --check
## 执行记录
- 待实现。
- 已实现采集范围确认框：默认“只采集状态正常的商品”，红色“采集所有状态的商品”，取消不启动 Worker。
- `CollectWorker` 将范围策略冻结在本轮运行日志中；逐条先读取并保存商品状态。默认范围仅采集正常商品，异常/未知商品保留旧标题和封面并按范围略过；全部范围继续采集四类状态。
- 汇总、行结果和运行日志分别统计四类状态与按范围略过数，账号预检略过统计不再把业务范围略过误报为“其他”。
- 已更新 `docs/routes.md`、`docs/04-architecture.md`、`docs/api.md`。
- 验证通过：`py -3.10 -m unittest tests.test_editor_login tests.test_workers tests.test_gui`（267 项）、`py -3.10 -m unittest discover -s tests`（605 项）、`py -3.10 -m ruff check app tests main.py`、`py -3.10 -m compileall app main.py`、`git diff --check`。
tests/test_editor_login.py
+27
View File
@@ -971,6 +971,33 @@ class EditorLoginTests(unittest.TestCase):
steps,
)
def test_collect_normal_only_skips_non_normal_before_reading_content(self):
cdp = FakeProductCDP(
"ws-status",
alerts=[{"title": "您的商品未上架", "description": "已下架"}],
)
with mock.patch("app.editor.open_product", return_value=cdp), mock.patch(
"app.editor.read_title"
) as read_title, mock.patch("app.editor.read_cover_src") as read_cover_src, mock.patch(
"app.editor.download_cover"
) as download_cover:
result = editor.collect(
{"debug_port": 9222},
{
"item_id": "51100639510",
"old_cover_path": "old.jpg",
"collection_scope": "normal_only",
},
)
self.assertTrue(result["collection_skipped"])
self.assertEqual("unlisted", result["product_status"])
self.assertEqual("未上架，按本轮范围略过", result["collection_skip_reason"])
read_title.assert_not_called()
read_cover_src.assert_not_called()
download_cover.assert_not_called()
def test_collect_keeps_reused_product_tab_open(self):
cdp = FakeProductCDP("ws-existing")
cdp.target_id = "target-existing"
tests/test_gui.py
+47 -2
View File
@@ -28,7 +28,7 @@ from app import (
if gui.QT_IMPORT_ERROR is not None:
raise unittest.SkipTest("PySide6 未安装")
from PySide6.QtCore import QItemSelectionModel, QModelIndex, QRect, QSize, Qt
from PySide6.QtCore import QItemSelectionModel, QModelIndex, QRect, QSize, QTimer, Qt
from PySide6.QtGui import QImage, QKeyEvent, QTextCursor
from PySide6.QtWidgets import (
QAbstractItemView,
@@ -38,6 +38,7 @@ from PySide6.QtWidgets import (
QListView,
QPlainTextEdit,
QProgressBar,
QPushButton,
QTableView,
)
@@ -7804,7 +7805,11 @@ class GuiTests(TempDirMixin, unittest.TestCase):
captured["started"] = True
captured = {}
with mock.patch("app.gui.CollectWorker", FakeWorker), mock.patch(
with mock.patch.object(
tab,
"_choose_collect_scope",
return_value="normal_only",
), mock.patch("app.gui.CollectWorker", FakeWorker), mock.patch(
"app.gui.run_worker",
return_value=FakeThread(),
):
@@ -7813,12 +7818,52 @@ class GuiTests(TempDirMixin, unittest.TestCase):
self.assertTrue(captured["started"])
self.assertEqual(["1001"], [task.item_id for task in captured["tasks"]])
self.assertEqual(cfg["db_path"], captured["kwargs"]["db_path"])
self.assertEqual("normal_only", captured["kwargs"]["collect_scope"])
self.assertFalse(tab.shop_filter.isEnabled())
self.assertFalse(tab.item_filter.isEnabled())
self.assertFalse(tab.status_filter.isEnabled())
self.assert_removed(temp_dir)
def test_collect_scope_dialog_uses_safe_default_and_red_all_status_choice(self):
with self.make_temp_dir() as temp_dir:
tab = CollectTab(config=self.make_config(temp_dir))
self.addCleanup(tab.close)
captured = {}
def click_all_statuses():
box = QApplication.activeModalWidget()
normal_button = box.findChild(QPushButton, "collectNormalOnlyButton")
all_button = box.findChild(QPushButton, "collectAllStatusesButton")
captured["default"] = box.defaultButton().objectName()
captured["all_style"] = all_button.styleSheet()
self.assertIsNotNone(normal_button)
self.assertIsNotNone(all_button)
all_button.click()
QTimer.singleShot(0, click_all_statuses)
scope = tab._choose_collect_scope()
self.assertEqual("all", scope)
self.assertEqual("collectNormalOnlyButton", captured["default"])
self.assertIn("#cf222e", captured["all_style"])
self.assert_removed(temp_dir)
def test_collect_without_candidates_does_not_open_scope_dialog(self):
with self.make_temp_dir() as temp_dir:
statuses = []
tab = CollectTab(config=self.make_config(temp_dir), status_callback=statuses.append)
self.addCleanup(tab.close)
with mock.patch.object(tab, "_choose_collect_scope") as choose_scope:
tab.collect_old_data()
choose_scope.assert_not_called()
self.assertEqual("没有可采集任务", statuses[-1])
self.assert_removed(temp_dir)
def test_collect_activity_tracks_step_resets_each_task_and_freezes_on_stop(self):
with self.make_temp_dir() as temp_dir:
tab = CollectTab(config=self.make_config(temp_dir))
tests/test_workers.py
+161
View File
@@ -24,6 +24,7 @@ from app.gui.workers import (
ProductSuiteAiWriteWorker,
ProductSuiteGenerateWorker,
ProductSuiteHistoryExportWorker,
CollectWorker,
)
@@ -119,6 +120,166 @@ class WorkerTests(unittest.TestCase):
self.assertEqual([(-1, "模拟失败")], failed)
self.assertEqual([{"ok": False, "error": "模拟失败"}], finished)
def test_collect_worker_scope_skips_non_normal_without_overwriting_old_content(self):
with tempfile.TemporaryDirectory() as temp_dir:
db_path = os.path.join(temp_dir, "cmshopee.db")
db.init_db(db_path)
batch_id = db.create_batch(["input.xlsx"], path=db_path)
db.insert_tasks(
batch_id,
[
{
"source_file_abs": os.path.join(temp_dir, "input.xlsx"),
"source_sheet": "Sheet1",
"source_row": index,
"account_name": "店铺",
"alias": "alias",
"item_id": str(51100639510 + index),
}
for index in range(2, 6)
],
path=db_path,
)
tasks = db.list_tasks(batch_id=batch_id, path=db_path)
connection = db.connect(db_path)
try:
with connection:
connection.execute(
"UPDATE tasks SET old_title = ?, old_cover_path = ? WHERE id = ?",
("历史标题", "history.jpg", tasks[1].id),
)
finally:
connection.close()
status_by_item = {
tasks[0].item_id: "normal",
tasks[1].item_id: "unlisted",
tasks[2].item_id: "reviewing",
tasks[3].item_id: "unknown",
}
account = SimpleNamespace(alias="alias", account_name="店铺", debug_port=9222)
def fake_collect(_account, task, on_step=None):
status = status_by_item[task["item_id"]]
self.assertEqual("normal_only", task["collection_scope"])
on_step("read_product_status")
if status != "normal":
labels = {
"unlisted": "未上架",
"reviewing": "审核中",
"unknown": "状态未知",
}
return {
"product_status": status,
"product_status_note": f"{labels[status]}提示",
"collection_skipped": True,
"collection_skip_reason": f"{labels[status]}，按本轮范围略过",
}
on_step("download_cover")
return {
"product_status": status,
"product_status_note": None,
"old_title": "新采集标题",
"old_cover_path": "new.jpg",
}
with mock.patch(
"app.gui.workers.accounts.list_accounts",
return_value=[account],
), mock.patch(
"app.gui.workers.accounts.detect_login",
return_value={"logged_in": True, "reason": None},
), mock.patch(
"app.gui.workers.editor.collect",
side_effect=fake_collect,
):
summary = CollectWorker(
tasks,
db_path=db_path,
preflight=False,
collect_scope="normal_only",
).execute()
self.assertEqual(1, summary["collected"])
self.assertEqual(3, summary["skipped"])
self.assertEqual(0, summary["failed"])
self.assertEqual(3, summary["status_scope_skipped"])
self.assertEqual(
{"normal": 1, "unlisted": 1, "reviewing": 1, "unknown": 1},
summary["product_status_counts"],
)
refreshed = db.list_tasks(batch_id=batch_id, path=db_path)
by_status = {task.product_status: task for task in refreshed}
self.assertEqual("collected", by_status["normal"].stage)
self.assertEqual("success", by_status["normal"].status)
self.assertEqual("skipped", by_status["unlisted"].status)
self.assertEqual("imported", by_status["unlisted"].stage)
self.assertEqual("历史标题", by_status["unlisted"].old_title)
self.assertEqual("history.jpg", by_status["unlisted"].old_cover_path)
def test_collect_worker_all_scope_collects_every_detected_status(self):
with tempfile.TemporaryDirectory() as temp_dir:
db_path = os.path.join(temp_dir, "cmshopee.db")
db.init_db(db_path)
batch_id = db.create_batch(["input.xlsx"], path=db_path)
db.insert_tasks(
batch_id,
[
{
"source_file_abs": os.path.join(temp_dir, "input.xlsx"),
"source_sheet": "Sheet1",
"source_row": index,
"account_name": "店铺",
"alias": "alias",
"item_id": str(51100639600 + index),
}
for index in range(2, 6)
],
path=db_path,
)
tasks = db.list_tasks(batch_id=batch_id, path=db_path)
statuses = ["normal", "unlisted", "reviewing", "unknown"]
account = SimpleNamespace(alias="alias", account_name="店铺", debug_port=9222)
def fake_collect(_account, task, on_step=None):
self.assertEqual("all", task["collection_scope"])
on_step("read_product_status")
status = statuses.pop(0)
return {
"product_status": status,
"product_status_note": None,
"old_title": f"标题{status}",
"old_cover_path": f"{status}.jpg",
}
with mock.patch(
"app.gui.workers.accounts.list_accounts",
return_value=[account],
), mock.patch(
"app.gui.workers.accounts.detect_login",
return_value={"logged_in": True, "reason": None},
), mock.patch(
"app.gui.workers.editor.collect",
side_effect=fake_collect,
):
summary = CollectWorker(
tasks,
db_path=db_path,
preflight=False,
collect_scope="all",
).execute()
self.assertEqual(4, summary["collected"])
self.assertEqual(0, summary["skipped"])
self.assertEqual(0, summary["status_scope_skipped"])
self.assertEqual(
{"normal": 1, "unlisted": 1, "reviewing": 1, "unknown": 1},
summary["product_status_counts"],
)
self.assertTrue(
all(task.stage == "collected" for task in db.list_tasks(batch_id=batch_id, path=db_path))
)
def test_run_worker_rejects_plain_object(self):
with self.assertRaises(TypeError):
run_worker(object(), start=False)