Files
cmshoppe/app/gui/workers.py
T

1773 lines
65 KiB
Python
Raw Normal View History

2026-07-02 16:47:37 +08:00
"""Concrete PySide6 workers used by GUI tabs."""
from __future__ import annotations
from .widgets import *
class GenerateWorker(BaseWorker):
"""Generate titles and covers for collected tasks."""
def __init__(
self,
tasks,
prompt_values,
db_path=None,
config=None,
diagnostic_log_dir=None,
):
super().__init__()
self.tasks = list(tasks)
self.prompt_values = dict(prompt_values or {})
self.db_path = db_path
self.config = config
self.diagnostic_log_dir = diagnostic_log_dir
self._run_id = None
self._account_by_alias = {}
self._task_positions = {}
self._eligible_total = 0
def execute(self):
account_rows = accounts.list_accounts(path=self.db_path, config=self.config)
account_by_alias = {
str(account.alias).strip(): account
for account in account_rows
if str(account.alias).strip()
}
self._account_by_alias = account_by_alias
eligible = [
task for task in self.tasks
if getattr(task, "stage", None) == "collected"
]
self._eligible_total = len(eligible)
self._task_positions = {
getattr(task, "id", None): index
for index, task in enumerate(eligible, start=1)
}
batch_ids = self._batch_ids(eligible)
self._run_id = self._create_run_log(eligible, batch_ids)
ai_cfg = appconfig.ai_config(self.config)
generate_cover = bool(ai_cfg.get("generate_cover", False))
if generate_cover:
start_message = "[开始] 本轮生成 {total} 条:标题{total},图片{total};标题并发{title_concurrency},图片并发{image_concurrency}".format(
total=len(eligible),
title_concurrency=ai_cfg.get("title_concurrency", 1),
image_concurrency=ai_cfg.get("image_concurrency", 1),
)
else:
start_message = "[开始] 本轮生成 {total} 条:本轮仅生成标题,不生成图片;标题并发{title_concurrency}".format(
total=len(eligible),
title_concurrency=ai_cfg.get("title_concurrency", 1),
)
self._log_run_event(start_message)
try:
summary = ai.generate_batch(
self.tasks,
self.prompt_values,
ai_cfg={
"config": self.config,
"db_path": self.db_path,
"image_dir": appconfig.image_dir(self.config),
"account_by_alias": account_by_alias,
"on_task_update": self._emit_row_update,
"on_event": self._on_generation_event,
"on_error": self._on_generation_error,
"generate_cover": generate_cover,
},
on_progress=self.progress.emit,
should_stop=self.should_cancel,
)
except Exception as exc:
error = diagnostics.redact_log_text(str(exc) or exc.__class__.__name__)
summary = {
"ok": False,
"error": error,
"total": len(eligible),
"title_done": 0,
"cover_done": 0,
"cover_total": len(eligible) if generate_cover else 0,
"generated_done": 0,
"failed": len(eligible),
"cancelled": self.should_cancel(),
"generate_cover": generate_cover,
}
self._log_run_event(
f"[失败] AI 生成运行失败:{error}",
level="error",
)
self._write_diagnostic_log(
"AI生成运行失败",
level="ERROR",
step="execute",
payload={"error": error},
exc=exc,
)
summary["run_id"] = self._run_id
summary["batch_ids"] = batch_ids
status = "cancelled" if summary.get("cancelled") else "done"
level = "warning" if summary.get("cancelled") or summary.get("error") else "info"
self._log_run_event(self._format_generate_completion(summary), level=level)
self._finish_run_log(status, summary)
return summary
def _emit_row_update(self, task_id, fields):
self.row_updated.emit(int(task_id), dict(fields or {}))
def _on_generation_event(self, payload):
task = payload.get("task")
message = self._format_generation_event(payload)
if not message:
return
self._log_run_event(message, task=task, level=payload.get("level") or "info")
def _format_generation_event(self, payload):
task = payload.get("task")
phase = payload.get("phase") or "generate"
step = payload.get("step") or "unknown"
result = payload.get("result") or "start"
detail = self._short_detail(payload.get("detail"))
if phase == "title":
if result == "start" and step == "title_submit":
return f"[标题] {self._task_progress_label(task)} 开始生成"
if result == "success" and step == "title_done":
return f"[标题] {self._task_progress_label(task)} 成功"
if result == "success" and step == "db_write":
suffix = f",{detail}" if detail else ""
return f"[标题] {self._task_progress_label(task)} 已保存{suffix}"
if result == "retry":
return self._retry_message("标题", task, payload, detail)
if result == "failed":
return f"[失败] {self._task_plain_label(task)} 标题生成失败:{detail or '未知错误'}"
if result == "cancelled":
return f"[停止] {self._task_plain_label(task)} 标题生成已取消"
return None
if phase == "cover":
if result == "start" and step == "cover_submit":
return f"[图片] {self._task_progress_label(task)} 开始生成"
if result == "success" and step == "db_write":
suffix = f",已保存 {detail}" if detail else ""
return f"[图片] {self._task_progress_label(task)} 成功{suffix}"
if result == "retry":
return self._retry_message("图片", task, payload, detail)
if result == "failed":
return f"[失败] {self._task_plain_label(task)} 图片生成失败:{detail or '未知错误'}"
if result == "cancelled":
return f"[停止] {self._task_plain_label(task)} 图片生成已取消"
return None
return None
def _retry_message(self, label, task, payload, detail):
attempt = int(payload.get("attempt", 0) or 0)
attempts = int(payload.get("attempts", 0) or 0)
max_retries = max(0, attempts - 1)
retry_text = f"准备重试 {attempt}/{max_retries}" if max_retries else "准备重试"
reason = f":{detail}" if detail else ""
return f"[{label}] {self._task_progress_label(task)} 调用失败,{retry_text}{reason}"
def _task_progress_label(self, task):
index = self._task_positions.get(getattr(task, "id", None), 0)
total = self._eligible_total or 0
item_id = getattr(task, "item_id", "") or "未知商品"
shop = self._task_shop_label(task)
shop_text = f"({shop})" if shop else ""
return f"{index}/{total} 商品 {item_id}{shop_text}"
def _task_plain_label(self, task):
item_id = getattr(task, "item_id", "") or "未知商品"
shop = self._task_shop_label(task)
return f"商品 {item_id}({shop})" if shop else f"商品 {item_id}"
def _task_shop_label(self, task):
alias = str(getattr(task, "alias", "") or "").strip()
account = self._account_by_alias.get(alias)
if account is not None:
return getattr(account, "account_name", None) or getattr(account, "alias", None) or alias
return getattr(task, "account_name", None) or alias
def _short_detail(self, detail):
if detail is None:
return ""
text = diagnostics.redact_log_text(str(detail)).replace("\r", " ").replace("\n", " ").strip()
if len(text) > 180:
return text[:177] + "..."
return text
def _format_generate_completion(self, summary):
progress = self._summary_text(summary)
if summary.get("cancelled"):
return f"[停止] AI 生成已停止:{progress}"
if summary.get("error"):
return f"[失败] AI 生成失败:{summary.get('error')},{progress}"
return f"[完成] AI 生成完成:{progress}"
def _summary_text(self, summary):
cover_total = summary.get("cover_total", summary.get("total", 0))
return "标题{title}/{total},图片{cover}/{cover_total},失败{failed}".format(
title=summary.get("title_done", 0),
cover=summary.get("cover_done", 0),
cover_total=cover_total,
total=summary.get("total", 0),
failed=summary.get("failed", 0),
)
def _on_generation_error(self, payload):
task = payload.get("task")
phase = payload.get("phase") or "generate"
step = payload.get("step") or "unknown"
error = diagnostics.redact_log_text(payload.get("error") or "未知错误")
self._write_diagnostic_log(
"AI生成任务失败",
level="ERROR",
step=step,
task=task,
payload={"phase": phase, "error": error},
exc=payload.get("exception"),
)
def _batch_ids(self, tasks):
batch_ids = []
for task in tasks:
batch_id = getattr(task, "batch_id", None)
if batch_id and batch_id not in batch_ids:
batch_ids.append(batch_id)
return batch_ids
def _create_run_log(self, eligible, batch_ids):
try:
ai_cfg = appconfig.ai_config(self.config)
return db.create_run_log(
"generate",
dry_run=False,
total=len(eligible),
options={
"batch_ids": batch_ids,
"default_text_model": ai_cfg.get("default_text_model"),
"default_image_model": ai_cfg.get("default_image_model"),
"resolution": ai_cfg.get("resolution"),
"title_concurrency": ai_cfg.get("title_concurrency"),
"image_concurrency": ai_cfg.get("image_concurrency"),
"generate_cover": ai_cfg.get("generate_cover", False),
},
path=self.db_path,
)
except Exception:
return None
def _finish_run_log(self, status, summary):
if self._run_id is None:
return
try:
generated_done = summary.get("generated_done")
if generated_done is None:
generated_done = summary.get("cover_done", 0)
if not summary.get("generate_cover", True) and not generated_done:
generated_done = summary.get("title_done", 0)
done = int(generated_done or 0) + int(summary.get("failed", 0) or 0)
db.finish_run_log(
self._run_id,
status=status,
done=done,
success_count=generated_done,
skipped_count=0,
failed_count=summary.get("failed", 0),
summary_json=summary,
path=self.db_path,
)
except Exception:
return
def _log_run_event(self, message, task=None, level="info"):
safe_message = diagnostics.redact_log_text(message)
self.log.emit(str(safe_message))
if self._run_id is None:
return
try:
db.add_run_log_event(
self._run_id,
safe_message,
task_id=getattr(task, "id", None),
alias=getattr(task, "alias", None),
item_id=getattr(task, "item_id", None),
level=level,
path=self.db_path,
)
except Exception:
return
def _write_diagnostic_log(
self,
message,
level="INFO",
step=None,
task=None,
payload=None,
exc=None,
):
try:
diagnostics.write_diagnostic_log(
message,
level=level,
step=step,
task_id=getattr(task, "id", None),
alias=getattr(task, "alias", None),
item_id=getattr(task, "item_id", None),
payload=payload,
exc=exc,
log_dir=self.diagnostic_log_dir,
)
except Exception:
return
class ApplyWorker(BaseWorker):
"""Apply generated title/cover changes, optionally previewing or grouping by account."""
def __init__(
self,
tasks,
db_path=None,
config=None,
preflight=True,
close_success_tab=False,
dry_run=False,
parallel_accounts=False,
max_parallel_accounts=1,
batch_size=None,
diagnostic_log_dir=None,
):
super().__init__()
self.tasks = list(tasks)
self.db_path = db_path
self.config = config
self.preflight = preflight
self.close_success_tab = close_success_tab
self.dry_run = bool(dry_run)
self.parallel_accounts = bool(parallel_accounts)
self.max_parallel_accounts = max(1, int(max_parallel_accounts or 1))
self.batch_size = None if batch_size is None else max(1, int(batch_size or 1))
self._current_batch_size = None
self._batch_count = 0
self._progress_lock = threading.Lock()
self.diagnostic_log_dir = diagnostic_log_dir
self._run_id = None
def execute(self):
account_rows = accounts.list_accounts(path=self.db_path, config=self.config)
account_by_alias = {
str(account.alias).strip(): account
for account in account_rows
if str(account.alias).strip()
}
eligible = [task for task in self.tasks if self._is_actionable_task(task)]
batch_ids = self._batch_ids(eligible)
total = len(eligible)
batch_size = self._effective_batch_size(total)
batches = self._task_batches(eligible, batch_size)
self._current_batch_size = batch_size
self._batch_count = len(batches)
counters = {
"done": 0,
"applied": 0,
"skipped": 0,
"failed": 0,
}
self._run_id = self._create_run_log(eligible, batch_ids)
self._log_run_event(
"step=start result=start detail=运行开始:{mode},任务{total},每批最多{batch_size},批次{batch_count},{parallel}".format(
mode="检查本轮更新" if self.dry_run else "真实更新",
total=total,
batch_size=batch_size,
batch_count=len(batches),
parallel=(
f"多账号并行最多{self.max_parallel_accounts}"
if self.parallel_accounts
else "串行"
),
)
)
if self.preflight and not self.dry_run:
self._log_run_event("step=preflight result=start detail=账号就绪检查")
blocked = self._preflight_block(eligible, account_rows, account_by_alias)
if blocked:
self._log_preflight_blocked(blocked)
summary = self._summary(
ok=False,
total=total,
counters=counters,
batch_ids=batch_ids,
blocked=True,
extra=blocked,
)
self._finish_run_log("blocked", summary)
return summary
self._log_run_event("step=preflight result=success detail=账号检查通过")
elif not self.preflight:
self._log_run_event(
"step=preflight result=skipped detail=测试模式跳过更新前检查",
level="warning",
)
for batch_index, batch_tasks in enumerate(batches, start=1):
if self.should_cancel():
break
self._log_batch_start(batch_index, len(batches), batch_tasks, counters, total)
if self.dry_run:
for task in batch_tasks:
if self.should_cancel():
break
outcome = self._preview_task(task, account_by_alias)
self._record_outcome(counters, total, outcome)
elif self.parallel_accounts and self.max_parallel_accounts > 1:
self._run_parallel_by_account(batch_tasks, account_by_alias, counters, total)
else:
for task in batch_tasks:
if self.should_cancel():
break
outcome = self._apply_one_task(task, account_by_alias)
self._record_outcome(counters, total, outcome)
summary = self._summary(
ok=counters["failed"] == 0,
total=total,
counters=counters,
batch_ids=batch_ids,
)
self._finish_run_log("cancelled" if self.should_cancel() else "done", summary)
return summary
def _is_actionable_task(self, task):
return (
getattr(task, "stage", None) == "generated"
and getattr(task, "status", None) in {"success", "pending", "failed"}
and bool(getattr(task, "new_title", None) or getattr(task, "new_cover_path", None))
)
def _preflight_block(self, eligible, account_rows, account_by_alias):
if not account_rows:
return {
"reason": "NO_ACCOUNTS",
"no_accounts": True,
}
duplicate_ports = self._duplicate_debug_ports(account_rows, eligible, account_by_alias)
if duplicate_ports:
return {
"reason": "DUPLICATE_DEBUG_PORT",
"duplicate_ports": duplicate_ports,
}
required_accounts = []
seen_aliases = set()
for task in eligible:
alias = str(task.alias).strip()
account = account_by_alias.get(alias)
if account is not None and alias not in seen_aliases:
required_accounts.append(account)
seen_aliases.add(alias)
not_running = []
logged_out = []
for account in required_accounts:
self._log_run_event(
f"step=check_chrome result=start detail=账号 {account.alias} debug_port={account.debug_port}",
level="info",
)
if not chrome.is_running(account.debug_port):
self._log_run_event(
f"step=check_chrome result=blocked detail=账号 {account.alias} CDP 端口未响应 debug_port={account.debug_port}",
level="warning",
)
not_running.append(self._account_payload(account, "CDP 端口未响应"))
continue
self._log_run_event(
f"step=check_chrome result=success detail=账号 {account.alias} debug_port={account.debug_port}",
level="info",
)
self._log_run_event(
f"step=login_check result=start detail=账号 {account.alias}",
level="info",
)
status = self._login_status(account)
if not status.get("logged_in"):
reason = self._login_skip_reason(status)
self._log_run_event(
f"step=login_check result=blocked detail=账号 {account.alias} {reason}",
level="warning",
)
logged_out.append(
self._account_payload(account, reason)
)
else:
self._log_run_event(
f"step=login_check result=success detail=账号 {account.alias}",
level="info",
)
if not_running or logged_out:
return {
"reason": "ACCOUNT_NOT_READY",
"not_running": not_running,
"logged_out": logged_out,
}
return None
def _duplicate_debug_ports(self, account_rows, eligible, account_by_alias):
required_aliases = {
str(task.alias).strip()
for task in eligible
if account_by_alias.get(str(task.alias).strip()) is not None
}
by_port = {}
for account in account_rows:
if account.alias not in required_aliases:
continue
by_port.setdefault(int(account.debug_port), []).append(account)
duplicates = []
for port, rows in by_port.items():
if len(rows) > 1:
duplicates.append(
{
"debug_port": port,
"aliases": [row.alias for row in rows],
}
)
return duplicates
def _effective_batch_size(self, total):
if self.batch_size is None:
return max(1, int(total or 1))
return self.batch_size
def _task_batches(self, tasks, batch_size):
if not tasks:
return []
return [
tasks[index:index + batch_size]
for index in range(0, len(tasks), batch_size)
]
def _log_batch_start(self, batch_index, batch_count, batch_tasks, counters, total):
first = counters["done"] + 1
last = min(first + len(batch_tasks) - 1, total)
label = "检查批次" if self.dry_run else "更新批次"
self._log_run_event(
f"step=batch result=start detail={label} {batch_index}/{batch_count} 开始:任务 {first}-{last}/{total}"
)
def _run_parallel_by_account(self, eligible, account_by_alias, counters, total):
groups = self._group_tasks_by_alias(eligible)
max_workers = min(self.max_parallel_accounts, len(groups))
if max_workers <= 1:
for group_tasks in groups:
self._run_task_group(group_tasks, account_by_alias, counters, total)
return
with ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = [
executor.submit(
self._run_task_group,
group_tasks,
account_by_alias,
counters,
total,
)
for group_tasks in groups
]
for future in as_completed(futures):
future.result()
def _group_tasks_by_alias(self, tasks):
groups = []
index_by_alias = {}
for task in tasks:
alias = str(task.alias).strip()
if alias not in index_by_alias:
index_by_alias[alias] = len(groups)
groups.append([])
groups[index_by_alias[alias]].append(task)
return groups
def _run_task_group(self, tasks, account_by_alias, counters, total):
for task in tasks:
if self.should_cancel():
break
outcome = self._apply_one_task(task, account_by_alias)
self._record_outcome(counters, total, outcome)
def _preview_task(self, task, account_by_alias):
account = account_by_alias.get(str(task.alias).strip())
if account is None:
reason = "别名未匹配账号"
self._log_run_event(
f"step=preview result=skipped detail=检查:任务 {task.id} 商品 {task.item_id} 将略过:{reason}",
task=task,
level="warning",
)
return "skipped"
action_parts = []
if getattr(task, "new_title", None):
action_parts.append("标题")
if getattr(task, "new_cover_path", None):
action_parts.append("封面")
action_text = "+".join(action_parts) or "无变更"
self._log_run_event(
"step=preview result=success detail=检查:任务 {task_id} 商品 {item_id} 账号 {alias} 将更新 {action}".format(
task_id=task.id,
item_id=task.item_id,
alias=account.alias,
action=action_text,
),
task=task,
)
return "applied"
def _apply_one_task(self, task, account_by_alias):
account = account_by_alias.get(str(task.alias).strip())
if account is None:
reason = "别名未匹配账号"
db.mark_skipped(task.id, reason, path=self.db_path)
self.row_updated.emit(task.id, {"status": "skipped", "last_error": reason})
self._log_run_event(
f"step=preflight result=skipped detail=任务 {task.id} 商品 {task.item_id} 已略过:{reason}",
task=task,
level="warning",
)
return "skipped"
started = time.monotonic()
current_step = "db_write"
def on_step(event):
nonlocal current_step
if isinstance(event, dict):
step = str(event.get("step") or "apply_task")
result = str(event.get("result") or "start")
detail = event.get("detail")
else:
step = str(event)
result = "start"
detail = None
current_step = step
level = "error" if result == "failed" else "info"
detail_text = "任务 {task_id} 商品 {item_id}".format(
task_id=task.id,
item_id=task.item_id,
)
if detail:
detail_text = f"{detail_text} {detail}"
self._log_run_event(
f"step={step} result={result} detail={detail_text}",
task=task,
level=level,
)
try:
self._log_run_event(
f"step=apply_task result=start detail=任务 {task.id} 商品 {task.item_id} 开始更新,账号 {account.alias}",
task=task,
)
current_step = "db_write"
self._log_run_event(
f"step=db_write result=start detail=任务 {task.id} 商品 {task.item_id} 标记更新运行",
task=task,
)
db.mark_running(task.id, "apply", path=self.db_path)
self.row_updated.emit(task.id, {"status": "running", "last_error": None})
result = editor.apply_task(
account,
task,
close_success_tab=self.close_success_tab,
on_step=on_step,
)
committed = bool(result.get("committed")) and not result.get("error")
error = result.get("error")
current_step = "db_write"
self._log_run_event(
f"step=db_write result=start detail=任务 {task.id} 商品 {task.item_id} 保存更新结果",
task=task,
)
if committed:
db.set_applied(task.id, True, path=self.db_path)
elapsed_ms = self._elapsed_ms(started)
self.row_updated.emit(
task.id,
{
"stage": "applied",
"status": "success",
"committed": 1,
"last_error": None,
},
)
self._log_run_event(
f"step=db_write result=success detail=任务 {task.id} 商品 {task.item_id} 更新成功 elapsed_ms={elapsed_ms}",
task=task,
)
return "applied"
error = diagnostics.redact_log_text(error or "更新未提交")
failed_step = self._failed_apply_step(result, current_step)
db.set_applied(task.id, False, error, path=self.db_path)
elapsed_ms = self._elapsed_ms(started)
self.failed.emit(task.id, str(error))
self.row_updated.emit(
task.id,
{"status": "failed", "last_error": str(error), "committed": 0},
)
self._log_run_event(
f"step={failed_step} result=failed detail={error} elapsed_ms={elapsed_ms}",
task=task,
level="error",
)
self._log_run_event(
f"step=db_write result=success detail=任务 {task.id} 商品 {task.item_id} 保存失败状态 elapsed_ms={elapsed_ms}",
task=task,
)
self._write_diagnostic_log(
"Shopee更新任务失败",
level="ERROR",
step=failed_step,
task=task,
elapsed_ms=elapsed_ms,
payload={"error": error, "result": result},
)
return "failed"
except Exception as exc:
error = diagnostics.redact_log_text(str(exc) or exc.__class__.__name__)
db.set_applied(task.id, False, error, path=self.db_path)
elapsed_ms = self._elapsed_ms(started)
self.failed.emit(task.id, error)
self.row_updated.emit(
task.id,
{"status": "failed", "last_error": error, "committed": 0},
)
self._log_run_event(
f"step={current_step} result=failed detail={error} elapsed_ms={elapsed_ms}",
task=task,
level="error",
)
self._write_diagnostic_log(
"Shopee更新任务异常",
level="ERROR",
step=current_step,
task=task,
elapsed_ms=elapsed_ms,
payload={"error": error},
exc=exc,
)
return "failed"
def _record_outcome(self, counters, total, outcome):
with self._progress_lock:
counters["done"] += 1
if outcome == "applied":
counters["applied"] += 1
elif outcome == "skipped":
counters["skipped"] += 1
else:
counters["failed"] += 1
self._emit_progress(
counters["done"],
total,
counters["applied"],
counters["skipped"],
counters["failed"],
)
def _account_payload(self, account, reason=None):
payload = {
"account_name": account.account_name,
"alias": account.alias,
"debug_port": account.debug_port,
}
if reason:
payload["reason"] = reason
return payload
def _emit_progress(self, done, total, applied, skipped, failed):
self.progress.emit(
{
"done": done,
"total": total,
"applied": applied,
"skipped": skipped,
"failed": failed,
"dry_run": self.dry_run,
"batch_size": self._current_batch_size,
"batch_count": self._batch_count,
}
)
def _login_status(self, account):
try:
return accounts.detect_login(account, path=self.db_path, config=self.config)
except Exception as exc:
return {
"logged_in": False,
"reason": f"LOGIN_CHECK_FAILED: {exc}",
}
def _login_skip_reason(self, status):
reason = status.get("reason")
return f"账号未登录: {reason}" if reason else "账号未登录"
def _batch_ids(self, tasks):
batch_ids = []
for task in tasks:
batch_id = getattr(task, "batch_id", None)
if batch_id and batch_id not in batch_ids:
batch_ids.append(batch_id)
return batch_ids
def _summary(self, ok, total, counters, batch_ids, blocked=False, extra=None):
summary = {
"ok": ok,
"total": total,
"done": counters["done"],
"applied": counters["applied"],
"skipped": counters["skipped"],
"failed": counters["failed"],
"batch_ids": batch_ids,
"dry_run": self.dry_run,
"parallel_accounts": self.parallel_accounts,
"batch_size": self._current_batch_size,
"batch_count": self._batch_count,
"run_id": self._run_id,
}
if blocked:
summary["blocked"] = True
if extra:
summary.update(extra)
return summary
def _create_run_log(self, eligible, batch_ids):
try:
return db.create_run_log(
"apply",
dry_run=self.dry_run,
total=len(eligible),
options={
"batch_ids": batch_ids,
"close_success_tab": self.close_success_tab,
"dry_run": self.dry_run,
"parallel_accounts": self.parallel_accounts,
"max_parallel_accounts": self.max_parallel_accounts,
"batch_size": self._current_batch_size,
"batch_count": self._batch_count,
},
path=self.db_path,
)
except Exception:
return None
def _finish_run_log(self, status, summary):
if self._run_id is None:
return
try:
db.finish_run_log(
self._run_id,
status=status,
done=summary.get("done", 0),
success_count=summary.get("applied", 0),
skipped_count=summary.get("skipped", 0),
failed_count=summary.get("failed", 0),
summary_json=summary,
path=self.db_path,
)
except Exception:
return
def _log_run_event(self, message, task=None, level="info"):
safe_message = diagnostics.redact_log_text(message)
self.log.emit(str(safe_message))
if self._run_id is None:
return
try:
db.add_run_log_event(
self._run_id,
safe_message,
task_id=getattr(task, "id", None),
alias=getattr(task, "alias", None),
item_id=getattr(task, "item_id", None),
level=level,
path=self.db_path,
)
except Exception:
return
def _log_preflight_blocked(self, blocked):
if blocked.get("no_accounts"):
self._log_run_event(
"step=preflight result=blocked detail=当前没有配置账号",
level="warning",
)
for item in blocked.get("duplicate_ports") or []:
self._log_run_event(
"step=preflight result=blocked detail=调试端口重复 debug_port={port} aliases={aliases}".format(
port=item.get("debug_port") or "",
aliases=",".join(item.get("aliases") or []),
),
level="warning",
)
for item in blocked.get("not_running") or []:
self._log_run_event(
"step=check_chrome result=blocked detail=账号 {alias} Chrome 未启动或调试端口不可访问: {reason}".format(
alias=item.get("alias") or "",
reason=item.get("reason") or "",
),
level="warning",
)
for item in blocked.get("logged_out") or []:
self._log_run_event(
"step=login_check result=blocked detail=账号 {alias} 未登录 Shopee: {reason}".format(
alias=item.get("alias") or "",
reason=item.get("reason") or "",
),
level="warning",
)
def _failed_apply_step(self, result, fallback):
if not isinstance(result, dict):
return fallback or "apply_task"
title = result.get("title")
if isinstance(title, dict) and not title.get("ok", True):
return "change_title"
cover = result.get("cover")
if isinstance(cover, dict) and not cover.get("ok", True):
return "replace_cover"
update = result.get("update")
if isinstance(update, dict):
return "click_update"
return fallback or "apply_task"
def _write_diagnostic_log(
self,
message,
level="INFO",
step=None,
task=None,
elapsed_ms=None,
payload=None,
exc=None,
):
_safe_write_diagnostic_log(
message,
level=level,
step=step,
task=task,
elapsed_ms=elapsed_ms,
payload=payload,
exc=exc,
log_dir=self.diagnostic_log_dir,
)
def _elapsed_ms(self, started):
return _elapsed_ms(started)
class CollectWorker(BaseWorker):
"""Collect old title and cover for imported tasks."""
def __init__(
self,
tasks,
db_path=None,
config=None,
preflight=True,
diagnostic_log_dir=None,
):
super().__init__()
self.tasks = list(tasks)
self.db_path = db_path
self.config = config
self.preflight = preflight
self.diagnostic_log_dir = diagnostic_log_dir
self._run_id = None
def execute(self):
account_rows = accounts.list_accounts(path=self.db_path, config=self.config)
account_by_alias = {
str(account.alias).strip(): account
for account in account_rows
if str(account.alias).strip()
}
eligible = [
task for task in self.tasks
if getattr(task, "stage", None) == "imported"
]
batch_ids = self._batch_ids(eligible)
total = len(eligible)
collected = 0
skipped = 0
failed = 0
done = 0
self._run_id = self._create_run_log(eligible, batch_ids)
self._log_run_event(
f"step=preflight result=start detail=采集运行开始 total={total}"
)
if self.preflight:
blocked = self._preflight_block(eligible, account_rows, account_by_alias)
if blocked:
self._log_preflight_blocked(blocked)
summary = self._summary(
ok=False,
total=total,
done=done,
collected=collected,
skipped=skipped,
failed=failed,
batch_ids=batch_ids,
blocked=True,
extra=blocked,
)
self._finish_run_log("blocked", summary)
return summary
self._log_run_event("step=preflight result=success detail=账号检查通过")
else:
self._log_run_event(
"step=preflight result=skipped detail=测试模式跳过采集前检查",
level="warning",
)
for task in eligible:
if self.should_cancel():
break
account = account_by_alias.get(str(task.alias).strip())
if account is None:
skipped += 1
done += 1
reason = "别名未匹配账号"
db.mark_skipped(task.id, reason, path=self.db_path)
self.row_updated.emit(task.id, {"status": "skipped", "last_error": reason})
self._log_run_event(
"step=preflight result=skipped detail=任务 {task_id} 商品 {item_id} {reason}".format(
task_id=task.id,
item_id=task.item_id,
reason=reason,
),
task=task,
level="warning",
)
self._emit_progress(done, total, collected, skipped, failed)
continue
status = self._login_status(account)
if not status.get("logged_in"):
skipped += 1
done += 1
reason = self._login_skip_reason(status)
db.mark_skipped(task.id, reason, path=self.db_path)
self.row_updated.emit(task.id, {"status": "skipped", "last_error": reason})
self._log_run_event(
"step=preflight result=skipped detail=任务 {task_id} 商品 {item_id} {reason}".format(
task_id=task.id,
item_id=task.item_id,
reason=reason,
),
task=task,
level="warning",
)
self._emit_progress(done, total, collected, skipped, failed)
continue
started = time.monotonic()
current_step = "db_write"
def on_step(step):
nonlocal current_step
current_step = str(step)
self._log_run_event(
"step={step} result=start detail=任务 {task_id} 商品 {item_id}".format(
step=current_step,
task_id=task.id,
item_id=task.item_id,
),
task=task,
)
try:
self._log_run_event(
"step=db_write result=start detail=任务 {task_id} 商品 {item_id} 标记采集运行".format(
task_id=task.id,
item_id=task.item_id,
),
task=task,
)
db.mark_running(task.id, "collect", path=self.db_path)
self.row_updated.emit(task.id, {"status": "running"})
result = editor.collect(
account,
{
"item_id": task.item_id,
"old_cover_path": self._old_cover_path(account, task),
},
on_step=on_step,
)
current_step = "db_write"
self._log_run_event(
"step=db_write result=start detail=任务 {task_id} 商品 {item_id} 保存采集结果".format(
task_id=task.id,
item_id=task.item_id,
),
task=task,
)
db.set_collected(
task.id,
result.get("old_title", ""),
result.get("old_cover_path", ""),
path=self.db_path,
)
collected += 1
elapsed_ms = self._elapsed_ms(started)
self.row_updated.emit(
task.id,
{
"stage": "collected",
"status": "success",
"old_title": result.get("old_title", ""),
"old_cover_path": result.get("old_cover_path", ""),
},
)
self._log_run_event(
"step=db_write result=success detail=任务 {task_id} 商品 {item_id} 采集成功 elapsed_ms={elapsed_ms}".format(
task_id=task.id,
item_id=task.item_id,
elapsed_ms=elapsed_ms,
),
task=task,
)
except Exception as exc:
failed += 1
error = str(exc) or exc.__class__.__name__
safe_error = diagnostics.redact_log_text(error)
elapsed_ms = self._elapsed_ms(started)
db.mark_failed(task.id, "collect", safe_error, path=self.db_path)
self.failed.emit(task.id, safe_error)
self.row_updated.emit(task.id, {"status": "failed", "last_error": safe_error})
self._log_run_event(
"step={step} result=failed detail={error} elapsed_ms={elapsed_ms}".format(
step=current_step,
error=safe_error,
elapsed_ms=elapsed_ms,
),
task=task,
level="error",
)
self._write_diagnostic_log(
"采集任务失败",
level="ERROR",
step=current_step,
task=task,
elapsed_ms=elapsed_ms,
payload={"error": safe_error},
exc=exc,
)
finally:
done += 1
self._emit_progress(done, total, collected, skipped, failed)
summary = self._summary(
ok=failed == 0,
total=total,
done=done,
collected=collected,
skipped=skipped,
failed=failed,
batch_ids=batch_ids,
)
self._finish_run_log("cancelled" if self.should_cancel() else "done", summary)
return summary
def _preflight_block(self, eligible, account_rows, account_by_alias):
if not account_rows:
return {
"reason": "NO_ACCOUNTS",
"no_accounts": True,
}
required_accounts = []
seen_aliases = set()
for task in eligible:
alias = str(task.alias).strip()
account = account_by_alias.get(alias)
if account is not None and alias not in seen_aliases:
required_accounts.append(account)
seen_aliases.add(alias)
not_running = []
logged_out = []
for account in required_accounts:
self._log_run_event(
f"step=check_chrome result=start detail=账号 {account.alias} debug_port={account.debug_port}",
level="info",
)
if not chrome.is_running(account.debug_port):
self._log_run_event(
f"step=check_chrome result=blocked detail=账号 {account.alias} CDP 端口未响应 debug_port={account.debug_port}",
level="warning",
)
not_running.append(self._account_payload(account, "CDP 端口未响应"))
continue
self._log_run_event(
f"step=check_chrome result=success detail=账号 {account.alias} debug_port={account.debug_port}",
level="info",
)
self._log_run_event(
f"step=login_check result=start detail=账号 {account.alias}",
level="info",
)
status = self._login_status(account)
if not status.get("logged_in"):
reason = self._login_skip_reason(status)
self._log_run_event(
f"step=login_check result=blocked detail=账号 {account.alias} {reason}",
level="warning",
)
logged_out.append(
self._account_payload(account, reason)
)
else:
self._log_run_event(
f"step=login_check result=success detail=账号 {account.alias}",
level="info",
)
if not_running or logged_out:
return {
"reason": "ACCOUNT_NOT_READY",
"not_running": not_running,
"logged_out": logged_out,
}
return None
def _account_payload(self, account, reason=None):
payload = {
"account_name": account.account_name,
"alias": account.alias,
"debug_port": account.debug_port,
}
if reason:
payload["reason"] = reason
return payload
def _emit_progress(self, done, total, collected, skipped, failed):
self.progress.emit(
{
"done": done,
"total": total,
"collected": collected,
"skipped": skipped,
"failed": failed,
}
)
def _login_status(self, account):
try:
return accounts.detect_login(account, path=self.db_path, config=self.config)
except Exception as exc:
return {
"logged_in": False,
"reason": f"LOGIN_CHECK_FAILED: {exc}",
}
def _login_skip_reason(self, status):
reason = status.get("reason")
return f"账号未登录: {reason}" if reason else "账号未登录"
def _old_cover_path(self, account, task):
image_root = appconfig.image_dir(self.config)
return image_paths.task_image_path(image_root, task, account, "old")
def _batch_ids(self, tasks):
batch_ids = []
for task in tasks:
batch_id = getattr(task, "batch_id", None)
if batch_id and batch_id not in batch_ids:
batch_ids.append(batch_id)
return batch_ids
def _summary(
self,
ok,
total,
done,
collected,
skipped,
failed,
batch_ids,
blocked=False,
extra=None,
):
summary = {
"ok": ok,
"total": total,
"done": done,
"collected": collected,
"skipped": skipped,
"failed": failed,
"batch_ids": batch_ids,
"run_id": self._run_id,
}
if blocked:
summary["blocked"] = True
if extra:
summary.update(extra)
return summary
def _create_run_log(self, eligible, batch_ids):
try:
return db.create_run_log(
"collect",
dry_run=False,
total=len(eligible),
options={
"batch_ids": batch_ids,
"preflight": self.preflight,
},
path=self.db_path,
)
except Exception:
return None
def _finish_run_log(self, status, summary):
if self._run_id is None:
return
try:
db.finish_run_log(
self._run_id,
status=status,
done=summary.get("done", 0),
success_count=summary.get("collected", 0),
skipped_count=summary.get("skipped", 0),
failed_count=summary.get("failed", 0),
summary_json=summary,
path=self.db_path,
)
except Exception:
return
def _log_run_event(self, message, task=None, level="info"):
safe_message = diagnostics.redact_log_text(message)
self.log.emit(str(safe_message))
if self._run_id is None:
return
try:
db.add_run_log_event(
self._run_id,
safe_message,
task_id=getattr(task, "id", None),
alias=getattr(task, "alias", None),
item_id=getattr(task, "item_id", None),
level=level,
path=self.db_path,
)
except Exception:
return
def _log_preflight_blocked(self, blocked):
if blocked.get("no_accounts"):
self._log_run_event(
"step=preflight result=blocked detail=当前没有配置账号",
level="warning",
)
for item in blocked.get("not_running") or []:
self._log_run_event(
"step=preflight result=blocked detail=账号 {alias} Chrome 未启动或调试端口不可访问: {reason}".format(
alias=item.get("alias") or "",
reason=item.get("reason") or "",
),
level="warning",
)
for item in blocked.get("logged_out") or []:
self._log_run_event(
"step=preflight result=blocked detail=账号 {alias} 未登录 Shopee: {reason}".format(
alias=item.get("alias") or "",
reason=item.get("reason") or "",
),
level="warning",
)
def _write_diagnostic_log(
self,
message,
level="INFO",
step=None,
task=None,
elapsed_ms=None,
payload=None,
exc=None,
):
try:
diagnostics.write_diagnostic_log(
message,
level=level,
step=step,
task_id=getattr(task, "id", None),
alias=getattr(task, "alias", None),
item_id=getattr(task, "item_id", None),
elapsed_ms=elapsed_ms,
payload=payload,
exc=exc,
log_dir=self.diagnostic_log_dir,
)
except Exception:
return
def _elapsed_ms(self, started):
return int((time.monotonic() - started) * 1000)
class WriteBackWorker(BaseWorker):
"""Write Excel fields back in a background thread."""
def __init__(self, batch_id, db_path=None, excel_path=None, mode="old", diagnostic_log_dir=None):
super().__init__()
self.batch_id = batch_id
self.db_path = db_path
self.excel_path = excel_path
self.mode = mode
self.diagnostic_log_dir = diagnostic_log_dir
self._run_id = None
def execute(self):
batch_ids = self._batch_ids()
self._run_id = _safe_create_run_log(
"write_back",
db_path=self.db_path,
total=len(batch_ids),
options={
"batch_ids": batch_ids,
"mode": self.mode,
"excel_path": self.excel_path,
},
)
self._log_run_event(
f"step=start result=start detail=Excel 回写开始 mode={self.mode} batch_count={len(batch_ids)}"
)
results = []
try:
for batch_id in batch_ids:
started = time.monotonic()
self._log_run_event(
f"step=write_excel result=start detail=batch_id={batch_id} mode={self.mode}"
)
result = self._write_one(batch_id)
results.append(result)
self._log_run_event(
"step=write_excel result=success detail=batch_id={batch_id} files={files} rows={rows} elapsed_ms={elapsed_ms}".format(
batch_id=batch_id,
files=result.get("files", 0),
rows=result.get("rows", 0),
elapsed_ms=self._elapsed_ms(started),
)
)
except Exception as exc:
error = diagnostics.redact_log_text(str(exc) or exc.__class__.__name__)
self._log_run_event(
f"step=write_excel result=failed detail={error}",
level="error",
)
self._write_diagnostic_log(
"Excel回写失败",
level="ERROR",
step="write_excel",
payload={"batch_ids": batch_ids, "mode": self.mode, "error": error},
exc=exc,
)
_safe_finish_run_log(
self._run_id,
db_path=self.db_path,
status="failed",
done=len(results),
success_count=sum(result.get("rows", 0) for result in results),
failed_count=1,
summary_json={"ok": False, "error": error, "mode": self.mode},
)
raise
result = results[0] if len(results) == 1 else self._combined_result(results)
self.progress.emit(
{
"done": result.get("rows", 0),
"total": result.get("rows", 0),
"files": result.get("files", 0),
}
)
_safe_finish_run_log(
self._run_id,
db_path=self.db_path,
status="done",
done=len(batch_ids),
success_count=result.get("rows", 0),
failed_count=0,
summary_json={"ok": result.get("ok", False), "mode": self.mode, "result": result},
)
return result
def _batch_ids(self):
if isinstance(self.batch_id, (list, tuple, set)):
return list(self.batch_id)
return [self.batch_id]
def _write_one(self, batch_id):
if self.mode == "results":
return excel.write_back_results(
batch_id,
excel_path=self.excel_path,
path=self.db_path,
)
return excel.write_back(
batch_id,
excel_path=self.excel_path,
path=self.db_path,
)
def _combined_result(self, results):
written_files = []
for result in results:
for file_path in result.get("written_files", []):
if file_path not in written_files:
written_files.append(file_path)
return {
"ok": all(result.get("ok", False) for result in results),
"batch_id": [result.get("batch_id") for result in results],
"files": sum(result.get("files", 0) for result in results),
"rows": sum(result.get("rows", 0) for result in results),
"written_files": written_files,
}
def _log_run_event(self, message, level="info"):
safe_message = _safe_add_run_log_event(
self._run_id,
message,
db_path=self.db_path,
level=level,
)
self.log.emit(str(safe_message))
def _write_diagnostic_log(self, message, level="INFO", step=None, payload=None, exc=None):
_safe_write_diagnostic_log(
message,
level=level,
step=step,
payload=payload,
exc=exc,
log_dir=self.diagnostic_log_dir,
)
def _elapsed_ms(self, started):
return _elapsed_ms(started)
class AccountLoginCheckWorker(BaseWorker):
def __init__(self, account, db_path=None, config=None, timeout=8, diagnostic_log_dir=None):
super().__init__()
self.account = account
self.db_path = db_path
self.config = config
self.timeout = timeout
self.diagnostic_log_dir = diagnostic_log_dir
self._run_id = None
def execute(self):
self._run_id = _safe_create_run_log(
"login_check",
db_path=self.db_path,
total=1,
options={
"alias": self.account.alias,
"debug_port": self.account.debug_port,
"timeout": self.timeout,
},
)
started = time.monotonic()
self._log_run_event(
f"step=detect_login result=start detail=账号 {self.account.alias} debug_port={self.account.debug_port}"
)
try:
status = accounts.detect_login(
self.account,
timeout=self.timeout,
path=self.db_path,
config=self.config,
)
except Exception as exc:
error = diagnostics.redact_log_text(str(exc) or exc.__class__.__name__)
elapsed_ms = self._elapsed_ms(started)
self._log_run_event(
f"step=detect_login result=failed detail={error} elapsed_ms={elapsed_ms}",
level="error",
)
self._write_diagnostic_log(
"登录检测失败",
level="ERROR",
step="detect_login",
elapsed_ms=elapsed_ms,
payload={"alias": self.account.alias, "error": error},
exc=exc,
)
_safe_finish_run_log(
self._run_id,
db_path=self.db_path,
status="failed",
done=0,
failed_count=1,
summary_json={"ok": False, "alias": self.account.alias, "error": error},
)
raise
elapsed_ms = self._elapsed_ms(started)
logged_in = bool(status.get("logged_in"))
result_text = "success" if logged_in else "failed"
level = "info" if logged_in else "warning"
self._log_run_event(
"step=detect_login result={result} detail=账号 {alias} logged_in={logged_in} reason={reason} elapsed_ms={elapsed_ms}".format(
result=result_text,
alias=self.account.alias,
logged_in=logged_in,
reason=status.get("reason") or "",
elapsed_ms=elapsed_ms,
),
level=level,
)
_safe_finish_run_log(
self._run_id,
db_path=self.db_path,
status="done",
done=1,
success_count=1 if logged_in else 0,
failed_count=0 if logged_in else 1,
summary_json={"ok": logged_in, "alias": self.account.alias, "status": status},
)
self.row_updated.emit(self.account.id, status)
return {"alias": self.account.alias, "status": status}
def _log_run_event(self, message, level="info"):
safe_message = _safe_add_run_log_event(
self._run_id,
message,
db_path=self.db_path,
account=self.account,
level=level,
)
self.log.emit(str(safe_message))
def _write_diagnostic_log(
self,
message,
level="INFO",
step=None,
elapsed_ms=None,
payload=None,
exc=None,
):
_safe_write_diagnostic_log(
message,
level=level,
step=step,
account=self.account,
elapsed_ms=elapsed_ms,
payload=payload,
exc=exc,
log_dir=self.diagnostic_log_dir,
)
def _elapsed_ms(self, started):
return _elapsed_ms(started)
class AIModelTestWorker(BaseWorker):
"""Test one AI model connection without blocking the GUI thread."""
def __init__(self, model_name, ai_models_path=None, db_path=None, diagnostic_log_dir=None):
super().__init__()
self.model_name = model_name
self.ai_models_path = ai_models_path or appconfig.AI_MODELS_PATH
self.db_path = db_path
self.diagnostic_log_dir = diagnostic_log_dir
self._run_id = None
def execute(self):
self._run_id = self._create_run_log()
started = time.monotonic()
self._log_run_event(
f"step=test_connection result=start detail=AI模型 {self.model_name}"
)
try:
result = appconfig.test_ai_model(self.model_name, path=self.ai_models_path)
except Exception as exc:
error = diagnostics.redact_log_text(str(exc) or exc.__class__.__name__)
elapsed_ms = self._elapsed_ms(started)
self._log_run_event(
f"step=test_connection result=failed detail={error} elapsed_ms={elapsed_ms}",
level="error",
)
self._write_diagnostic_log(
"AI模型测试连接异常",
level="ERROR",
step="test_connection",
elapsed_ms=elapsed_ms,
payload={"model_name": self.model_name, "error": error},
exc=exc,
)
_safe_finish_run_log(
self._run_id,
db_path=self.db_path,
status="failed",
done=0,
failed_count=1,
summary_json={"ok": False, "name": self.model_name, "error": error},
)
raise
elapsed_ms = self._elapsed_ms(started)
payload = dict(appconfig.sanitize_for_log(result or {}))
payload["name"] = self.model_name
ok = bool(payload.get("ok"))
self._log_run_event(
"step=test_connection result={result} detail=AI模型 {name} status={status} error={error} elapsed_ms={elapsed_ms}".format(
result="success" if ok else "failed",
name=self.model_name,
status=payload.get("status") or "",
error=payload.get("error") or "",
elapsed_ms=elapsed_ms,
),
level="info" if ok else "warning",
)
_safe_finish_run_log(
self._run_id,
db_path=self.db_path,
status="done",
done=1,
success_count=1 if ok else 0,
failed_count=0 if ok else 1,
summary_json=payload,
)
return payload
def _create_run_log(self):
if not self.db_path:
return None
return _safe_create_run_log(
"ai_model_test",
db_path=self.db_path,
total=1,
options={"model_name": self.model_name},
)
def _log_run_event(self, message, level="info"):
safe_message = _safe_add_run_log_event(
self._run_id,
message,
db_path=self.db_path,
level=level,
)
self.log.emit(str(safe_message))
def _write_diagnostic_log(
self,
message,
level="INFO",
step=None,
elapsed_ms=None,
payload=None,
exc=None,
):
_safe_write_diagnostic_log(
message,
level=level,
step=step,
elapsed_ms=elapsed_ms,
payload=payload,
exc=exc,
log_dir=self.diagnostic_log_dir,
)
def _elapsed_ms(self, started):
return _elapsed_ms(started)