feat: polish generate run logs

This commit is contained in:
chengma
2026-07-08 12:07:38 +08:00
parent 089a2f5545
commit 40cf79b930
3 changed files with 224 additions and 13 deletions
+108 -11
View File
@@ -2,7 +2,18 @@
from __future__ import annotations from __future__ import annotations
import datetime as _dt
import re
from .widgets import * from .widgets import *
_USER_LOG_URL_RE = re.compile(r"https?://[^\s,,;;))\]]+", re.IGNORECASE)
_USER_LOG_PATH_RE = re.compile(
r"(?i)(/api(?:/v\d+)?/[^\s,,;;))\]]*|/generated/images/[^\s,,;;))\]]*)"
)
class GenerateWorker(BaseWorker): class GenerateWorker(BaseWorker):
"""Generate titles and covers for eligible collected or failed generation tasks.""" """Generate titles and covers for eligible collected or failed generation tasks."""
@@ -28,8 +39,12 @@ class GenerateWorker(BaseWorker):
self._cmhub_points_balance = None self._cmhub_points_balance = None
self._billing_error = None self._billing_error = None
self._billing_stop_requested = False self._billing_stop_requested = False
self._run_started_at_text = ""
self._run_started_monotonic = None
def execute(self): def execute(self):
self._run_started_at_text = self._format_local_time()
self._run_started_monotonic = time.monotonic()
account_rows = accounts.list_accounts(path=self.db_path, config=self.config) account_rows = accounts.list_accounts(path=self.db_path, config=self.config)
account_by_alias = { account_by_alias = {
str(account.alias).strip(): account str(account.alias).strip(): account
@@ -57,7 +72,7 @@ class GenerateWorker(BaseWorker):
if generate_cover: if generate_cover:
if appconfig.ai_backend(self.config) == "cmhub": if appconfig.ai_backend(self.config) == "cmhub":
concurrency = ai.cmhub_image_concurrency_plan(ai_cfg) concurrency = ai.cmhub_image_concurrency_plan(ai_cfg)
start_message = "[开始] 本轮生成 {total} 条:标题{title_total},图片{cover_total};标题并发{title_concurrency},图片并发{image_concurrency},cmhub实际生图并发{request_concurrency},下载并发{download_concurrency}".format( start_message = "[开始] 本轮生成 {total} 条:标题{title_total},图片{cover_total};标题并发{title_concurrency},图片并发{image_concurrency},cmhub实际生图并发{request_concurrency},下载并发{download_concurrency};开始时间 {started_at}".format(
total=len(eligible), total=len(eligible),
title_total=component_totals["title_total"], title_total=component_totals["title_total"],
cover_total=component_totals["cover_total"], cover_total=component_totals["cover_total"],
@@ -65,20 +80,23 @@ class GenerateWorker(BaseWorker):
image_concurrency=concurrency["configured_image_concurrency"], image_concurrency=concurrency["configured_image_concurrency"],
request_concurrency=concurrency["request_concurrency"], request_concurrency=concurrency["request_concurrency"],
download_concurrency=concurrency["download_concurrency"], download_concurrency=concurrency["download_concurrency"],
started_at=self._run_started_at_text,
) )
else: else:
start_message = "[开始] 本轮生成 {total} 条:标题{title_total},图片{cover_total};标题并发{title_concurrency},图片并发{image_concurrency}".format( start_message = "[开始] 本轮生成 {total} 条:标题{title_total},图片{cover_total};标题并发{title_concurrency},图片并发{image_concurrency};开始时间 {started_at}".format(
total=len(eligible), total=len(eligible),
title_total=component_totals["title_total"], title_total=component_totals["title_total"],
cover_total=component_totals["cover_total"], cover_total=component_totals["cover_total"],
title_concurrency=ai_cfg.get("title_concurrency", 1), title_concurrency=ai_cfg.get("title_concurrency", 1),
image_concurrency=ai_cfg.get("image_concurrency", 1), image_concurrency=ai_cfg.get("image_concurrency", 1),
started_at=self._run_started_at_text,
) )
else: else:
start_message = "[开始] 本轮生成 {total} 条:本轮仅生成标题,不生成图片;标题{title_total};标题并发{title_concurrency}".format( start_message = "[开始] 本轮生成 {total} 条:本轮仅生成标题,不生成图片;标题{title_total};标题并发{title_concurrency};开始时间 {started_at}".format(
total=len(eligible), total=len(eligible),
title_total=component_totals["title_total"], title_total=component_totals["title_total"],
title_concurrency=ai_cfg.get("title_concurrency", 1), title_concurrency=ai_cfg.get("title_concurrency", 1),
started_at=self._run_started_at_text,
) )
self._log_run_event(start_message) self._log_run_event(start_message)
try: try:
@@ -113,8 +131,9 @@ class GenerateWorker(BaseWorker):
"cancelled": self.should_cancel(), "cancelled": self.should_cancel(),
"generate_cover": generate_cover, "generate_cover": generate_cover,
} }
user_error = self._user_log_detail(error) or "未知错误"
self._log_run_event( self._log_run_event(
f"[失败] AI 生成运行失败:{error}", f"[失败] AI 生成运行失败:{user_error}",
level="error", level="error",
) )
self._write_diagnostic_log( self._write_diagnostic_log(
@@ -179,7 +198,13 @@ class GenerateWorker(BaseWorker):
phase = payload.get("phase") or "generate" phase = payload.get("phase") or "generate"
step = payload.get("step") or "unknown" step = payload.get("step") or "unknown"
result = payload.get("result") or "start" result = payload.get("result") or "start"
detail = self._short_detail(payload.get("detail")) detail = self._user_log_detail(
payload.get("detail"),
phase=phase,
step=step,
code=payload.get("code"),
status=payload.get("status"),
)
if isinstance(payload.get("metadata"), dict): if isinstance(payload.get("metadata"), dict):
return self._format_cmhub_billing_event(task, phase, payload.get("metadata")) return self._format_cmhub_billing_event(task, phase, payload.get("metadata"))
if phase == "title": if phase == "title":
@@ -206,7 +231,7 @@ class GenerateWorker(BaseWorker):
if result == "start" and step == "cover_submit": if result == "start" and step == "cover_submit":
return f"[图片] {self._task_progress_label(task)} 开始生成" return f"[图片] {self._task_progress_label(task)} 开始生成"
if result == "success" and step == "cover_request": if result == "success" and step == "cover_request":
return f"[图片] {self._task_progress_label(task)} {detail or 'cmhub 已返回,耗时未知'}" return f"[图片] {self._task_progress_label(task)} {detail or 'cmhub 已返回图片,耗时未知'}"
if result == "warning" and step == "cover_download": if result == "warning" and step == "cover_download":
return f"[图片] {self._task_progress_label(task)} {detail or '图片下载较慢'}" return f"[图片] {self._task_progress_label(task)} {detail or '图片下载较慢'}"
if result == "success" and step == "cover_download": if result == "success" and step == "cover_download":
@@ -214,7 +239,7 @@ class GenerateWorker(BaseWorker):
if result == "success" and step == "cover_save": if result == "success" and step == "cover_save":
return f"[图片] {self._task_progress_label(task)} 本地保存完成,{detail or '耗时未知'}" return f"[图片] {self._task_progress_label(task)} 本地保存完成,{detail or '耗时未知'}"
if result == "success" and step == "db_write": if result == "success" and step == "db_write":
suffix = f",已保存 {detail}" if detail else "" suffix = ",已保存到本地" if detail else ""
return f"[图片] {self._task_progress_label(task)} 成功{suffix}" return f"[图片] {self._task_progress_label(task)} 成功{suffix}"
if result == "retry": if result == "retry":
return self._retry_message("图片", task, payload, detail) return self._retry_message("图片", task, payload, detail)
@@ -285,16 +310,88 @@ class GenerateWorker(BaseWorker):
return text[:1997] + "..." return text[:1997] + "..."
return text return text
def _user_log_detail(self, detail, phase=None, step=None, code=None, status=None):
if detail is None:
return ""
raw = diagnostics.redact_log_text(str(detail)).replace("\r", " ").replace("\n", " ").strip()
lowered = raw.lower()
code_text = str(code or "").strip().lower()
if code_text == "insufficient_points" or "点数不足" in raw:
return "点数不足,请先充值"
if code_text == "unauthorized" or "api key 无效" in raw.lower():
return "cmhub API Key 无效,请去⑤设置重填"
if code_text in {"model_not_allowed", "no_pricing_rule"} or "模型别名" in raw or "模型配置不可用" in raw:
return "cmhub 模型别名不可用,请去⑤设置刷新别名并保存"
if code_text == "content_blocked" or "内容安全" in raw:
return "cmhub 内容安全策略拒绝本次生成"
if code_text == "rate_limited" or "rate_limited" in lowered or "请求过于频繁" in raw:
return "cmhub 请求过于频繁,请稍后重试"
if code_text == "connect_timeout" or "连接 cmhub 超时" in raw:
return "连接 cmhub 超时,请检查网络或稍后重试"
if code_text == "read_timeout" or "等待 cmhub 返回超时" in raw:
return "等待 cmhub 返回超时,本条已失败;可稍后重试"
if (
code_text == "not_found"
or "not_found" in lowered
or "接口不存在" in raw
or str(status or "") == "404"
):
return "cmhub 网关接口不可用,请检查⑤设置中的 Base URL,或联系服务方确认网关版本"
if code_text == "upstream_error" or "upstream_error" in lowered or "上游" in raw:
return "cmhub 上游生成失败,请稍后重试"
if "下载 cmhub 图片失败" in raw:
return "下载 cmhub 图片失败,请检查网络后稍后重试"
text = raw.replace("image_url", "图片")
text = text.replace("返回 图片", "返回图片")
text = _USER_LOG_URL_RE.sub("[链接已隐藏]", text)
text = _USER_LOG_PATH_RE.sub("[接口路径已隐藏]", text)
text = text.replace("GET [链接已隐藏]", "请求 cmhub")
text = text.replace("POST [链接已隐藏]", "请求 cmhub")
if len(text) > 180:
return text[:177] + "..."
return text
def _format_generate_completion(self, summary): def _format_generate_completion(self, summary):
progress = self._summary_text(summary) progress = self._summary_text(summary)
billing_error = summary.get("billing_error") or {} billing_error = summary.get("billing_error") or {}
finished_at = self._format_local_time()
elapsed = self._format_run_elapsed()
suffix = f";{self._completion_time_label(summary)} {finished_at},总用时 {elapsed}"
if billing_error: if billing_error:
return f"[失败] AI 生成已中止:{billing_error.get('message') or '点数不足,请先充值'},{progress}" message = self._user_log_detail(billing_error.get("message") or "点数不足,请先充值") or "点数不足,请先充值"
return f"[失败] AI 生成已中止:{message},{progress}{suffix}"
if summary.get("cancelled"): if summary.get("cancelled"):
return f"[停止] AI 生成已停止:{progress}" return f"[停止] AI 生成已停止:{progress}{suffix}"
if summary.get("error"): if summary.get("error"):
return f"[失败] AI 生成失败:{summary.get('error')},{progress}" error = self._user_log_detail(summary.get("error")) or "未知错误"
return f"[完成] AI 生成完成:{progress}" return f"[失败] AI 生成失败:{error},{progress}{suffix}"
return f"[完成] AI 生成完成:{progress}{suffix}"
def _completion_time_label(self, summary):
if summary.get("billing_error") or summary.get("error"):
return "失败时间"
if summary.get("cancelled"):
return "停止时间"
return "完成时间"
def _format_local_time(self):
return _dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
def _format_run_elapsed(self):
if self._run_started_monotonic is None:
return "0秒"
return self._format_duration(time.monotonic() - self._run_started_monotonic)
def _format_duration(self, seconds):
total = max(0, int(seconds or 0))
hours, remainder = divmod(total, 3600)
minutes, seconds = divmod(remainder, 60)
if hours:
return f"{hours}小时{minutes:02d}分{seconds:02d}秒"
if minutes:
return f"{minutes}分{seconds:02d}秒"
return f"{seconds}秒"
def _summary_text(self, summary): def _summary_text(self, summary):
title_total = summary.get("title_total", summary.get("total", 0)) title_total = summary.get("title_total", summary.get("total", 0))
+9 -1
View File
@@ -3,7 +3,7 @@ id: T-556
title: ②AI生成运行日志运营化与错误脱敏 title: ②AI生成运行日志运营化与错误脱敏
phase: 7 phase: 7
deps: [T-519, T-553, T-555] deps: [T-519, T-553, T-555]
status: TODO status: DONE
created: 2026-07-08 created: 2026-07-08
--- ---
@@ -96,3 +96,11 @@ created: 2026-07-08
## 边界(不改什么) ## 边界(不改什么)
本任务只优化②AI生成“用户可见滚动日志/业务运行日志”的文案、脱敏与总结信息;不新增日志窗口、不改进度条布局、不改 T-555 用时标签、不改 cmhub API、下载逻辑、计费 metadata、图片保存格式、SQLite schema、Excel 回写或蝦皮更新流程。 本任务只优化②AI生成“用户可见滚动日志/业务运行日志”的文案、脱敏与总结信息;不新增日志窗口、不改进度条布局、不改 T-555 用时标签、不改 cmhub API、下载逻辑、计费 metadata、图片保存格式、SQLite schema、Excel 回写或蝦皮更新流程。
## 执行记录
- 2026-07-08:完成 `GenerateWorker` 用户日志优化:本轮开始日志增加开始时间,完成/停止/失败总结增加对应时间和总用时。
- 2026-07-08:新增用户可见日志详情清洗逻辑,重试/失败/总结中的 cmhub 错误不再暴露完整 URL、接口路径、媒体路径、`image_url`、query token 或 API Key;`not_found`、超时、上游失败、别名不可用、点数不足等错误映射为可行动中文文案。
- 2026-07-08:保留本地诊断日志分层;`CMSHOPEE_DEBUG_CMHUB_IMAGE_URL=1` 产生的 debug-only 图片 URL 仍只作为本机调试输出,不持久化到 SQLite 运行日志。
- 2026-07-08:更新 GUI worker 单测,覆盖开始/完成时间、总用时、cmhub URL/接口路径脱敏,以及可行动中文错误文案。
- 验证通过:`python -m ruff check app tests main.py`、`py -3.10 -m compileall app main.py`、`py -3.10 -m unittest discover -s tests`、`git diff --check`。
+107 -1
View File
@@ -2375,9 +2375,10 @@ class GuiTests(TempDirMixin, unittest.TestCase):
self.assertEqual(summary["run_id"], run_log.id) self.assertEqual(summary["run_id"], run_log.id)
joined_logs = "\n".join(logs) joined_logs = "\n".join(logs)
self.assertIn("[开始] 本轮生成 1 条", joined_logs) self.assertIn("[开始] 本轮生成 1 条", joined_logs)
self.assertRegex(joined_logs, r"开始时间 \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}")
self.assertIn("[图片] 1/1 商品 51100639510", joined_logs) self.assertIn("[图片] 1/1 商品 51100639510", joined_logs)
self.assertIn("准备重试 1/2", joined_logs) self.assertIn("准备重试 1/2", joined_logs)
self.assertIn("cmhub 已返回 image_url,耗时 91.2秒", joined_logs) self.assertIn("cmhub 已返回图片,耗时 91.2秒", joined_logs)
self.assertIn("下载完成,1.3MB,耗时 12.4秒", joined_logs) self.assertIn("下载完成,1.3MB,耗时 12.4秒", joined_logs)
self.assertIn("图片下载较慢,已用 24.0秒,大小 1.3MB", joined_logs) self.assertIn("图片下载较慢,已用 24.0秒,大小 1.3MB", joined_logs)
self.assertIn("cmhub 图片 URL:https://cdn.example.com/generated.png?token=***", joined_logs) self.assertIn("cmhub 图片 URL:https://cdn.example.com/generated.png?token=***", joined_logs)
@@ -2385,6 +2386,7 @@ class GuiTests(TempDirMixin, unittest.TestCase):
self.assertIn("token=***", joined_logs) self.assertIn("token=***", joined_logs)
self.assertNotIn("SECRET-TOKEN", joined_logs) self.assertNotIn("SECRET-TOKEN", joined_logs)
self.assertIn("[完成] AI 生成完成:标题1/1,图片1/1,失败0", joined_logs) self.assertIn("[完成] AI 生成完成:标题1/1,图片1/1,失败0", joined_logs)
self.assertRegex(joined_logs, r"完成时间 \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},总用时 \d+秒")
events = db.list_run_log_events(summary["run_id"], path=cfg["db_path"]) events = db.list_run_log_events(summary["run_id"], path=cfg["db_path"])
event_messages = "\n".join(event.message for event in events) event_messages = "\n".join(event.message for event in events)
self.assertIn("[图片] 1/1 商品 51100639510", event_messages) self.assertIn("[图片] 1/1 商品 51100639510", event_messages)
@@ -2395,6 +2397,110 @@ class GuiTests(TempDirMixin, unittest.TestCase):
self.assert_removed(temp_dir) self.assert_removed(temp_dir)
def test_generate_worker_user_log_hides_cmhub_request_urls(self):
with self.make_temp_dir() as temp_dir:
cfg = self.make_config(temp_dir)
cfg["ai"] = appconfig.ai_config(cfg)
cfg["ai"]["backend"] = "cmhub"
accounts.create_account("主店", "alias-a", debug_port=9222, config=cfg)
batch_id = db.create_batch(["input.xlsx"], path=cfg["db_path"])
db.insert_tasks(
batch_id,
[
{
"source_file_abs": os.path.join(temp_dir, "input.xlsx"),
"source_sheet": "商品",
"source_row": 2,
"account_name": "Excel主店",
"alias": "alias-a",
"item_id": "51100639510",
}
],
path=cfg["db_path"],
)
task = db.list_tasks(batch_id=batch_id, path=cfg["db_path"])[0]
db.set_collected(task.id, "旧标题", "old.jpg", path=cfg["db_path"])
tasks = db.list_tasks(batch_id=batch_id, path=cfg["db_path"])
logs = []
def fake_generate_batch(tasks_arg, prompt_values, ai_cfg=None, on_progress=None, should_stop=None):
ai_cfg["on_event"](
{
"task": tasks_arg[0],
"phase": "title",
"step": "title_request",
"result": "retry",
"attempt": 1,
"attempts": 3,
"detail": (
"cmhub upstream_error: POST "
"https://cmhub.example.com/api/v1/generate/title "
"failed image_url=http://43.128.3.240:8080/generated/images/a.png?token=SECRET-TOKEN"
),
"level": "warning",
}
)
ai_cfg["on_event"](
{
"task": tasks_arg[0],
"phase": "title",
"step": "title_request",
"result": "failed",
"detail": (
"cmhub not_found: cmhub 接口不存在,请检查 Base URL "
"或该实例是否已部署 /api/v1/models "
"https://cmhub.example.com/api/v1/models"
),
"level": "error",
}
)
return {
"ok": False,
"total": 1,
"title_total": 1,
"title_done": 0,
"cover_done": 0,
"cover_total": 0,
"generated_done": 0,
"failed": 1,
"generate_cover": False,
"error": (
"cmhub not_found: GET "
"https://cmhub.example.com/api/v1/models?token=SECRET-TOKEN"
),
}
worker = GenerateWorker(
tasks,
{"title": "标题提示", "cover": "封面提示"},
db_path=cfg["db_path"],
config=cfg,
)
worker.log.connect(logs.append)
with mock.patch("app.gui.ai.generate_batch", side_effect=fake_generate_batch):
summary = worker.execute()
self.assertFalse(summary["ok"])
joined_logs = "\n".join(logs)
self.assertIn("cmhub 上游生成失败,请稍后重试", joined_logs)
self.assertIn("cmhub 网关接口不可用,请检查⑤设置中的 Base URL", joined_logs)
self.assertRegex(joined_logs, r"失败时间 \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},总用时 \d+秒")
events = db.list_run_log_events(summary["run_id"], path=cfg["db_path"])
event_messages = "\n".join(event.message for event in events)
combined = joined_logs + "\n" + event_messages
for forbidden in [
"https://",
"http://",
"/api/v1/",
"/generated/images/",
"image_url",
"SECRET-TOKEN",
]:
self.assertNotIn(forbidden, combined)
self.assert_removed(temp_dir)
def test_generate_worker_records_cmhub_billing_metadata(self): def test_generate_worker_records_cmhub_billing_metadata(self):
with self.make_temp_dir() as temp_dir: with self.make_temp_dir() as temp_dir:
cfg = self.make_config(temp_dir) cfg = self.make_config(temp_dir)