T-564 async cmhub image tasks
This commit is contained in:
@@ -24,6 +24,11 @@ from requests.adapters import HTTPAdapter
|
||||
from . import appconfig, db, diagnostics, image_paths
|
||||
from . import prompts as prompt_module
|
||||
|
||||
try:
|
||||
from .version import APP_VERSION
|
||||
except Exception:
|
||||
APP_VERSION = "dev"
|
||||
|
||||
|
||||
|
||||
class AIError(RuntimeError):
|
||||
@@ -43,6 +48,9 @@ class CMHubError(AIError):
|
||||
CMHUB_IMAGE_MAX_BYTES = 20 * 1024 * 1024
|
||||
CMHUB_TITLE_READ_TIMEOUT_SECONDS = 600
|
||||
CMHUB_IMAGE_READ_TIMEOUT_SECONDS = 900
|
||||
CMHUB_IMAGE_SUBMIT_READ_TIMEOUT_SECONDS = 30
|
||||
CMHUB_IMAGE_POLL_READ_TIMEOUT_SECONDS = 15
|
||||
CMHUB_IMAGE_POLL_DELAYS_SECONDS = (3.0, 5.0, 8.0, 10.0)
|
||||
CMHUB_IMAGE_CONCURRENCY_LIMIT = 5
|
||||
CMHUB_IMAGE_DOWNLOAD_ATTEMPTS = 3
|
||||
CMHUB_IMAGE_SLOW_DOWNLOAD_SECONDS = 20.0
|
||||
@@ -685,7 +693,7 @@ def generate_batch(tasks, prompts, ai_cfg=None, on_progress=None, should_stop=No
|
||||
try:
|
||||
rendered_cover_prompt, new_cover_path = prepare_cover_task(task, new_title)
|
||||
future = request_executor.submit(
|
||||
_request_cmhub_cover_image,
|
||||
_request_cmhub_cover_image_async,
|
||||
rendered_cover_prompt,
|
||||
getattr(task, "old_cover_path", "") or "",
|
||||
new_cover_path,
|
||||
@@ -696,6 +704,9 @@ def generate_batch(tasks, prompts, ai_cfg=None, on_progress=None, should_stop=No
|
||||
cmhub_config_path=cmhub_config_path,
|
||||
on_step=step_callback(task, "cover"),
|
||||
on_event=step_callback(task, "cover"),
|
||||
task=task,
|
||||
db_path=db_path,
|
||||
should_stop=should_stop,
|
||||
)
|
||||
request_futures[future] = (task, new_title)
|
||||
return True
|
||||
@@ -881,7 +892,7 @@ def _gen_cover_cmhub(
|
||||
on_step=None,
|
||||
on_event=None,
|
||||
):
|
||||
request_result = _request_cmhub_cover_image(
|
||||
request_result = _request_cmhub_cover_image_sync(
|
||||
cover_prompt,
|
||||
old_cover_path,
|
||||
out_path,
|
||||
@@ -909,6 +920,34 @@ def _request_cmhub_cover_image(
|
||||
on_step=None,
|
||||
on_event=None,
|
||||
validate_input=True,
|
||||
):
|
||||
return _request_cmhub_cover_image_sync(
|
||||
cover_prompt,
|
||||
old_cover_path,
|
||||
out_path,
|
||||
resolution=resolution,
|
||||
jpg_quality=jpg_quality,
|
||||
retry=retry,
|
||||
config=config,
|
||||
cmhub_config_path=cmhub_config_path,
|
||||
on_step=on_step,
|
||||
on_event=on_event,
|
||||
validate_input=validate_input,
|
||||
)
|
||||
|
||||
|
||||
def _request_cmhub_cover_image_sync(
|
||||
cover_prompt,
|
||||
old_cover_path,
|
||||
out_path,
|
||||
resolution,
|
||||
jpg_quality,
|
||||
retry,
|
||||
config,
|
||||
cmhub_config_path,
|
||||
on_step=None,
|
||||
on_event=None,
|
||||
validate_input=True,
|
||||
):
|
||||
if validate_input:
|
||||
old_cover_path, out_path = _prepare_cover_input(old_cover_path, out_path, on_step)
|
||||
@@ -977,6 +1016,249 @@ def _request_cmhub_cover_image(
|
||||
}
|
||||
|
||||
|
||||
def _request_cmhub_cover_image_async(
|
||||
cover_prompt,
|
||||
old_cover_path,
|
||||
out_path,
|
||||
resolution,
|
||||
jpg_quality,
|
||||
retry,
|
||||
config,
|
||||
cmhub_config_path,
|
||||
on_step=None,
|
||||
on_event=None,
|
||||
task=None,
|
||||
db_path=None,
|
||||
should_stop=None,
|
||||
validate_input=True,
|
||||
):
|
||||
if validate_input:
|
||||
old_cover_path, out_path = _prepare_cover_input(old_cover_path, out_path, on_step)
|
||||
should_stop = should_stop or (lambda: False)
|
||||
task_id = int(getattr(task, "id", 0) or 0)
|
||||
if task_id <= 0:
|
||||
raise AIError("缺少本地任务ID,无法提交 cmhub 生图任务")
|
||||
ai_cfg = appconfig.ai_config(config)
|
||||
_notify_step(on_step, "load_image_model")
|
||||
runtime = _cmhub_runtime(config, "image", cmhub_config_path)
|
||||
resolution = str(resolution or ai_cfg.get("resolution", "1k"))
|
||||
quality = _jpg_quality(jpg_quality if jpg_quality is not None else ai_cfg.get("jpg_quality", 90))
|
||||
_notify_step(on_step, "cover_build_request")
|
||||
payload = {
|
||||
"prompt": str(cover_prompt or ""),
|
||||
"model": runtime["alias"],
|
||||
"image_base64": _image_data_url(old_cover_path),
|
||||
"resolution": _normalize_cmhub_resolution(resolution),
|
||||
"aspect_ratio": "1:1",
|
||||
}
|
||||
request_result = {
|
||||
"connect_timeout": runtime["connect_timeout"],
|
||||
"read_timeout": _cmhub_read_timeout(config, resolution),
|
||||
"out_path": out_path,
|
||||
"resolution": resolution,
|
||||
"quality": quality,
|
||||
"use_system_proxy": runtime["use_system_proxy"],
|
||||
"download_with_curl": runtime["download_with_curl"],
|
||||
}
|
||||
existing_image_task_id = str(getattr(task, "image_task_id", "") or "").strip()
|
||||
if existing_image_task_id:
|
||||
_notify_step_event(
|
||||
on_step,
|
||||
"cover_request",
|
||||
detail="发现未完成的 cmhub 生图任务,继续查询结果",
|
||||
)
|
||||
return _poll_cmhub_cover_image_task(
|
||||
existing_image_task_id,
|
||||
runtime,
|
||||
request_result,
|
||||
task_id=task_id,
|
||||
db_path=db_path,
|
||||
should_stop=should_stop,
|
||||
on_step=on_step,
|
||||
)
|
||||
|
||||
image_task_key = str(getattr(task, "image_task_key", "") or "").strip()
|
||||
if not image_task_key:
|
||||
image_task_key = db.ensure_image_task_key(task_id, path=db_path)
|
||||
_raise_if_cmhub_cover_cancelled(should_stop, on_step)
|
||||
attempts = _attempt_count(ai_cfg, retry)
|
||||
_notify_step(on_step, "cover_request")
|
||||
request_started = time.perf_counter()
|
||||
try:
|
||||
data = _cmhub_call_with_retry(
|
||||
"POST",
|
||||
appconfig.cmhub_request_url(runtime["base_url"], "/api/v1/generate/image/tasks"),
|
||||
runtime["api_key"],
|
||||
payload=payload,
|
||||
connect_timeout=runtime["connect_timeout"],
|
||||
read_timeout=CMHUB_IMAGE_SUBMIT_READ_TIMEOUT_SECONDS,
|
||||
attempts=attempts,
|
||||
headers_extra={
|
||||
"Idempotency-Key": image_task_key,
|
||||
"X-Client-Version": str(APP_VERSION),
|
||||
},
|
||||
on_retry=lambda attempt, total_attempts, exc: _notify_cmhub_retry(
|
||||
on_step,
|
||||
"cover_request",
|
||||
attempt,
|
||||
total_attempts,
|
||||
exc,
|
||||
),
|
||||
)
|
||||
except CMHubError as exc:
|
||||
if exc.code in {
|
||||
"bad_request",
|
||||
"content_blocked",
|
||||
"insufficient_points",
|
||||
"idempotency_conflict",
|
||||
"unauthorized",
|
||||
"account_disabled",
|
||||
"model_not_allowed",
|
||||
"no_pricing_rule",
|
||||
"not_found",
|
||||
}:
|
||||
db.clear_image_task(task_id, path=db_path)
|
||||
raise
|
||||
request_elapsed = time.perf_counter() - request_started
|
||||
_emit_cmhub_metadata(on_event, data, "cover_request")
|
||||
image_task_id = str(data.get("task_id") or "").strip()
|
||||
if not image_task_id:
|
||||
raise CMHubError("bad_response", "cmhub 生图任务提交返回格式错误", retryable=False)
|
||||
db.set_image_task_submitted(task_id, image_task_id, image_task_key, path=db_path)
|
||||
_notify_step_event(
|
||||
on_step,
|
||||
"cover_request",
|
||||
detail="cmhub 已提交生图任务 %s,耗时 %s"
|
||||
% (image_task_id, _format_seconds(request_elapsed)),
|
||||
)
|
||||
return _poll_cmhub_cover_image_task(
|
||||
image_task_id,
|
||||
runtime,
|
||||
request_result,
|
||||
task_id=task_id,
|
||||
db_path=db_path,
|
||||
should_stop=should_stop,
|
||||
on_step=on_step,
|
||||
)
|
||||
|
||||
|
||||
def _poll_cmhub_cover_image_task(
|
||||
image_task_id,
|
||||
runtime,
|
||||
request_result,
|
||||
task_id,
|
||||
db_path,
|
||||
should_stop,
|
||||
on_step=None,
|
||||
):
|
||||
poll_url = appconfig.cmhub_request_url(
|
||||
runtime["base_url"],
|
||||
"/api/v1/generate/image/tasks/%s" % urllib.parse.quote(str(image_task_id), safe=""),
|
||||
)
|
||||
started = time.perf_counter()
|
||||
deadline = started + max(1, int(CMHUB_IMAGE_READ_TIMEOUT_SECONDS))
|
||||
poll_index = 0
|
||||
while True:
|
||||
_raise_if_cmhub_cover_cancelled(should_stop, on_step)
|
||||
if time.perf_counter() >= deadline:
|
||||
raise CMHubError(
|
||||
"read_timeout",
|
||||
"等待 cmhub 生图任务完成超时,下次可继续查询",
|
||||
retryable=False,
|
||||
)
|
||||
_notify_step(on_step, "cover_poll")
|
||||
try:
|
||||
data = _cmhub_call_once(
|
||||
"GET",
|
||||
poll_url,
|
||||
runtime["api_key"],
|
||||
payload=None,
|
||||
connect_timeout=runtime["connect_timeout"],
|
||||
read_timeout=CMHUB_IMAGE_POLL_READ_TIMEOUT_SECONDS,
|
||||
headers_extra={"X-Client-Version": str(APP_VERSION)},
|
||||
)
|
||||
except CMHubError as exc:
|
||||
if exc.code in {"connect_timeout", "read_timeout", "network_error", "rate_limited"}:
|
||||
_notify_step_event(
|
||||
on_step,
|
||||
"cover_poll",
|
||||
result="retry",
|
||||
detail=str(exc),
|
||||
level="warning",
|
||||
)
|
||||
_sleep_cmhub_poll(poll_index, should_stop, on_step)
|
||||
poll_index += 1
|
||||
continue
|
||||
db.clear_image_task(task_id, path=db_path)
|
||||
raise
|
||||
status = str(data.get("status") or "").strip().lower()
|
||||
if status in {"queued", "running"}:
|
||||
_notify_step_event(
|
||||
on_step,
|
||||
"cover_poll",
|
||||
detail="cmhub 生图任务%s,继续等待" % ("排队中" if status == "queued" else "生成中"),
|
||||
)
|
||||
_sleep_cmhub_poll(poll_index, should_stop, on_step)
|
||||
poll_index += 1
|
||||
continue
|
||||
if status == "succeeded":
|
||||
result = data.get("result") if isinstance(data.get("result"), dict) else {}
|
||||
image_url = str(result.get("image_url") or data.get("image_url") or "").strip()
|
||||
if not image_url:
|
||||
raise CMHubError("bad_response", "cmhub 生图任务成功但没有图片地址", retryable=False)
|
||||
_notify_step(on_step, "cover_parse_response")
|
||||
elapsed = time.perf_counter() - started
|
||||
_notify_step_event(
|
||||
on_step,
|
||||
"cover_request",
|
||||
detail="cmhub 已返回 image_url,耗时 %s" % _format_seconds(elapsed),
|
||||
)
|
||||
if _debug_cmhub_image_url_enabled():
|
||||
_notify_step_event(
|
||||
on_step,
|
||||
"cover_image_url",
|
||||
result="debug",
|
||||
detail="cmhub 图片 URL:%s" % image_url,
|
||||
level="warning",
|
||||
debug_only=True,
|
||||
)
|
||||
merged = dict(request_result)
|
||||
merged["image_url"] = image_url
|
||||
return merged
|
||||
if status in {"failed", "expired"}:
|
||||
error = data.get("error") if isinstance(data.get("error"), dict) else {}
|
||||
code = _normalize_cmhub_error_code(error.get("code") or status)
|
||||
message = _cmhub_user_message(code, str(error.get("message") or status))
|
||||
db.clear_image_task(task_id, path=db_path)
|
||||
raise CMHubError(code, message, retryable=_cmhub_retryable(code))
|
||||
raise CMHubError("bad_response", "cmhub 生图任务状态返回格式错误", retryable=False)
|
||||
|
||||
|
||||
def _sleep_cmhub_poll(poll_index, should_stop, on_step=None):
|
||||
_raise_if_cmhub_cover_cancelled(should_stop, on_step)
|
||||
delays = CMHUB_IMAGE_POLL_DELAYS_SECONDS
|
||||
delay = delays[min(max(0, int(poll_index)), len(delays) - 1)]
|
||||
time.sleep(max(0.0, float(delay)))
|
||||
_raise_if_cmhub_cover_cancelled(should_stop, on_step)
|
||||
|
||||
|
||||
def _raise_if_cmhub_cover_cancelled(should_stop, on_step=None):
|
||||
try:
|
||||
cancelled = bool(should_stop and should_stop())
|
||||
except Exception:
|
||||
cancelled = False
|
||||
if not cancelled:
|
||||
return
|
||||
_notify_step_event(
|
||||
on_step,
|
||||
"cover_poll",
|
||||
result="cancelled",
|
||||
detail="已停止等待生图结果;服务端任务可能仍在完成,下次可继续查询",
|
||||
level="warning",
|
||||
)
|
||||
raise CancelledError()
|
||||
|
||||
|
||||
def _download_and_save_cmhub_cover(request_result, on_step=None):
|
||||
image_url = request_result["image_url"]
|
||||
connect_timeout = request_result["connect_timeout"]
|
||||
@@ -1155,6 +1437,7 @@ def _cmhub_call_with_retry(
|
||||
read_timeout,
|
||||
attempts,
|
||||
on_retry=None,
|
||||
headers_extra=None,
|
||||
):
|
||||
attempts = max(1, int(attempts or 1))
|
||||
last_exc = None
|
||||
@@ -1167,6 +1450,7 @@ def _cmhub_call_with_retry(
|
||||
payload,
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
headers_extra=headers_extra,
|
||||
)
|
||||
except CMHubError as exc:
|
||||
last_exc = exc
|
||||
@@ -1181,11 +1465,13 @@ def _cmhub_call_with_retry(
|
||||
raise last_exc
|
||||
|
||||
|
||||
def _cmhub_call_once(method, url, api_key, payload, connect_timeout, read_timeout):
|
||||
def _cmhub_call_once(method, url, api_key, payload, connect_timeout, read_timeout, headers_extra=None):
|
||||
headers = {
|
||||
"Authorization": "Bearer " + str(api_key),
|
||||
"Accept": "application/json",
|
||||
}
|
||||
if headers_extra:
|
||||
headers.update({str(key): str(value) for key, value in dict(headers_extra).items()})
|
||||
request_kwargs = {
|
||||
"headers": headers,
|
||||
"timeout": (max(1, int(connect_timeout)), max(1, int(read_timeout))),
|
||||
@@ -1262,7 +1548,13 @@ def _cmhub_code_for_status(status):
|
||||
|
||||
|
||||
def _cmhub_retryable(code):
|
||||
return str(code) in {"upstream_error", "rate_limited", "connect_timeout"}
|
||||
return str(code) in {
|
||||
"upstream_error",
|
||||
"upstream_timeout",
|
||||
"task_timeout",
|
||||
"rate_limited",
|
||||
"connect_timeout",
|
||||
}
|
||||
|
||||
|
||||
def _cmhub_user_message(code, message):
|
||||
@@ -1275,6 +1567,9 @@ def _cmhub_user_message(code, message):
|
||||
"no_pricing_rule": "cmhub 模型别名未配置价格",
|
||||
"content_blocked": "cmhub 内容安全策略拒绝本次生成",
|
||||
"upstream_error": "cmhub 上游生成失败,请稍后重试",
|
||||
"upstream_timeout": "cmhub 上游生成超时,点数已退回,请稍后重试",
|
||||
"task_timeout": "cmhub 生图任务超时,点数已退回,请稍后重试",
|
||||
"idempotency_conflict": "cmhub 生图幂等键冲突,请重新生成",
|
||||
"rate_limited": "cmhub 请求过于频繁,请稍后重试",
|
||||
"not_found": "cmhub 接口不存在,请检查 Base URL 或该实例是否已部署 /api/v1/models",
|
||||
}
|
||||
|
||||
@@ -46,6 +46,7 @@ FAILURE_STEP_LABELS = {
|
||||
"cover_submit": "提交封面任务",
|
||||
"cover_build_request": "构建封面请求",
|
||||
"cover_request": "请求生成封面",
|
||||
"cover_poll": "等待封面结果",
|
||||
"cover_download": "下载新封面",
|
||||
"cover_save": "保存新封面",
|
||||
"cover_parse_response": "解析封面响应",
|
||||
@@ -138,6 +139,8 @@ class Task:
|
||||
old_cover_path: Optional[str]
|
||||
new_title: Optional[str]
|
||||
new_cover_path: Optional[str]
|
||||
image_task_id: Optional[str]
|
||||
image_task_key: Optional[str]
|
||||
committed: int
|
||||
stage: str
|
||||
status: str
|
||||
@@ -231,6 +234,8 @@ CREATE TABLE IF NOT EXISTS tasks (
|
||||
old_cover_path TEXT,
|
||||
new_title TEXT,
|
||||
new_cover_path TEXT,
|
||||
image_task_id TEXT,
|
||||
image_task_key TEXT,
|
||||
committed INTEGER NOT NULL DEFAULT 0,
|
||||
stage TEXT NOT NULL DEFAULT 'imported',
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
@@ -355,6 +360,7 @@ def init_db(path=None, conn=None) -> None:
|
||||
with database:
|
||||
database.executescript(SCHEMA_SQL)
|
||||
_ensure_batch_delete_columns(database)
|
||||
_ensure_task_image_task_columns(database)
|
||||
|
||||
|
||||
def _ensure_batch_delete_columns(database):
|
||||
@@ -364,6 +370,14 @@ def _ensure_batch_delete_columns(database):
|
||||
if "deleted_reason" not in columns:
|
||||
database.execute("ALTER TABLE batches ADD COLUMN deleted_reason TEXT")
|
||||
|
||||
|
||||
def _ensure_task_image_task_columns(database):
|
||||
columns = {row["name"] for row in database.execute("PRAGMA table_info(tasks)").fetchall()}
|
||||
if "image_task_id" not in columns:
|
||||
database.execute("ALTER TABLE tasks ADD COLUMN image_task_id TEXT")
|
||||
if "image_task_key" not in columns:
|
||||
database.execute("ALTER TABLE tasks ADD COLUMN image_task_key TEXT")
|
||||
|
||||
def create_batch(file_paths: Iterable[str], note=None, path=None, conn=None) -> str:
|
||||
batch_id = datetime.now().strftime("%Y%m%d_%H%M%S_") + uuid.uuid4().hex[:8]
|
||||
files = [os.path.abspath(file_path) for file_path in file_paths]
|
||||
@@ -688,6 +702,79 @@ def mark_skipped(task_id, reason, path=None, conn=None) -> None:
|
||||
)
|
||||
|
||||
|
||||
def ensure_image_task_key(task_id, path=None, conn=None) -> str:
|
||||
"""Return a stable cmhub image idempotency key for the current cover attempt."""
|
||||
|
||||
task_id = int(task_id)
|
||||
with _connection(conn, path) as database:
|
||||
with database:
|
||||
row = database.execute(
|
||||
"SELECT image_task_key FROM tasks WHERE id = ?",
|
||||
(task_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise DbError(f"任务不存在: {task_id}")
|
||||
key = str(row["image_task_key"] or "").strip()
|
||||
if not key:
|
||||
key = f"cmshopee-task-{task_id}-{uuid.uuid4().hex}"
|
||||
database.execute(
|
||||
"""
|
||||
UPDATE tasks
|
||||
SET image_task_key = ?, updated_at = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
(key, _now(), task_id),
|
||||
)
|
||||
return key
|
||||
|
||||
|
||||
def set_image_task_submitted(task_id, image_task_id, image_task_key=None, path=None, conn=None) -> None:
|
||||
"""Persist the cmhub image task id immediately after submit succeeds."""
|
||||
|
||||
task_id = int(task_id)
|
||||
image_task_id = str(image_task_id or "").strip()
|
||||
if not image_task_id:
|
||||
raise DbError("缺少 cmhub 生图任务ID")
|
||||
with _connection(conn, path) as database:
|
||||
with database:
|
||||
row = database.execute(
|
||||
"SELECT image_task_key FROM tasks WHERE id = ?",
|
||||
(task_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise DbError(f"任务不存在: {task_id}")
|
||||
key = str(image_task_key or row["image_task_key"] or "").strip()
|
||||
if not key:
|
||||
key = f"cmshopee-task-{task_id}-{uuid.uuid4().hex}"
|
||||
database.execute(
|
||||
"""
|
||||
UPDATE tasks
|
||||
SET image_task_id = ?,
|
||||
image_task_key = ?,
|
||||
updated_at = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
(image_task_id, key, _now(), task_id),
|
||||
)
|
||||
|
||||
|
||||
def clear_image_task(task_id, path=None, conn=None) -> None:
|
||||
"""Clear persisted cmhub image task state so the next cover run can submit anew."""
|
||||
|
||||
with _connection(conn, path) as database:
|
||||
with database:
|
||||
database.execute(
|
||||
"""
|
||||
UPDATE tasks
|
||||
SET image_task_id = NULL,
|
||||
image_task_key = NULL,
|
||||
updated_at = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
(_now(), int(task_id)),
|
||||
)
|
||||
|
||||
|
||||
def set_collected(task_id, old_title, old_cover_path, path=None, conn=None) -> None:
|
||||
now = _now()
|
||||
with _connection(conn, path) as database:
|
||||
@@ -821,19 +908,23 @@ def reset_generated(
|
||||
now = _now()
|
||||
new_title = None if reset_title else before.new_title
|
||||
new_cover = None if reset_cover else before.new_cover_path
|
||||
image_task_id = None if reset_cover else before.image_task_id
|
||||
image_task_key = None if reset_cover else before.image_task_key
|
||||
with database:
|
||||
database.execute(
|
||||
"""
|
||||
UPDATE tasks
|
||||
SET new_title = ?,
|
||||
new_cover_path = ?,
|
||||
image_task_id = ?,
|
||||
image_task_key = ?,
|
||||
stage = 'generated',
|
||||
status = 'success',
|
||||
last_error = NULL,
|
||||
updated_at = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
(new_title, new_cover, now, int(task_id)),
|
||||
(new_title, new_cover, image_task_id, image_task_key, now, int(task_id)),
|
||||
)
|
||||
after = get_task(task_id, conn=database)
|
||||
return {
|
||||
|
||||
@@ -253,6 +253,8 @@ CREATE TABLE tasks (
|
||||
-- AI 输出
|
||||
new_title TEXT,
|
||||
new_cover_path TEXT, -- 新封面本地图片路径
|
||||
image_task_id TEXT, -- cmhub 异步生图任务ID;用于重启/重试续查
|
||||
image_task_key TEXT, -- 本次生图提交幂等键;用于 submit 网络抖动重发
|
||||
-- 应用
|
||||
committed INTEGER NOT NULL DEFAULT 0, -- 是否成功点「更新」提交
|
||||
stage TEXT NOT NULL DEFAULT 'imported', -- imported/collected/generated/applied
|
||||
@@ -403,9 +405,9 @@ data/images/<batch_id>/<slug>/<task_id>_<item_id>_new.<ext> # AI 生成的新
|
||||
单个「开始生成」按钮,标题必生成,封面按本轮成本开关可选生成:
|
||||
|
||||
1. **并发生成标题**:线程池大小 = `title_concurrency`,调 `gen_title(标题提示词, old_title)` → new_title。默认 `backend=cmhub`,调用 `POST /api/v1/generate/title` 并使用 `title_alias`;`backend=direct` 仅内部兼容时使用 `default_text_model`。
|
||||
2. **若②勾选「生成封面图片(成本较高)」**:接着并发生成图片,线程池大小 = `image_concurrency`,调 `gen_cover(封面提示词, old_cover_path, resolution, jpg_quality)` → 新图存 `data/images/<batch_id>/<slug>/<task_id>_<item_id>_new.jpg`。默认 `backend=cmhub`,调用 `POST /api/v1/generate/image`,拿 `image_url` 后立即安全下载并转本地 JPEG;`backend=direct` 仅内部兼容时使用 `default_image_model`。
|
||||
- T-545 已实现:cmhub 模式下不再直接按用户填写的 `image_concurrency` 全量打到网关;实际生图请求并发 = `min(image_concurrency, 5)`。拿到 `image_url` 后交给独立下载/保存线程池,下载线程数量与实际生图请求并发一致,同样最大 5。这样批量时“下一批生图请求”和“上一批图片下载/保存”可以流水线并行,但不会对 cmhub 生图接口或 `/media/generated/images/*.png` 静态下载打出超过 5 的并发。direct 兼容路径暂不改变。
|
||||
- cmhub 生图连接超时取 `ai.cmhub.connect_timeout`(默认 66 秒);生图请求和随后 `image_url` 下载的读取等待统一固定 900 秒,读超时不自动重发,避免重复扣点。下载层最多安全重试 3 次,只复用同一个 `image_url`,不重新请求 cmhub 生图;下载总耗时超过 20 秒时写“图片下载较慢”警告。`backend=direct` 兼容路径仍按模型 `timeout_seconds` 或 `resolution_timeouts[resolution]`(512→180/1k→240/2k→360/4k→600)取返回超时。
|
||||
2. **若②勾选「生成封面图片(成本较高)」**:接着并发生成图片,线程池大小 = `image_concurrency`,调 `generate_batch()` 的封面阶段 → 新图存 `data/images/<batch_id>/<slug>/<task_id>_<item_id>_new.jpg`。默认 `backend=cmhub` 的②批量生成走异步任务接口:先 `POST /api/v1/generate/image/tasks` 提交并预扣点,带 `Idempotency-Key` 与 `X-Client-Version`,submit 成功后立即把 `task_id` 写入 `tasks.image_task_id`;再循环 `GET /api/v1/generate/image/tasks/{task_id}` 轮询,`succeeded` 后取 `result.image_url` 并安全下载转本地 JPEG。若本地已有 `image_task_id`,下次开始生成直接续查,不重新 submit、不重复扣点。`gen_cover()` 单独直接调用无 DB 上下文,仍保留旧同步 `POST /api/v1/generate/image` 兼容路径;`backend=direct` 仅内部兼容时使用 `default_image_model`。
|
||||
- T-545/T-564 已实现:cmhub 模式下不再直接按用户填写的 `image_concurrency` 全量打到网关;实际 submit+poll 在途并发 = `min(image_concurrency, 5)`。拿到 `image_url` 后交给独立下载/保存线程池,下载线程数量与实际生图并发一致,同样最大 5。这样批量时“下一批生图任务”和“上一批图片下载/保存”可以流水线并行,但不会对 cmhub 生图任务接口或 `/media/generated/images/*.png` 静态下载打出超过 5 的并发。direct 兼容路径暂不改变。
|
||||
- cmhub 生图连接超时取 `ai.cmhub.connect_timeout`(默认 66 秒);异步 submit 读取等待 30 秒,poll 单次读取等待 15 秒,本地总等待预算 900 秒,撞预算或用户停止时保留 `image_task_id/image_task_key` 供下次续查;poll 返回 `failed/expired`(cmhub 已退点)时清空 `image_task_id/image_task_key`,后续重试会生成新的幂等键并重新 submit。下载层读取等待 900 秒,最多安全重试 3 次,只复用同一个 `image_url`,不重新请求 cmhub 生图;下载总耗时超过 20 秒时写“图片下载较慢”警告。`backend=direct` 兼容路径仍按模型 `timeout_seconds` 或 `resolution_timeouts[resolution]`(512→180/1k→240/2k→360/4k→600)取返回超时。
|
||||
- T-548 已实现:cmhub 图片下载后端新增 `ai.cmhub.download_with_curl`(`auto`/`true`/`false`,默认 `auto`)。Windows 且检测到系统 curl 时优先用 curl 下载,否则或 curl 执行失败时回退 requests;生成、models、balance 仍走共享 requests Session。curl 下载前仍执行公网 URL 校验;URL 写入临时 curl 配置文件并通过 `-K` 传入,避免带 token 的 `image_url` 出现在进程命令行;`use_system_proxy=false` 时 curl 加 `--noproxy "*"`。
|
||||
- cmhub 返回的完整 `image_url` 默认只在内存中临时用于下载,不写入 `tasks` 或 `run_log_events`。本机调试时可设置环境变量 `CMSHOPEE_DEBUG_CMHUB_IMAGE_URL=1`,②本轮可见运行日志会显示脱敏后的 URL 调试行,且该行不持久化到 SQLite。
|
||||
3. **若未勾选生成封面**:标题成功后立即写 `new_title`,`new_cover_path=NULL`,不渲染封面提示词、不调用 `gen_cover()`、不创建本地新封面文件。
|
||||
|
||||
+10
-5
@@ -110,6 +110,10 @@ mark_failed(task_id, phase, error) -> None # status=failed,stage 不前进
|
||||
mark_skipped(task_id, reason) -> None # status=skipped,stage 不前进
|
||||
set_collected(task_id, old_title, old_cover_path) -> None # stage=collected,status=success,collect_attempts+1
|
||||
set_generated(task_id, new_title, new_cover_path) -> None # stage=generated,status=success,generate_attempts+1;new_cover_path 可为 None 表示只生成标题
|
||||
ensure_image_task_key(task_id) -> str # T-564 cmhub 异步生图:无幂等键则生成并写 tasks.image_task_key
|
||||
set_image_task_submitted(task_id, image_task_id, image_task_key=None) -> None
|
||||
# T-564 submit 202 后立即写 tasks.image_task_id/image_task_key,供重启/重试续查
|
||||
clear_image_task(task_id) -> None # poll failed/expired 或重置封面后清空异步生图状态
|
||||
update_generated_title(task_id, new_title, path=None) -> None # T-509 本地人工微调新标题;仅 generated/未提交/非运行中;清空 last_error 并回到 pending,不触碰 Shopee/CDP/Excel/封面
|
||||
set_applied(task_id, committed, error=None) -> None # 成功 stage=applied;失败 status=failed 且 stage 不前进
|
||||
# T-404a 已实现:选中记录重置
|
||||
@@ -293,12 +297,13 @@ gen_title(title_prompt, old_title, retry=None, config=None, models_path="data/co
|
||||
# 文本生成:按 backend 分流;direct 走 chat JSON,cmhub 走 /generate/title;提示词 + 旧标题 → 新标题
|
||||
|
||||
gen_cover(cover_prompt, old_cover_path, out_path, resolution=None, jpg_quality=None, retry=None, config=None, models_path="data/config/ai_models.json", on_step=None, on_event=None, cmhub_config_path="data/config/cmhub.json") -> str
|
||||
# 图像生成(image-to-image):按 backend 分流;direct 走 chat/images_edits,cmhub 走 /generate/image;
|
||||
# 图像生成(image-to-image):按 backend 分流;direct 走 chat/images_edits;cmhub 单独调用走旧同步 /generate/image 兼容路径;
|
||||
# 支持返回 url / data URL / b64_json,按 resolution resize 并以 jpg_quality 保存 JPEG,返回路径;新生成默认写入 `image_dir/<batch_id>/<slug>/<task_id>_<item_id>_new.jpg`,历史 DB 已存路径继续按原路径读取
|
||||
|
||||
generate_batch(tasks, prompts, ai_cfg=None, on_progress=None, should_stop=None) -> dict
|
||||
fetch_cmhub_models(base_url, api_key, connect_timeout=10, read_timeout=30) -> list[dict]
|
||||
# 编排:先以 title_concurrency 线程池并发跑 gen_title;ai.generate_cover 为 true 时再以 image_concurrency 并发跑 gen_cover
|
||||
# cmhub 批量生图由 generate_batch 直接走异步 /generate/image/tasks:submit 落库 task_id,再 poll 续查,成功后仍返回本地 JPEG 路径
|
||||
# 标题-only 模式标题成功即 db.set_generated(task_id, new_title, None);标题+封面模式图片成功后写 new_cover_path;should_stop() 为真则取消未开始项
|
||||
# on_progress({"total","title_done","cover_done","failed","cancelled","ok"}) 回调刷新进度
|
||||
# ai_cfg 可传 on_event/on_error 回调,逐条报告 title/cover 阶段 step/result/detail,供 GUI run_logs 与本地诊断日志使用
|
||||
@@ -308,13 +313,13 @@ fetch_cmhub_models(base_url, api_key, connect_timeout=10, read_timeout=30) -> li
|
||||
要点:
|
||||
|
||||
- `backend=direct`:内部兼容/手工回滚路径;标题用 `default_text_model`、封面用 `default_image_model`(`appconfig.get_model` 取定义,含 url/key/api_type)。
|
||||
- `backend=cmhub`:普通产品默认路径;标题调用 `POST /api/v1/generate/title`,封面调用 `POST /api/v1/generate/image`,模型字段使用 `ai.cmhub.title_alias/image_alias`,Key 来自 `data/config/cmhub.json`。
|
||||
- `backend=cmhub`:普通产品默认路径;标题调用 `POST /api/v1/generate/title`;②批量封面生成调用 `POST /api/v1/generate/image/tasks` + `GET /api/v1/generate/image/tasks/{task_id}`,模型字段使用 `ai.cmhub.title_alias/image_alias`,Key 来自 `data/config/cmhub.json`。`gen_cover()` 单独调用没有任务/DB 上下文,第一版保留旧同步 `POST /api/v1/generate/image` 兼容路径。
|
||||
- 标题提示词组装:`gen_title()` 的 direct 与 cmhub 路径共用标题 prompt 规则。若标题提示词包含 `{旧标题}`,生成前替换为该任务旧标题,不再自动追加旧标题块;若不包含 `{旧标题}`,保持旧行为自动追加“旧标题:...”块。两种情况都会追加“请只返回新标题,不要解释。”输出约束;其它 `{...}` 原样保留。
|
||||
- `fetch_cmhub_models()` 调 `GET /api/v1/models` 返回别名清单,供⑤设置页动态下拉使用;Base URL 会先规整为网关根,HTTP 404 映射为 `not_found` 并提示检查 Base URL 或实例是否部署 `/api/v1/models`。
|
||||
- `api_type=chat/auto` 走 OpenAI-compatible chat JSON;`api_type=images_edits` 走 multipart form。
|
||||
- direct 连接超时参考模型 `connect_timeout_seconds`;**返回超时 = 模型 `timeout_seconds` 或 `appconfig.response_timeout()`(随分辨率:512/1k/2k/4k → 180/240/360/600)**。cmhub 使用 `requests timeout=(connect, read)`,connect 来自 `ai.cmhub.connect_timeout`;cmhub 生文读取等待固定 600s,cmhub 生图请求和图片下载读取等待固定 650s,且生图读超时不自动重发。
|
||||
- direct 连接超时参考模型 `connect_timeout_seconds`;**返回超时 = 模型 `timeout_seconds` 或 `appconfig.response_timeout()`(随分辨率:512/1k/2k/4k → 180/240/360/600)**。cmhub 使用 `requests timeout=(connect, read)`,connect 来自 `ai.cmhub.connect_timeout`;cmhub 生文读取等待固定 600s。②批量生图异步 submit 读取等待 30s、poll 单次读取等待 15s、本地总预算 900s,图片下载读取等待 900s;`gen_cover()` 旧同步兼容路径仍用 900s 读取等待。
|
||||
- 并发数/重试/分辨率/jpg 质量来自 `appconfig.ai_config()`;标题/图片并发会被夹到 1..5,失败重试次数会被夹到 0..10,兼容旧配置中的超限值;Key 本地明文存储,但不入日志、不导出。cmhub 响应的 `points_cost/points_balance/call_id` 不改变返回值,通过 `on_event` metadata 事件上报,GUI 余额/计费展示留给 T-528。
|
||||
- 标题快、图片慢:分两段、各用各自并发数;失败按 `retry` 重试,仍失败记 error 不阻塞其余。cmhub 生图使用内部实际并发 `min(image_concurrency, 5)`,拿到 `image_url` 后交给独立下载/保存线程池,下载线程数与实际生图并发一致且最大 5;T-548 后图片下载后端由 `ai.cmhub.download_with_curl` 控制,默认 `auto`:Windows 且检测到系统 curl 时优先用 curl 下载,否则回退 requests;curl 失败也会自动回退 requests。下载前仍执行公网 URL 校验,curl 使用 `-K` 临时配置文件传入 URL,不把带 token 的 `image_url` 放进命令行参数;`use_system_proxy=false` 时 curl 加 `--noproxy "*"`。下载失败最多安全重试 3 次,只复用同一个 `image_url`,不会重新调用 cmhub 生图接口;下载总耗时超过 20 秒时写“图片下载较慢”警告;direct 兼容路径暂保持原 `image_concurrency` 语义。
|
||||
- 标题快、图片慢:分两段、各用各自并发数;失败按 `retry` 重试,仍失败记 error 不阻塞其余。cmhub 批量生图使用内部实际并发 `min(image_concurrency, 5)` 约束 submit+poll 在途数;已有 `tasks.image_task_id` 时直接 GET 续查,不二次 submit;submit 前先持久化 `image_task_key`,submit 成功立即持久化 `image_task_id`;poll `failed/expired` 会清空二者,poll 超时/用户停止/程序退出则保留二者供下次续查。拿到 `image_url` 后交给独立下载/保存线程池,下载线程数与实际生图并发一致且最大 5;T-548 后图片下载后端由 `ai.cmhub.download_with_curl` 控制,默认 `auto`:Windows 且检测到系统 curl 时优先用 curl 下载,否则回退 requests;curl 失败也会自动回退 requests。下载前仍执行公网 URL 校验,curl 使用 `-K` 临时配置文件传入 URL,不把带 token 的 `image_url` 放进命令行参数;`use_system_proxy=false` 时 curl 加 `--noproxy "*"`。下载失败最多安全重试 3 次,只复用同一个 `image_url`,不会重新调用 cmhub 生图接口;下载总耗时超过 20 秒时写“图片下载较慢”警告;direct 兼容路径暂保持原 `image_concurrency` 语义。
|
||||
- 调试 cmhub 图片 URL:默认不在日志或 DB 中记录完整 `image_url`。若本机调试需要查看,启动程序前设置环境变量 `CMSHOPEE_DEBUG_CMHUB_IMAGE_URL=1`,②本轮可见运行日志会显示脱敏后的“cmhub 图片 URL”,但该调试行不写入 SQLite `run_log_events`。
|
||||
- 调用有成本与失败可能:超时、限流、内容安全拒绝都要返回明确错误。
|
||||
- 生成结果**直接进入 ③ 更新候选**;③ 点击「开始更新」后弹窗批量确认,确认后提交线上。本地留档 + 回写 Excel 供追溯。
|
||||
@@ -396,7 +401,7 @@ T-523 后 GUI 已从旧 `app/gui.py` 拆为 `app/gui/` 包:`__init__.py` 负
|
||||
- 删除按钮在当前类别只剩 1 个模型时禁用;后端仍以“至少启用一个 text 和 image 模型”为硬约束。
|
||||
- 「测试连接」创建 `AIModelTestWorker` 后台调用 `appconfig.test_ai_model()`,GUI 主线程不直接发网络请求。
|
||||
- 角色与生成参数读写 `config.json`,并按 3 个组件一组排列:标题大模型(仅 text)、图片大模型(仅 image)、标题/图片并发、失败重试、分辨率、jpg 质量。
|
||||
- 分辨率下拉固定 `512/1k/2k/4k`;普通默认 cmhub 模式下,返回超时标签只读展示实际等待口径「标题 600 秒 / 图片 650 秒」,分辨率只控制生成图片尺寸。direct 兼容路径仍使用 `resolution_timeouts[resolution]`。
|
||||
- 分辨率下拉固定 `512/1k/2k/4k`;普通默认 cmhub 模式下,返回超时标签只读展示实际等待口径「标题 600 秒 / 图片 900 秒」,分辨率只控制生成图片尺寸。direct 兼容路径仍使用 `resolution_timeouts[resolution]`。
|
||||
- 路径与端口读写 `config.json`,并按 3 个组件一组排列:默认调试端口、调试端口范围、CDP 就绪超时等短字段一格;Chrome 路径、账号数据根目录、图片目录、DB 路径等长字段跨整行或跨 2/3 列。保存时校验端口范围和默认端口。
|
||||
- 蝦皮更新安全读写 `config.json` 的 `shopee_update` 段,并按 3 个组件一组排列:允许真实提交、允许更新封面、每批最大更新条数、成功后关闭本次新开编辑页、多账号并行、最大并行账号数;其中「多账号并行更新」与「最大并行账号数」必须合并为同一个横向组件,最大并行账号数紧跟在多账号并行更新后面,不允许被三列表单排到下一行。`dry_run` 字段可保留为内部兼容,但普通用户界面不再展示 dry-run 开关,③ 使用「检查本轮更新」按钮触发检查模式;测试商品 ID 仅作为历史/调试兼容字段保留,不参与普通正式更新安全检查,普通设置页已隐藏该入口。
|
||||
- 真实提交与封面更新默认关闭;用户在 ⑤ 保存开启后,③ 仍需要通过安全开关检查并弹窗确认,才会创建更新 worker。
|
||||
|
||||
@@ -12,12 +12,13 @@
|
||||
> **v3.4 修订(2026-07-07,T-538 路径收敛)**:打包版和源码运行的用户数据统一落在 `data/` 下;本文早期提到的 `config.json`、`config/ai_models.json`、`config/cmhub.json`、`images/`,当前默认路径分别为 `data/config.json`、`data/config/ai_models.json`、`data/config/cmhub.json`、`data/images/`。文件名和 schema 不变,旧布局由启动迁移逻辑处理。
|
||||
> **v3.4 核对(2026-07-06,刷新别名 notfound)**:核对对接文档 §4.4 与 cmhub `ModelsView` 路由,确认 cmshopee `GET /api/v1/models` + Bearer 请求**已达标**;「notfound」为 404,根因是 Base URL 带多余 `/api(/v1)` 路径(双拼)或所连实例未部署 `/api/v1/models`(见 `docs/troubleshooting.md`)。**T-530 已落地**:保存/请求前规整 Base URL 到网关根,HTTP 404 映射为 `not_found` 并给出中文排障提示。
|
||||
> **v3.5 修订(2026-07-08,T-553 待实现)**:cmhub 生图稳定性口径调整为连接超时默认 66 秒、生图请求和 `image_url` 下载读取等待统一 900 秒,与线上 Nginx/Gunicorn 的长等待窗口对齐;生图读超时仍不自动重发,避免重复扣点。
|
||||
> **v3.6 修订(2026-07-08,T-564)**:cmhub 已新增异步生图任务接口,②批量生图改为 `POST /api/v1/generate/image/tasks` submit + `GET /api/v1/generate/image/tasks/{task_id}` poll;cmshopee 持久化 `tasks.image_task_id/image_task_key`,支持停止/超时/重启后续查,避免 900 秒同步长连接和读超时重复扣点。旧同步 `POST /api/v1/generate/image` 仅保留给单独 `gen_cover()` 兼容/回滚路径。
|
||||
|
||||
## 1. 背景与目标
|
||||
|
||||
**现状**:`app/ai.py` 支持 cmhub 网关和 direct 兼容路径。direct 模式下每个模型在 `data/config/ai_models.json` 配 `url/model/api_key/api_type`,`gen_title` 自拼 chat `messages` 解析文本,`gen_cover` 走 `images_edits` multipart 或 vision chat 直接拿图片字节本地存。
|
||||
|
||||
**目标**:改为对接 cmhub 的生成、余额和模型发现接口:`POST /api/v1/generate/title`、`POST /api/v1/generate/image`、`GET /api/v1/balance`、`GET /api/v1/models`,一把 `Authorization: Bearer <API_KEY>` 即可调用。上游 provider、密钥、计费、SSRF 防护、分辨率映射、对象存储与别名发现由 cmhub 承担。
|
||||
**目标**:改为对接 cmhub 的生成、余额和模型发现接口:`POST /api/v1/generate/title`、②批量生图 `POST /api/v1/generate/image/tasks` + `GET /api/v1/generate/image/tasks/{task_id}`、兼容生图 `POST /api/v1/generate/image`、`GET /api/v1/balance`、`GET /api/v1/models`,一把 `Authorization: Bearer <API_KEY>` 即可调用。上游 provider、密钥、计费、SSRF 防护、分辨率映射、对象存储与别名发现由 cmhub 承担。
|
||||
|
||||
**收益**:密钥收敛(本地只留一把 cmhub Key);换上游模型对 cmshopee 零改动(cmhub 用能力别名);可删除大量 provider 适配代码;计费/额度统一。
|
||||
|
||||
@@ -29,9 +30,9 @@
|
||||
| 生文请求 | 自拼 chat `messages`(system+user) | `{prompt, model:别名, image_url?/image_base64?, resolution?, parameters?}` |
|
||||
| 生文响应 | chat completion → 取单条文本 | `{titles:[...], alias, model_used, points_cost, points_balance, call_id}` |
|
||||
| 生图请求 | `images_edits` multipart 或 vision chat | `{prompt, model:别名, image_base64?/image_url?, resolution?, aspect_ratio?, parameters?}` |
|
||||
| 生图响应 | 直接返回 image bytes | `{image_url, ...}` → 需再下载 |
|
||||
| 生图响应 | 直接返回 image bytes | ②批量:submit 返回 `{task_id, ...}`,poll 成功返回 `{result:{image_url}}`;兼容同步:`{image_url, ...}` → 需再下载 |
|
||||
| 错误 | HTTP error 文本 | `{error:{code,message}}`:`insufficient_points`(402)/`upstream_error`(502)/`rate_limited`(429)/`unauthorized`(401)/`account_disabled`(403)/`bad_request`(400)/`model_not_allowed`/`no_pricing_rule` |
|
||||
| 超时 | 按 `resolution_timeouts` | 生图同步且慢;T-553 目标为 cmshopee 生图请求和图片下载读取等待统一固定 900s,不再按分辨率变化;生图读超时仍不自动重发 |
|
||||
| 超时 | 按 `resolution_timeouts` | ②批量生图 submit 短读超时、poll 总预算 900s、下载 900s;旧同步兼容路径仍 900s |
|
||||
| 幂等 | 直连一次成功一次 | **非幂等、无幂等键**:客户端超时 ≠ 未扣点,读超时后不可无脑重发 |
|
||||
|
||||
关键差异(决定改造点):
|
||||
@@ -89,16 +90,18 @@
|
||||
|
||||
返回值不变(仍返回已存 JPEG 路径),现有调用保持兼容;允许新增可选事件回调参数承载计费元数据。内层:
|
||||
|
||||
- **请求**:`POST {base_url}/api/v1/generate/image`,体:
|
||||
- **②批量请求(T-564)**:`POST {base_url}/api/v1/generate/image/tasks`,头带 `Idempotency-Key` 与 `X-Client-Version`,体与旧生图一致:
|
||||
```jsonc
|
||||
{ "prompt": <cover_prompt>, "model": <image_alias>,
|
||||
"image_base64": <旧封面转 data URL>, "resolution": <resolution>, "aspect_ratio": <可选> }
|
||||
```
|
||||
旧封面必传(改图类),复用现有 `_image_data_url(old_cover_path)` 生成 base64。
|
||||
- **②批量响应(T-564)**:submit 返回 `202 {task_id,status:"queued",call_id,points_cost,points_balance,expires_at}`;cmshopee 立即写 `tasks.image_task_id`,随后 `GET /api/v1/generate/image/tasks/{task_id}` 轮询。`queued/running` 继续等待;`succeeded` 取 `result.image_url`;`failed/expired` 视为终态失败且 cmhub 已退点。
|
||||
- **单独 `gen_cover()` 兼容**:公开函数没有本地 `task.id/db_path` 上下文,第一版继续调用旧同步 `POST /api/v1/generate/image`,保持返回值和旧调用方兼容;②批量生成才使用完整持久化、幂等键和重启续查。
|
||||
- `resolution` 归一为**大写** `512/1K/2K/4K`(cmshopee 内部用小写 `1k`,发请求前转 `1K`);`aspect_ratio` 默认 `1:1`(Shopee 封面)。
|
||||
- **响应**:拿 `image_url` → **新增一步下载**该图字节(cmhub 自家对象存储公网 URL)→ 交给现有 `_save_jpeg(image_bytes, out_path, resolution, quality)` 落盘。下载 helper 必须校验 URL scheme 只允许 `http/https`,拒绝内网/回环/本机地址,并校验域名解析后的 IP 仍不属于内网/回环/本机地址,设置超时和大小上限;生成后**立即下载**(对象存储 URL 可能有有效期)。`points_cost`/`points_balance`/`call_id` 同样通过事件回调传播,不改变 `gen_cover` 返回值。
|
||||
- **超时(关键)**:生图同步且慢。T-553 目标为 cmshopee 的 cmhub 连接超时默认 66 秒,生图请求和随后 `image_url` 下载读取等待统一固定 900 秒,不再按分辨率变化,绝不用 30s/60s 调生图——否则客户端超时但服务端仍在算并扣点(见 §4.4 幂等)。
|
||||
- **并发(T-545 已实现)**:最近实测 `/media/generated/images/*.png` 下载链路在 10 并发下明显慢且有连接失败。cmhub 模式下采用内置保护:实际生图请求并发 = `min(ai.image_concurrency, 5)`;下载/保存使用独立线程池,线程数与实际生图请求并发一致,同样最大 5;不新增用户可见配置项。运行日志必须同时显示用户设置和实际并发,避免用户误解设置 10 就会对 cmhub 打 10 并发。下载失败记为该任务失败,不得重新调用生图接口导致重复扣点;读超时仍按 §4.4 的非幂等规则处理。
|
||||
- **下载**:拿 `image_url` 后下载该图字节(cmhub 自家对象存储公网 URL)→ 交给现有 `_save_jpeg(image_bytes, out_path, resolution, quality)` 落盘。下载 helper 必须校验 URL scheme 只允许 `http/https`,拒绝内网/回环/本机地址,并校验域名解析后的 IP 仍不属于内网/回环/本机地址,设置超时和大小上限;生成后**立即下载**(对象存储 URL 可能有有效期)。`points_cost`/`points_balance`/`call_id` 通过事件回调传播,不改变 `gen_cover` 返回值。
|
||||
- **超时(关键)**:②批量生图 submit 读取等待约 30 秒,poll 单次读取等待约 15 秒,本地总预算 900 秒;撞预算、用户停止或程序退出都保留 `image_task_id/image_task_key`,下次直接续查,不重新 submit。下载读取等待 900 秒;旧同步兼容路径也使用 900 秒。绝不用 30s/60s 同步死等旧生图,否则客户端超时但服务端仍在算并扣点。
|
||||
- **并发(T-545/T-564 已实现)**:最近实测 `/media/generated/images/*.png` 下载链路在 10 并发下明显慢且有连接失败。cmhub 模式下采用内置保护:实际 submit+poll 在途并发 = `min(ai.image_concurrency, 5)`;下载/保存使用独立线程池,线程数与实际生图并发一致,同样最大 5;不新增用户可见配置项。运行日志必须同时显示用户设置和实际并发,避免用户误解设置 10 就会对 cmhub 打 10 并发。下载失败记为该任务失败,不得重新调用生图接口导致重复扣点。
|
||||
- **下载后端(T-548 已实现)**:cmhub 生成/models/balance 仍走共享 requests Session;仅 `image_url` 图片下载可按 `ai.cmhub.download_with_curl` 选择系统 curl。默认 `auto` 在 Windows 且检测到系统 curl 时优先 curl,非 Windows、无 curl 或 curl 失败自动回退 requests。curl 调用前仍做公网 URL 校验,用 `-K` 临时配置文件传 URL,避免 token 出现在进程命令行;`use_system_proxy=false` 时加 `--noproxy "*"`。
|
||||
|
||||
### 4.4 错误映射与重试策略
|
||||
@@ -112,14 +115,19 @@ cmhub 返回结构化 `{error:{code}}`。映射层**按 `code` 优先分支**(
|
||||
| `account_disabled` | 403 | 提示 Key 被吊销/账号禁用,去网页端重生成 | 否 |
|
||||
| `bad_request` / `model_not_allowed` / `no_pricing_rule` / `content_blocked` | 400 | 记录具体 code,判为配置/参数/内容错 | 否 |
|
||||
| `upstream_error` | 502 | 上游失败(cmhub **已自动退点**),可提示稍后重试 | 是(安全)|
|
||||
| `upstream_timeout` / `task_timeout` | 200/5xx | 上游或任务超时(cmhub **已自动退点**),可提示稍后重试 | 是(安全)|
|
||||
| `idempotency_conflict` | 409/400 | 本地幂等键被不同 payload 复用,清空本地 key 后失败提示 | 否 |
|
||||
| `rate_limited` | 429 | 退避重试,读 `Retry-After`(默认限流 60次/分)| 是 |
|
||||
| **未列出的 code** | 任意 | 展示 `message`,当不可重试错误(前向兼容)| 否 |
|
||||
|
||||
**幂等与超时——生图重试必须特别处理(会亏钱)**:cmhub 生成接口**非幂等、无幂等键**,客户端超时 ≠ 未扣点。
|
||||
**幂等与超时——②批量生图以 task_id 续查为准**:旧同步生图接口非幂等,客户端超时 ≠ 未扣点;T-564 后②批量生图必须使用任务化接口和本地持久化规避这个问题。
|
||||
|
||||
- **生图(`gen_cover`)**:**读超时后绝不自动重发**——服务端可能已算完并扣点,重发 = 重复扣点。首选办法是把读超时设够大(§4.3,T-553 目标固定 900s)从源头避免歧义;只对**连接超时**(请求根本没送达服务端)安全重试。
|
||||
- submit 前先生成并写入 `tasks.image_task_key`,同一次网络抖动重发复用同一个 `Idempotency-Key`;submit 成功后立即写入 `tasks.image_task_id`。
|
||||
- 只要本地已有 `image_task_id`,下一轮直接 GET 续查,不重新 submit;`succeeded` 下载保存并 `set_generated()`;本地总预算超时、用户停止、程序退出都保留 `image_task_id/image_task_key`。
|
||||
- poll 返回 `failed/expired`(已退点)时清空 `image_task_id/image_task_key`,后续重试会生成新的 key 并重新 submit;submit 阶段 `content_blocked`、`insufficient_points`、`bad_request`、`idempotency_conflict` 等没有有效任务时也清空 key。
|
||||
- **单独 `gen_cover()` 兼容路径**:没有 DB 上下文,仍走旧同步接口;读超时后仍不自动重发,避免重复扣点。
|
||||
- **生文(`gen_title`)**:秒级返回、点数低,读超时重试风险小,但仍建议同样区分连接超时/读超时;重试次数可小。
|
||||
- 只对 `502`/`429`/**连接**超时重试;`402/401/403/400`/读超时立即失败。现状 `_call_with_retry` 是**一刀切重试**,cmhub 模式必须替换为这套区分策略。
|
||||
- 只对 `502`/`429`/**连接**超时以及已退点的 `upstream_timeout/task_timeout` 重试;`402/401/403/400`/读超时立即失败或进入续查。现状 `_call_with_retry` 是**一刀切重试**,cmhub 模式必须替换为这套区分策略。
|
||||
- `insufficient_points` 是新的用户可见态:② 生成页应弹明确提示并引导去网页端充值,不当普通失败淹没在计数里。
|
||||
|
||||
### 4.5 余额展示与额度预检
|
||||
@@ -140,13 +148,14 @@ cmhub 返回结构化 `{error:{code}}`。映射层**按 `code` 优先分支**(
|
||||
|
||||
| 模块 | 改动 | 量 |
|
||||
| --- | --- | --- |
|
||||
| `app/ai.py` | `gen_title`/`gen_cover` 加 cmhub 分支(请求体+解析+生图下载);抽 backend 选择;错误映射 + 区分重试。`direct` 分支保留现有代码 | M |
|
||||
| `app/ai.py` | `gen_title`/`gen_cover` 加 cmhub 分支(请求体+解析+生图下载);T-564 后②批量生图走 submit+poll+续查,单独 `gen_cover()` 保留旧同步兼容;抽 backend 选择;错误映射 + 区分重试。`direct` 分支保留现有代码 | M |
|
||||
| `app/db.py` | T-564 新增 `tasks.image_task_id/image_task_key`,提供 `ensure_image_task_key()`、`set_image_task_submitted()`、`clear_image_task()`;重置封面时清空异步生图状态,只重置标题时保留 | S |
|
||||
| `app/appconfig.py` | `ai` 段加 `backend`/`cmhub` 子段默认值与校验;cmhub Key 的读写与打码(复用脱敏工具);新增 `cmhub_request_url()` 类 helper | S |
|
||||
| `app/gui/tabs/settings.py` / `app/gui/workers.py` | ⑤ AI 设置按 backend 切换:cmhub 模式显示「网关地址 + API Key + 生文/生图别名 + 测试连接/查余额」;direct 模式保留现有 master-detail;测试连接/查余额走后台 worker。**最大 UI 触点** | M |
|
||||
| 测试 | `tests/test_ai.py` 增 cmhub mock(titles 列表、image_url 下载、安全下载、各错误码与重试、连接/读超时差异);`test_appconfig` 加 schema 与 `data/config/cmhub.json` helper;⑤ gui 设置测试跟随 | M |
|
||||
| 文档 | `docs/04-architecture.md` §5.1b/§6.2、`docs/api.md`、`docs/03-tech-stack.md`、`current-state.md` 同步 | S |
|
||||
|
||||
**明确不动(T-526)**:`editor.py`/`cdp.py`/`chrome.py`/`accounts.py`/`excel.py`/`db.py`,以及 ①采集/③更新/④账号全流程;`generate_batch` 主编排、图片本地路径方案和 T-520 封面开关保持原语义。T-527/T-528 可按任务边界修改 `app/gui/tabs/settings.py`、`app/gui/tabs/generate.py`、`app/gui/workers.py` 的设置与用户提示层。
|
||||
**明确不动(T-564)**:不改 CDP/Shopee/Excel/账号流程,不改图片本地路径方案和 T-520 封面开关;不引入 cancel 接口;不做 DB 版本化迁移重构。`generate_batch` 仍保持“先标题、后封面、失败不阻塞其余”的骨架,只替换 cmhub 批量封面请求内核。
|
||||
|
||||
## 6. 安全与合规
|
||||
|
||||
@@ -158,8 +167,8 @@ cmhub 返回结构化 `{error:{code}}`。映射层**按 `code` 优先分支**(
|
||||
|
||||
- **单元(mock cmhub,不连真实网关)**:
|
||||
- 生文:`titles` 多条取首条;空 `titles`/空串 → `AIError`。
|
||||
- 生图:`image_url` → mock 下载字节 → `_save_jpeg` 落盘校验分辨率/质量;覆盖 scheme、内网/回环字符串地址、域名解析到内网 IP 的拒绝路径。
|
||||
- 错误码矩阵:402/401/403/400 不重试且原因正确;502/429/连接超时按 attempts 重试;生图读超时不重发;未知 code 不重试。
|
||||
- 生图:②批量 mock submit 202 → `task_id` 落库 → poll succeeded → `image_url` → mock 下载字节 → `_save_jpeg` 落盘校验分辨率/质量;覆盖已有 `image_task_id` 续查不 POST、failed/expired 清 key、取消/超时保留 task_id。单独 `gen_cover()` 继续覆盖旧同步 `image_url` 下载路径。
|
||||
- 错误码矩阵:402/401/403/400 不重试且原因正确;502/429/连接超时按 attempts 重试;`upstream_timeout/task_timeout` 可重试且清 key;未知 code 不重试。
|
||||
- 配置:`backend=cmhub` 走 cmhub 分支、显式 `direct` 走旧分支;T-529 后缺 `backend` 的配置按 `DEFAULT_CONFIG` 补为 cmhub;缺 `base_url`/Key/别名时明确报错。
|
||||
- **GUI**:⑤ cmhub 面板读写、测试连接 worker、别名下拉;② 生成在 `insufficient_points` 时的提示路径。
|
||||
- **回归**:`direct` 模式现有 `test_ai.py` 用例保持绿。
|
||||
@@ -178,7 +187,7 @@ cmhub 返回结构化 `{error:{code}}`。映射层**按 `code` 优先分支**(
|
||||
3. ~~`resolution` 取值~~ **已解决**:`512/1K/2K/4K`(大写 K),默认 `1K`;cmshopee 小写值发请求前归一。
|
||||
4. ~~`aspect_ratio`~~ **已解决**:默认 `1:1`,Shopee 封面用 `1:1`。
|
||||
5. **`image_url` 有效期(按最坏处理)**:对象存储 URL 可能过期——本设计已是**生成后立即下载落盘**,无需长期持有。
|
||||
6. ~~超时上限~~ **已解决**:T-553 目标为 cmshopee 生图请求和图片下载读取等待统一固定 900s,连接超时默认 66s。
|
||||
6. ~~超时上限~~ **已解决**:T-553/T-564 后②批量生图 submit 短等待、poll 总预算 900s,图片下载读取等待 900s,连接超时默认 66s。
|
||||
7. **Base URL / API Key 形态**:域名待部署方提供;Key 形如 `sk_cmhub_xxx`,仅网页端生成时显示一次——⑤设置需提示用户从网页端复制粘贴,本地保存。
|
||||
|
||||
## 10. 落地拆分与任务顺序
|
||||
@@ -188,6 +197,7 @@ cmhub 返回结构化 `{error:{code}}`。映射层**按 `code` 优先分支**(
|
||||
- **第三步**:② 计费错误提示(`insufficient_points` 引导充值)+ 可选余额展示。
|
||||
- **第四步(T-529)**:产品默认 cmhub,⑤去掉 AI 后端选择,保存固定 `backend=cmhub`。
|
||||
- **第五步(T-530)**:Base URL 规整到网关根,404 给出明确中文提示。
|
||||
- **第六步(T-564)**:②批量生图切到 cmhub 异步任务接口,持久化 `image_task_id/image_task_key` 支持续查;旧同步生图只保留给单独 `gen_cover()` 兼容。
|
||||
- 文档随每步同步。
|
||||
|
||||
> 当前已在 `docs/06-tasks.md` 落成 T-526~T-535,且 T-525 工程基础设施已完成;下一步按看板进入 T-539 设置简化,仍遵守 `docs/05-coding-rules.md` 验证清单。
|
||||
> T-526~T-535、T-545、T-548、T-549 与 T-564 已按当前任务文件体系落地;后续新任务继续写入 `docs/tasks/T-<编号>.md`,不再新增到冻结的 `docs/06-tasks.md`。
|
||||
|
||||
+1
-1
@@ -164,7 +164,7 @@
|
||||
- T-531 已完成:⑤设置页任意可编辑控件变更都进入未保存状态,保存按钮旁显示“● 未保存更改”;切换到其它 Tab 或关闭窗口时弹出保存/放弃/取消。放弃会重新从本地配置文件回填控件,避免未保存的 URL/API Key 留在界面上;程序化回填、保存后重载和刷新别名填充下拉不会误触发未保存状态。
|
||||
- direct 模型清单和 `data/config/ai_models.json` 代码路径保留为内部兼容/手工回滚,不在普通 UI 暴露。
|
||||
- AI 生成参数:标题并发、图片并发、失败重试、分辨率、返回超时、jpg 质量等短字段按三列排列;标题/图片并发可选 1..5,失败重试可选 0..10,旧配置超限值会自动夹紧。
|
||||
- 分辨率为 `512 / 1k / 2k / 4k`,在 cmhub 默认模式下只控制生成图片尺寸;⑤「返回超时」只读展示当前实际等待口径:标题 600 秒、图片 650 秒,不再随分辨率切换显示 180/240/360/600,避免用户误解生图等待时间。
|
||||
- 分辨率为 `512 / 1k / 2k / 4k`,在 cmhub 默认模式下只控制生成图片尺寸;⑤「返回超时」只读展示当前实际等待口径:标题 600 秒、图片 900 秒,不再随分辨率切换显示 180/240/360/600,避免用户误解生图等待时间。
|
||||
- 保存写入 `config.json` 的 `ai` 段,供 ② AI生成复用;标题/图片模型角色下拉随 direct UI 一起隐藏。
|
||||
- 路径与端口(T-501b/T-506/T-539 已接入):组件组改为 3 个组件一组;普通设置页只显示 Chrome 路径、默认端口、端口起止、CDP 就绪超时。T-538 后账号数据根目录、图片目录、DB 路径固定解析到 `data/` 下,普通 UI 不再提供输入框,避免用户误改后数据分裂;`config.json` 中 `user_data_root` / `image_dir` / `db_path` 字段继续作为内部兼容字段保留,手工配置值仍会被读取和保存。
|
||||
- 蝦皮更新安全(T-501c/T-506 已接入):组件组改为 3 个组件一组;允许真实提交、允许更新封面、每批最大更新条数、成功后关闭本次新开编辑页等短字段三列排列;「多账号并行更新」与「最大并行账号数」必须合并为同一个横向组件,最大并行账号数紧跟在多账号并行更新后面,不允许换到下一行;测试商品 ID 仅作为历史/调试兼容字段保留,不参与普通正式更新安全检查,普通设置页已隐藏该入口。
|
||||
|
||||
+8
-2
@@ -3,7 +3,7 @@ id: T-564
|
||||
title: 生图对接 cmhub 异步任务接口(submit + 轮询 + task_id 续查),替代 900s 同步长等待
|
||||
phase: 7
|
||||
deps: [T-526, T-529, T-546]
|
||||
status: TODO
|
||||
status: DONE
|
||||
created: 2026-07-08
|
||||
---
|
||||
|
||||
@@ -92,4 +92,10 @@ cmhub 侧已实现并测试通过异步任务化接口(新增 `/generate/image
|
||||
|
||||
## 执行记录
|
||||
|
||||
(做完在这里写:改了什么文件、跑了什么验证命令及结果、遇到的阻塞、关键决策。)
|
||||
- 2026-07-08:完成 T-564。
|
||||
- `app/db.py`:新增 `tasks.image_task_id` / `tasks.image_task_key` schema 与 ad-hoc 迁移,补 `Task` dataclass 字段;新增 `ensure_image_task_key()`、`set_image_task_submitted()`、`clear_image_task()`;`reset_generated(..., reset_cover=True)` 会清空异步生图状态,只重置标题时保留。
|
||||
- `app/ai.py`:②批量 cmhub 生图切到 `POST /api/v1/generate/image/tasks` + `GET /api/v1/generate/image/tasks/{task_id}`;submit 带 `Idempotency-Key` / `X-Client-Version`,submit 成功后立即落库 `image_task_id`;已有 `image_task_id` 时直接续查;poll `succeeded` 后复用原图片下载/保存;poll `failed/expired` 清空 key/id;poll 超时或用户停止保留 key/id;单独 `gen_cover()` 保留旧同步接口兼容。
|
||||
- `tests/test_db.py`:覆盖新列、幂等键稳定、submit 落库、清空、重置标题/封面生命周期。
|
||||
- `tests/test_ai.py`:覆盖 submit 后 poll 前已落库、已有 task_id 续查不 POST、failed 任务清 key、停止轮询保留 task_id,以及既有下载/并发回归。
|
||||
- 文档:同步 `docs/04-architecture.md`、`docs/api.md`、`docs/cmhub-integration-design.md`、`docs/routes.md`、`docs/ui/tab5-settings.svg`;按仓库规则未手动覆盖 `docs/current-state.md`,未修改冻结 `docs/06-tasks.md`。
|
||||
- 验证通过:`py -3.10 -m unittest tests.test_ai tests.test_db`、`python -m ruff check app tests main.py`、`py -3.10 -m compileall app main.py`、`py -3.10 -m unittest discover -s tests`、`git diff --check`。
|
||||
|
||||
@@ -50,7 +50,7 @@
|
||||
<text class="lbl" x="14" y="54">标题大模型</text><rect class="inp" x="100" y="40" width="190" height="24" rx="4"/><text class="cell" x="110" y="57">GPT-5.5 文本 ▼</text>
|
||||
<text class="lbl" x="320" y="54">图片大模型</text><rect class="inp" x="406" y="40" width="190" height="24" rx="4"/><text class="cell" x="416" y="57">Nano Banana 2 ▼</text>
|
||||
<text class="lbl" x="630" y="54">分辨率</text><rect class="inp" x="690" y="40" width="90" height="24" rx="4"/><text class="cell" x="700" y="57">1k ▼</text>
|
||||
<text class="mut" x="790" y="57">→ 标题600s / 图片650s</text>
|
||||
<text class="mut" x="790" y="57">→ 标题600s / 图片900s</text>
|
||||
|
||||
<text class="lbl" x="14" y="90">标题并发</text><rect class="inp" x="80" y="76" width="50" height="24" rx="4"/><text class="cell" x="98" y="93">4</text>
|
||||
<text class="lbl" x="150" y="90">图片并发</text><rect class="inp" x="216" y="76" width="50" height="24" rx="4"/><text class="cell" x="234" y="93">4</text>
|
||||
|
||||
|
Before Width: | Height: | Size: 7.0 KiB After Width: | Height: | Size: 7.0 KiB |
+252
-4
@@ -1005,6 +1005,19 @@ class AITests(TempDirMixin, unittest.TestCase):
|
||||
downloads = []
|
||||
|
||||
def fake_request(method, url, **kwargs):
|
||||
if str(method).upper() == "GET":
|
||||
task_id = url.rsplit("/", 1)[-1]
|
||||
index = int(task_id.rsplit("-", 1)[-1])
|
||||
return _RequestsResponse(
|
||||
{
|
||||
"task_id": task_id,
|
||||
"status": "succeeded",
|
||||
"result": {
|
||||
"image_url": "https://cdn.example.com/generated-%s.png"
|
||||
% index
|
||||
},
|
||||
}
|
||||
)
|
||||
with lock:
|
||||
counters["request_count"] += 1
|
||||
request_index = counters["request_count"]
|
||||
@@ -1018,8 +1031,8 @@ class AITests(TempDirMixin, unittest.TestCase):
|
||||
try:
|
||||
return _RequestsResponse(
|
||||
{
|
||||
"image_url": "https://cdn.example.com/generated-%s.png"
|
||||
% request_index
|
||||
"task_id": "cmhub-task-%s" % request_index,
|
||||
"status": "queued",
|
||||
}
|
||||
)
|
||||
finally:
|
||||
@@ -1082,10 +1095,16 @@ class AITests(TempDirMixin, unittest.TestCase):
|
||||
downloads_seen = []
|
||||
|
||||
def fake_request(method, url, **kwargs):
|
||||
requests_seen.append((method, url, kwargs))
|
||||
if str(method).upper() == "GET":
|
||||
return _RequestsResponse(
|
||||
{"image_url": "https://cdn.example.com/generated.png"}
|
||||
{
|
||||
"task_id": "cmhub-task-1",
|
||||
"status": "succeeded",
|
||||
"result": {"image_url": "https://cdn.example.com/generated.png"},
|
||||
}
|
||||
)
|
||||
requests_seen.append((method, url, kwargs))
|
||||
return _RequestsResponse({"task_id": "cmhub-task-1", "status": "queued"})
|
||||
|
||||
def fake_get(url, **kwargs):
|
||||
downloads_seen.append((url, kwargs))
|
||||
@@ -1115,6 +1134,235 @@ class AITests(TempDirMixin, unittest.TestCase):
|
||||
|
||||
self.assert_removed(temp_dir)
|
||||
|
||||
def test_generate_batch_cmhub_async_submit_persists_task_id_before_poll(self):
|
||||
try:
|
||||
from PIL import Image # noqa: F401
|
||||
except ImportError:
|
||||
self.skipTest("Pillow not installed")
|
||||
|
||||
with self.make_temp_dir() as temp_dir:
|
||||
cfg, key_path = self._cmhub_config(temp_dir)
|
||||
cfg["db_path"] = os.path.join(temp_dir, "cmshopee.db")
|
||||
cfg["image_dir"] = os.path.join(temp_dir, "images")
|
||||
cfg["ai"]["generate_cover"] = True
|
||||
batch_id, tasks = self._collected_tasks(temp_dir, cfg, ["旧标题"])
|
||||
self._write_old_cover_files(tasks)
|
||||
db.set_generated(tasks[0].id, "已有标题", None, path=cfg["db_path"])
|
||||
task = db.list_tasks(batch_id=batch_id, path=cfg["db_path"])[0]
|
||||
generated_png = self._png_bytes()
|
||||
public_dns = [
|
||||
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 443))
|
||||
]
|
||||
poll_db_values = []
|
||||
headers_seen = []
|
||||
|
||||
def fake_request(method, url, **kwargs):
|
||||
if str(method).upper() == "POST":
|
||||
headers_seen.append(dict(kwargs["headers"]))
|
||||
return _RequestsResponse(
|
||||
{
|
||||
"task_id": "cmhub-task-1",
|
||||
"status": "queued",
|
||||
"points_cost": 2,
|
||||
"points_balance": 80,
|
||||
"call_id": "call-image-1",
|
||||
},
|
||||
status_code=202,
|
||||
)
|
||||
poll_db_values.append(db.get_task(task.id, path=cfg["db_path"]).image_task_id)
|
||||
return _RequestsResponse(
|
||||
{
|
||||
"task_id": "cmhub-task-1",
|
||||
"status": "succeeded",
|
||||
"result": {"image_url": "https://cdn.example.com/generated.png"},
|
||||
}
|
||||
)
|
||||
|
||||
def fake_get(url, **kwargs):
|
||||
return _RequestsResponse(content=generated_png)
|
||||
|
||||
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request), \
|
||||
mock.patch.object(ai._cmhub_session(), "get", side_effect=fake_get), \
|
||||
mock.patch("app.ai.socket.getaddrinfo", return_value=public_dns):
|
||||
summary = ai.generate_batch(
|
||||
[task],
|
||||
{"title": "标题提示", "cover": "封面 {新标题}"},
|
||||
ai_cfg={
|
||||
"config": cfg,
|
||||
"db_path": cfg["db_path"],
|
||||
"cmhub_config_path": key_path,
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(summary["ok"])
|
||||
self.assertEqual(["cmhub-task-1"], poll_db_values)
|
||||
self.assertTrue(headers_seen[0]["Idempotency-Key"].startswith("cmshopee-task-"))
|
||||
self.assertTrue(headers_seen[0]["X-Client-Version"])
|
||||
updated = db.get_task(task.id, path=cfg["db_path"])
|
||||
self.assertEqual("cmhub-task-1", updated.image_task_id)
|
||||
self.assertTrue(updated.image_task_key)
|
||||
self.assertTrue(os.path.exists(updated.new_cover_path))
|
||||
|
||||
self.assert_removed(temp_dir)
|
||||
|
||||
def test_generate_batch_cmhub_resumes_existing_image_task_without_submit(self):
|
||||
try:
|
||||
from PIL import Image # noqa: F401
|
||||
except ImportError:
|
||||
self.skipTest("Pillow not installed")
|
||||
|
||||
with self.make_temp_dir() as temp_dir:
|
||||
cfg, key_path = self._cmhub_config(temp_dir)
|
||||
cfg["db_path"] = os.path.join(temp_dir, "cmshopee.db")
|
||||
cfg["image_dir"] = os.path.join(temp_dir, "images")
|
||||
cfg["ai"]["generate_cover"] = True
|
||||
batch_id, tasks = self._collected_tasks(temp_dir, cfg, ["旧标题"])
|
||||
self._write_old_cover_files(tasks)
|
||||
db.set_generated(tasks[0].id, "已有标题", None, path=cfg["db_path"])
|
||||
key = db.ensure_image_task_key(tasks[0].id, path=cfg["db_path"])
|
||||
db.set_image_task_submitted(tasks[0].id, "cmhub-task-resume", key, path=cfg["db_path"])
|
||||
task = db.list_tasks(batch_id=batch_id, path=cfg["db_path"])[0]
|
||||
generated_png = self._png_bytes()
|
||||
public_dns = [
|
||||
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 443))
|
||||
]
|
||||
posts = []
|
||||
|
||||
def fake_request(method, url, **kwargs):
|
||||
if str(method).upper() == "POST":
|
||||
posts.append((method, url, kwargs))
|
||||
return _RequestsResponse({"task_id": "unexpected", "status": "queued"})
|
||||
return _RequestsResponse(
|
||||
{
|
||||
"task_id": "cmhub-task-resume",
|
||||
"status": "succeeded",
|
||||
"result": {"image_url": "https://cdn.example.com/resume.png"},
|
||||
}
|
||||
)
|
||||
|
||||
def fake_get(url, **kwargs):
|
||||
return _RequestsResponse(content=generated_png)
|
||||
|
||||
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request), \
|
||||
mock.patch.object(ai._cmhub_session(), "get", side_effect=fake_get), \
|
||||
mock.patch("app.ai.socket.getaddrinfo", return_value=public_dns):
|
||||
summary = ai.generate_batch(
|
||||
[task],
|
||||
{"title": "标题提示", "cover": "封面 {新标题}"},
|
||||
ai_cfg={
|
||||
"config": cfg,
|
||||
"db_path": cfg["db_path"],
|
||||
"cmhub_config_path": key_path,
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(summary["ok"])
|
||||
self.assertEqual([], posts)
|
||||
updated = db.get_task(task.id, path=cfg["db_path"])
|
||||
self.assertEqual("cmhub-task-resume", updated.image_task_id)
|
||||
self.assertEqual(key, updated.image_task_key)
|
||||
self.assertTrue(os.path.exists(updated.new_cover_path))
|
||||
|
||||
self.assert_removed(temp_dir)
|
||||
|
||||
def test_generate_batch_cmhub_failed_task_clears_image_task_state(self):
|
||||
try:
|
||||
from PIL import Image # noqa: F401
|
||||
except ImportError:
|
||||
self.skipTest("Pillow not installed")
|
||||
|
||||
with self.make_temp_dir() as temp_dir:
|
||||
cfg, key_path = self._cmhub_config(temp_dir)
|
||||
cfg["db_path"] = os.path.join(temp_dir, "cmshopee.db")
|
||||
cfg["image_dir"] = os.path.join(temp_dir, "images")
|
||||
cfg["ai"]["generate_cover"] = True
|
||||
batch_id, tasks = self._collected_tasks(temp_dir, cfg, ["旧标题"])
|
||||
self._write_old_cover_files(tasks)
|
||||
db.set_generated(tasks[0].id, "已有标题", None, path=cfg["db_path"])
|
||||
key = db.ensure_image_task_key(tasks[0].id, path=cfg["db_path"])
|
||||
db.set_image_task_submitted(tasks[0].id, "cmhub-task-failed", key, path=cfg["db_path"])
|
||||
task = db.list_tasks(batch_id=batch_id, path=cfg["db_path"])[0]
|
||||
|
||||
def fake_request(method, url, **kwargs):
|
||||
return _RequestsResponse(
|
||||
{
|
||||
"task_id": "cmhub-task-failed",
|
||||
"status": "failed",
|
||||
"error": {"code": "upstream_timeout", "message": "上游超时"},
|
||||
}
|
||||
)
|
||||
|
||||
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request):
|
||||
summary = ai.generate_batch(
|
||||
[task],
|
||||
{"title": "标题提示", "cover": "封面 {新标题}"},
|
||||
ai_cfg={
|
||||
"config": cfg,
|
||||
"db_path": cfg["db_path"],
|
||||
"cmhub_config_path": key_path,
|
||||
},
|
||||
)
|
||||
|
||||
self.assertFalse(summary["ok"])
|
||||
self.assertEqual(1, summary["failed"])
|
||||
updated = db.get_task(task.id, path=cfg["db_path"])
|
||||
self.assertIsNone(updated.image_task_id)
|
||||
self.assertIsNone(updated.image_task_key)
|
||||
self.assertIn("上游生成超时", updated.last_error)
|
||||
|
||||
self.assert_removed(temp_dir)
|
||||
|
||||
def test_generate_batch_cmhub_cancelled_poll_keeps_image_task_state(self):
|
||||
try:
|
||||
from PIL import Image # noqa: F401
|
||||
except ImportError:
|
||||
self.skipTest("Pillow not installed")
|
||||
|
||||
with self.make_temp_dir() as temp_dir:
|
||||
cfg, key_path = self._cmhub_config(temp_dir)
|
||||
cfg["db_path"] = os.path.join(temp_dir, "cmshopee.db")
|
||||
cfg["image_dir"] = os.path.join(temp_dir, "images")
|
||||
cfg["ai"]["generate_cover"] = True
|
||||
batch_id, tasks = self._collected_tasks(temp_dir, cfg, ["旧标题"])
|
||||
self._write_old_cover_files(tasks)
|
||||
db.set_generated(tasks[0].id, "已有标题", None, path=cfg["db_path"])
|
||||
task = db.list_tasks(batch_id=batch_id, path=cfg["db_path"])[0]
|
||||
events = []
|
||||
|
||||
def fake_request(method, url, **kwargs):
|
||||
if str(method).upper() == "POST":
|
||||
return _RequestsResponse({"task_id": "cmhub-task-cancel", "status": "queued"})
|
||||
return _RequestsResponse({"task_id": "cmhub-task-cancel", "status": "running"})
|
||||
|
||||
def should_stop():
|
||||
current = db.get_task(task.id, path=cfg["db_path"])
|
||||
return bool(current and current.image_task_id)
|
||||
|
||||
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request):
|
||||
summary = ai.generate_batch(
|
||||
[task],
|
||||
{"title": "标题提示", "cover": "封面 {新标题}"},
|
||||
ai_cfg={
|
||||
"config": cfg,
|
||||
"db_path": cfg["db_path"],
|
||||
"cmhub_config_path": key_path,
|
||||
"on_event": events.append,
|
||||
},
|
||||
should_stop=should_stop,
|
||||
)
|
||||
|
||||
self.assertFalse(summary["ok"])
|
||||
self.assertTrue(summary["cancelled"])
|
||||
updated = db.get_task(task.id, path=cfg["db_path"])
|
||||
self.assertEqual("cmhub-task-cancel", updated.image_task_id)
|
||||
self.assertTrue(updated.image_task_key)
|
||||
self.assertIsNone(updated.new_cover_path)
|
||||
self.assertTrue(
|
||||
any("服务端任务可能仍在完成" in event.get("detail", "") for event in events)
|
||||
)
|
||||
|
||||
self.assert_removed(temp_dir)
|
||||
|
||||
def test_generate_batch_forwards_cmhub_metadata_event(self):
|
||||
with self.make_temp_dir() as temp_dir:
|
||||
cfg, key_path = self._cmhub_config(temp_dir)
|
||||
|
||||
@@ -29,6 +29,11 @@ class DbTests(TempDirMixin, unittest.TestCase):
|
||||
).fetchall()
|
||||
}
|
||||
self.assertTrue({"batches", "accounts", "tasks"}.issubset(tables))
|
||||
task_columns = {
|
||||
row["name"]
|
||||
for row in conn.execute("PRAGMA table_info(tasks)").fetchall()
|
||||
}
|
||||
self.assertTrue({"image_task_id", "image_task_key"}.issubset(task_columns))
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
@@ -79,6 +84,8 @@ class DbTests(TempDirMixin, unittest.TestCase):
|
||||
task = db.list_tasks(batch_id=batch_id, alias="alias", path=db_path)[0]
|
||||
self.assertEqual("imported", task.stage)
|
||||
self.assertEqual("pending", task.status)
|
||||
self.assertIsNone(task.image_task_id)
|
||||
self.assertIsNone(task.image_task_key)
|
||||
|
||||
db.mark_running(task.id, "collect", path=db_path)
|
||||
self.assertEqual("running", db.list_tasks(path=db_path)[0].status)
|
||||
@@ -126,6 +133,65 @@ class DbTests(TempDirMixin, unittest.TestCase):
|
||||
|
||||
self.assert_removed(temp_dir)
|
||||
|
||||
def test_image_task_helpers_and_reset_lifecycle(self):
|
||||
with self.make_temp_dir() as temp_dir:
|
||||
db_path = os.path.join(temp_dir, "cmshopee.db")
|
||||
db.init_db(db_path)
|
||||
batch_id = db.create_batch(["input.xlsx"], path=db_path)
|
||||
db.insert_tasks(
|
||||
batch_id,
|
||||
[
|
||||
{
|
||||
"source_file_abs": os.path.join(temp_dir, "input.xlsx"),
|
||||
"source_sheet": "Sheet1",
|
||||
"source_row": 2,
|
||||
"account_name": "shop",
|
||||
"alias": "alias",
|
||||
"item_id": "51100639510",
|
||||
}
|
||||
],
|
||||
path=db_path,
|
||||
)
|
||||
task = db.list_tasks(batch_id=batch_id, path=db_path)[0]
|
||||
db.set_collected(task.id, "旧标题", "old.jpg", path=db_path)
|
||||
db.set_generated(task.id, "新标题", "new.jpg", path=db_path)
|
||||
|
||||
key = db.ensure_image_task_key(task.id, path=db_path)
|
||||
self.assertTrue(key.startswith(f"cmshopee-task-{task.id}-"))
|
||||
self.assertEqual(key, db.ensure_image_task_key(task.id, path=db_path))
|
||||
|
||||
db.set_image_task_submitted(task.id, "cmhub-task-1", key, path=db_path)
|
||||
submitted = db.get_task(task.id, path=db_path)
|
||||
self.assertEqual("cmhub-task-1", submitted.image_task_id)
|
||||
self.assertEqual(key, submitted.image_task_key)
|
||||
|
||||
title_only = db.reset_generated(
|
||||
task.id,
|
||||
reset_title=True,
|
||||
reset_cover=False,
|
||||
path=db_path,
|
||||
)["after"]
|
||||
self.assertEqual("cmhub-task-1", title_only.image_task_id)
|
||||
self.assertEqual(key, title_only.image_task_key)
|
||||
|
||||
cover_reset = db.reset_generated(
|
||||
task.id,
|
||||
reset_title=False,
|
||||
reset_cover=True,
|
||||
path=db_path,
|
||||
)["after"]
|
||||
self.assertIsNone(cover_reset.image_task_id)
|
||||
self.assertIsNone(cover_reset.image_task_key)
|
||||
|
||||
new_key = db.ensure_image_task_key(task.id, path=db_path)
|
||||
self.assertNotEqual(key, new_key)
|
||||
db.clear_image_task(task.id, path=db_path)
|
||||
cleared = db.get_task(task.id, path=db_path)
|
||||
self.assertIsNone(cleared.image_task_id)
|
||||
self.assertIsNone(cleared.image_task_key)
|
||||
|
||||
self.assert_removed(temp_dir)
|
||||
|
||||
def test_mark_failed_prepends_chinese_step_without_duplicate_prefix(self):
|
||||
with self.make_temp_dir() as temp_dir:
|
||||
db_path = os.path.join(temp_dir, "cmshopee.db")
|
||||
|
||||
Reference in New Issue
Block a user