feat: use system curl for cmhub image downloads
This commit is contained in:
@@ -7,7 +7,10 @@ import copy
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
@@ -973,6 +976,8 @@ def _request_cmhub_cover_image(
|
||||
"out_path": out_path,
|
||||
"resolution": resolution,
|
||||
"quality": quality,
|
||||
"use_system_proxy": runtime["use_system_proxy"],
|
||||
"download_with_curl": runtime["download_with_curl"],
|
||||
}
|
||||
|
||||
|
||||
@@ -988,6 +993,8 @@ def _download_and_save_cmhub_cover(request_result, on_step=None):
|
||||
image_url,
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
use_system_proxy=request_result.get("use_system_proxy", False),
|
||||
download_with_curl=request_result.get("download_with_curl", "false"),
|
||||
on_step=on_step,
|
||||
)
|
||||
_notify_step_event(
|
||||
@@ -1034,6 +1041,7 @@ def _cmhub_runtime(config, operation, cmhub_config_path):
|
||||
"alias": hub[alias_key],
|
||||
"connect_timeout": max(1, int(hub.get("connect_timeout", 10) or 10)),
|
||||
"use_system_proxy": use_system_proxy,
|
||||
"download_with_curl": str(hub.get("download_with_curl", "auto") or "auto"),
|
||||
}
|
||||
|
||||
|
||||
@@ -1075,6 +1083,8 @@ def _download_cmhub_image_with_retry(
|
||||
url,
|
||||
connect_timeout,
|
||||
read_timeout,
|
||||
use_system_proxy=False,
|
||||
download_with_curl="false",
|
||||
on_step=None,
|
||||
attempts=CMHUB_IMAGE_DOWNLOAD_ATTEMPTS,
|
||||
slow_threshold=CMHUB_IMAGE_SLOW_DOWNLOAD_SECONDS,
|
||||
@@ -1088,6 +1098,8 @@ def _download_cmhub_image_with_retry(
|
||||
url,
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
use_system_proxy=use_system_proxy,
|
||||
download_with_curl=download_with_curl,
|
||||
)
|
||||
elapsed = time.perf_counter() - total_started
|
||||
if elapsed >= float(slow_threshold or 0):
|
||||
@@ -1346,8 +1358,35 @@ def _format_bytes(size):
|
||||
return "%.1f%s" % (value, unit)
|
||||
|
||||
|
||||
def _download_cmhub_image(url, connect_timeout, read_timeout, max_bytes=CMHUB_IMAGE_MAX_BYTES):
|
||||
def _download_cmhub_image(
|
||||
url,
|
||||
connect_timeout,
|
||||
read_timeout,
|
||||
max_bytes=CMHUB_IMAGE_MAX_BYTES,
|
||||
use_system_proxy=False,
|
||||
download_with_curl="false",
|
||||
):
|
||||
_assert_public_http_url(url)
|
||||
if _should_use_curl_for_cmhub_download(download_with_curl):
|
||||
try:
|
||||
return _download_cmhub_image_with_curl(
|
||||
url,
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
max_bytes=max_bytes,
|
||||
use_system_proxy=use_system_proxy,
|
||||
)
|
||||
except AIError:
|
||||
pass
|
||||
return _download_cmhub_image_with_requests(
|
||||
url,
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
max_bytes=max_bytes,
|
||||
)
|
||||
|
||||
|
||||
def _download_cmhub_image_with_requests(url, connect_timeout, read_timeout, max_bytes):
|
||||
try:
|
||||
response = _cmhub_session().get(
|
||||
url,
|
||||
@@ -1372,6 +1411,119 @@ def _download_cmhub_image(url, connect_timeout, read_timeout, max_bytes=CMHUB_IM
|
||||
return b"".join(chunks)
|
||||
|
||||
|
||||
def _should_use_curl_for_cmhub_download(mode):
|
||||
normalized = _normalize_curl_download_mode(mode)
|
||||
if normalized == "false":
|
||||
return False
|
||||
if normalized == "auto" and os.name != "nt":
|
||||
return False
|
||||
return bool(_find_system_curl())
|
||||
|
||||
|
||||
def _normalize_curl_download_mode(mode):
|
||||
if isinstance(mode, bool):
|
||||
return "true" if mode else "false"
|
||||
normalized = str(mode or "auto").strip().lower()
|
||||
if normalized in {"auto", "true", "false"}:
|
||||
return normalized
|
||||
return "auto"
|
||||
|
||||
|
||||
def _find_system_curl():
|
||||
candidates = []
|
||||
if os.name == "nt":
|
||||
system_root = os.environ.get("SystemRoot") or r"C:\Windows"
|
||||
candidates.append(os.path.join(system_root, "System32", "curl.exe"))
|
||||
discovered = shutil.which("curl")
|
||||
if discovered:
|
||||
candidates.append(discovered)
|
||||
seen = set()
|
||||
for candidate in candidates:
|
||||
if not candidate:
|
||||
continue
|
||||
path = os.path.abspath(candidate)
|
||||
lowered = path.lower()
|
||||
if lowered in seen:
|
||||
continue
|
||||
seen.add(lowered)
|
||||
if os.path.isfile(path):
|
||||
return path
|
||||
return ""
|
||||
|
||||
|
||||
def _download_cmhub_image_with_curl(
|
||||
url,
|
||||
connect_timeout,
|
||||
read_timeout,
|
||||
max_bytes,
|
||||
use_system_proxy=False,
|
||||
):
|
||||
curl_path = _find_system_curl()
|
||||
if not curl_path:
|
||||
raise AIError("下载 cmhub 图片失败: 未找到系统 curl")
|
||||
temp_config_path = None
|
||||
temp_output_path = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(
|
||||
"w",
|
||||
encoding="utf-8",
|
||||
suffix=".curlrc",
|
||||
delete=False,
|
||||
) as config_file:
|
||||
temp_config_path = config_file.name
|
||||
config_file.write("url = %s\n" % _curl_config_quote(url))
|
||||
with tempfile.NamedTemporaryFile("wb", suffix=".img", delete=False) as output_file:
|
||||
temp_output_path = output_file.name
|
||||
args = [
|
||||
curl_path,
|
||||
"-K",
|
||||
temp_config_path,
|
||||
"--fail",
|
||||
"--silent",
|
||||
"--show-error",
|
||||
"--connect-timeout",
|
||||
str(max(1, int(connect_timeout))),
|
||||
"--max-time",
|
||||
str(max(1, int(read_timeout))),
|
||||
"--max-filesize",
|
||||
str(max(1, int(max_bytes))),
|
||||
"--output",
|
||||
temp_output_path,
|
||||
]
|
||||
if not bool(use_system_proxy):
|
||||
args.extend(["--noproxy", "*"])
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
args,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=max(2, int(connect_timeout) + int(read_timeout) + 10),
|
||||
check=False,
|
||||
shell=False,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
raise AIError("下载 cmhub 图片失败: curl 执行失败") from exc
|
||||
if completed.returncode != 0:
|
||||
raise AIError("下载 cmhub 图片失败: curl 退出码 %s" % completed.returncode)
|
||||
size = os.path.getsize(temp_output_path)
|
||||
if size > max_bytes:
|
||||
raise AIError("下载 cmhub 图片失败: 图片超过大小上限")
|
||||
with open(temp_output_path, "rb") as fh:
|
||||
return fh.read()
|
||||
finally:
|
||||
for path in (temp_config_path, temp_output_path):
|
||||
if path:
|
||||
try:
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _curl_config_quote(value):
|
||||
text = str(value or "")
|
||||
return '"' + text.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
||||
|
||||
|
||||
def _assert_public_http_url(url):
|
||||
parts = urllib.parse.urlsplit(str(url or ""))
|
||||
if parts.scheme not in {"http", "https"}:
|
||||
|
||||
@@ -83,6 +83,7 @@ DEFAULT_CONFIG = {
|
||||
"image_alias": "",
|
||||
"connect_timeout": 10,
|
||||
"use_system_proxy": False,
|
||||
"download_with_curl": "auto",
|
||||
"check_balance_before_batch": False,
|
||||
},
|
||||
"title_concurrency": 4,
|
||||
@@ -375,6 +376,9 @@ def _normalize_config_values(config):
|
||||
cmhub = ai.get("cmhub")
|
||||
if isinstance(cmhub, dict):
|
||||
cmhub["base_url"] = normalize_cmhub_base_url(cmhub.get("base_url", ""))
|
||||
cmhub["download_with_curl"] = _normalize_cmhub_download_with_curl(
|
||||
cmhub.get("download_with_curl", "auto")
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
@@ -386,6 +390,15 @@ def _clamp_int(value, minimum, maximum, default):
|
||||
return min(int(maximum), max(int(minimum), number))
|
||||
|
||||
|
||||
def _normalize_cmhub_download_with_curl(value):
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
text = str(value or "auto").strip().lower()
|
||||
if text in {"auto", "true", "false"}:
|
||||
return text
|
||||
return "auto"
|
||||
|
||||
|
||||
def _assert_no_secrets(config):
|
||||
def visit(value, path):
|
||||
if isinstance(value, dict):
|
||||
@@ -621,6 +634,9 @@ def cmhub_config(config=None) -> dict:
|
||||
merged["title_alias"] = str(merged.get("title_alias", "") or "").strip()
|
||||
merged["image_alias"] = str(merged.get("image_alias", "") or "").strip()
|
||||
merged["connect_timeout"] = int(merged.get("connect_timeout", 10) or 10)
|
||||
merged["download_with_curl"] = _normalize_cmhub_download_with_curl(
|
||||
merged.get("download_with_curl", "auto")
|
||||
)
|
||||
merged["check_balance_before_batch"] = bool(merged.get("check_balance_before_batch", False))
|
||||
if merged["connect_timeout"] <= 0:
|
||||
raise ConfigError("ai.cmhub.connect_timeout 必须大于 0")
|
||||
|
||||
@@ -403,6 +403,7 @@ data/images/<batch_id>/<slug>/<task_id>_<item_id>_new.<ext> # AI 生成的新
|
||||
2. **若②勾选「生成封面图片(成本较高)」**:接着并发生成图片,线程池大小 = `image_concurrency`,调 `gen_cover(封面提示词, old_cover_path, resolution, jpg_quality)` → 新图存 `data/images/<batch_id>/<slug>/<task_id>_<item_id>_new.jpg`。默认 `backend=cmhub`,调用 `POST /api/v1/generate/image`,拿 `image_url` 后立即安全下载并转本地 JPEG;`backend=direct` 仅内部兼容时使用 `default_image_model`。
|
||||
- T-545 已实现:cmhub 模式下不再直接按用户填写的 `image_concurrency` 全量打到网关;实际生图请求并发 = `min(image_concurrency, 5)`。拿到 `image_url` 后交给独立下载/保存线程池,下载线程数量与实际生图请求并发一致,同样最大 5。这样批量时“下一批生图请求”和“上一批图片下载/保存”可以流水线并行,但不会对 cmhub 生图接口或 `/media/generated/images/*.png` 静态下载打出超过 5 的并发。direct 兼容路径暂不改变。
|
||||
- cmhub 生图连接超时取 `ai.cmhub.connect_timeout`;生图请求和随后 `image_url` 下载的读取等待统一固定 650 秒,读超时不自动重发,避免重复扣点。下载层最多安全重试 3 次,只复用同一个 `image_url`,不重新请求 cmhub 生图;下载总耗时超过 20 秒时写“图片下载较慢”警告。`backend=direct` 兼容路径仍按模型 `timeout_seconds` 或 `resolution_timeouts[resolution]`(512→180/1k→240/2k→360/4k→600)取返回超时。
|
||||
- T-548 已实现:cmhub 图片下载后端新增 `ai.cmhub.download_with_curl`(`auto`/`true`/`false`,默认 `auto`)。Windows 且检测到系统 curl 时优先用 curl 下载,否则或 curl 执行失败时回退 requests;生成、models、balance 仍走共享 requests Session。curl 下载前仍执行公网 URL 校验;URL 写入临时 curl 配置文件并通过 `-K` 传入,避免带 token 的 `image_url` 出现在进程命令行;`use_system_proxy=false` 时 curl 加 `--noproxy "*"`。
|
||||
- cmhub 返回的完整 `image_url` 默认只在内存中临时用于下载,不写入 `tasks` 或 `run_log_events`。本机调试时可设置环境变量 `CMSHOPEE_DEBUG_CMHUB_IMAGE_URL=1`,②本轮可见运行日志会显示脱敏后的 URL 调试行,且该行不持久化到 SQLite。
|
||||
3. **若未勾选生成封面**:标题成功后立即写 `new_title`,`new_cover_path=NULL`,不渲染封面提示词、不调用 `gen_cover()`、不创建本地新封面文件。
|
||||
|
||||
|
||||
+1
-1
@@ -138,7 +138,7 @@
|
||||
| T-545 | ② cmhub 生图和下载并发上限 5 + 独立下载线程池 | T-535, T-533, T-519 | 背景:实测 10 图片并发时,cmhub 后台单条生成小于 200 秒,但本地下载 `/media/generated/images/*.png` 常见 24~185 秒且有连接失败;如果每个生图线程同时负责“等待 cmhub 返回 + 下载 + 转 JPEG + 保存”,下载慢会占住生图线程,后续任务排队。方案:cmhub 模式下实际生图请求并发 = `min(ai.image_concurrency, 5)`;拿到 `image_url` 后把下载/转 JPEG/保存交给独立下载线程池,下载线程数与实际生图请求并发一致、同样最大 5;不新增用户可见配置项,⑤仍只保留「图片并发」。②开始日志必须显示用户设置图片并发、cmhub 实际生图并发和下载并发;分段日志继续记录“cmhub 已返回 image_url / 下载完成 / 本地保存完成”的耗时。下载失败按当前任务失败记录,但不得重新调用 cmhub 生图接口导致重复扣点;`cover_done/generated_done` 必须等下载保存成功并写 DB 后才计数;停止逻辑继续取消未开始项,运行中请求/下载允许自然完成或失败。direct 兼容路径暂不改变。补 `tests/test_ai.py` 覆盖并发上限和下载线程池不阻塞后续生图提交、下载失败不重复扣点;补 `tests/test_gui.py` 覆盖开始日志显示实际并发 | DONE |
|
||||
| T-536 | GUI 按钮圆角全局统一 | T-512, T-513, T-523 | 现象:只有③「开始更新」及少数上色按钮(删除批次/删除账号/未匹配(n))有 `border-radius: 4px`,其余按钮走原生渲染显直角——不一致。根因:圆角是 T-512/T-513 给按钮上色时顺带写进 QSS 的副产品,不是全局形状决策;一旦给 `QPushButton` 设 stylesheet 就放弃原生渲染,才补了 radius/border。方案(**方式 A:全局统一圆角**):① 在 `app/gui/widgets.py` 抽一个**共享按钮基础样式常量/helper**(统一 `border-radius`,如 4px,与卡片 6px 圆角语言协调),并**接管按钮的全部视觉状态**——normal/hover/pressed/disabled/focus 的背景、边框都定义好,避免全局设 QSS 后按钮变扁平方块、丢 hover 反馈;② 在主窗口/app 级用全局 `QPushButton` QSS 应用该基础样式,让**所有按钮共享同一圆角**;③ warning(`startUpdateButton`)与 danger(`_danger_outline_button_style`:删除批次/删除账号/未匹配)按钮改为**只叠加颜色**,复用共享基础样式的圆角/内边距/状态,不再各自重写 radius/border——杜绝“上色=顺带圆角”的隐性耦合;④ 以 Windows 为主目标做一次视觉自测(hover/按下/禁用不劣于原生)。只改 GUI 样式层(`widgets.py` + 主窗口全局 QSS + 各上色按钮引用),不改任何按钮的启用/禁用逻辑、行为、业务流程、DB、Excel、Shopee/CDP。GUI 单测至少断言上色按钮仍带各自语义色且不再各自硬写 radius(改为引用共享样式);圆角外观本身以人工视觉验收为准 | DONE |
|
||||
| T-546 | cmhub 客户端共享 Session + 连接池 + 代理处理(补 T-545 未覆盖的下载慢根因) | T-545, T-526 | 现象:实测生图拿到 `image_url` 后本地下载单条约 191 秒、而同一 URL 用 curl 约 10 秒;日志伴随 `connect_timeout: 连接 cmhub 超时`。T-545 已做「生图并发上限 5 + 独立下载线程池」解决“下载堵住生图槽位”的流水线问题,但**未根治单条下载在并发争抢下变慢**。根因(代码核实):① cmhub 所有 HTTP(`_cmhub_call_once` 的生成/models/balance、`_download_cmhub_image` 的下载)都是**裸 `requests.get`/`requests.request`、无共享 `Session`**,每个任务全新 TCP+TLS,叠加同步生成长连接(单条 60~260s)与 `connect_timeout` 触发的重试,形成连接风暴/连接饥饿;② `requests` 默认 `trust_env=True` 读 `HTTP(S)_PROXY`/`ALL_PROXY`,若系统带慢代理会拖累,而干净窗口的 curl 直连快。方案:① cmhub 所有 HTTP 统一走一个**模块级共享 `requests.Session`**,挂 `HTTPAdapter(pool_connections/pool_maxsize)`,池大小 ≥(实际生图并发 + 下载并发)以免连接不足排队、复用连接减少握手与 connect_timeout;生成与下载共享该 Session(`requests.Session` 跨线程发请求安全,但连接池要够大);② 代理处理——诊断日志脱敏记录 cmhub 请求是否经代理;对 cmhub 请求提供明确策略(可配 `trust_env=False` 或显式 `proxies`),避免误走慢代理,默认可先保持读环境但提供关闭开关;③ 复测口径——Session+连接池到位后单条下载耗时应回落到与 curl 同量级;若在 T-545 的并发上限 5 下仍显著慢,评估把 cmhub 生图默认并发再降(1~2);④ 确认 connect_timeout 重试退避不加剧连接风暴。诊断建议:可先临时把图片并发设 1 复测以区分“并发争抢”与“代理”。边界:只改 `app/ai.py` 的 cmhub HTTP 客户端层(+ 可选 config 代理项)+ 诊断日志 + 相关文档;不改 cmhub 协议、T-545 的并发/下载池语义、生成编排、DB、Excel、CDP/Shopee。测试:`tests/test_ai.py` 覆盖共享 Session 被复用(mock 同一 session 多次调用)、代理配置被尊重、连接池大小设定;现有 cmhub 用例保持绿。落地:`app/ai.py` 加模块级 `_cmhub_session()`(`HTTPAdapter` 池 `CMHUB_HTTP_POOL_SIZE=32`)+ `_apply_cmhub_proxy()`;生成/下载/models/balance 统一走该 Session;`ai.cmhub.use_system_proxy` 默认 `false`(绕过系统代理,公网网关直连),`_cmhub_runtime` 读取并应用、`fetch_cmhub_models/balance` 加同名参数;`test_ai.py` 补 3 项新测试并把原 18 处 `requests` mock 改到共享 Session;243 单测全绿。**待 Windows 实网复测确认下载提速**(troubleshooting 已加「cmhub 图片下载很慢」排障节与代理定位命令) | DONE |
|
||||
| T-548 | cmhub 图片下载改用系统 curl(Win10+ 自带) | T-546, T-538 | 背景:图片下载慢的另一手段——用 curl(用户的快基准)下载。curl 在 **Windows 10+ 自带**(`C:\Windows\System32\curl.exe`),**无需捆绑**(不用项目里 `wget/` 那个 2022 老 wget,其 OpenSSL 太旧且捆绑二进制易被杀软误报);应用底线本就是 Win10+(PySide6/Qt6 不支持 Win7)。**前置说明**:若 T-546 的代理绕过已让 requests 下载回落到 curl 同量级,本任务可作可选/降级;此处按用户要求落地。方案:① **只改图片下载这一步**(`app/ai.py` `_download_cmhub_image`),cmhub 生成/models/balance 仍走 requests 共享 Session。② 后端选择:新增 config `ai.cmhub.download_with_curl`(`auto`/`true`/`false`,默认 `auto`=Windows 且检测到 curl 用 curl,否则回退 requests);`shutil.which("curl")` 或固定 `System32\curl.exe` 检测;**非 Windows / 无 curl / curl 执行失败 → 自动回退现有 requests 下载**(保证 Linux/Mac/CI/无 curl 环境和现有测试不受影响)。③ **安全防护(硬要求)**:a) SSRF 不可绕过——调 curl 前仍执行 `_assert_public_http_url()` 拦内网/回环 IP,用 `subprocess.run([...], shell=False)` 参数列表、不拼 shell;b) token 不泄漏——`image_url` 带 `?token=...`,直接进命令行会出现在进程列表(tasklist)和日志,改用 curl `-K <临时配置文件>`(把 `url = "..."` 写进临时文件传入、用完删)避免 URL 进程列表暴露,日志一律脱敏、不打印完整 URL/命令;c) 绕过代理——按 `use_system_proxy`:false 时加 `--noproxy "*"`(这才是 curl 快的关键,与 T-546 一致),true 时不加。④ 超时/大小:`--connect-timeout <connect>` + `--max-time <read≈650>`;下载到临时文件、读回字节后**仍复用现有 `_save_jpeg` 重编码**;保留 20MB 上限(`--max-filesize` + 落盘后大小复核)。⑤ 重试:沿用现有 `_download_cmhub_image_with_retry`(3 次),只替换内层下载实现;curl 非零退出=失败、走现有重试语义。⑥ 打包:curl 系统自带、不捆绑、不改 spec;`docs/packaging.md` 注明依赖 Win10+ 自带 curl。测试:`tests/test_ai.py` mock `subprocess.run`——断言 curl 参数含 `--noproxy`/超时/`-K` 临时配置(URL 不在 argv)、SSRF 校验先于 curl、curl 失败回退 requests、非 Windows/无 curl 自动回退;现有下载用例保持绿。边界:只改 `_download_cmhub_image` 下载后端 + config 项 + 诊断脱敏 + 打包文档;不改生成/API 调用、`_save_jpeg`、并发/重试语义、DB、Excel、CDP/Shopee | TODO |
|
||||
| T-548 | cmhub 图片下载改用系统 curl(Win10+ 自带) | T-546, T-538 | 背景:图片下载慢的另一手段——用 curl(用户的快基准)下载。curl 在 **Windows 10+ 自带**(`C:\Windows\System32\curl.exe`),**无需捆绑**(不用项目里 `wget/` 那个 2022 老 wget,其 OpenSSL 太旧且捆绑二进制易被杀软误报);应用底线本就是 Win10+(PySide6/Qt6 不支持 Win7)。**前置说明**:若 T-546 的代理绕过已让 requests 下载回落到 curl 同量级,本任务可作可选/降级;此处按用户要求落地。方案:① **只改图片下载这一步**(`app/ai.py` `_download_cmhub_image`),cmhub 生成/models/balance 仍走 requests 共享 Session。② 后端选择:新增 config `ai.cmhub.download_with_curl`(`auto`/`true`/`false`,默认 `auto`=Windows 且检测到 curl 用 curl,否则回退 requests);`shutil.which("curl")` 或固定 `System32\curl.exe` 检测;**非 Windows / 无 curl / curl 执行失败 → 自动回退现有 requests 下载**(保证 Linux/Mac/CI/无 curl 环境和现有测试不受影响)。③ **安全防护(硬要求)**:a) SSRF 不可绕过——调 curl 前仍执行 `_assert_public_http_url()` 拦内网/回环 IP,用 `subprocess.run([...], shell=False)` 参数列表、不拼 shell;b) token 不泄漏——`image_url` 带 `?token=...`,直接进命令行会出现在进程列表(tasklist)和日志,改用 curl `-K <临时配置文件>`(把 `url = "..."` 写进临时文件传入、用完删)避免 URL 进程列表暴露,日志一律脱敏、不打印完整 URL/命令;c) 绕过代理——按 `use_system_proxy`:false 时加 `--noproxy "*"`(这才是 curl 快的关键,与 T-546 一致),true 时不加。④ 超时/大小:`--connect-timeout <connect>` + `--max-time <read≈650>`;下载到临时文件、读回字节后**仍复用现有 `_save_jpeg` 重编码**;保留 20MB 上限(`--max-filesize` + 落盘后大小复核)。⑤ 重试:沿用现有 `_download_cmhub_image_with_retry`(3 次),只替换内层下载实现;curl 非零退出=失败、走现有重试语义。⑥ 打包:curl 系统自带、不捆绑、不改 spec;`docs/packaging.md` 注明依赖 Win10+ 自带 curl。测试:`tests/test_ai.py` mock `subprocess.run`——断言 curl 参数含 `--noproxy`/超时/`-K` 临时配置(URL 不在 argv)、SSRF 校验先于 curl、curl 失败回退 requests、非 Windows/无 curl 自动回退;现有下载用例保持绿。边界:只改 `_download_cmhub_image` 下载后端 + config 项 + 诊断脱敏 + 打包文档;不改生成/API 调用、`_save_jpeg`、并发/重试语义、DB、Excel、CDP/Shopee | DONE |
|
||||
|
||||
## Phase 8 · 工程基础设施后续(`docs/engineering-review.md`)
|
||||
|
||||
|
||||
+2
-2
@@ -33,7 +33,7 @@ ai_config(config=None) -> dict # default_text_model/default_image
|
||||
# title_concurrency/image_concurrency/retry/jpg_quality/
|
||||
# resolution/resolution_timeouts
|
||||
ai_backend(config=None) -> str # 默认 cmhub;direct 仅内部兼容/手工回滚
|
||||
cmhub_config(config=None) -> dict # base_url/title_alias/image_alias/connect_timeout
|
||||
cmhub_config(config=None) -> dict # base_url/title_alias/image_alias/connect_timeout/download_with_curl
|
||||
normalize_cmhub_base_url(base_url) -> str # 规整为 cmhub 网关根:scheme+host(+port)
|
||||
cmhub_request_url(base_url, endpoint) -> str # 先规整 base_url,再拼 /api/v1/...
|
||||
response_timeout(config=None) -> int # = resolution_timeouts[resolution](返回超时,随分辨率)
|
||||
@@ -313,7 +313,7 @@ fetch_cmhub_models(base_url, api_key, connect_timeout=10, read_timeout=30) -> li
|
||||
- `api_type=chat/auto` 走 OpenAI-compatible chat JSON;`api_type=images_edits` 走 multipart form。
|
||||
- direct 连接超时参考模型 `connect_timeout_seconds`;**返回超时 = 模型 `timeout_seconds` 或 `appconfig.response_timeout()`(随分辨率:512/1k/2k/4k → 180/240/360/600)**。cmhub 使用 `requests timeout=(connect, read)`,connect 来自 `ai.cmhub.connect_timeout`;cmhub 生文读取等待固定 600s,cmhub 生图请求和图片下载读取等待固定 650s,且生图读超时不自动重发。
|
||||
- 并发数/重试/分辨率/jpg 质量来自 `appconfig.ai_config()`;标题/图片并发会被夹到 1..5,失败重试次数会被夹到 0..10,兼容旧配置中的超限值;Key 本地明文存储,但不入日志、不导出。cmhub 响应的 `points_cost/points_balance/call_id` 不改变返回值,通过 `on_event` metadata 事件上报,GUI 余额/计费展示留给 T-528。
|
||||
- 标题快、图片慢:分两段、各用各自并发数;失败按 `retry` 重试,仍失败记 error 不阻塞其余。cmhub 生图使用内部实际并发 `min(image_concurrency, 5)`,拿到 `image_url` 后交给独立下载/保存线程池,下载线程数与实际生图并发一致且最大 5;下载失败最多安全重试 3 次,只复用同一个 `image_url`,不会重新调用 cmhub 生图接口;下载总耗时超过 20 秒时写“图片下载较慢”警告;direct 兼容路径暂保持原 `image_concurrency` 语义。
|
||||
- 标题快、图片慢:分两段、各用各自并发数;失败按 `retry` 重试,仍失败记 error 不阻塞其余。cmhub 生图使用内部实际并发 `min(image_concurrency, 5)`,拿到 `image_url` 后交给独立下载/保存线程池,下载线程数与实际生图并发一致且最大 5;T-548 后图片下载后端由 `ai.cmhub.download_with_curl` 控制,默认 `auto`:Windows 且检测到系统 curl 时优先用 curl 下载,否则回退 requests;curl 失败也会自动回退 requests。下载前仍执行公网 URL 校验,curl 使用 `-K` 临时配置文件传入 URL,不把带 token 的 `image_url` 放进命令行参数;`use_system_proxy=false` 时 curl 加 `--noproxy "*"`。下载失败最多安全重试 3 次,只复用同一个 `image_url`,不会重新调用 cmhub 生图接口;下载总耗时超过 20 秒时写“图片下载较慢”警告;direct 兼容路径暂保持原 `image_concurrency` 语义。
|
||||
- 调试 cmhub 图片 URL:默认不在日志或 DB 中记录完整 `image_url`。若本机调试需要查看,启动程序前设置环境变量 `CMSHOPEE_DEBUG_CMHUB_IMAGE_URL=1`,②本轮可见运行日志会显示脱敏后的“cmhub 图片 URL”,但该调试行不写入 SQLite `run_log_events`。
|
||||
- 调用有成本与失败可能:超时、限流、内容安全拒绝都要返回明确错误。
|
||||
- 生成结果**直接进入 ③ 更新候选**;③ 点击「开始更新」后弹窗批量确认,确认后提交线上。本地留档 + 回写 Excel 供追溯。
|
||||
|
||||
@@ -98,6 +98,7 @@
|
||||
- **响应**:拿 `image_url` → **新增一步下载**该图字节(cmhub 自家对象存储公网 URL)→ 交给现有 `_save_jpeg(image_bytes, out_path, resolution, quality)` 落盘。下载 helper 必须校验 URL scheme 只允许 `http/https`,拒绝内网/回环/本机地址,并校验域名解析后的 IP 仍不属于内网/回环/本机地址,设置超时和大小上限;生成后**立即下载**(对象存储 URL 可能有有效期)。`points_cost`/`points_balance`/`call_id` 同样通过事件回调传播,不改变 `gen_cover` 返回值。
|
||||
- **超时(关键)**:生图同步且慢。当前 cmshopee 的 cmhub 生图请求和随后 `image_url` 下载读取等待统一固定 650 秒,不再按分辨率变化,绝不用 30s/60s 调生图——否则客户端超时但服务端仍在算并扣点(见 §4.4 幂等)。
|
||||
- **并发(T-545 已实现)**:最近实测 `/media/generated/images/*.png` 下载链路在 10 并发下明显慢且有连接失败。cmhub 模式下采用内置保护:实际生图请求并发 = `min(ai.image_concurrency, 5)`;下载/保存使用独立线程池,线程数与实际生图请求并发一致,同样最大 5;不新增用户可见配置项。运行日志必须同时显示用户设置和实际并发,避免用户误解设置 10 就会对 cmhub 打 10 并发。下载失败记为该任务失败,不得重新调用生图接口导致重复扣点;读超时仍按 §4.4 的非幂等规则处理。
|
||||
- **下载后端(T-548 已实现)**:cmhub 生成/models/balance 仍走共享 requests Session;仅 `image_url` 图片下载可按 `ai.cmhub.download_with_curl` 选择系统 curl。默认 `auto` 在 Windows 且检测到系统 curl 时优先 curl,非 Windows、无 curl 或 curl 失败自动回退 requests。curl 调用前仍做公网 URL 校验,用 `-K` 临时配置文件传 URL,避免 token 出现在进程命令行;`use_system_proxy=false` 时加 `--noproxy "*"`。
|
||||
|
||||
### 4.4 错误映射与重试策略
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -194,6 +194,8 @@ T-544 第一版目标是**启动时检查是否必须升级**,但仍不做自
|
||||
|
||||
`cmshopee.spec` 不声明任何 `datas`;`scripts/build_exe.ps1` 会在打包后检查 `dist\cmshopee\`,如果发现上述路径会直接失败。
|
||||
|
||||
T-548 后 cmhub 图片下载可在 Windows 上优先调用系统自带 curl。curl 在 Windows 10+ 通常位于 `C:\Windows\System32\curl.exe`,发布包不捆绑 curl、wget 或其它下载二进制;如果系统无 curl 或 curl 执行失败,程序会自动回退到内置 requests 下载。
|
||||
|
||||
## 五、首次运行与本地数据位置
|
||||
|
||||
T-538 后,打包版不再把全进程工作目录切到 `cmshopee.exe` 所在目录;路径由 `app/appconfig.py` 按显式数据根解析。
|
||||
|
||||
@@ -1491,3 +1491,12 @@
|
||||
- 测试:`tests/test_appconfig.py` 覆盖旧配置超限值会归一到 1/5/10、正常值不变;`tests/test_gui.py` 覆盖三个 spin 的新范围。
|
||||
- 文档:`docs/06-tasks.md` 将 T-547 标为 DONE;同步 `docs/04-architecture.md`、`docs/api.md`、`docs/routes.md` 和 `docs/current-state.md`。
|
||||
- 验证:`py -3.10 -m unittest discover -s tests -p "test_appconfig.py"` 通过(15 tests);`py -3.10 -m unittest discover -s tests -p "test_gui.py"` 通过(102 tests);`python -m ruff check app tests main.py` 通过;`py -3.10 -m compileall app main.py` 通过;`git diff --check` 通过;`py -3.10 -m unittest discover -s tests` 通过(244 tests)。
|
||||
|
||||
## 【2026-07-08】T-548 · cmhub 图片下载优先系统 curl
|
||||
|
||||
- 需求:cmhub 生图拿到 `image_url` 后,图片下载可优先使用 Windows 10+ 系统自带 curl,贴近用户用 curl 实测较快的下载路径;不得捆绑 `wget/` 或其它下载二进制。
|
||||
- 代码:`app/appconfig.py` 新增 `ai.cmhub.download_with_curl`,默认 `auto`,并归一为 `auto/true/false`;`app/ai.py` 只替换 `image_url` 下载后端,生成、models、balance 仍走共享 requests Session。`auto` 在 Windows 且检测到系统 curl 时优先 curl,非 Windows、无 curl 或 curl 执行失败时自动回退 requests。
|
||||
- 安全:curl 调用前仍执行 `_assert_public_http_url()`,避免 SSRF 绕过;用 `subprocess.run([...], shell=False)`;带 token 的 `image_url` 写入临时 curl 配置文件并通过 `-K` 传入,不出现在命令行参数;`use_system_proxy=false` 时追加 `--noproxy "*"`;下载后仍走现有 JPEG 重编码和大小上限。
|
||||
- 测试:`tests/test_ai.py` 覆盖 curl 参数、URL 不进 argv、SSRF 校验先于 subprocess、curl 失败回退 requests、无 curl 自动回退 requests;现有 cmhub 测试 helper 显式关闭 curl,避免单测误连真实网络。`tests/test_appconfig.py` 覆盖 `download_with_curl` 默认值和归一。
|
||||
- 文档:同步 `docs/06-tasks.md`、`docs/04-architecture.md`、`docs/api.md`、`docs/cmhub-integration-design.md`、`docs/packaging.md` 和 `docs/current-state.md`。
|
||||
- 验证:`py -3.10 -m unittest discover -s tests -p "test_ai.py"` 通过(38 tests);`py -3.10 -m unittest discover -s tests -p "test_appconfig.py"` 通过(16 tests);`python -m ruff check app tests main.py` 通过;`py -3.10 -m compileall app main.py` 通过;`git diff --check` 通过;`py -3.10 -m unittest discover -s tests` 通过(250 tests)。
|
||||
|
||||
@@ -93,6 +93,7 @@ class AITests(TempDirMixin, unittest.TestCase):
|
||||
"title_alias": "title-standard",
|
||||
"image_alias": "image-hd",
|
||||
"connect_timeout": 3,
|
||||
"download_with_curl": "false",
|
||||
"check_balance_before_batch": False,
|
||||
}
|
||||
key_path = os.path.join(temp_dir, "cmhub.json")
|
||||
@@ -637,6 +638,136 @@ class AITests(TempDirMixin, unittest.TestCase):
|
||||
with self.assertRaises(ai.AIError):
|
||||
ai._download_cmhub_image("https://cdn.example.com/a.png", 1, 1)
|
||||
|
||||
def test_cmhub_image_download_uses_curl_without_url_in_argv(self):
|
||||
generated_png = self._png_bytes()
|
||||
url = "https://cdn.example.com/generated.png?token=secret-token"
|
||||
public_dns = [
|
||||
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 443))
|
||||
]
|
||||
calls = []
|
||||
|
||||
def fake_run(args, **kwargs):
|
||||
calls.append((args, kwargs))
|
||||
self.assertIn("-K", args)
|
||||
config_path = args[args.index("-K") + 1]
|
||||
with open(config_path, "r", encoding="utf-8") as fh:
|
||||
self.assertIn(url, fh.read())
|
||||
self.assertNotIn(url, args)
|
||||
self.assertIn("--connect-timeout", args)
|
||||
self.assertEqual("3", args[args.index("--connect-timeout") + 1])
|
||||
self.assertIn("--max-time", args)
|
||||
self.assertEqual("650", args[args.index("--max-time") + 1])
|
||||
self.assertIn("--max-filesize", args)
|
||||
self.assertEqual(str(ai.CMHUB_IMAGE_MAX_BYTES), args[args.index("--max-filesize") + 1])
|
||||
self.assertIn("--noproxy", args)
|
||||
self.assertEqual("*", args[args.index("--noproxy") + 1])
|
||||
self.assertFalse(kwargs["shell"])
|
||||
output_path = args[args.index("--output") + 1]
|
||||
with open(output_path, "wb") as fh:
|
||||
fh.write(generated_png)
|
||||
return SimpleNamespace(returncode=0, stdout=b"", stderr=b"")
|
||||
|
||||
with mock.patch("app.ai._find_system_curl", return_value=r"C:\Windows\System32\curl.exe"), \
|
||||
mock.patch("app.ai.subprocess.run", side_effect=fake_run), \
|
||||
mock.patch("app.ai.socket.getaddrinfo", return_value=public_dns):
|
||||
image_bytes = ai._download_cmhub_image(
|
||||
url,
|
||||
connect_timeout=3,
|
||||
read_timeout=650,
|
||||
use_system_proxy=False,
|
||||
download_with_curl="true",
|
||||
)
|
||||
|
||||
self.assertEqual(generated_png, image_bytes)
|
||||
self.assertEqual(1, len(calls))
|
||||
|
||||
def test_cmhub_image_download_skips_curl_for_private_url(self):
|
||||
with mock.patch("app.ai._find_system_curl", return_value=r"C:\Windows\System32\curl.exe"), \
|
||||
mock.patch("app.ai.subprocess.run") as run:
|
||||
with self.assertRaises(ai.AIError):
|
||||
ai._download_cmhub_image(
|
||||
"http://127.0.0.1/a.png",
|
||||
connect_timeout=3,
|
||||
read_timeout=650,
|
||||
download_with_curl="true",
|
||||
)
|
||||
run.assert_not_called()
|
||||
|
||||
def test_cmhub_image_download_falls_back_to_requests_when_curl_fails(self):
|
||||
generated_png = self._png_bytes()
|
||||
public_dns = [
|
||||
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 443))
|
||||
]
|
||||
|
||||
def fake_get(url, **kwargs):
|
||||
return _RequestsResponse(content=generated_png)
|
||||
|
||||
with mock.patch("app.ai._find_system_curl", return_value=r"C:\Windows\System32\curl.exe"), \
|
||||
mock.patch(
|
||||
"app.ai.subprocess.run",
|
||||
return_value=SimpleNamespace(returncode=28, stdout=b"", stderr=b"timeout"),
|
||||
) as run, \
|
||||
mock.patch.object(ai._cmhub_session(), "get", side_effect=fake_get) as get, \
|
||||
mock.patch("app.ai.socket.getaddrinfo", return_value=public_dns):
|
||||
image_bytes = ai._download_cmhub_image(
|
||||
"https://cdn.example.com/generated.png",
|
||||
connect_timeout=3,
|
||||
read_timeout=650,
|
||||
download_with_curl="true",
|
||||
)
|
||||
|
||||
self.assertEqual(generated_png, image_bytes)
|
||||
self.assertEqual(1, run.call_count)
|
||||
self.assertEqual(1, get.call_count)
|
||||
|
||||
def test_cmhub_image_download_auto_without_curl_uses_requests(self):
|
||||
generated_png = self._png_bytes()
|
||||
public_dns = [
|
||||
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 443))
|
||||
]
|
||||
|
||||
def fake_get(url, **kwargs):
|
||||
return _RequestsResponse(content=generated_png)
|
||||
|
||||
with mock.patch("app.ai.os.name", "nt"), \
|
||||
mock.patch("app.ai._find_system_curl", return_value=""), \
|
||||
mock.patch("app.ai.subprocess.run") as run, \
|
||||
mock.patch.object(ai._cmhub_session(), "get", side_effect=fake_get), \
|
||||
mock.patch("app.ai.socket.getaddrinfo", return_value=public_dns):
|
||||
image_bytes = ai._download_cmhub_image(
|
||||
"https://cdn.example.com/generated.png",
|
||||
connect_timeout=3,
|
||||
read_timeout=650,
|
||||
download_with_curl="auto",
|
||||
)
|
||||
|
||||
self.assertEqual(generated_png, image_bytes)
|
||||
run.assert_not_called()
|
||||
|
||||
def test_cmhub_image_download_auto_on_non_windows_uses_requests(self):
|
||||
generated_png = self._png_bytes()
|
||||
public_dns = [
|
||||
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 443))
|
||||
]
|
||||
|
||||
def fake_get(url, **kwargs):
|
||||
return _RequestsResponse(content=generated_png)
|
||||
|
||||
with mock.patch("app.ai.os.name", "posix"), \
|
||||
mock.patch("app.ai._find_system_curl", return_value="/usr/bin/curl"), \
|
||||
mock.patch("app.ai.subprocess.run") as run, \
|
||||
mock.patch.object(ai._cmhub_session(), "get", side_effect=fake_get), \
|
||||
mock.patch("app.ai.socket.getaddrinfo", return_value=public_dns):
|
||||
image_bytes = ai._download_cmhub_image(
|
||||
"https://cdn.example.com/generated.png",
|
||||
connect_timeout=3,
|
||||
read_timeout=650,
|
||||
download_with_curl="auto",
|
||||
)
|
||||
|
||||
self.assertEqual(generated_png, image_bytes)
|
||||
run.assert_not_called()
|
||||
|
||||
def test_cmhub_upstream_error_retries_and_keeps_metadata(self):
|
||||
with self.make_temp_dir() as temp_dir:
|
||||
cfg, key_path = self._cmhub_config(temp_dir)
|
||||
|
||||
@@ -101,6 +101,39 @@ class AppConfigTests(TempDirMixin, unittest.TestCase):
|
||||
|
||||
self.assert_removed(temp_dir)
|
||||
|
||||
def test_cmhub_download_with_curl_mode_is_normalized(self):
|
||||
with self.make_temp_dir() as temp_dir:
|
||||
config_path = os.path.join(temp_dir, "config.json")
|
||||
|
||||
config = appconfig.load_config(config_path)
|
||||
self.assertEqual("auto", appconfig.cmhub_config(config)["download_with_curl"])
|
||||
|
||||
saved = appconfig.save_config(
|
||||
{
|
||||
"ai": {
|
||||
"cmhub": {
|
||||
"download_with_curl": True,
|
||||
}
|
||||
}
|
||||
},
|
||||
path=config_path,
|
||||
)
|
||||
self.assertEqual("true", appconfig.cmhub_config(saved)["download_with_curl"])
|
||||
|
||||
saved = appconfig.save_config(
|
||||
{
|
||||
"ai": {
|
||||
"cmhub": {
|
||||
"download_with_curl": "invalid",
|
||||
}
|
||||
}
|
||||
},
|
||||
path=config_path,
|
||||
)
|
||||
self.assertEqual("auto", appconfig.cmhub_config(saved)["download_with_curl"])
|
||||
|
||||
self.assert_removed(temp_dir)
|
||||
|
||||
def test_data_paths_resolve_under_default_data_dir(self):
|
||||
with self.make_temp_dir() as temp_dir:
|
||||
data_root = os.path.join(temp_dir, "data")
|
||||
|
||||
Reference in New Issue
Block a user