feat: cmhub shared Session + connection pool + proxy handling (T-546)
- app/ai.py: 模块级共享 requests.Session + HTTPAdapter 连接池 (CMHUB_HTTP_POOL_SIZE=32)；_apply_cmhub_proxy() 切换 trust_env； 生成/下载/models/balance 统一复用该 Session。 - app/appconfig.py: ai.cmhub.use_system_proxy 默认 false（绕过系统代理）。 - tests/test_ai.py: 补 Session 复用/代理/连接池测试；原 18 处 cmhub mock 从模块级 requests 改到共享 Session。 - docs: troubleshooting 加「图片下载很慢」排障节；06-tasks 标 T-546 DONE。 附带：把 app/ai.py 行尾从 CRLF 归一为 LF；并一并纳入工作区中此前未提交、 已在运行构建里的 cmhub 下载诊断代码（调试图片 URL 日志、下载重试事件）， 无法与 T-546 在同一文件内拆分单独提交。243 单测全绿。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
chengma
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
b257fb5c83
commit
5a09c6662e
@@ -16,6 +16,7 @@ import urllib.request
|
||||
import uuid
|
||||
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
|
||||
from . import appconfig, db, diagnostics, image_paths
|
||||
from . import prompts as prompt_module
|
||||
@@ -40,6 +41,50 @@ CMHUB_IMAGE_MAX_BYTES = 20 * 1024 * 1024
|
||||
CMHUB_TITLE_READ_TIMEOUT_SECONDS = 600
|
||||
CMHUB_IMAGE_READ_TIMEOUT_SECONDS = 650
|
||||
CMHUB_IMAGE_CONCURRENCY_LIMIT = 5
|
||||
CMHUB_IMAGE_DOWNLOAD_ATTEMPTS = 3
|
||||
CMHUB_IMAGE_SLOW_DOWNLOAD_SECONDS = 20.0
|
||||
# 连接池要覆盖“生图并发 + 下载并发”（各上限 5）再留余量，避免连接不足排队或
|
||||
# urllib3 "Connection pool is full" 警告。
|
||||
CMHUB_HTTP_POOL_SIZE = 32
|
||||
|
||||
_CMHUB_SESSION = None
|
||||
_CMHUB_SESSION_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _cmhub_session():
|
||||
"""返回 cmhub 专用的共享 `requests.Session`（连接复用 + 连接池）。
|
||||
|
||||
所有 cmhub HTTP（生成/下载/models/balance）都走同一个 Session，避免每次调用
|
||||
新建 TCP+TLS 造成握手风暴与 connect_timeout；跨线程发请求安全，连接池足够大即可。
|
||||
"""
|
||||
|
||||
global _CMHUB_SESSION
|
||||
if _CMHUB_SESSION is None:
|
||||
with _CMHUB_SESSION_LOCK:
|
||||
if _CMHUB_SESSION is None:
|
||||
session = requests.Session()
|
||||
adapter = HTTPAdapter(
|
||||
pool_connections=CMHUB_HTTP_POOL_SIZE,
|
||||
pool_maxsize=CMHUB_HTTP_POOL_SIZE,
|
||||
max_retries=0,
|
||||
)
|
||||
session.mount("http://", adapter)
|
||||
session.mount("https://", adapter)
|
||||
_CMHUB_SESSION = session
|
||||
return _CMHUB_SESSION
|
||||
|
||||
|
||||
def _apply_cmhub_proxy(use_system_proxy):
|
||||
"""设置 cmhub Session 是否读取系统代理环境变量。
|
||||
|
||||
`use_system_proxy=False` → `trust_env=False`，忽略 `HTTP(S)_PROXY`/`ALL_PROXY` 直连；
|
||||
公网 cmhub 网关默认直连，避免误走慢代理导致明文图片下载卡住。需要代理的环境可在
|
||||
`config.json` 的 `ai.cmhub.use_system_proxy` 打开。
|
||||
"""
|
||||
|
||||
session = _cmhub_session()
|
||||
session.trust_env = bool(use_system_proxy)
|
||||
return session
|
||||
|
||||
_RESOLUTION_SIZES = {
|
||||
"512": (512, 512),
|
||||
@@ -233,7 +278,16 @@ def _notify_step(callback, step):
|
||||
pass
|
||||
|
||||
|
||||
def _notify_step_event(callback, step, result="success", detail=None, level="info"):
|
||||
def _notify_step_event(
|
||||
callback,
|
||||
step,
|
||||
result="success",
|
||||
detail=None,
|
||||
level="info",
|
||||
debug_only=False,
|
||||
attempt=None,
|
||||
attempts=None,
|
||||
):
|
||||
if callback is None:
|
||||
return
|
||||
payload = {
|
||||
@@ -243,6 +297,12 @@ def _notify_step_event(callback, step, result="success", detail=None, level="inf
|
||||
}
|
||||
if detail is not None:
|
||||
payload["detail"] = detail
|
||||
if attempt is not None:
|
||||
payload["attempt"] = attempt
|
||||
if attempts is not None:
|
||||
payload["attempts"] = attempts
|
||||
if debug_only:
|
||||
payload["debug_only"] = True
|
||||
try:
|
||||
callback(payload)
|
||||
except Exception:
|
||||
@@ -421,6 +481,7 @@ def generate_batch(tasks, prompts, ai_cfg=None, on_progress=None, should_stop=No
|
||||
attempt=event.get("attempt"),
|
||||
attempts=event.get("attempts"),
|
||||
metadata=event.get("metadata"),
|
||||
debug_only=event.get("debug_only"),
|
||||
)
|
||||
return
|
||||
set_step(task, event)
|
||||
@@ -703,7 +764,7 @@ def generate_batch(tasks, prompts, ai_cfg=None, on_progress=None, should_stop=No
|
||||
return summary
|
||||
|
||||
|
||||
def fetch_cmhub_models(base_url, api_key, connect_timeout=10, read_timeout=30):
|
||||
def fetch_cmhub_models(base_url, api_key, connect_timeout=10, read_timeout=30, use_system_proxy=False):
|
||||
"""Fetch cmhub model aliases for settings UI."""
|
||||
|
||||
base_url = str(base_url or "").strip()
|
||||
@@ -712,6 +773,7 @@ def fetch_cmhub_models(base_url, api_key, connect_timeout=10, read_timeout=30):
|
||||
raise CMHubError("cmhub_not_configured", "请去⑤设置配置 cmhub Base URL")
|
||||
if not api_key:
|
||||
raise CMHubError("cmhub_not_configured", "请去⑤设置配置 cmhub API Key")
|
||||
_apply_cmhub_proxy(use_system_proxy)
|
||||
data = _cmhub_call_with_retry(
|
||||
"GET",
|
||||
appconfig.cmhub_request_url(base_url, "/api/v1/models"),
|
||||
@@ -728,7 +790,7 @@ def fetch_cmhub_models(base_url, api_key, connect_timeout=10, read_timeout=30):
|
||||
return [copy.deepcopy(model) for model in models if isinstance(model, dict)]
|
||||
|
||||
|
||||
def fetch_cmhub_balance(base_url, api_key, connect_timeout=10, read_timeout=30):
|
||||
def fetch_cmhub_balance(base_url, api_key, connect_timeout=10, read_timeout=30, use_system_proxy=False):
|
||||
"""Fetch cmhub point balance for settings UI."""
|
||||
|
||||
base_url = str(base_url or "").strip()
|
||||
@@ -737,6 +799,7 @@ def fetch_cmhub_balance(base_url, api_key, connect_timeout=10, read_timeout=30):
|
||||
raise CMHubError("cmhub_not_configured", "请去⑤设置配置 cmhub Base URL")
|
||||
if not api_key:
|
||||
raise CMHubError("cmhub_not_configured", "请去⑤设置配置 cmhub API Key")
|
||||
_apply_cmhub_proxy(use_system_proxy)
|
||||
data = _cmhub_call_with_retry(
|
||||
"GET",
|
||||
appconfig.cmhub_request_url(base_url, "/api/v1/balance"),
|
||||
@@ -894,6 +957,15 @@ def _request_cmhub_cover_image(
|
||||
"cover_request",
|
||||
detail="cmhub 已返回 image_url，耗时 %s" % _format_seconds(request_elapsed),
|
||||
)
|
||||
if _debug_cmhub_image_url_enabled():
|
||||
_notify_step_event(
|
||||
on_step,
|
||||
"cover_image_url",
|
||||
result="debug",
|
||||
detail="cmhub 图片 URL：%s" % image_url,
|
||||
level="warning",
|
||||
debug_only=True,
|
||||
)
|
||||
return {
|
||||
"image_url": image_url,
|
||||
"connect_timeout": runtime["connect_timeout"],
|
||||
@@ -912,13 +984,12 @@ def _download_and_save_cmhub_cover(request_result, on_step=None):
|
||||
resolution = request_result["resolution"]
|
||||
quality = request_result["quality"]
|
||||
_notify_step(on_step, "cover_download")
|
||||
download_started = time.perf_counter()
|
||||
image_bytes = _download_cmhub_image(
|
||||
image_bytes, download_elapsed = _download_cmhub_image_with_retry(
|
||||
image_url,
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
on_step=on_step,
|
||||
)
|
||||
download_elapsed = time.perf_counter() - download_started
|
||||
_notify_step_event(
|
||||
on_step,
|
||||
"cover_download",
|
||||
@@ -955,11 +1026,14 @@ def _cmhub_runtime(config, operation, cmhub_config_path):
|
||||
"请去⑤设置配置 cmhub：缺少 " + "、".join(missing),
|
||||
retryable=False,
|
||||
)
|
||||
use_system_proxy = bool(hub.get("use_system_proxy", False))
|
||||
_apply_cmhub_proxy(use_system_proxy)
|
||||
return {
|
||||
"base_url": hub["base_url"].rstrip("/"),
|
||||
"api_key": api_key,
|
||||
"alias": hub[alias_key],
|
||||
"connect_timeout": max(1, int(hub.get("connect_timeout", 10) or 10)),
|
||||
"use_system_proxy": use_system_proxy,
|
||||
}
|
||||
|
||||
|
||||
@@ -992,6 +1066,73 @@ def _cmhub_read_timeout(config, resolution):
|
||||
return CMHUB_IMAGE_READ_TIMEOUT_SECONDS
|
||||
|
||||
|
||||
def _debug_cmhub_image_url_enabled():
|
||||
value = str(os.environ.get("CMSHOPEE_DEBUG_CMHUB_IMAGE_URL", "") or "")
|
||||
return value.strip().lower() in {"1", "true", "yes", "on", "debug"}
|
||||
|
||||
|
||||
def _download_cmhub_image_with_retry(
|
||||
url,
|
||||
connect_timeout,
|
||||
read_timeout,
|
||||
on_step=None,
|
||||
attempts=CMHUB_IMAGE_DOWNLOAD_ATTEMPTS,
|
||||
slow_threshold=CMHUB_IMAGE_SLOW_DOWNLOAD_SECONDS,
|
||||
):
|
||||
total_attempts = max(1, int(attempts or 1))
|
||||
total_started = time.perf_counter()
|
||||
last_exc = None
|
||||
for index in range(total_attempts):
|
||||
try:
|
||||
image_bytes = _download_cmhub_image(
|
||||
url,
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
)
|
||||
elapsed = time.perf_counter() - total_started
|
||||
if elapsed >= float(slow_threshold or 0):
|
||||
_notify_step_event(
|
||||
on_step,
|
||||
"cover_download",
|
||||
result="warning",
|
||||
detail="图片下载较慢，已用 %s，大小 %s"
|
||||
% (_format_seconds(elapsed), _format_bytes(len(image_bytes))),
|
||||
level="warning",
|
||||
)
|
||||
return image_bytes, elapsed
|
||||
except Exception as exc:
|
||||
last_exc = exc
|
||||
if index + 1 >= total_attempts or not _cmhub_download_retryable(exc):
|
||||
break
|
||||
_notify_step_event(
|
||||
on_step,
|
||||
"cover_download",
|
||||
result="retry",
|
||||
detail=str(exc),
|
||||
level="warning",
|
||||
attempt=index + 1,
|
||||
attempts=total_attempts,
|
||||
)
|
||||
time.sleep(min(2.0, 0.5 * (index + 1)))
|
||||
if total_attempts > 1 and _cmhub_download_retryable(last_exc):
|
||||
raise AIError(
|
||||
"下载 cmhub 图片失败（已尝试 %s 次）: %s"
|
||||
% (total_attempts, str(last_exc))
|
||||
) from last_exc
|
||||
raise last_exc
|
||||
|
||||
|
||||
def _cmhub_download_retryable(exc):
|
||||
if exc is None:
|
||||
return False
|
||||
message = str(exc or "")
|
||||
if "图片超过大小上限" in message:
|
||||
return False
|
||||
if "HTTP 4" in message:
|
||||
return False
|
||||
return message.startswith("下载 cmhub 图片失败")
|
||||
|
||||
|
||||
def _cmhub_call_with_retry(
|
||||
method,
|
||||
url,
|
||||
@@ -1039,7 +1180,7 @@ def _cmhub_call_once(method, url, api_key, payload, connect_timeout, read_timeou
|
||||
if str(method).upper() != "GET":
|
||||
request_kwargs["json"] = payload or {}
|
||||
try:
|
||||
response = requests.request(str(method).upper(), url, **request_kwargs)
|
||||
response = _cmhub_session().request(str(method).upper(), url, **request_kwargs)
|
||||
except requests.exceptions.ConnectTimeout as exc:
|
||||
raise CMHubError("connect_timeout", "连接 cmhub 超时", retryable=True) from exc
|
||||
except requests.exceptions.ReadTimeout as exc:
|
||||
@@ -1208,7 +1349,7 @@ def _format_bytes(size):
|
||||
def _download_cmhub_image(url, connect_timeout, read_timeout, max_bytes=CMHUB_IMAGE_MAX_BYTES):
|
||||
_assert_public_http_url(url)
|
||||
try:
|
||||
response = requests.get(
|
||||
response = _cmhub_session().get(
|
||||
url,
|
||||
stream=True,
|
||||
timeout=(max(1, int(connect_timeout)), max(1, int(read_timeout))),
|
||||
@@ -1379,6 +1520,7 @@ def _emit_generation_event(
|
||||
attempt=None,
|
||||
attempts=None,
|
||||
metadata=None,
|
||||
debug_only=None,
|
||||
):
|
||||
if callback is None:
|
||||
return
|
||||
@@ -1397,6 +1539,8 @@ def _emit_generation_event(
|
||||
payload["attempts"] = attempts
|
||||
if metadata is not None:
|
||||
payload["metadata"] = appconfig.sanitize_for_log(metadata)
|
||||
if debug_only:
|
||||
payload["debug_only"] = True
|
||||
try:
|
||||
callback(payload)
|
||||
except Exception:
|
||||
|
||||
@@ -78,6 +78,7 @@ DEFAULT_CONFIG = {
|
||||
"title_alias": "",
|
||||
"image_alias": "",
|
||||
"connect_timeout": 10,
|
||||
"use_system_proxy": False,
|
||||
"check_balance_before_batch": False,
|
||||
},
|
||||
"title_concurrency": 4,
|
||||
|
||||
+1
-1
@@ -137,7 +137,7 @@
|
||||
| T-535 | ② cmhub 生成标题读取等待固定 600 秒 | T-526, T-533 | 问题：当前 cmhub 生文请求的读取等待时间复用 `ai.resolution_timeouts`，会跟随当前分辨率变化；默认 `1k=240s`，若上游模型排队或响应较慢，标题生成容易先超时。方案：cmhub `gen_title()` 的 `title_request` 读取等待固定使用 600 秒（等同 4k 上限），不再跟随当前分辨率；连接超时仍使用 `ai.cmhub.connect_timeout`，重试次数仍使用 `ai.retry`。2026-07-07 补充：cmhub 生图请求与图片下载读取等待统一固定 650 秒，读超时仍不自动重发；⑤设置页「返回超时」标签在普通默认 cmhub 模式下展示「标题 600 秒 / 图片 650 秒」，避免用户误以为分辨率下拉仍会改变 cmhub 等待时间；direct 兼容路径保持现状。补 `tests/test_ai.py` 断言标题请求 timeout 为 `(connect_timeout, 600)`、生图请求和下载 timeout 为 `(connect_timeout, 650)`，补 `tests/test_gui.py` 断言 cmhub 标签固定展示实际口径、direct 仍展示分辨率映射；不改配置 schema、cmhub HTTP 协议、DB、Excel 或 Shopee/CDP | DONE |
|
||||
| T-545 | ② cmhub 生图和下载并发上限 5 + 独立下载线程池 | T-535, T-533, T-519 | 背景：实测 10 图片并发时，cmhub 后台单条生成小于 200 秒，但本地下载 `/media/generated/images/*.png` 常见 24~185 秒且有连接失败；如果每个生图线程同时负责“等待 cmhub 返回 + 下载 + 转 JPEG + 保存”，下载慢会占住生图线程，后续任务排队。方案：cmhub 模式下实际生图请求并发 = `min(ai.image_concurrency, 5)`；拿到 `image_url` 后把下载/转 JPEG/保存交给独立下载线程池，下载线程数与实际生图请求并发一致、同样最大 5；不新增用户可见配置项，⑤仍只保留「图片并发」。②开始日志必须显示用户设置图片并发、cmhub 实际生图并发和下载并发；分段日志继续记录“cmhub 已返回 image_url / 下载完成 / 本地保存完成”的耗时。下载失败按当前任务失败记录，但不得重新调用 cmhub 生图接口导致重复扣点；`cover_done/generated_done` 必须等下载保存成功并写 DB 后才计数；停止逻辑继续取消未开始项，运行中请求/下载允许自然完成或失败。direct 兼容路径暂不改变。补 `tests/test_ai.py` 覆盖并发上限和下载线程池不阻塞后续生图提交、下载失败不重复扣点；补 `tests/test_gui.py` 覆盖开始日志显示实际并发 | DONE |
|
||||
| T-536 | GUI 按钮圆角全局统一 | T-512, T-513, T-523 | 现象：只有③「开始更新」及少数上色按钮（删除批次/删除账号/未匹配(n)）有 `border-radius: 4px`，其余按钮走原生渲染显直角——不一致。根因：圆角是 T-512/T-513 给按钮上色时顺带写进 QSS 的副产品，不是全局形状决策；一旦给 `QPushButton` 设 stylesheet 就放弃原生渲染，才补了 radius/border。方案（**方式 A：全局统一圆角**）：① 在 `app/gui/widgets.py` 抽一个**共享按钮基础样式常量/helper**（统一 `border-radius`，如 4px，与卡片 6px 圆角语言协调），并**接管按钮的全部视觉状态**——normal/hover/pressed/disabled/focus 的背景、边框都定义好，避免全局设 QSS 后按钮变扁平方块、丢 hover 反馈；② 在主窗口/app 级用全局 `QPushButton` QSS 应用该基础样式，让**所有按钮共享同一圆角**；③ warning（`startUpdateButton`）与 danger（`_danger_outline_button_style`：删除批次/删除账号/未匹配）按钮改为**只叠加颜色**，复用共享基础样式的圆角/内边距/状态，不再各自重写 radius/border——杜绝“上色=顺带圆角”的隐性耦合；④ 以 Windows 为主目标做一次视觉自测（hover/按下/禁用不劣于原生）。只改 GUI 样式层（`widgets.py` + 主窗口全局 QSS + 各上色按钮引用），不改任何按钮的启用/禁用逻辑、行为、业务流程、DB、Excel、Shopee/CDP。GUI 单测至少断言上色按钮仍带各自语义色且不再各自硬写 radius（改为引用共享样式）；圆角外观本身以人工视觉验收为准 | DONE |
|
||||
| T-546 | cmhub 客户端共享 Session + 连接池 + 代理处理（补 T-545 未覆盖的下载慢根因） | T-545, T-526 | 现象：实测生图拿到 `image_url` 后本地下载单条约 191 秒、而同一 URL 用 curl 约 10 秒；日志伴随 `connect_timeout: 连接 cmhub 超时`。T-545 已做「生图并发上限 5 + 独立下载线程池」解决“下载堵住生图槽位”的流水线问题，但**未根治单条下载在并发争抢下变慢**。根因（代码核实）：① cmhub 所有 HTTP（`_cmhub_call_once` 的生成/models/balance、`_download_cmhub_image` 的下载）都是**裸 `requests.get`/`requests.request`、无共享 `Session`**，每个任务全新 TCP+TLS，叠加同步生成长连接（单条 60~260s）与 `connect_timeout` 触发的重试，形成连接风暴/连接饥饿；② `requests` 默认 `trust_env=True` 读 `HTTP(S)_PROXY`/`ALL_PROXY`，若系统带慢代理会拖累，而干净窗口的 curl 直连快。方案：① cmhub 所有 HTTP 统一走一个**模块级共享 `requests.Session`**，挂 `HTTPAdapter(pool_connections/pool_maxsize)`，池大小 ≥（实际生图并发 + 下载并发）以免连接不足排队、复用连接减少握手与 connect_timeout；生成与下载共享该 Session（`requests.Session` 跨线程发请求安全，但连接池要够大）；② 代理处理——诊断日志脱敏记录 cmhub 请求是否经代理；对 cmhub 请求提供明确策略（可配 `trust_env=False` 或显式 `proxies`），避免误走慢代理，默认可先保持读环境但提供关闭开关；③ 复测口径——Session+连接池到位后单条下载耗时应回落到与 curl 同量级；若在 T-545 的并发上限 5 下仍显著慢，评估把 cmhub 生图默认并发再降（1~2）；④ 确认 connect_timeout 重试退避不加剧连接风暴。诊断建议：可先临时把图片并发设 1 复测以区分“并发争抢”与“代理”。边界：只改 `app/ai.py` 的 cmhub HTTP 客户端层（+ 可选 config 代理项）+ 诊断日志 + 相关文档；不改 cmhub 协议、T-545 的并发/下载池语义、生成编排、DB、Excel、CDP/Shopee。测试：`tests/test_ai.py` 覆盖共享 Session 被复用（mock 同一 session 多次调用）、代理配置被尊重、连接池大小设定；现有 cmhub 用例保持绿 | TODO |
|
||||
| T-546 | cmhub 客户端共享 Session + 连接池 + 代理处理（补 T-545 未覆盖的下载慢根因） | T-545, T-526 | 现象：实测生图拿到 `image_url` 后本地下载单条约 191 秒、而同一 URL 用 curl 约 10 秒；日志伴随 `connect_timeout: 连接 cmhub 超时`。T-545 已做「生图并发上限 5 + 独立下载线程池」解决“下载堵住生图槽位”的流水线问题，但**未根治单条下载在并发争抢下变慢**。根因（代码核实）：① cmhub 所有 HTTP（`_cmhub_call_once` 的生成/models/balance、`_download_cmhub_image` 的下载）都是**裸 `requests.get`/`requests.request`、无共享 `Session`**，每个任务全新 TCP+TLS，叠加同步生成长连接（单条 60~260s）与 `connect_timeout` 触发的重试，形成连接风暴/连接饥饿；② `requests` 默认 `trust_env=True` 读 `HTTP(S)_PROXY`/`ALL_PROXY`，若系统带慢代理会拖累，而干净窗口的 curl 直连快。方案：① cmhub 所有 HTTP 统一走一个**模块级共享 `requests.Session`**，挂 `HTTPAdapter(pool_connections/pool_maxsize)`，池大小 ≥（实际生图并发 + 下载并发）以免连接不足排队、复用连接减少握手与 connect_timeout；生成与下载共享该 Session（`requests.Session` 跨线程发请求安全，但连接池要够大）；② 代理处理——诊断日志脱敏记录 cmhub 请求是否经代理；对 cmhub 请求提供明确策略（可配 `trust_env=False` 或显式 `proxies`），避免误走慢代理，默认可先保持读环境但提供关闭开关；③ 复测口径——Session+连接池到位后单条下载耗时应回落到与 curl 同量级；若在 T-545 的并发上限 5 下仍显著慢，评估把 cmhub 生图默认并发再降（1~2）；④ 确认 connect_timeout 重试退避不加剧连接风暴。诊断建议：可先临时把图片并发设 1 复测以区分“并发争抢”与“代理”。边界：只改 `app/ai.py` 的 cmhub HTTP 客户端层（+ 可选 config 代理项）+ 诊断日志 + 相关文档；不改 cmhub 协议、T-545 的并发/下载池语义、生成编排、DB、Excel、CDP/Shopee。测试：`tests/test_ai.py` 覆盖共享 Session 被复用（mock 同一 session 多次调用）、代理配置被尊重、连接池大小设定；现有 cmhub 用例保持绿。落地：`app/ai.py` 加模块级 `_cmhub_session()`（`HTTPAdapter` 池 `CMHUB_HTTP_POOL_SIZE=32`）+ `_apply_cmhub_proxy()`；生成/下载/models/balance 统一走该 Session；`ai.cmhub.use_system_proxy` 默认 `false`（绕过系统代理，公网网关直连），`_cmhub_runtime` 读取并应用、`fetch_cmhub_models/balance` 加同名参数；`test_ai.py` 补 3 项新测试并把原 18 处 `requests` mock 改到共享 Session；243 单测全绿。**待 Windows 实网复测确认下载提速**（troubleshooting 已加「cmhub 图片下载很慢」排障节与代理定位命令） | DONE |
|
||||
|
||||
## Phase 8 · 工程基础设施后续（`docs/engineering-review.md`）
|
||||
|
||||
|
||||
@@ -271,3 +271,35 @@ T-541 已在主窗口启动时读取 `QApplication.primaryScreen().availableGeom
|
||||
- 不保存坏的历史窗口坐标，避免下次继续打开到屏幕外。
|
||||
|
||||
修复后需要在普通桌面和 Windows 10 虚拟机/小分辨率环境各启动一次打包版，确认标题栏完整可见。
|
||||
|
||||
## ② cmhub 图片下载很慢（几十秒~几分钟），而 curl 只需几秒
|
||||
|
||||
### 现象
|
||||
|
||||
② 生图时，日志显示 `cmhub 已返回 image_url` 后，本地「下载完成」耗时几十秒甚至上百秒；用 curl 下载同一个 `image_url`（如 `http://<ip>:8080/generated/images/...png`）只需几秒。
|
||||
|
||||
### 原因
|
||||
|
||||
图片是从 cmhub 的**媒体服务**（常见形如 `http://<公网IP>:8080/...`，明文 HTTP、非标准端口）下载，与生成 API（HTTPS 域名）是不同端点。慢的常见根因两类：
|
||||
|
||||
1. **系统代理**（最常见）：`requests` 默认读 `HTTP(S)_PROXY`/`ALL_PROXY` 环境变量，会把明文 HTTP 到 `:8080` 的下载塞进代理；慢/不支持非标端口的代理转发会让下载卡顿，而干净窗口的 curl 直连很快。
|
||||
2. **并发争抢**：多图并发时，同步生成的长连接 + 并发下载挤同一主机，单条下载被拖慢（T-545 已限制生图/下载并发上限 5；T-546 已加共享连接池缓解）。
|
||||
|
||||
### 定位（一条命令区分代理 vs 并发）
|
||||
|
||||
```powershell
|
||||
# curl 直连基准
|
||||
curl -o NUL -w "curl %{time_total}s`n" "<image_url>"
|
||||
# python 关掉代理直连
|
||||
py -3.10 -c "import requests,time; s=requests.Session(); s.trust_env=False; t=time.time(); r=s.get('<image_url>'); print('no-proxy', len(r.content), round(time.time()-t,1),'s')"
|
||||
```
|
||||
|
||||
若 `no-proxy` 明显变快 → 就是代理。
|
||||
|
||||
### 修复（T-546 后）
|
||||
|
||||
cmshopee 的 cmhub 请求默认**绕过系统代理**（`ai.cmhub.use_system_proxy` 默认 `false`），并用共享 `requests.Session` + 连接池减少握手/连接饥饿。
|
||||
|
||||
- 若你的机器**必须走代理**才能上网，编辑 `data/config.json` 把 `ai.cmhub.use_system_proxy` 改为 `true`（注意：慢代理仍可能拖累图片下载）。
|
||||
- 若下载仍慢且 curl 也慢，则是 cmhub **媒体服务器本身慢**（如 Django 直接服媒体、单线程），属服务端问题，需在 cmhub 侧用 nginx/对象存储服 `/generated/images/`。
|
||||
- 调试期可临时把 ⑤「图片并发数」设 1 复测单张，排除并发因素。
|
||||
|
||||
+226
-14
@@ -311,7 +311,7 @@ class AITests(TempDirMixin, unittest.TestCase):
|
||||
}
|
||||
)
|
||||
|
||||
with mock.patch("app.ai.requests.request", side_effect=fake_request):
|
||||
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request):
|
||||
title = ai.gen_title(
|
||||
"优化标题",
|
||||
"旧标题",
|
||||
@@ -398,8 +398,8 @@ class AITests(TempDirMixin, unittest.TestCase):
|
||||
downloads.append((url, kwargs))
|
||||
return _RequestsResponse(content=generated.getvalue())
|
||||
|
||||
with mock.patch("app.ai.requests.request", side_effect=fake_request), \
|
||||
mock.patch("app.ai.requests.get", side_effect=fake_get), \
|
||||
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request), \
|
||||
mock.patch.object(ai._cmhub_session(), "get", side_effect=fake_get), \
|
||||
mock.patch(
|
||||
"app.ai.socket.getaddrinfo",
|
||||
return_value=[
|
||||
@@ -454,6 +454,176 @@ class AITests(TempDirMixin, unittest.TestCase):
|
||||
|
||||
self.assert_removed(temp_dir)
|
||||
|
||||
def test_cmhub_gen_cover_emits_debug_image_url_when_enabled(self):
|
||||
try:
|
||||
from PIL import Image
|
||||
except ImportError:
|
||||
self.skipTest("Pillow not installed")
|
||||
|
||||
with self.make_temp_dir() as temp_dir:
|
||||
cfg, key_path = self._cmhub_config(temp_dir)
|
||||
old_cover = os.path.join(temp_dir, "old.jpg")
|
||||
output = os.path.join(temp_dir, "new.jpg")
|
||||
Image.new("RGB", (16, 16), (20, 30, 40)).save(old_cover, "JPEG")
|
||||
generated_png = self._png_bytes()
|
||||
steps = []
|
||||
|
||||
def fake_request(method, url, **kwargs):
|
||||
return _RequestsResponse(
|
||||
{"image_url": "https://cdn.example.com/generated.png?token=secret"}
|
||||
)
|
||||
|
||||
def fake_get(url, **kwargs):
|
||||
return _RequestsResponse(content=generated_png)
|
||||
|
||||
with mock.patch.dict(
|
||||
os.environ,
|
||||
{"CMSHOPEE_DEBUG_CMHUB_IMAGE_URL": "1"},
|
||||
), mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request), \
|
||||
mock.patch.object(ai._cmhub_session(), "get", side_effect=fake_get), \
|
||||
mock.patch(
|
||||
"app.ai.socket.getaddrinfo",
|
||||
return_value=[
|
||||
(
|
||||
socket.AF_INET,
|
||||
socket.SOCK_STREAM,
|
||||
6,
|
||||
"",
|
||||
("93.184.216.34", 443),
|
||||
)
|
||||
],
|
||||
):
|
||||
ai.gen_cover(
|
||||
"生成封面",
|
||||
old_cover,
|
||||
output,
|
||||
config=cfg,
|
||||
cmhub_config_path=key_path,
|
||||
on_step=steps.append,
|
||||
)
|
||||
|
||||
debug_events = [
|
||||
event for event in steps
|
||||
if isinstance(event, dict) and event.get("step") == "cover_image_url"
|
||||
]
|
||||
self.assertEqual(1, len(debug_events))
|
||||
self.assertEqual("debug", debug_events[0]["result"])
|
||||
self.assertTrue(debug_events[0]["debug_only"])
|
||||
self.assertIn("https://cdn.example.com/generated.png", debug_events[0]["detail"])
|
||||
|
||||
self.assert_removed(temp_dir)
|
||||
|
||||
def test_cmhub_gen_cover_retries_image_download_without_new_generation(self):
|
||||
try:
|
||||
from PIL import Image
|
||||
except ImportError:
|
||||
self.skipTest("Pillow not installed")
|
||||
|
||||
with self.make_temp_dir() as temp_dir:
|
||||
cfg, key_path = self._cmhub_config(temp_dir)
|
||||
old_cover = os.path.join(temp_dir, "old.jpg")
|
||||
output = os.path.join(temp_dir, "new.jpg")
|
||||
Image.new("RGB", (16, 16), (20, 30, 40)).save(old_cover, "JPEG")
|
||||
generated_png = self._png_bytes()
|
||||
request_calls = []
|
||||
download_calls = []
|
||||
steps = []
|
||||
|
||||
def fake_request(method, url, **kwargs):
|
||||
request_calls.append((method, url, kwargs))
|
||||
return _RequestsResponse(
|
||||
{"image_url": "https://cdn.example.com/generated.png"}
|
||||
)
|
||||
|
||||
def fake_get(url, **kwargs):
|
||||
download_calls.append((url, kwargs))
|
||||
if len(download_calls) == 1:
|
||||
raise ai.requests.exceptions.ConnectionError("temporary")
|
||||
return _RequestsResponse(content=generated_png)
|
||||
|
||||
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request), \
|
||||
mock.patch.object(ai._cmhub_session(), "get", side_effect=fake_get), \
|
||||
mock.patch(
|
||||
"app.ai.socket.getaddrinfo",
|
||||
return_value=[
|
||||
(
|
||||
socket.AF_INET,
|
||||
socket.SOCK_STREAM,
|
||||
6,
|
||||
"",
|
||||
("93.184.216.34", 443),
|
||||
)
|
||||
],
|
||||
), mock.patch("app.ai.time.sleep"):
|
||||
result = ai.gen_cover(
|
||||
"生成封面",
|
||||
old_cover,
|
||||
output,
|
||||
config=cfg,
|
||||
cmhub_config_path=key_path,
|
||||
on_step=steps.append,
|
||||
)
|
||||
|
||||
self.assertEqual(os.path.abspath(output), result)
|
||||
self.assertEqual(1, len(request_calls))
|
||||
self.assertEqual(2, len(download_calls))
|
||||
retry_events = [
|
||||
event for event in steps
|
||||
if isinstance(event, dict)
|
||||
and event.get("step") == "cover_download"
|
||||
and event.get("result") == "retry"
|
||||
]
|
||||
self.assertEqual(1, len(retry_events))
|
||||
self.assertEqual(1, retry_events[0]["attempt"])
|
||||
self.assertEqual(ai.CMHUB_IMAGE_DOWNLOAD_ATTEMPTS, retry_events[0]["attempts"])
|
||||
|
||||
self.assert_removed(temp_dir)
|
||||
|
||||
def test_cmhub_download_slow_warning_event(self):
|
||||
try:
|
||||
from PIL import Image # noqa: F401
|
||||
except ImportError:
|
||||
self.skipTest("Pillow not installed")
|
||||
|
||||
generated_png = self._png_bytes()
|
||||
steps = []
|
||||
|
||||
def fake_get(url, **kwargs):
|
||||
return _RequestsResponse(content=generated_png)
|
||||
|
||||
with mock.patch.object(ai._cmhub_session(), "get", side_effect=fake_get), \
|
||||
mock.patch(
|
||||
"app.ai.socket.getaddrinfo",
|
||||
return_value=[
|
||||
(
|
||||
socket.AF_INET,
|
||||
socket.SOCK_STREAM,
|
||||
6,
|
||||
"",
|
||||
("93.184.216.34", 443),
|
||||
)
|
||||
],
|
||||
):
|
||||
image_bytes, elapsed = ai._download_cmhub_image_with_retry(
|
||||
"https://cdn.example.com/generated.png",
|
||||
connect_timeout=3,
|
||||
read_timeout=650,
|
||||
on_step=steps.append,
|
||||
slow_threshold=0,
|
||||
)
|
||||
|
||||
self.assertEqual(generated_png, image_bytes)
|
||||
self.assertGreaterEqual(elapsed, 0)
|
||||
self.assertTrue(
|
||||
any(
|
||||
isinstance(event, dict)
|
||||
and event.get("step") == "cover_download"
|
||||
and event.get("result") == "warning"
|
||||
and "图片下载较慢" in event.get("detail", "")
|
||||
for event in steps
|
||||
)
|
||||
)
|
||||
|
||||
def test_cmhub_image_url_rejects_private_and_private_dns(self):
|
||||
with self.assertRaises(ai.AIError):
|
||||
ai._download_cmhub_image("http://127.0.0.1/a.png", 1, 1)
|
||||
@@ -481,7 +651,7 @@ class AITests(TempDirMixin, unittest.TestCase):
|
||||
)
|
||||
return _RequestsResponse({"titles": ["新标题"], "points_balance": 10})
|
||||
|
||||
with mock.patch("app.ai.requests.request", side_effect=fake_request), \
|
||||
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request), \
|
||||
mock.patch("app.ai.time.sleep"):
|
||||
title = ai.gen_title(
|
||||
"prompt",
|
||||
@@ -512,7 +682,7 @@ class AITests(TempDirMixin, unittest.TestCase):
|
||||
calls.append((method, url, kwargs))
|
||||
raise ai.requests.exceptions.ReadTimeout("slow")
|
||||
|
||||
with mock.patch("app.ai.requests.request", side_effect=fake_request):
|
||||
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request):
|
||||
with self.assertRaises(ai.CMHubError) as raised:
|
||||
ai.gen_cover(
|
||||
"prompt",
|
||||
@@ -548,7 +718,7 @@ class AITests(TempDirMixin, unittest.TestCase):
|
||||
}
|
||||
)
|
||||
|
||||
with mock.patch("app.ai.requests.request", side_effect=fake_request):
|
||||
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request):
|
||||
models = ai.fetch_cmhub_models("https://cmhub.example.com", "sk-cmhub-secret")
|
||||
|
||||
self.assertEqual("GET", calls[0][0])
|
||||
@@ -562,7 +732,7 @@ class AITests(TempDirMixin, unittest.TestCase):
|
||||
calls.append((method, url, kwargs))
|
||||
return _RequestsResponse({"detail": "notfound"}, status_code=404)
|
||||
|
||||
with mock.patch("app.ai.requests.request", side_effect=fake_request):
|
||||
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request):
|
||||
with self.assertRaises(ai.CMHubError) as raised:
|
||||
ai.fetch_cmhub_models(
|
||||
"https://cmhub.example.com/api/v1/",
|
||||
@@ -584,7 +754,7 @@ class AITests(TempDirMixin, unittest.TestCase):
|
||||
calls.append((method, url, kwargs))
|
||||
return _RequestsResponse({"user": {"id": "u1"}, "points_balance": 42})
|
||||
|
||||
with mock.patch("app.ai.requests.request", side_effect=fake_request):
|
||||
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request):
|
||||
balance = ai.fetch_cmhub_balance("https://cmhub.example.com", "sk-cmhub-secret")
|
||||
|
||||
self.assertEqual("GET", calls[0][0])
|
||||
@@ -657,8 +827,8 @@ class AITests(TempDirMixin, unittest.TestCase):
|
||||
downloads.append((url, kwargs))
|
||||
return _RequestsResponse(content=generated_png)
|
||||
|
||||
with mock.patch("app.ai.requests.request", side_effect=fake_request), \
|
||||
mock.patch("app.ai.requests.get", side_effect=fake_get), \
|
||||
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request), \
|
||||
mock.patch.object(ai._cmhub_session(), "get", side_effect=fake_get), \
|
||||
mock.patch("app.ai.socket.getaddrinfo", return_value=public_dns):
|
||||
summary = ai.generate_batch(
|
||||
tasks,
|
||||
@@ -713,8 +883,8 @@ class AITests(TempDirMixin, unittest.TestCase):
|
||||
downloads_seen.append((url, kwargs))
|
||||
raise ai.requests.exceptions.ConnectionError("download failed")
|
||||
|
||||
with mock.patch("app.ai.requests.request", side_effect=fake_request), \
|
||||
mock.patch("app.ai.requests.get", side_effect=fake_get), \
|
||||
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request), \
|
||||
mock.patch.object(ai._cmhub_session(), "get", side_effect=fake_get), \
|
||||
mock.patch("app.ai.socket.getaddrinfo", return_value=public_dns):
|
||||
summary = ai.generate_batch(
|
||||
tasks,
|
||||
@@ -730,7 +900,7 @@ class AITests(TempDirMixin, unittest.TestCase):
|
||||
self.assertEqual(0, summary["cover_done"])
|
||||
self.assertEqual(1, summary["failed"])
|
||||
self.assertEqual(1, len(requests_seen))
|
||||
self.assertEqual(1, len(downloads_seen))
|
||||
self.assertEqual(ai.CMHUB_IMAGE_DOWNLOAD_ATTEMPTS, len(downloads_seen))
|
||||
updated = db.list_tasks(batch_id=batch_id, path=cfg["db_path"])[0]
|
||||
self.assertEqual("failed", updated.status)
|
||||
self.assertIn("下载 cmhub 图片失败", updated.last_error)
|
||||
@@ -756,7 +926,7 @@ class AITests(TempDirMixin, unittest.TestCase):
|
||||
}
|
||||
)
|
||||
|
||||
with mock.patch("app.ai.requests.request", side_effect=fake_request):
|
||||
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request):
|
||||
summary = ai.generate_batch(
|
||||
tasks,
|
||||
{"title": "标题提示", "cover": "封面"},
|
||||
@@ -1190,6 +1360,48 @@ class AITests(TempDirMixin, unittest.TestCase):
|
||||
|
||||
self.assert_removed(temp_dir)
|
||||
|
||||
def test_cmhub_session_is_shared_singleton_with_pool(self):
|
||||
session_a = ai._cmhub_session()
|
||||
session_b = ai._cmhub_session()
|
||||
self.assertIs(session_a, session_b)
|
||||
# 连接池要覆盖生图并发 + 下载并发（各上限 5）
|
||||
self.assertGreaterEqual(
|
||||
ai.CMHUB_HTTP_POOL_SIZE, ai.CMHUB_IMAGE_CONCURRENCY_LIMIT * 2
|
||||
)
|
||||
adapter = session_a.get_adapter("https://cmhub.example.com")
|
||||
self.assertEqual(ai.CMHUB_HTTP_POOL_SIZE, adapter._pool_maxsize)
|
||||
|
||||
def test_apply_cmhub_proxy_toggles_trust_env(self):
|
||||
try:
|
||||
session = ai._apply_cmhub_proxy(False)
|
||||
self.assertIs(session, ai._cmhub_session())
|
||||
self.assertFalse(session.trust_env)
|
||||
ai._apply_cmhub_proxy(True)
|
||||
self.assertTrue(ai._cmhub_session().trust_env)
|
||||
finally:
|
||||
# 复位为默认（绕过系统代理），避免影响其它测试
|
||||
ai._apply_cmhub_proxy(False)
|
||||
|
||||
def test_cmhub_runtime_applies_and_returns_use_system_proxy(self):
|
||||
with mock.patch(
|
||||
"app.ai.appconfig.cmhub_config",
|
||||
return_value={
|
||||
"base_url": "https://cmhub.example.com",
|
||||
"title_alias": "title-standard",
|
||||
"image_alias": "image-hd",
|
||||
"connect_timeout": 10,
|
||||
"use_system_proxy": True,
|
||||
},
|
||||
), mock.patch(
|
||||
"app.ai.appconfig.get_cmhub_api_key", return_value="sk_cmhub_test"
|
||||
):
|
||||
try:
|
||||
runtime = ai._cmhub_runtime({}, "image", cmhub_config_path=None)
|
||||
self.assertTrue(runtime["use_system_proxy"])
|
||||
self.assertTrue(ai._cmhub_session().trust_env)
|
||||
finally:
|
||||
ai._apply_cmhub_proxy(False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user