chengma/cmshoppe
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: cmhub shared Session + connection pool + proxy handling (T-546)

Browse Source 
- app/ai.py: 模块级共享 requests.Session + HTTPAdapter 连接池
  (CMHUB_HTTP_POOL_SIZE=32)；_apply_cmhub_proxy() 切换 trust_env；
  生成/下载/models/balance 统一复用该 Session。
- app/appconfig.py: ai.cmhub.use_system_proxy 默认 false（绕过系统代理）。
- tests/test_ai.py: 补 Session 复用/代理/连接池测试；原 18 处 cmhub mock
  从模块级 requests 改到共享 Session。
- docs: troubleshooting 加「图片下载很慢」排障节；06-tasks 标 T-546 DONE。

附带：把 app/ai.py 行尾从 CRLF 归一为 LF；并一并纳入工作区中此前未提交、
已在运行构建里的 cmhub 下载诊断代码（调试图片 URL 日志、下载重试事件），
无法与 T-546 在同一文件内拆分单独提交。243 单测全绿。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
chengma
2026-07-08 00:15:43 +08:00
co-authored by Claude Opus 4.8
parent b257fb5c83
commit 5a09c6662e
5 changed files with 412 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_ai.py
+226 -14
View File
@@ -311,7 +311,7 @@ class AITests(TempDirMixin, unittest.TestCase):
}
)
with mock.patch("app.ai.requests.request", side_effect=fake_request):
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request):
title = ai.gen_title(
"优化标题",
"旧标题",
@@ -398,8 +398,8 @@ class AITests(TempDirMixin, unittest.TestCase):
downloads.append((url, kwargs))
return _RequestsResponse(content=generated.getvalue())
with mock.patch("app.ai.requests.request", side_effect=fake_request), \
mock.patch("app.ai.requests.get", side_effect=fake_get), \
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request), \
mock.patch.object(ai._cmhub_session(), "get", side_effect=fake_get), \
mock.patch(
"app.ai.socket.getaddrinfo",
return_value=[
@@ -454,6 +454,176 @@ class AITests(TempDirMixin, unittest.TestCase):
self.assert_removed(temp_dir)
def test_cmhub_gen_cover_emits_debug_image_url_when_enabled(self):
try:
from PIL import Image
except ImportError:
self.skipTest("Pillow not installed")
with self.make_temp_dir() as temp_dir:
cfg, key_path = self._cmhub_config(temp_dir)
old_cover = os.path.join(temp_dir, "old.jpg")
output = os.path.join(temp_dir, "new.jpg")
Image.new("RGB", (16, 16), (20, 30, 40)).save(old_cover, "JPEG")
generated_png = self._png_bytes()
steps = []
def fake_request(method, url, **kwargs):
return _RequestsResponse(
{"image_url": "https://cdn.example.com/generated.png?token=secret"}
)
def fake_get(url, **kwargs):
return _RequestsResponse(content=generated_png)
with mock.patch.dict(
os.environ,
{"CMSHOPEE_DEBUG_CMHUB_IMAGE_URL": "1"},
), mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request), \
mock.patch.object(ai._cmhub_session(), "get", side_effect=fake_get), \
mock.patch(
"app.ai.socket.getaddrinfo",
return_value=[
(
socket.AF_INET,
socket.SOCK_STREAM,
6,
"",
("93.184.216.34", 443),
)
],
):
ai.gen_cover(
"生成封面",
old_cover,
output,
config=cfg,
cmhub_config_path=key_path,
on_step=steps.append,
)
debug_events = [
event for event in steps
if isinstance(event, dict) and event.get("step") == "cover_image_url"
]
self.assertEqual(1, len(debug_events))
self.assertEqual("debug", debug_events[0]["result"])
self.assertTrue(debug_events[0]["debug_only"])
self.assertIn("https://cdn.example.com/generated.png", debug_events[0]["detail"])
self.assert_removed(temp_dir)
def test_cmhub_gen_cover_retries_image_download_without_new_generation(self):
try:
from PIL import Image
except ImportError:
self.skipTest("Pillow not installed")
with self.make_temp_dir() as temp_dir:
cfg, key_path = self._cmhub_config(temp_dir)
old_cover = os.path.join(temp_dir, "old.jpg")
output = os.path.join(temp_dir, "new.jpg")
Image.new("RGB", (16, 16), (20, 30, 40)).save(old_cover, "JPEG")
generated_png = self._png_bytes()
request_calls = []
download_calls = []
steps = []
def fake_request(method, url, **kwargs):
request_calls.append((method, url, kwargs))
return _RequestsResponse(
{"image_url": "https://cdn.example.com/generated.png"}
)
def fake_get(url, **kwargs):
download_calls.append((url, kwargs))
if len(download_calls) == 1:
raise ai.requests.exceptions.ConnectionError("temporary")
return _RequestsResponse(content=generated_png)
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request), \
mock.patch.object(ai._cmhub_session(), "get", side_effect=fake_get), \
mock.patch(
"app.ai.socket.getaddrinfo",
return_value=[
(
socket.AF_INET,
socket.SOCK_STREAM,
6,
"",
("93.184.216.34", 443),
)
],
), mock.patch("app.ai.time.sleep"):
result = ai.gen_cover(
"生成封面",
old_cover,
output,
config=cfg,
cmhub_config_path=key_path,
on_step=steps.append,
)
self.assertEqual(os.path.abspath(output), result)
self.assertEqual(1, len(request_calls))
self.assertEqual(2, len(download_calls))
retry_events = [
event for event in steps
if isinstance(event, dict)
and event.get("step") == "cover_download"
and event.get("result") == "retry"
]
self.assertEqual(1, len(retry_events))
self.assertEqual(1, retry_events[0]["attempt"])
self.assertEqual(ai.CMHUB_IMAGE_DOWNLOAD_ATTEMPTS, retry_events[0]["attempts"])
self.assert_removed(temp_dir)
def test_cmhub_download_slow_warning_event(self):
try:
from PIL import Image # noqa: F401
except ImportError:
self.skipTest("Pillow not installed")
generated_png = self._png_bytes()
steps = []
def fake_get(url, **kwargs):
return _RequestsResponse(content=generated_png)
with mock.patch.object(ai._cmhub_session(), "get", side_effect=fake_get), \
mock.patch(
"app.ai.socket.getaddrinfo",
return_value=[
(
socket.AF_INET,
socket.SOCK_STREAM,
6,
"",
("93.184.216.34", 443),
)
],
):
image_bytes, elapsed = ai._download_cmhub_image_with_retry(
"https://cdn.example.com/generated.png",
connect_timeout=3,
read_timeout=650,
on_step=steps.append,
slow_threshold=0,
)
self.assertEqual(generated_png, image_bytes)
self.assertGreaterEqual(elapsed, 0)
self.assertTrue(
any(
isinstance(event, dict)
and event.get("step") == "cover_download"
and event.get("result") == "warning"
and "图片下载较慢" in event.get("detail", "")
for event in steps
)
)
def test_cmhub_image_url_rejects_private_and_private_dns(self):
with self.assertRaises(ai.AIError):
ai._download_cmhub_image("http://127.0.0.1/a.png", 1, 1)
@@ -481,7 +651,7 @@ class AITests(TempDirMixin, unittest.TestCase):
)
return _RequestsResponse({"titles": ["新标题"], "points_balance": 10})
with mock.patch("app.ai.requests.request", side_effect=fake_request), \
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request), \
mock.patch("app.ai.time.sleep"):
title = ai.gen_title(
"prompt",
@@ -512,7 +682,7 @@ class AITests(TempDirMixin, unittest.TestCase):
calls.append((method, url, kwargs))
raise ai.requests.exceptions.ReadTimeout("slow")
with mock.patch("app.ai.requests.request", side_effect=fake_request):
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request):
with self.assertRaises(ai.CMHubError) as raised:
ai.gen_cover(
"prompt",
@@ -548,7 +718,7 @@ class AITests(TempDirMixin, unittest.TestCase):
}
)
with mock.patch("app.ai.requests.request", side_effect=fake_request):
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request):
models = ai.fetch_cmhub_models("https://cmhub.example.com", "sk-cmhub-secret")
self.assertEqual("GET", calls[0][0])
@@ -562,7 +732,7 @@ class AITests(TempDirMixin, unittest.TestCase):
calls.append((method, url, kwargs))
return _RequestsResponse({"detail": "notfound"}, status_code=404)
with mock.patch("app.ai.requests.request", side_effect=fake_request):
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request):
with self.assertRaises(ai.CMHubError) as raised:
ai.fetch_cmhub_models(
"https://cmhub.example.com/api/v1/",
@@ -584,7 +754,7 @@ class AITests(TempDirMixin, unittest.TestCase):
calls.append((method, url, kwargs))
return _RequestsResponse({"user": {"id": "u1"}, "points_balance": 42})
with mock.patch("app.ai.requests.request", side_effect=fake_request):
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request):
balance = ai.fetch_cmhub_balance("https://cmhub.example.com", "sk-cmhub-secret")
self.assertEqual("GET", calls[0][0])
@@ -657,8 +827,8 @@ class AITests(TempDirMixin, unittest.TestCase):
downloads.append((url, kwargs))
return _RequestsResponse(content=generated_png)
with mock.patch("app.ai.requests.request", side_effect=fake_request), \
mock.patch("app.ai.requests.get", side_effect=fake_get), \
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request), \
mock.patch.object(ai._cmhub_session(), "get", side_effect=fake_get), \
mock.patch("app.ai.socket.getaddrinfo", return_value=public_dns):
summary = ai.generate_batch(
tasks,
@@ -713,8 +883,8 @@ class AITests(TempDirMixin, unittest.TestCase):
downloads_seen.append((url, kwargs))
raise ai.requests.exceptions.ConnectionError("download failed")
with mock.patch("app.ai.requests.request", side_effect=fake_request), \
mock.patch("app.ai.requests.get", side_effect=fake_get), \
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request), \
mock.patch.object(ai._cmhub_session(), "get", side_effect=fake_get), \
mock.patch("app.ai.socket.getaddrinfo", return_value=public_dns):
summary = ai.generate_batch(
tasks,
@@ -730,7 +900,7 @@ class AITests(TempDirMixin, unittest.TestCase):
self.assertEqual(0, summary["cover_done"])
self.assertEqual(1, summary["failed"])
self.assertEqual(1, len(requests_seen))
self.assertEqual(1, len(downloads_seen))
self.assertEqual(ai.CMHUB_IMAGE_DOWNLOAD_ATTEMPTS, len(downloads_seen))
updated = db.list_tasks(batch_id=batch_id, path=cfg["db_path"])[0]
self.assertEqual("failed", updated.status)
self.assertIn("下载 cmhub 图片失败", updated.last_error)
@@ -756,7 +926,7 @@ class AITests(TempDirMixin, unittest.TestCase):
}
)
with mock.patch("app.ai.requests.request", side_effect=fake_request):
with mock.patch.object(ai._cmhub_session(), "request", side_effect=fake_request):
summary = ai.generate_batch(
tasks,
{"title": "标题提示", "cover": "封面"},
@@ -1190,6 +1360,48 @@ class AITests(TempDirMixin, unittest.TestCase):
self.assert_removed(temp_dir)
def test_cmhub_session_is_shared_singleton_with_pool(self):
session_a = ai._cmhub_session()
session_b = ai._cmhub_session()
self.assertIs(session_a, session_b)
# 连接池要覆盖生图并发 + 下载并发（各上限 5）
self.assertGreaterEqual(
ai.CMHUB_HTTP_POOL_SIZE, ai.CMHUB_IMAGE_CONCURRENCY_LIMIT * 2
)
adapter = session_a.get_adapter("https://cmhub.example.com")
self.assertEqual(ai.CMHUB_HTTP_POOL_SIZE, adapter._pool_maxsize)
def test_apply_cmhub_proxy_toggles_trust_env(self):
try:
session = ai._apply_cmhub_proxy(False)
self.assertIs(session, ai._cmhub_session())
self.assertFalse(session.trust_env)
ai._apply_cmhub_proxy(True)
self.assertTrue(ai._cmhub_session().trust_env)
finally:
# 复位为默认（绕过系统代理），避免影响其它测试
ai._apply_cmhub_proxy(False)
def test_cmhub_runtime_applies_and_returns_use_system_proxy(self):
with mock.patch(
"app.ai.appconfig.cmhub_config",
return_value={
"base_url": "https://cmhub.example.com",
"title_alias": "title-standard",
"image_alias": "image-hd",
"connect_timeout": 10,
"use_system_proxy": True,
},
), mock.patch(
"app.ai.appconfig.get_cmhub_api_key", return_value="sk_cmhub_test"
):
try:
runtime = ai._cmhub_runtime({}, "image", cmhub_config_path=None)
self.assertTrue(runtime["use_system_proxy"])
self.assertTrue(ai._cmhub_session().trust_env)
finally:
ai._apply_cmhub_proxy(False)
if __name__ == "__main__":
unittest.main()