feat: use system curl for cmhub image downloads
This commit is contained in:
@@ -7,7 +7,10 @@ import copy
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
@@ -973,6 +976,8 @@ def _request_cmhub_cover_image(
|
||||
"out_path": out_path,
|
||||
"resolution": resolution,
|
||||
"quality": quality,
|
||||
"use_system_proxy": runtime["use_system_proxy"],
|
||||
"download_with_curl": runtime["download_with_curl"],
|
||||
}
|
||||
|
||||
|
||||
@@ -988,6 +993,8 @@ def _download_and_save_cmhub_cover(request_result, on_step=None):
|
||||
image_url,
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
use_system_proxy=request_result.get("use_system_proxy", False),
|
||||
download_with_curl=request_result.get("download_with_curl", "false"),
|
||||
on_step=on_step,
|
||||
)
|
||||
_notify_step_event(
|
||||
@@ -1034,6 +1041,7 @@ def _cmhub_runtime(config, operation, cmhub_config_path):
|
||||
"alias": hub[alias_key],
|
||||
"connect_timeout": max(1, int(hub.get("connect_timeout", 10) or 10)),
|
||||
"use_system_proxy": use_system_proxy,
|
||||
"download_with_curl": str(hub.get("download_with_curl", "auto") or "auto"),
|
||||
}
|
||||
|
||||
|
||||
@@ -1075,6 +1083,8 @@ def _download_cmhub_image_with_retry(
|
||||
url,
|
||||
connect_timeout,
|
||||
read_timeout,
|
||||
use_system_proxy=False,
|
||||
download_with_curl="false",
|
||||
on_step=None,
|
||||
attempts=CMHUB_IMAGE_DOWNLOAD_ATTEMPTS,
|
||||
slow_threshold=CMHUB_IMAGE_SLOW_DOWNLOAD_SECONDS,
|
||||
@@ -1088,6 +1098,8 @@ def _download_cmhub_image_with_retry(
|
||||
url,
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
use_system_proxy=use_system_proxy,
|
||||
download_with_curl=download_with_curl,
|
||||
)
|
||||
elapsed = time.perf_counter() - total_started
|
||||
if elapsed >= float(slow_threshold or 0):
|
||||
@@ -1346,8 +1358,35 @@ def _format_bytes(size):
|
||||
return "%.1f%s" % (value, unit)
|
||||
|
||||
|
||||
def _download_cmhub_image(url, connect_timeout, read_timeout, max_bytes=CMHUB_IMAGE_MAX_BYTES):
|
||||
def _download_cmhub_image(
|
||||
url,
|
||||
connect_timeout,
|
||||
read_timeout,
|
||||
max_bytes=CMHUB_IMAGE_MAX_BYTES,
|
||||
use_system_proxy=False,
|
||||
download_with_curl="false",
|
||||
):
|
||||
_assert_public_http_url(url)
|
||||
if _should_use_curl_for_cmhub_download(download_with_curl):
|
||||
try:
|
||||
return _download_cmhub_image_with_curl(
|
||||
url,
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
max_bytes=max_bytes,
|
||||
use_system_proxy=use_system_proxy,
|
||||
)
|
||||
except AIError:
|
||||
pass
|
||||
return _download_cmhub_image_with_requests(
|
||||
url,
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
max_bytes=max_bytes,
|
||||
)
|
||||
|
||||
|
||||
def _download_cmhub_image_with_requests(url, connect_timeout, read_timeout, max_bytes):
|
||||
try:
|
||||
response = _cmhub_session().get(
|
||||
url,
|
||||
@@ -1372,6 +1411,119 @@ def _download_cmhub_image(url, connect_timeout, read_timeout, max_bytes=CMHUB_IM
|
||||
return b"".join(chunks)
|
||||
|
||||
|
||||
def _should_use_curl_for_cmhub_download(mode):
|
||||
normalized = _normalize_curl_download_mode(mode)
|
||||
if normalized == "false":
|
||||
return False
|
||||
if normalized == "auto" and os.name != "nt":
|
||||
return False
|
||||
return bool(_find_system_curl())
|
||||
|
||||
|
||||
def _normalize_curl_download_mode(mode):
|
||||
if isinstance(mode, bool):
|
||||
return "true" if mode else "false"
|
||||
normalized = str(mode or "auto").strip().lower()
|
||||
if normalized in {"auto", "true", "false"}:
|
||||
return normalized
|
||||
return "auto"
|
||||
|
||||
|
||||
def _find_system_curl():
|
||||
candidates = []
|
||||
if os.name == "nt":
|
||||
system_root = os.environ.get("SystemRoot") or r"C:\Windows"
|
||||
candidates.append(os.path.join(system_root, "System32", "curl.exe"))
|
||||
discovered = shutil.which("curl")
|
||||
if discovered:
|
||||
candidates.append(discovered)
|
||||
seen = set()
|
||||
for candidate in candidates:
|
||||
if not candidate:
|
||||
continue
|
||||
path = os.path.abspath(candidate)
|
||||
lowered = path.lower()
|
||||
if lowered in seen:
|
||||
continue
|
||||
seen.add(lowered)
|
||||
if os.path.isfile(path):
|
||||
return path
|
||||
return ""
|
||||
|
||||
|
||||
def _download_cmhub_image_with_curl(
|
||||
url,
|
||||
connect_timeout,
|
||||
read_timeout,
|
||||
max_bytes,
|
||||
use_system_proxy=False,
|
||||
):
|
||||
curl_path = _find_system_curl()
|
||||
if not curl_path:
|
||||
raise AIError("下载 cmhub 图片失败: 未找到系统 curl")
|
||||
temp_config_path = None
|
||||
temp_output_path = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(
|
||||
"w",
|
||||
encoding="utf-8",
|
||||
suffix=".curlrc",
|
||||
delete=False,
|
||||
) as config_file:
|
||||
temp_config_path = config_file.name
|
||||
config_file.write("url = %s\n" % _curl_config_quote(url))
|
||||
with tempfile.NamedTemporaryFile("wb", suffix=".img", delete=False) as output_file:
|
||||
temp_output_path = output_file.name
|
||||
args = [
|
||||
curl_path,
|
||||
"-K",
|
||||
temp_config_path,
|
||||
"--fail",
|
||||
"--silent",
|
||||
"--show-error",
|
||||
"--connect-timeout",
|
||||
str(max(1, int(connect_timeout))),
|
||||
"--max-time",
|
||||
str(max(1, int(read_timeout))),
|
||||
"--max-filesize",
|
||||
str(max(1, int(max_bytes))),
|
||||
"--output",
|
||||
temp_output_path,
|
||||
]
|
||||
if not bool(use_system_proxy):
|
||||
args.extend(["--noproxy", "*"])
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
args,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=max(2, int(connect_timeout) + int(read_timeout) + 10),
|
||||
check=False,
|
||||
shell=False,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
raise AIError("下载 cmhub 图片失败: curl 执行失败") from exc
|
||||
if completed.returncode != 0:
|
||||
raise AIError("下载 cmhub 图片失败: curl 退出码 %s" % completed.returncode)
|
||||
size = os.path.getsize(temp_output_path)
|
||||
if size > max_bytes:
|
||||
raise AIError("下载 cmhub 图片失败: 图片超过大小上限")
|
||||
with open(temp_output_path, "rb") as fh:
|
||||
return fh.read()
|
||||
finally:
|
||||
for path in (temp_config_path, temp_output_path):
|
||||
if path:
|
||||
try:
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _curl_config_quote(value):
|
||||
text = str(value or "")
|
||||
return '"' + text.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
||||
|
||||
|
||||
def _assert_public_http_url(url):
|
||||
parts = urllib.parse.urlsplit(str(url or ""))
|
||||
if parts.scheme not in {"http", "https"}:
|
||||
|
||||
@@ -83,6 +83,7 @@ DEFAULT_CONFIG = {
|
||||
"image_alias": "",
|
||||
"connect_timeout": 10,
|
||||
"use_system_proxy": False,
|
||||
"download_with_curl": "auto",
|
||||
"check_balance_before_batch": False,
|
||||
},
|
||||
"title_concurrency": 4,
|
||||
@@ -375,6 +376,9 @@ def _normalize_config_values(config):
|
||||
cmhub = ai.get("cmhub")
|
||||
if isinstance(cmhub, dict):
|
||||
cmhub["base_url"] = normalize_cmhub_base_url(cmhub.get("base_url", ""))
|
||||
cmhub["download_with_curl"] = _normalize_cmhub_download_with_curl(
|
||||
cmhub.get("download_with_curl", "auto")
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
@@ -386,6 +390,15 @@ def _clamp_int(value, minimum, maximum, default):
|
||||
return min(int(maximum), max(int(minimum), number))
|
||||
|
||||
|
||||
def _normalize_cmhub_download_with_curl(value):
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
text = str(value or "auto").strip().lower()
|
||||
if text in {"auto", "true", "false"}:
|
||||
return text
|
||||
return "auto"
|
||||
|
||||
|
||||
def _assert_no_secrets(config):
|
||||
def visit(value, path):
|
||||
if isinstance(value, dict):
|
||||
@@ -621,6 +634,9 @@ def cmhub_config(config=None) -> dict:
|
||||
merged["title_alias"] = str(merged.get("title_alias", "") or "").strip()
|
||||
merged["image_alias"] = str(merged.get("image_alias", "") or "").strip()
|
||||
merged["connect_timeout"] = int(merged.get("connect_timeout", 10) or 10)
|
||||
merged["download_with_curl"] = _normalize_cmhub_download_with_curl(
|
||||
merged.get("download_with_curl", "auto")
|
||||
)
|
||||
merged["check_balance_before_batch"] = bool(merged.get("check_balance_before_batch", False))
|
||||
if merged["connect_timeout"] <= 0:
|
||||
raise ConfigError("ai.cmhub.connect_timeout 必须大于 0")
|
||||
|
||||
Reference in New Issue
Block a user