chengma/cmhub
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add multi-image vision analysis

Browse Source
This commit is contained in:
QiuSW
2026-07-16 14:13:19 +08:00
parent 0ea65d4df4
commit 5325eacdfe
29 changed files with 1013 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/generation.py
+153 -18
View File
@@ -6,7 +6,7 @@ import ipaddress
import socket
from dataclasses import dataclass, field
from time import perf_counter
from typing import Any, Callable, Mapping
from typing import Any, Callable, Mapping, Sequence
from urllib.parse import urljoin, urlsplit
import requests
@@ -19,7 +19,12 @@ from apps.ai.aliases import (
REQUIRED_CAPABILITIES,
resolve_model_alias,
)
from apps.ai.providers import AiCapabilityError, AiProviderError, get_provider
from apps.ai.providers import (
AiCapabilityError,
AiProviderError,
MultimodalImage,
get_provider,
)
from apps.billing.models import CallRecord, normalize_resolution
from apps.billing.pricing import NoPricingRuleError, calculate_points_cost
from apps.billing.services import (
@@ -68,6 +73,7 @@ class GenerationInput:
image_url: str = ""
image_base64: str = ""
aspect_ratio: str = "1:1"
images: tuple[Mapping[str, Any], ...] = field(default_factory=tuple)
@dataclass(frozen=True)
@@ -80,6 +86,7 @@ class PreparedGeneration:
resolution: str
parameters: dict[str, Any]
image_input: ImageInput | None
image_inputs: tuple[ImageInput, ...]
model_alias: Any
resolved_model: Any
provider: Any
@@ -105,6 +112,7 @@ class GenerationResult:
call_record: CallRecord
titles: tuple[str, ...] = field(default_factory=tuple)
image_url: str = ""
text: str = ""
def as_response_data(self) -> dict[str, Any]:
common = {
@@ -116,6 +124,8 @@ class GenerationResult:
}
if self.operation_type == CallRecord.OperationType.TITLE:
return {"titles": list(self.titles), **common}
if self.operation_type == CallRecord.OperationType.VISION:
return {"text": self.text, **common}
return {"image_url": self.image_url, **common}
@@ -165,6 +175,21 @@ def generate_image_response(
return result.as_response_data()
def analyze_images_response(*, user, api_key, request_data: Mapping[str, Any]) -> dict:
result = run_synchronous_generation(
GenerationInput(
user=user,
api_key=api_key,
operation_type=CallRecord.OperationType.VISION,
prompt=request_data["prompt"],
alias=request_data.get("model") or None,
parameters=dict(request_data.get("parameters") or {}),
images=tuple(dict(item) for item in request_data.get("images") or ()),
)
)
return result.as_response_data()
def run_synchronous_generation(
generation_input: GenerationInput,
*,
@@ -180,7 +205,11 @@ def run_synchronous_generation(
def prepare_generation(generation_input: GenerationInput) -> PreparedGeneration:
operation_type = normalize_operation_type(generation_input.operation_type)
resolution = normalize_resolution(generation_input.resolution or "1K") or "1K"
resolution = (
""
if operation_type == CallRecord.OperationType.VISION
else normalize_resolution(generation_input.resolution or "1K") or "1K"
)
parameters = dict(generation_input.parameters or {})
prompt = str(generation_input.prompt or "")
@@ -189,12 +218,17 @@ def prepare_generation(generation_input: GenerationInput) -> PreparedGeneration:
api_key=generation_input.api_key,
prompt=prompt,
)
image_input = load_image_input(
{
"image_base64": generation_input.image_base64,
"image_url": generation_input.image_url,
}
)
if operation_type == CallRecord.OperationType.VISION:
image_input = None
image_inputs = load_vision_image_inputs(generation_input.images)
else:
image_input = load_image_input(
{
"image_base64": generation_input.image_base64,
"image_url": generation_input.image_url,
}
)
image_inputs = ()
model_alias = resolve_model_alias_or_raise(operation_type, generation_input.alias)
resolved_model = resolved_model_or_raise(model_alias)
@@ -215,6 +249,7 @@ def prepare_generation(generation_input: GenerationInput) -> PreparedGeneration:
resolution=resolution,
parameters=parameters,
image_input=image_input,
image_inputs=image_inputs,
model_alias=model_alias,
resolved_model=resolved_model,
provider=provider,
@@ -253,6 +288,8 @@ def execute_precharged_generation(
started = perf_counter()
if prepared.operation_type == CallRecord.OperationType.TITLE:
result = execute_title_generation(precharged, started)
elif prepared.operation_type == CallRecord.OperationType.VISION:
result = execute_vision_generation(precharged, started)
else:
result = execute_image_generation(
precharged,
@@ -358,11 +395,44 @@ def execute_image_generation(
)
def execute_vision_generation(
precharged: PrechargedGeneration,
started: float,
) -> GenerationResult:
prepared = precharged.prepared
generation = prepared.provider.analyze_images(
prepared.prompt,
prepared.resolved_model,
images=tuple(
MultimodalImage(data=image.data, mime_type=image.mime_type)
for image in prepared.image_inputs
),
parameters=prepared.parameters,
)
latency_ms = elapsed_ms(started)
text = str(generation.text or "").strip()
call_record = mark_call_success(
precharged.call_record,
result_summary=summarize_text(text),
upstream_latency_ms=latency_ms,
)
return GenerationResult(
operation_type=prepared.operation_type,
alias=prepared.alias,
model_used=generation.model_used,
points_cost=precharged.points_cost,
points_balance=precharged.points_balance_after_charge,
call_record=call_record,
text=text,
)
def normalize_operation_type(operation_type: str) -> str:
normalized = str(operation_type or "").strip()
if normalized not in {
CallRecord.OperationType.TITLE,
CallRecord.OperationType.IMAGE,
CallRecord.OperationType.VISION,
}:
raise ValueError(f"Unsupported generation operation type: {operation_type}")
return normalized
@@ -392,8 +462,8 @@ def resolve_model_alias_or_raise(operation_type: str, alias: str | None):
def ensure_provider_supports(provider, operation_type: str) -> None:
required_capability = REQUIRED_CAPABILITIES[operation_type]
if required_capability not in provider.capabilities():
required_capabilities = REQUIRED_CAPABILITIES[operation_type]
if not required_capabilities.issubset(provider.capabilities()):
raise ApiRequestError(
"model_not_allowed",
"该模型不支持此操作",
@@ -471,7 +541,57 @@ def load_image_input(data: Mapping[str, Any]) -> ImageInput | None:
return None
def decode_image_input(value: str) -> ImageInput:
def load_vision_image_inputs(
items: Sequence[Mapping[str, Any]],
) -> tuple[ImageInput, ...]:
max_images = max(1, int(getattr(settings, "VISION_MAX_IMAGES", 8)))
max_image_bytes = max(
1,
int(getattr(settings, "VISION_MAX_IMAGE_BYTES", 10 * 1024 * 1024)),
)
max_total_bytes = max(
1,
int(getattr(settings, "VISION_MAX_TOTAL_BYTES", 32 * 1024 * 1024)),
)
if not items:
raise ApiRequestError("bad_request", "images 至少需要一张图片", status.HTTP_400_BAD_REQUEST)
if len(items) > max_images:
raise ApiRequestError(
"bad_request",
f"单次最多上传 {max_images} 张图片",
status.HTTP_400_BAD_REQUEST,
)
image_inputs = []
total_bytes = 0
for item in items:
raw_base64 = str(item.get("image_base64") or "").strip()
image_url = str(item.get("image_url") or "").strip()
if bool(raw_base64) == bool(image_url):
raise ApiRequestError(
"bad_request",
"每张图片必须且只能提供 image_url 或 image_base64",
status.HTTP_400_BAD_REQUEST,
)
image_input = (
decode_image_input(raw_base64, max_bytes=max_image_bytes)
if raw_base64
else download_image_input(image_url, max_bytes=max_image_bytes)
)
if not image_input.mime_type.lower().startswith("image/"):
raise ApiRequestError("bad_request", "图片格式无效", status.HTTP_400_BAD_REQUEST)
total_bytes += len(image_input.data)
if total_bytes > max_total_bytes:
raise ApiRequestError(
"bad_request",
"图片总大小超过限制",
status.HTTP_400_BAD_REQUEST,
)
image_inputs.append(image_input)
return tuple(image_inputs)
def decode_image_input(value: str, *, max_bytes: int | None = None) -> ImageInput:
mime_type = "image/png"
encoded = value
if value.startswith("data:"):
@@ -479,16 +599,20 @@ def decode_image_input(value: str) -> ImageInput:
raise ApiRequestError("bad_request", "image_base64 格式无效", status.HTTP_400_BAD_REQUEST)
prefix, encoded = value.split(",", 1)
mime_type = prefix[len("data:") :].split(";", 1)[0] or mime_type
if max_bytes is not None and len(encoded) > ((max_bytes + 2) // 3) * 4:
raise ApiRequestError("bad_request", "图片过大", status.HTTP_400_BAD_REQUEST)
try:
image = base64.b64decode(encoded, validate=True)
except (binascii.Error, ValueError) as exc:
raise ApiRequestError("bad_request", "image_base64 格式无效", status.HTTP_400_BAD_REQUEST) from exc
if not image:
raise ApiRequestError("bad_request", "image_base64 不能为空", status.HTTP_400_BAD_REQUEST)
if max_bytes is not None and len(image) > max_bytes:
raise ApiRequestError("bad_request", "图片过大", status.HTTP_400_BAD_REQUEST)
return ImageInput(data=image, mime_type=mime_type, filename=filename_for_mime(mime_type))
def download_image_input(url: str) -> ImageInput:
def download_image_input(url: str, *, max_bytes: int | None = None) -> ImageInput:
session = requests.Session()
session.trust_env = False
current_url = validated_image_url(url)
@@ -522,7 +646,7 @@ def download_image_input(url: str) -> ImageInput:
content_type = response.headers.get("Content-Type", "image/png").split(";", 1)[0].strip().lower()
if not content_type.startswith("image/"):
raise ApiRequestError("bad_request", "image_url 不是图片资源", status.HTTP_400_BAD_REQUEST)
image = read_limited_image_response(response)
image = read_limited_image_response(response, max_bytes=max_bytes)
if not image:
raise ApiRequestError("bad_request", "image_url 图片内容为空", status.HTTP_400_BAD_REQUEST)
return ImageInput(
@@ -593,12 +717,19 @@ def is_redirect_response(response) -> bool:
return 300 <= int(getattr(response, "status_code", 0)) < 400
def read_limited_image_response(response) -> bytes:
max_bytes = max(1, int(getattr(settings, "IMAGE_URL_MAX_BYTES", 10 * 1024 * 1024)))
def read_limited_image_response(response, *, max_bytes: int | None = None) -> bytes:
byte_limit = max(
1,
int(
max_bytes
if max_bytes is not None
else getattr(settings, "IMAGE_URL_MAX_BYTES", 10 * 1024 * 1024)
),
)
content_length = response.headers.get("Content-Length")
if content_length:
try:
if int(content_length) > max_bytes:
if int(content_length) > byte_limit:
raise ApiRequestError("bad_request", "image_url 图片过大", status.HTTP_400_BAD_REQUEST)
except ValueError:
pass
@@ -609,7 +740,7 @@ def read_limited_image_response(response) -> bytes:
if not chunk:
continue
total += len(chunk)
if total > max_bytes:
if total > byte_limit:
raise ApiRequestError("bad_request", "image_url 图片过大", status.HTTP_400_BAD_REQUEST)
chunks.append(chunk)
return b"".join(chunks)
@@ -629,5 +760,9 @@ def summarize_titles(titles: list[str], text: str) -> str:
return summary[:500]
def summarize_text(text: str) -> str:
return str(text or "").strip()[:500]
def elapsed_ms(started: float) -> int:
return int((perf_counter() - started) * 1000)