chengma/cmhub
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add multi-image vision analysis

Browse Source
This commit is contained in:
QiuSW
2026-07-16 14:13:19 +08:00
parent 0ea65d4df4
commit 5325eacdfe
29 changed files with 1013 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/ai/aliases.py
+8 -6
View File
@@ -18,15 +18,16 @@ class ModelCapabilityError(AliasResolutionError):
REQUIRED_CAPABILITIES = {
ModelAlias.OperationType.TITLE: "text",
ModelAlias.OperationType.IMAGE: "image",
ModelAlias.OperationType.TITLE: frozenset({"text"}),
ModelAlias.OperationType.IMAGE: frozenset({"image"}),
ModelAlias.OperationType.VISION: frozenset({"text", "vision"}),
}
def resolve_model_alias(operation_type: str, alias: str | None = None) -> ModelAlias:
"""Resolve an external capability alias to an active ModelAlias row."""
required_capability = REQUIRED_CAPABILITIES.get(operation_type)
if required_capability is None:
required_capabilities = REQUIRED_CAPABILITIES.get(operation_type)
if required_capabilities is None:
raise AliasResolutionError(f"unsupported operation_type: {operation_type}")
queryset = ModelAlias.objects.select_related("ai_model").filter(
@@ -47,10 +48,11 @@ def resolve_model_alias(operation_type: str, alias: str | None = None) -> ModelA
ai_model: AiModel = model_alias.ai_model
capabilities = ai_model.capabilities_set()
if required_capability not in capabilities:
missing_capabilities = required_capabilities - capabilities
if missing_capabilities:
raise ModelCapabilityError(
f"alias {model_alias.alias} maps to model {ai_model.name} without "
f"{required_capability} capability"
f"{', '.join(sorted(missing_capabilities))} capability"
)
return model_alias
apps/ai/catalog.py
+5 -3
View File
@@ -14,13 +14,15 @@ PUBLIC_DEFAULT_RESOLUTION = "default"
def _has_required_capability(model_alias: ModelAlias) -> bool:
required_capability = REQUIRED_CAPABILITIES.get(model_alias.operation_type)
if required_capability is None:
required_capabilities = REQUIRED_CAPABILITIES.get(model_alias.operation_type)
if required_capabilities is None:
return False
return required_capability in model_alias.ai_model.capabilities_set()
return required_capabilities.issubset(model_alias.ai_model.capabilities_set())
def _requires_image_input(ai_model: AiModel, operation_type: str) -> bool:
if operation_type == ModelAlias.OperationType.VISION:
return True
if operation_type != ModelAlias.OperationType.IMAGE:
return False
try:
apps/ai/migrations/0005_alter_modelalias_operation_type.py
+18
View File
@@ -0,0 +1,18 @@
# Generated by Django 5.2.15 on 2026-07-16 03:52
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('ai', '0004_alter_aiconfigauditlog_action_and_more'),
]
operations = [
migrations.AlterField(
model_name='modelalias',
name='operation_type',
field=models.CharField(choices=[('title', '生成标题'), ('image', '生成图片'), ('vision', '图片理解')], max_length=32, verbose_name='操作类型'),
),
]
apps/ai/models.py
+1
View File
@@ -92,6 +92,7 @@ class ModelAlias(models.Model):
class OperationType(models.TextChoices):
TITLE = "title", "生成标题"
IMAGE = "image", "生成图片"
VISION = "vision", "图片理解"
alias = models.SlugField("能力别名", max_length=64)
operation_type = models.CharField("操作类型", max_length=32, choices=OperationType.choices)
apps/ai/providers/__init__.py
+2
View File
@@ -4,6 +4,7 @@ from .base import (
AiProviderError,
AiResponseParseError,
ImageGenerationResult,
MultimodalImage,
Provider,
ResolvedModel,
TextGenerationResult,
@@ -16,6 +17,7 @@ __all__ = [
"AiProviderError",
"AiResponseParseError",
"ImageGenerationResult",
"MultimodalImage",
"Provider",
"ResolvedModel",
"TextGenerationResult",
apps/ai/providers/base.py
+17 -1
View File
@@ -1,7 +1,7 @@
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Mapping, Protocol
from typing import Any, Mapping, Protocol, Sequence
class AiProviderError(RuntimeError):
@@ -78,6 +78,12 @@ class ImageGenerationResult:
raw: Mapping[str, Any]
@dataclass(frozen=True)
class MultimodalImage:
data: bytes
mime_type: str = "image/png"
class Provider(Protocol):
def capabilities(self) -> set[str]:
...
@@ -108,6 +114,16 @@ class Provider(Protocol):
) -> ImageGenerationResult:
...
def analyze_images(
self,
prompt: str,
model: ResolvedModel,
*,
images: Sequence[MultimodalImage],
parameters: Mapping[str, Any] | None = None,
) -> TextGenerationResult:
...
def validate_model_config(model: ResolvedModel) -> None:
errors = []
apps/ai/providers/openai_compatible.py
+117 -1
View File
@@ -1,6 +1,6 @@
from __future__ import annotations
from typing import Any, Mapping
from typing import Any, Mapping, Sequence
import requests
@@ -8,6 +8,7 @@ from .base import (
AiCapabilityError,
AiResponseParseError,
ImageGenerationResult,
MultimodalImage,
ResolvedModel,
TextGenerationResult,
validate_model_config,
@@ -18,6 +19,7 @@ from .utils import (
API_IMAGES,
API_IMAGES_EDITS,
extract_image_from_response,
extract_raw_text,
extract_text_from_response,
extract_titles_from_response,
image_request_timeout,
@@ -166,6 +168,37 @@ class ChatCompletionsProvider(BaseHttpProvider):
raise AiResponseParseError("AI response did not contain an image")
return ImageGenerationResult(image=image_bytes, model_used=model.model, raw=raw)
def analyze_images(
self,
prompt: str,
model: ResolvedModel,
*,
images: Sequence[MultimodalImage],
parameters: Mapping[str, Any] | None = None,
) -> TextGenerationResult:
validate_model_config(model)
if not images:
raise AiCapabilityError("vision analysis requires at least one image")
url = normalize_api_url(model.url, API_CHAT)
payload = build_chat_vision_payload(
model,
prompt,
images=images,
parameters=parameters,
)
response = self.session.post(
url,
headers=self._headers(model, json=True),
json=payload,
timeout=self._timeout(model, "1K"),
)
response.raise_for_status()
raw = response.json()
text = extract_raw_text(raw).strip()
if not text:
raise AiResponseParseError("AI response did not contain text")
return TextGenerationResult(text=text, titles=(), model_used=model.model, raw=raw)
class GeminiProvider(ChatCompletionsProvider):
def generate_text(
@@ -241,6 +274,37 @@ class GeminiProvider(ChatCompletionsProvider):
raise AiResponseParseError("AI response did not contain an image")
return ImageGenerationResult(image=image_bytes, model_used=model.model, raw=raw)
def analyze_images(
self,
prompt: str,
model: ResolvedModel,
*,
images: Sequence[MultimodalImage],
parameters: Mapping[str, Any] | None = None,
) -> TextGenerationResult:
validate_model_config(model)
if not images:
raise AiCapabilityError("vision analysis requires at least one image")
url = normalize_api_url(model.url, API_GEMINI).replace("{model}", model.model)
payload = build_gemini_vision_payload(
model,
prompt,
images=images,
parameters=parameters,
)
response = self.session.post(
url,
headers=self._headers(model, json=True),
json=payload,
timeout=self._timeout(model, "1K"),
)
response.raise_for_status()
raw = response.json()
text = extract_raw_text(raw).strip()
if not text:
raise AiResponseParseError("AI response did not contain text")
return TextGenerationResult(text=text, titles=(), model_used=model.model, raw=raw)
class ImagesGenerationProvider(BaseHttpProvider):
def capabilities(self) -> set[str]:
@@ -249,6 +313,9 @@ class ImagesGenerationProvider(BaseHttpProvider):
def generate_text(self, *args: Any, **kwargs: Any) -> TextGenerationResult:
raise AiCapabilityError("images generation provider cannot generate text")
def analyze_images(self, *args: Any, **kwargs: Any) -> TextGenerationResult:
raise AiCapabilityError("images generation provider cannot analyze images")
def generate_image(
self,
prompt: str,
@@ -298,6 +365,9 @@ class ImagesEditsProvider(BaseHttpProvider):
def generate_text(self, *args: Any, **kwargs: Any) -> TextGenerationResult:
raise AiCapabilityError("images edits provider cannot generate text")
def analyze_images(self, *args: Any, **kwargs: Any) -> TextGenerationResult:
raise AiCapabilityError("images edits provider cannot analyze images")
def generate_image(
self,
prompt: str,
@@ -384,6 +454,32 @@ def build_chat_image_payload(
)
def build_chat_vision_payload(
model: ResolvedModel,
prompt: str,
*,
images: Sequence[MultimodalImage],
parameters: Mapping[str, Any] | None = None,
) -> dict[str, Any]:
content: list[dict[str, Any]] = [{"type": "text", "text": prompt}]
content.extend(
{
"type": "image_url",
"image_url": {
"url": image_bytes_to_data_url(image.data, image.mime_type),
},
}
for image in images
)
payload: dict[str, Any] = {
"model": model.model,
"messages": [{"role": "user", "content": content}],
"stream": False,
}
apply_extra_body(payload, model, parameters)
return payload
def build_gemini_payload(
model: ResolvedModel,
prompt: str,
@@ -406,6 +502,26 @@ def build_gemini_payload(
return payload
def build_gemini_vision_payload(
model: ResolvedModel,
prompt: str,
*,
images: Sequence[MultimodalImage],
parameters: Mapping[str, Any] | None = None,
) -> dict[str, Any]:
parts: list[dict[str, Any]] = [{"text": prompt}]
for image in images:
data_url = image_bytes_to_data_url(image.data, image.mime_type)
mime_type, data = split_data_url(data_url)
parts.append({"inlineData": {"mimeType": mime_type, "data": data}})
payload: dict[str, Any] = {
"contents": [{"parts": parts}],
"generationConfig": {"responseModalities": ["TEXT"]},
}
apply_extra_body(payload, model, parameters)
return payload
def apply_extra_body(
payload: dict[str, Any],
model: ResolvedModel,
apps/ai/tests.py
+121 -2
View File
@@ -11,8 +11,18 @@ from apps.ai.admin import AiConfigAuditLogAdmin, AiModelAdmin, ModelAliasAdmin
from apps.ai.aliases import AliasNotFoundError, ModelCapabilityError, resolve_alias
from apps.ai.importers import import_ai_models_config
from apps.ai.models import AiConfigAuditLog, AiModel, ModelAlias
from apps.ai.providers import AiCapabilityError, ResolvedModel, get_provider, resolve_api_type
from apps.ai.providers.openai_compatible import ChatCompletionsProvider, ImagesEditsProvider
from apps.ai.providers import (
AiCapabilityError,
MultimodalImage,
ResolvedModel,
get_provider,
resolve_api_type,
)
from apps.ai.providers.openai_compatible import (
ChatCompletionsProvider,
GeminiProvider,
ImagesEditsProvider,
)
from apps.ai.providers.utils import image_request_timeout, resolution_to_size
@@ -255,6 +265,85 @@ class ChatCompletionsProviderTests(SimpleTestCase):
self.assertEqual(session.posts[0]["timeout"], (30, 600))
def test_analyze_images_builds_ordered_payload_and_preserves_full_text(self):
session = FakeSession(
FakeResponse(
{
"choices": [
{
"message": {
"content": "第一张是正面图。\n第二张是细节图。"
}
}
]
}
)
)
provider = ChatCompletionsProvider(session=session)
model = ResolvedModel(
name="Vision text",
url="https://api.vectorengine.ai/v1",
model="vision-model",
api_key="test-key",
api_type="chat",
)
result = provider.analyze_images(
"比较两张商品图",
model,
images=(
MultimodalImage(b"first-image", "image/jpeg"),
MultimodalImage(b"second-image", "image/png"),
),
parameters={"temperature": 0.2, "messages": []},
)
self.assertEqual(result.text, "第一张是正面图。\n第二张是细节图。")
self.assertEqual(result.titles, ())
content = session.posts[0]["json"]["messages"][0]["content"]
self.assertEqual(content[0], {"type": "text", "text": "比较两张商品图"})
self.assertTrue(content[1]["image_url"]["url"].startswith("data:image/jpeg;base64,"))
self.assertTrue(content[2]["image_url"]["url"].startswith("data:image/png;base64,"))
self.assertEqual(session.posts[0]["json"]["temperature"], 0.2)
def test_gemini_analyze_images_builds_ordered_inline_data(self):
session = FakeSession(
FakeResponse(
{
"candidates": [
{"content": {"parts": [{"text": "多图分析结果"}]}}
]
}
)
)
provider = GeminiProvider(session=session)
model = ResolvedModel(
name="Gemini vision",
url="https://gemini.example.com",
model="gemini-vision",
api_key="test-key",
api_type="gemini",
)
result = provider.analyze_images(
"理解这些图片",
model,
images=(
MultimodalImage(b"one", "image/webp"),
MultimodalImage(b"two", "image/jpeg"),
),
)
self.assertEqual(result.text, "多图分析结果")
parts = session.posts[0]["json"]["contents"][0]["parts"]
self.assertEqual(parts[0], {"text": "理解这些图片"})
self.assertEqual(parts[1]["inlineData"]["mimeType"], "image/webp")
self.assertEqual(parts[2]["inlineData"]["mimeType"], "image/jpeg")
self.assertEqual(
session.posts[0]["json"]["generationConfig"]["responseModalities"],
["TEXT"],
)
class ImagesEditsProviderTests(SimpleTestCase):
def test_generate_image_builds_multipart_request_and_parses_base64(self):
@@ -344,6 +433,13 @@ class ImagesEditsProviderTests(SimpleTestCase):
with self.assertRaises(AiCapabilityError):
provider.generate_text("Generate title", model)
with self.assertRaises(AiCapabilityError):
provider.analyze_images(
"Analyze image",
model,
images=(MultimodalImage(b"source-image"),),
)
@override_settings(AI_KEY_ENCRYPTION_KEY=TEST_ENCRYPTION_KEY)
class AiModelEncryptionTests(TestCase):
@@ -440,6 +536,29 @@ class AliasResolutionTests(TestCase):
self.assertEqual(resolved.model, "gpt-image-2")
self.assertIn("image", resolved.capabilities)
def test_resolve_vision_alias_requires_text_and_vision_capabilities(self):
ModelAlias.objects.create(
operation_type=ModelAlias.OperationType.VISION,
alias="vision-standard",
ai_model=self.text_model,
is_default=True,
)
resolved = resolve_alias(ModelAlias.OperationType.VISION)
self.assertEqual(resolved.model, "gpt-5.5")
self.assertTrue({"text", "vision"}.issubset(resolved.capabilities))
def test_resolve_vision_alias_rejects_vision_model_without_text(self):
ModelAlias.objects.create(
operation_type=ModelAlias.OperationType.VISION,
alias="bad-vision",
ai_model=self.image_model,
)
with self.assertRaises(ModelCapabilityError):
resolve_alias(ModelAlias.OperationType.VISION, "bad-vision")
def test_resolve_alias_rejects_capability_mismatch(self):
ModelAlias.objects.create(
operation_type=ModelAlias.OperationType.TITLE,
apps/api/generation.py
+153 -18
View File
@@ -6,7 +6,7 @@ import ipaddress
import socket
from dataclasses import dataclass, field
from time import perf_counter
from typing import Any, Callable, Mapping
from typing import Any, Callable, Mapping, Sequence
from urllib.parse import urljoin, urlsplit
import requests
@@ -19,7 +19,12 @@ from apps.ai.aliases import (
REQUIRED_CAPABILITIES,
resolve_model_alias,
)
from apps.ai.providers import AiCapabilityError, AiProviderError, get_provider
from apps.ai.providers import (
AiCapabilityError,
AiProviderError,
MultimodalImage,
get_provider,
)
from apps.billing.models import CallRecord, normalize_resolution
from apps.billing.pricing import NoPricingRuleError, calculate_points_cost
from apps.billing.services import (
@@ -68,6 +73,7 @@ class GenerationInput:
image_url: str = ""
image_base64: str = ""
aspect_ratio: str = "1:1"
images: tuple[Mapping[str, Any], ...] = field(default_factory=tuple)
@dataclass(frozen=True)
@@ -80,6 +86,7 @@ class PreparedGeneration:
resolution: str
parameters: dict[str, Any]
image_input: ImageInput | None
image_inputs: tuple[ImageInput, ...]
model_alias: Any
resolved_model: Any
provider: Any
@@ -105,6 +112,7 @@ class GenerationResult:
call_record: CallRecord
titles: tuple[str, ...] = field(default_factory=tuple)
image_url: str = ""
text: str = ""
def as_response_data(self) -> dict[str, Any]:
common = {
@@ -116,6 +124,8 @@ class GenerationResult:
}
if self.operation_type == CallRecord.OperationType.TITLE:
return {"titles": list(self.titles), **common}
if self.operation_type == CallRecord.OperationType.VISION:
return {"text": self.text, **common}
return {"image_url": self.image_url, **common}
@@ -165,6 +175,21 @@ def generate_image_response(
return result.as_response_data()
def analyze_images_response(*, user, api_key, request_data: Mapping[str, Any]) -> dict:
result = run_synchronous_generation(
GenerationInput(
user=user,
api_key=api_key,
operation_type=CallRecord.OperationType.VISION,
prompt=request_data["prompt"],
alias=request_data.get("model") or None,
parameters=dict(request_data.get("parameters") or {}),
images=tuple(dict(item) for item in request_data.get("images") or ()),
)
)
return result.as_response_data()
def run_synchronous_generation(
generation_input: GenerationInput,
*,
@@ -180,7 +205,11 @@ def run_synchronous_generation(
def prepare_generation(generation_input: GenerationInput) -> PreparedGeneration:
operation_type = normalize_operation_type(generation_input.operation_type)
resolution = normalize_resolution(generation_input.resolution or "1K") or "1K"
resolution = (
""
if operation_type == CallRecord.OperationType.VISION
else normalize_resolution(generation_input.resolution or "1K") or "1K"
)
parameters = dict(generation_input.parameters or {})
prompt = str(generation_input.prompt or "")
@@ -189,12 +218,17 @@ def prepare_generation(generation_input: GenerationInput) -> PreparedGeneration:
api_key=generation_input.api_key,
prompt=prompt,
)
image_input = load_image_input(
{
"image_base64": generation_input.image_base64,
"image_url": generation_input.image_url,
}
)
if operation_type == CallRecord.OperationType.VISION:
image_input = None
image_inputs = load_vision_image_inputs(generation_input.images)
else:
image_input = load_image_input(
{
"image_base64": generation_input.image_base64,
"image_url": generation_input.image_url,
}
)
image_inputs = ()
model_alias = resolve_model_alias_or_raise(operation_type, generation_input.alias)
resolved_model = resolved_model_or_raise(model_alias)
@@ -215,6 +249,7 @@ def prepare_generation(generation_input: GenerationInput) -> PreparedGeneration:
resolution=resolution,
parameters=parameters,
image_input=image_input,
image_inputs=image_inputs,
model_alias=model_alias,
resolved_model=resolved_model,
provider=provider,
@@ -253,6 +288,8 @@ def execute_precharged_generation(
started = perf_counter()
if prepared.operation_type == CallRecord.OperationType.TITLE:
result = execute_title_generation(precharged, started)
elif prepared.operation_type == CallRecord.OperationType.VISION:
result = execute_vision_generation(precharged, started)
else:
result = execute_image_generation(
precharged,
@@ -358,11 +395,44 @@ def execute_image_generation(
)
def execute_vision_generation(
precharged: PrechargedGeneration,
started: float,
) -> GenerationResult:
prepared = precharged.prepared
generation = prepared.provider.analyze_images(
prepared.prompt,
prepared.resolved_model,
images=tuple(
MultimodalImage(data=image.data, mime_type=image.mime_type)
for image in prepared.image_inputs
),
parameters=prepared.parameters,
)
latency_ms = elapsed_ms(started)
text = str(generation.text or "").strip()
call_record = mark_call_success(
precharged.call_record,
result_summary=summarize_text(text),
upstream_latency_ms=latency_ms,
)
return GenerationResult(
operation_type=prepared.operation_type,
alias=prepared.alias,
model_used=generation.model_used,
points_cost=precharged.points_cost,
points_balance=precharged.points_balance_after_charge,
call_record=call_record,
text=text,
)
def normalize_operation_type(operation_type: str) -> str:
normalized = str(operation_type or "").strip()
if normalized not in {
CallRecord.OperationType.TITLE,
CallRecord.OperationType.IMAGE,
CallRecord.OperationType.VISION,
}:
raise ValueError(f"Unsupported generation operation type: {operation_type}")
return normalized
@@ -392,8 +462,8 @@ def resolve_model_alias_or_raise(operation_type: str, alias: str | None):
def ensure_provider_supports(provider, operation_type: str) -> None:
required_capability = REQUIRED_CAPABILITIES[operation_type]
if required_capability not in provider.capabilities():
required_capabilities = REQUIRED_CAPABILITIES[operation_type]
if not required_capabilities.issubset(provider.capabilities()):
raise ApiRequestError(
"model_not_allowed",
"该模型不支持此操作",
@@ -471,7 +541,57 @@ def load_image_input(data: Mapping[str, Any]) -> ImageInput | None:
return None
def decode_image_input(value: str) -> ImageInput:
def load_vision_image_inputs(
items: Sequence[Mapping[str, Any]],
) -> tuple[ImageInput, ...]:
max_images = max(1, int(getattr(settings, "VISION_MAX_IMAGES", 8)))
max_image_bytes = max(
1,
int(getattr(settings, "VISION_MAX_IMAGE_BYTES", 10 * 1024 * 1024)),
)
max_total_bytes = max(
1,
int(getattr(settings, "VISION_MAX_TOTAL_BYTES", 32 * 1024 * 1024)),
)
if not items:
raise ApiRequestError("bad_request", "images 至少需要一张图片", status.HTTP_400_BAD_REQUEST)
if len(items) > max_images:
raise ApiRequestError(
"bad_request",
f"单次最多上传 {max_images} 张图片",
status.HTTP_400_BAD_REQUEST,
)
image_inputs = []
total_bytes = 0
for item in items:
raw_base64 = str(item.get("image_base64") or "").strip()
image_url = str(item.get("image_url") or "").strip()
if bool(raw_base64) == bool(image_url):
raise ApiRequestError(
"bad_request",
"每张图片必须且只能提供 image_url 或 image_base64",
status.HTTP_400_BAD_REQUEST,
)
image_input = (
decode_image_input(raw_base64, max_bytes=max_image_bytes)
if raw_base64
else download_image_input(image_url, max_bytes=max_image_bytes)
)
if not image_input.mime_type.lower().startswith("image/"):
raise ApiRequestError("bad_request", "图片格式无效", status.HTTP_400_BAD_REQUEST)
total_bytes += len(image_input.data)
if total_bytes > max_total_bytes:
raise ApiRequestError(
"bad_request",
"图片总大小超过限制",
status.HTTP_400_BAD_REQUEST,
)
image_inputs.append(image_input)
return tuple(image_inputs)
def decode_image_input(value: str, *, max_bytes: int | None = None) -> ImageInput:
mime_type = "image/png"
encoded = value
if value.startswith("data:"):
@@ -479,16 +599,20 @@ def decode_image_input(value: str) -> ImageInput:
raise ApiRequestError("bad_request", "image_base64 格式无效", status.HTTP_400_BAD_REQUEST)
prefix, encoded = value.split(",", 1)
mime_type = prefix[len("data:") :].split(";", 1)[0] or mime_type
if max_bytes is not None and len(encoded) > ((max_bytes + 2) // 3) * 4:
raise ApiRequestError("bad_request", "图片过大", status.HTTP_400_BAD_REQUEST)
try:
image = base64.b64decode(encoded, validate=True)
except (binascii.Error, ValueError) as exc:
raise ApiRequestError("bad_request", "image_base64 格式无效", status.HTTP_400_BAD_REQUEST) from exc
if not image:
raise ApiRequestError("bad_request", "image_base64 不能为空", status.HTTP_400_BAD_REQUEST)
if max_bytes is not None and len(image) > max_bytes:
raise ApiRequestError("bad_request", "图片过大", status.HTTP_400_BAD_REQUEST)
return ImageInput(data=image, mime_type=mime_type, filename=filename_for_mime(mime_type))
def download_image_input(url: str) -> ImageInput:
def download_image_input(url: str, *, max_bytes: int | None = None) -> ImageInput:
session = requests.Session()
session.trust_env = False
current_url = validated_image_url(url)
@@ -522,7 +646,7 @@ def download_image_input(url: str) -> ImageInput:
content_type = response.headers.get("Content-Type", "image/png").split(";", 1)[0].strip().lower()
if not content_type.startswith("image/"):
raise ApiRequestError("bad_request", "image_url 不是图片资源", status.HTTP_400_BAD_REQUEST)
image = read_limited_image_response(response)
image = read_limited_image_response(response, max_bytes=max_bytes)
if not image:
raise ApiRequestError("bad_request", "image_url 图片内容为空", status.HTTP_400_BAD_REQUEST)
return ImageInput(
@@ -593,12 +717,19 @@ def is_redirect_response(response) -> bool:
return 300 <= int(getattr(response, "status_code", 0)) < 400
def read_limited_image_response(response) -> bytes:
max_bytes = max(1, int(getattr(settings, "IMAGE_URL_MAX_BYTES", 10 * 1024 * 1024)))
def read_limited_image_response(response, *, max_bytes: int | None = None) -> bytes:
byte_limit = max(
1,
int(
max_bytes
if max_bytes is not None
else getattr(settings, "IMAGE_URL_MAX_BYTES", 10 * 1024 * 1024)
),
)
content_length = response.headers.get("Content-Length")
if content_length:
try:
if int(content_length) > max_bytes:
if int(content_length) > byte_limit:
raise ApiRequestError("bad_request", "image_url 图片过大", status.HTTP_400_BAD_REQUEST)
except ValueError:
pass
@@ -609,7 +740,7 @@ def read_limited_image_response(response) -> bytes:
if not chunk:
continue
total += len(chunk)
if total > max_bytes:
if total > byte_limit:
raise ApiRequestError("bad_request", "image_url 图片过大", status.HTTP_400_BAD_REQUEST)
chunks.append(chunk)
return b"".join(chunks)
@@ -629,5 +760,9 @@ def summarize_titles(titles: list[str], text: str) -> str:
return summary[:500]
def summarize_text(text: str) -> str:
return str(text or "").strip()[:500]
def elapsed_ms(started: float) -> int:
return int((perf_counter() - started) * 1000)
apps/api/serializers.py
+32
View File
@@ -50,6 +50,38 @@ class GenerateImageRequestSerializer(serializers.Serializer):
parameters = serializers.DictField(required=False, default=dict)
class VisionImageInputSerializer(serializers.Serializer):
image_url = serializers.URLField(required=False, allow_blank=True)
image_base64 = serializers.CharField(required=False, allow_blank=True)
def validate(self, attrs):
has_url = bool(str(attrs.get("image_url") or "").strip())
has_base64 = bool(str(attrs.get("image_base64") or "").strip())
if has_url == has_base64:
raise serializers.ValidationError(
"每张图片必须且只能提供 image_url 或 image_base64。"
)
return attrs
class AnalyzeImagesRequestSerializer(serializers.Serializer):
prompt = serializers.CharField(trim_whitespace=True, allow_blank=False)
model = serializers.CharField(
required=False,
allow_blank=True,
trim_whitespace=True,
max_length=64,
)
images = VisionImageInputSerializer(many=True, allow_empty=False)
parameters = serializers.DictField(required=False, default=dict)
def validate_images(self, value):
max_images = max(1, int(settings.VISION_MAX_IMAGES))
if len(value) > max_images:
raise serializers.ValidationError(f"单次最多上传 {max_images} 张图片。")
return value
class RechargeCreateRequestSerializer(serializers.Serializer):
amount = serializers.DecimalField(
max_digits=12,
apps/api/tests.py
+328 -1
View File
@@ -331,6 +331,12 @@ class ModelsCatalogApiTests(TestCase):
url="https://provider-secret.example/v1/images/edits",
model_sku="secret-sku-image-2",
)
vision_alias = self.create_alias(
alias="vision-standard",
operation_type=ModelAlias.OperationType.VISION,
capabilities=["text", "vision"],
model_sku="secret-sku-vision",
)
PricingRule.objects.create(
operation_type=title_alias.operation_type,
alias=title_alias.alias,
@@ -349,13 +355,19 @@ class ModelsCatalogApiTests(TestCase):
resolution="1k",
points_cost=12,
)
PricingRule.objects.create(
operation_type=vision_alias.operation_type,
alias=vision_alias.alias,
resolution="",
points_cost=3,
)
response = self.client.get(self.url, **self.auth_header())
self.assertEqual(response.status_code, 200)
self.assertNotIn(GenerateRateThrottle, ModelsView.throttle_classes)
models = {item["alias"]: item for item in response.data["models"]}
self.assertEqual(set(models), {"title-standard", "image-edit"})
self.assertEqual(set(models), {"title-standard", "image-edit", "vision-standard"})
self.assertEqual(
set(models["title-standard"]),
{
@@ -384,6 +396,14 @@ class ModelsCatalogApiTests(TestCase):
{"resolution": "1K", "points_cost": 12},
],
)
self.assertEqual(models["vision-standard"]["operation_type"], "vision")
self.assertEqual(models["vision-standard"]["capabilities"], ["text", "vision"])
self.assertTrue(models["vision-standard"]["requires_image"])
self.assertEqual(models["vision-standard"]["pricing_status"], "priced")
self.assertEqual(
models["vision-standard"]["prices"],
[{"resolution": "default", "points_cost": 3}],
)
response_body = json.dumps(response.data, ensure_ascii=False)
self.assertNotIn("secret-sku", response_body)
self.assertNotIn("provider-secret.example", response_body)
@@ -1056,8 +1076,10 @@ class FakeGenerationProvider:
self._capabilities = set(capabilities or {"text", "image", "vision"})
self.text_calls = []
self.image_calls = []
self.vision_calls = []
self.text_error = None
self.image_error = None
self.vision_error = None
def capabilities(self):
return set(self._capabilities)
@@ -1083,6 +1105,17 @@ class FakeGenerationProvider:
raw={"b64_json": "SECRET_RAW_SHOULD_NOT_BE_STORED"},
)
def analyze_images(self, prompt, model, **kwargs):
self.vision_calls.append({"prompt": prompt, "model": model, **kwargs})
if self.vision_error is not None:
raise self.vision_error
return TextGenerationResult(
text="第一张展示商品正面。\n第二张展示商品细节。",
titles=(),
model_used=model.model,
raw={"secret": "SECRET_RAW_SHOULD_NOT_BE_STORED"},
)
class FakeImageUrlResponse:
def __init__(self, *, status_code=200, headers=None, chunks=()):
@@ -1143,14 +1176,26 @@ class GenerateApiTests(TestCase):
model=f"gpt-image-{suffix}",
capabilities=["image", "vision"],
)
self.vision_model = self.create_ai_model(
name=f"vision-model-{suffix}",
model=f"gpt-vision-{suffix}",
capabilities=["text", "vision"],
)
self.title_alias = f"title-standard-{suffix}"
self.image_alias = f"image-hd-{suffix}"
self.vision_alias = f"vision-standard-{suffix}"
ModelAlias.objects.create(
operation_type=ModelAlias.OperationType.TITLE,
alias=self.title_alias,
ai_model=self.title_model,
is_default=True,
)
ModelAlias.objects.create(
operation_type=ModelAlias.OperationType.VISION,
alias=self.vision_alias,
ai_model=self.vision_model,
is_default=True,
)
ModelAlias.objects.create(
operation_type=ModelAlias.OperationType.IMAGE,
alias=self.image_alias,
@@ -1168,6 +1213,11 @@ class GenerateApiTests(TestCase):
resolution="1K",
points_cost=10,
)
PricingRule.objects.create(
operation_type=CallRecord.OperationType.VISION,
alias=self.vision_alias,
points_cost=3,
)
def create_ai_model(self, *, name, model, capabilities):
ai_model = AiModel(
@@ -1314,6 +1364,283 @@ class GenerateApiTests(TestCase):
1,
)
def test_analyze_images_supports_single_image_with_explicit_alias(self):
encoded = base64.b64encode(b"single-image").decode("ascii")
response = self.post_with_provider(
"/api/v1/analyze/images",
{
"prompt": "描述这张商品图",
"model": self.vision_alias,
"images": [{"image_base64": encoded}],
},
)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.data["alias"], self.vision_alias)
self.assertEqual(response.data["points_cost"], 3)
self.assertEqual(len(self.provider.vision_calls), 1)
self.assertEqual(
[image.data for image in self.provider.vision_calls[0]["images"]],
[b"single-image"],
)
def test_analyze_images_supports_ordered_mixed_sources_and_charges_once(self):
first = base64.b64encode(b"first-image").decode("ascii")
response_from_url = FakeImageUrlResponse(
headers={"Content-Type": "image/jpeg"},
chunks=(b"second-", b"image"),
)
with (
patch(
"apps.api.generation.socket.getaddrinfo",
return_value=dns_result("93.184.216.34"),
),
patch(
"apps.api.generation.requests.Session.get",
return_value=response_from_url,
),
):
response = self.post_with_provider(
"/api/v1/analyze/images",
{
"prompt": "比较两张商品图",
"images": [
{"image_base64": f"data:image/png;base64,{first}"},
{"image_url": "https://images.example.test/detail.jpg"},
],
"parameters": {"temperature": 0.2},
},
)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.data["text"], "第一张展示商品正面。\n第二张展示商品细节。")
self.assertEqual(response.data["alias"], self.vision_alias)
self.assertEqual(response.data["model_used"], self.vision_model.model)
self.assertEqual(response.data["points_cost"], 3)
self.assertEqual(response.data["points_balance"], 97)
self.assertEqual(len(self.provider.vision_calls), 1)
images = self.provider.vision_calls[0]["images"]
self.assertEqual([image.data for image in images], [b"first-image", b"second-image"])
self.assertEqual(
[image.mime_type for image in images],
["image/png", "image/jpeg"],
)
self.wallet.refresh_from_db()
self.assertEqual(self.wallet.points_balance, 97)
call = CallRecord.objects.get(pk=response.data["call_id"])
self.assertEqual(call.operation_type, CallRecord.OperationType.VISION)
self.assertEqual(call.status, CallRecord.Status.SUCCESS)
self.assertEqual(call.resolution, "")
self.assertEqual(call.result_summary, response.data["text"])
self.assertNotIn("first-image", call.result_summary)
self.assertNotIn("SECRET_RAW", call.result_summary)
self.assertEqual(
PointsLedger.objects.filter(
ref_call=call,
change_type=PointsLedger.ChangeType.CONSUME,
).count(),
1,
)
def test_analyze_images_requires_nonempty_exclusive_image_sources(self):
empty = self.post_with_provider(
"/api/v1/analyze/images",
{"prompt": "分析图片", "images": []},
)
both = self.post_with_provider(
"/api/v1/analyze/images",
{
"prompt": "分析图片",
"images": [
{
"image_url": "https://images.example.test/input.jpg",
"image_base64": "aW1hZ2U=",
}
],
},
)
self.assertEqual(empty.status_code, 400)
self.assertEqual(empty.data["error"]["code"], "bad_request")
self.assertEqual(both.status_code, 400)
self.assertEqual(both.data["error"]["code"], "bad_request")
self.assertEqual(self.provider.vision_calls, [])
self.assert_generation_not_charged()
@override_settings(VISION_MAX_IMAGES=1)
def test_analyze_images_rejects_too_many_images_before_charge(self):
encoded = base64.b64encode(b"image").decode("ascii")
response = self.post_with_provider(
"/api/v1/analyze/images",
{
"prompt": "分析图片",
"images": [
{"image_base64": encoded},
{"image_base64": encoded},
],
},
)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.data["error"]["code"], "bad_request")
self.assert_generation_not_charged()
@override_settings(VISION_MAX_IMAGE_BYTES=3, VISION_MAX_TOTAL_BYTES=10)
def test_analyze_images_rejects_oversized_single_image_before_charge(self):
encoded = base64.b64encode(b"four").decode("ascii")
response = self.post_with_provider(
"/api/v1/analyze/images",
{"prompt": "分析图片", "images": [{"image_base64": encoded}]},
)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.data["error"]["code"], "bad_request")
self.assert_generation_not_charged()
@override_settings(VISION_MAX_IMAGE_BYTES=10, VISION_MAX_TOTAL_BYTES=5)
def test_analyze_images_rejects_oversized_total_before_charge(self):
encoded = base64.b64encode(b"abc").decode("ascii")
response = self.post_with_provider(
"/api/v1/analyze/images",
{
"prompt": "分析图片",
"images": [
{"image_base64": encoded},
{"image_base64": encoded},
],
},
)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.data["error"]["code"], "bad_request")
self.assert_generation_not_charged()
def test_analyze_images_rejects_private_image_url_before_charge(self):
with patch(
"apps.api.generation.socket.getaddrinfo",
return_value=dns_result("127.0.0.1"),
):
response = self.post_with_provider(
"/api/v1/analyze/images",
{
"prompt": "分析图片",
"images": [{"image_url": "http://internal.example.test/input.jpg"}],
},
)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.data["error"]["code"], "bad_request")
self.assertEqual(self.provider.vision_calls, [])
self.assert_generation_not_charged()
def test_analyze_images_requires_api_key(self):
encoded = base64.b64encode(b"image").decode("ascii")
response = self.client.post(
"/api/v1/analyze/images",
{"prompt": "分析图片", "images": [{"image_base64": encoded}]},
format="json",
)
self.assertEqual(response.status_code, 401)
self.assertEqual(response.data["error"]["code"], "unauthorized")
self.assert_generation_not_charged()
@override_settings(
MODERATION_ENABLED=True,
MODERATION_PROVIDER="keyword",
MODERATION_CACHE_VERSION_KEY="test:api:vision:moderation:version",
)
def test_analyze_images_blocks_prompt_before_loading_images_or_charge(self):
SensitiveWord.objects.create(word="敏感词", category="policy")
with (
patch("apps.api.generation.decode_image_input") as decode_image,
patch("apps.api.generation.download_image_input") as download_image,
):
response = self.post_with_provider(
"/api/v1/analyze/images",
{
"prompt": "分析敏-感​词图片",
"images": [
{"image_url": "https://images.example.test/input.jpg"}
],
},
)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.data["error"]["code"], "content_blocked")
decode_image.assert_not_called()
download_image.assert_not_called()
self.assert_generation_not_charged()
def test_analyze_images_rejects_model_or_provider_without_text_vision(self):
bad_alias = f"vision-without-text-{uuid.uuid4().hex[:8]}"
ModelAlias.objects.create(
operation_type=ModelAlias.OperationType.VISION,
alias=bad_alias,
ai_model=self.image_model,
)
encoded = base64.b64encode(b"image").decode("ascii")
payload = {
"prompt": "分析图片",
"model": bad_alias,
"images": [{"image_base64": encoded}],
}
model_rejected = self.post_with_provider("/api/v1/analyze/images", payload)
provider_rejected = self.post_with_provider(
"/api/v1/analyze/images",
{**payload, "model": self.vision_alias},
provider=FakeGenerationProvider(capabilities={"vision"}),
)
self.assertEqual(model_rejected.status_code, 400)
self.assertEqual(model_rejected.data["error"]["code"], "model_not_allowed")
self.assertEqual(provider_rejected.status_code, 400)
self.assertEqual(provider_rejected.data["error"]["code"], "model_not_allowed")
self.assert_generation_not_charged()
def test_analyze_images_upstream_failure_refunds_once(self):
encoded = base64.b64encode(b"image").decode("ascii")
self.provider.vision_error = requests.Timeout("vision timeout")
response = self.post_with_provider(
"/api/v1/analyze/images",
{
"prompt": "分析图片",
"model": self.vision_alias,
"images": [{"image_base64": encoded}],
},
)
self.assertEqual(response.status_code, 502)
self.assertEqual(response.data["error"]["code"], "upstream_timeout")
self.wallet.refresh_from_db()
self.assertEqual(self.wallet.points_balance, 100)
call = CallRecord.objects.get(operation_type=CallRecord.OperationType.VISION)
self.assertEqual(call.status, CallRecord.Status.FAILED)
self.assertEqual(
PointsLedger.objects.filter(
ref_call=call,
change_type=PointsLedger.ChangeType.CONSUME,
).count(),
1,
)
self.assertEqual(
PointsLedger.objects.filter(
ref_call=call,
change_type=PointsLedger.ChangeType.REFUND,
).count(),
1,
)
def test_generate_image_stores_file_returns_url_and_does_not_store_raw_base64(self):
encoded = base64.b64encode(b"input-image").decode("ascii")
apps/api/urls.py
+2
View File
@@ -2,6 +2,7 @@ from django.urls import path
from .views import (
AlipayRechargeCallbackView,
AnalyzeImagesView,
BalanceView,
ClientLatestReleaseView,
GenerateImageTaskDetailView,
@@ -23,6 +24,7 @@ urlpatterns = [
name="api-client-release-latest",
),
path("v1/generate/title", GenerateTitleView.as_view(), name="api-generate-title"),
path("v1/analyze/images", AnalyzeImagesView.as_view(), name="api-analyze-images"),
path("v1/generate/image", GenerateImageView.as_view(), name="api-generate-image"),
path(
"v1/generate/image/tasks",
apps/api/views.py
+23
View File
@@ -15,6 +15,7 @@ from apps.api.authentication import ApiKeyAuthentication
from apps.api.errors import api_error
from apps.api.generation import (
ApiRequestError,
analyze_images_response,
generate_image_response,
generate_title_response,
)
@@ -25,6 +26,7 @@ from apps.api.image_tasks import (
)
from apps.api.models import ImageGenerationTask
from apps.api.serializers import (
AnalyzeImagesRequestSerializer,
GenerateImageRequestSerializer,
GenerateTitleRequestSerializer,
RechargeCreateRequestSerializer,
@@ -97,6 +99,27 @@ class GenerateTitleView(ExternalApiView):
return Response(data, status=status.HTTP_200_OK)
class AnalyzeImagesView(ExternalApiView):
throttle_classes = (GenerateRateThrottle,)
def post(self, request):
serializer = AnalyzeImagesRequestSerializer(data=request.data)
if not serializer.is_valid():
return Response(
api_error("bad_request", "参数错误"),
status=status.HTTP_400_BAD_REQUEST,
)
try:
data = analyze_images_response(
user=request.user,
api_key=request.auth,
request_data=serializer.validated_data,
)
except ApiRequestError as exc:
return Response(exc.as_response_data(), status=exc.http_status)
return Response(data, status=status.HTTP_200_OK)
class GenerateImageView(ExternalApiView):
throttle_classes = (GenerateRateThrottle,)
apps/billing/migrations/0008_alter_callrecord_operation_type_and_more.py
+23
View File
@@ -0,0 +1,23 @@
# Generated by Django 5.2.15 on 2026-07-16 03:52
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('billing', '0007_alter_pointsledger_change_type_signupbonusgrant'),
]
operations = [
migrations.AlterField(
model_name='callrecord',
name='operation_type',
field=models.CharField(choices=[('title', '生成标题'), ('image', '生成图片'), ('vision', '图片理解')], max_length=32, verbose_name='操作类型'),
),
migrations.AlterField(
model_name='pricingrule',
name='operation_type',
field=models.CharField(choices=[('title', '生成标题'), ('image', '生成图片'), ('vision', '图片理解')], max_length=32, verbose_name='操作类型'),
),
]
apps/billing/models.py
+1
View File
@@ -14,6 +14,7 @@ class CallRecord(models.Model):
class OperationType(models.TextChoices):
TITLE = "title", "生成标题"
IMAGE = "image", "生成图片"
VISION = "vision", "图片理解"
class Status(models.TextChoices):
PENDING = "pending", "待处理"
apps/portal/templates/portal/models.html
+2
View File
@@ -30,6 +30,8 @@
生成标题
{% elif item.operation_type == "image" %}
生成图片
{% elif item.operation_type == "vision" %}
图片理解
{% else %}
{{ item.operation_type }}
{% endif %}
apps/portal/tests.py
+16 -1
View File
@@ -503,6 +503,12 @@ class PortalAccountFlowTests(TestCase):
url="https://provider-secret.example/v1/images/edits",
model_sku="secret-sku-image-2",
)
vision_alias = self.create_model_alias(
alias="vision-standard",
operation_type=ModelAlias.OperationType.VISION,
capabilities=["text", "vision"],
model_sku="secret-sku-vision",
)
PricingRule.objects.create(
operation_type=title_alias.operation_type,
alias=title_alias.alias,
@@ -516,12 +522,18 @@ class PortalAccountFlowTests(TestCase):
points_cost=12,
is_active=False,
)
PricingRule.objects.create(
operation_type=vision_alias.operation_type,
alias=vision_alias.alias,
resolution="",
points_cost=3,
)
self.client.force_login(user)
response = self.client.get("/models")
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.context["models"]), 2)
self.assertEqual(len(response.context["models"]), 3)
self.assertContains(response, "可用模型")
self.assertContains(response, "title-standard")
self.assertContains(response, "生成标题")
@@ -529,6 +541,9 @@ class PortalAccountFlowTests(TestCase):
self.assertContains(response, "2 点")
self.assertContains(response, "image-edit")
self.assertContains(response, "生成图片")
self.assertContains(response, "vision-standard")
self.assertContains(response, "图片理解")
self.assertContains(response, "3 点")
self.assertContains(response, "需要")
self.assertContains(response, "暂未定价")
self.assertNotContains(response, "secret-sku")