feat: add multi-image vision analysis
This commit is contained in:
@@ -51,6 +51,9 @@ IMAGE_URL_MAX_BYTES=10485760
|
||||
IMAGE_URL_MAX_REDIRECTS=3
|
||||
IMAGE_URL_CONNECT_TIMEOUT_SECONDS=10
|
||||
IMAGE_URL_READ_TIMEOUT_SECONDS=60
|
||||
VISION_MAX_IMAGES=8
|
||||
VISION_MAX_IMAGE_BYTES=10485760
|
||||
VISION_MAX_TOTAL_BYTES=33554432
|
||||
RECHARGE_MAX_AMOUNT_CNY=100000.00
|
||||
|
||||
# Content moderation
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
# cmhub
|
||||
|
||||
`cmhub` 是一个**自助用户端 + 计费型 AI 能力网关 + 运营后台**三合一服务:终端用户自助注册、扫码充值、管理 API Key;用 API Key 调用把桌面工具 `cmbot` 的「生成标题」「生成图片」能力封装成的 HTTP API,按**点数计费**;运营用 django-admin 管理用户、点数和记录。
|
||||
`cmhub` 是一个**自助用户端 + 计费型 AI 能力网关 + 运营后台**三合一服务:终端用户自助注册、扫码充值、管理 API Key;用 API Key 调用「生成标题」「生成图片」「多图理解」等 HTTP API,按**点数计费**;运营用 django-admin 管理用户、点数和记录。
|
||||
|
||||
## 它做什么
|
||||
|
||||
- 用户端(自助):公开首页、注册/登录、扫码充值、查看充值记录/剩余点数/消费记录、查看可用模型、生成与删除 API Key、下载桌面端(新用户注册成功赠送 100 点试用点数)。
|
||||
- 对外提供 HTTP API:生成标题、生成图片(旧同步接口 + 新异步提交/轮询接口)、查询点数余额、查询可用能力别名。
|
||||
- 对外提供 HTTP API:生成标题、生成图片(旧同步接口 + 新异步提交/轮询接口)、多张图片理解并返回文字、查询点数余额、查询可用能力别名。
|
||||
- 按「操作类型 + 能力别名(+ 可选分辨率)」计费,调用消耗不同点数;余额不足返回「点数不足,请先充值」。
|
||||
- **预付费点数模型**:用户在外部支付系统充值,付款成功由支付系统回调本服务,按汇率把金额转成点数存在本地;之后调用直接扣本地点数,扣费与上游解耦、低延迟。
|
||||
- django-admin 运营后台:管理注册用户、点数余额、计费规则、充值订单、点数流水、调用记录。
|
||||
@@ -27,6 +27,8 @@ Python 3.12 / Django 5.2 LTS + DRF / django-admin / 用户端 Django 模板 SSR
|
||||
|
||||
Phase 2 计费核心已完成,Phase 3 对外 API 与充值已完成到 T-306,Phase 4 用户端已完成 T-501~T-505,Phase 5 已完成 T-401 运营后台完善、T-402 MVP 完整验收与 T-403 部署 / 运行文档,Phase 6 已完成 T-601 可用别名发现、T-602/T-603 django-admin 中文化、T-604 中文敏感词本地过滤、T-605 免邮箱验证策略落地、T-606 公开首页 + 客户端下载入口、T-607 桌面端最新版本检查接口、T-608 新用户注册赠送 100 点试用点数、T-609 桌面端版本强制更新标记、T-610 首页导入模板下载入口、T-611 用户端品牌名统一为“虾皮圈”、T-612 生图同步接口止血、T-613 抽生成核心 service、T-614 生图异步任务化接口、T-615 旧同步生图接口遥测 / 弃用口径、T-616 生图失败自动重试 2 次、T-617 桌面端版本文件大小字段与 T-618 客户端发布版本后台必填校验元数据。用户可通过公开首页进入注册、登录、下载客户端和下载导入模板;新用户注册后经计费层自动获得 100 点并写注册赠点流水;登录后可扫码充值并轮询到账,生成 / 删除(吊销)API Key,查看余额、充值总额、分页充值记录、分页点数记录与可用模型;桌面端可匿名请求最新客户端版本 JSON,并读取 `release.force_update` 判断是否必须升级、读取 `release.size_bytes` 校验安装包大小;新版桌面端可用异步生图提交 / 轮询接口,旧同步生图接口继续兼容并写结构化用量日志;运营可在 django-admin 检索用户、钱包、API Key、计费规则、汇率、充值订单、点数流水、注册赠点记录、调用记录、图片生成任务、客户端发布版本和导入模板,客户端发布版本新增 / 编辑时必须填写 SHA256 与文件大小,并通过计费层带原因手工调点。生产侧仍需补真实支付回调到账闭环,并按日志观察旧同步生图接口迁移进度。详见 [`docs/current-state.md`](docs/current-state.md)。
|
||||
|
||||
T-619 已完成:新增独立 `vision` 能力和同步 `POST /api/v1/analyze/images` 多图理解接口,按一次请求固定扣点;既有标题、生图和异步生图接口保持兼容。生产启用前需在 admin 配置同时具备 `text`、`vision` 能力的模型、`vision-standard` 默认别名及默认计费规则。
|
||||
|
||||
> ⚠️ 涉及资金/点数。改动充值、扣费、退款、对账相关代码前,先读 [`docs/05-coding-rules.md`](docs/05-coding-rules.md) 第 8 节与 [`docs/04-architecture.md`](docs/04-architecture.md) 第四节计费时序。
|
||||
|
||||
## 启动
|
||||
|
||||
+8
-6
@@ -18,15 +18,16 @@ class ModelCapabilityError(AliasResolutionError):
|
||||
|
||||
|
||||
REQUIRED_CAPABILITIES = {
|
||||
ModelAlias.OperationType.TITLE: "text",
|
||||
ModelAlias.OperationType.IMAGE: "image",
|
||||
ModelAlias.OperationType.TITLE: frozenset({"text"}),
|
||||
ModelAlias.OperationType.IMAGE: frozenset({"image"}),
|
||||
ModelAlias.OperationType.VISION: frozenset({"text", "vision"}),
|
||||
}
|
||||
|
||||
|
||||
def resolve_model_alias(operation_type: str, alias: str | None = None) -> ModelAlias:
|
||||
"""Resolve an external capability alias to an active ModelAlias row."""
|
||||
required_capability = REQUIRED_CAPABILITIES.get(operation_type)
|
||||
if required_capability is None:
|
||||
required_capabilities = REQUIRED_CAPABILITIES.get(operation_type)
|
||||
if required_capabilities is None:
|
||||
raise AliasResolutionError(f"unsupported operation_type: {operation_type}")
|
||||
|
||||
queryset = ModelAlias.objects.select_related("ai_model").filter(
|
||||
@@ -47,10 +48,11 @@ def resolve_model_alias(operation_type: str, alias: str | None = None) -> ModelA
|
||||
|
||||
ai_model: AiModel = model_alias.ai_model
|
||||
capabilities = ai_model.capabilities_set()
|
||||
if required_capability not in capabilities:
|
||||
missing_capabilities = required_capabilities - capabilities
|
||||
if missing_capabilities:
|
||||
raise ModelCapabilityError(
|
||||
f"alias {model_alias.alias} maps to model {ai_model.name} without "
|
||||
f"{required_capability} capability"
|
||||
f"{', '.join(sorted(missing_capabilities))} capability"
|
||||
)
|
||||
return model_alias
|
||||
|
||||
|
||||
+5
-3
@@ -14,13 +14,15 @@ PUBLIC_DEFAULT_RESOLUTION = "default"
|
||||
|
||||
|
||||
def _has_required_capability(model_alias: ModelAlias) -> bool:
|
||||
required_capability = REQUIRED_CAPABILITIES.get(model_alias.operation_type)
|
||||
if required_capability is None:
|
||||
required_capabilities = REQUIRED_CAPABILITIES.get(model_alias.operation_type)
|
||||
if required_capabilities is None:
|
||||
return False
|
||||
return required_capability in model_alias.ai_model.capabilities_set()
|
||||
return required_capabilities.issubset(model_alias.ai_model.capabilities_set())
|
||||
|
||||
|
||||
def _requires_image_input(ai_model: AiModel, operation_type: str) -> bool:
|
||||
if operation_type == ModelAlias.OperationType.VISION:
|
||||
return True
|
||||
if operation_type != ModelAlias.OperationType.IMAGE:
|
||||
return False
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
# Generated by Django 5.2.15 on 2026-07-16 03:52
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('ai', '0004_alter_aiconfigauditlog_action_and_more'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='modelalias',
|
||||
name='operation_type',
|
||||
field=models.CharField(choices=[('title', '生成标题'), ('image', '生成图片'), ('vision', '图片理解')], max_length=32, verbose_name='操作类型'),
|
||||
),
|
||||
]
|
||||
@@ -92,6 +92,7 @@ class ModelAlias(models.Model):
|
||||
class OperationType(models.TextChoices):
|
||||
TITLE = "title", "生成标题"
|
||||
IMAGE = "image", "生成图片"
|
||||
VISION = "vision", "图片理解"
|
||||
|
||||
alias = models.SlugField("能力别名", max_length=64)
|
||||
operation_type = models.CharField("操作类型", max_length=32, choices=OperationType.choices)
|
||||
|
||||
@@ -4,6 +4,7 @@ from .base import (
|
||||
AiProviderError,
|
||||
AiResponseParseError,
|
||||
ImageGenerationResult,
|
||||
MultimodalImage,
|
||||
Provider,
|
||||
ResolvedModel,
|
||||
TextGenerationResult,
|
||||
@@ -16,6 +17,7 @@ __all__ = [
|
||||
"AiProviderError",
|
||||
"AiResponseParseError",
|
||||
"ImageGenerationResult",
|
||||
"MultimodalImage",
|
||||
"Provider",
|
||||
"ResolvedModel",
|
||||
"TextGenerationResult",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Mapping, Protocol
|
||||
from typing import Any, Mapping, Protocol, Sequence
|
||||
|
||||
|
||||
class AiProviderError(RuntimeError):
|
||||
@@ -78,6 +78,12 @@ class ImageGenerationResult:
|
||||
raw: Mapping[str, Any]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MultimodalImage:
|
||||
data: bytes
|
||||
mime_type: str = "image/png"
|
||||
|
||||
|
||||
class Provider(Protocol):
|
||||
def capabilities(self) -> set[str]:
|
||||
...
|
||||
@@ -108,6 +114,16 @@ class Provider(Protocol):
|
||||
) -> ImageGenerationResult:
|
||||
...
|
||||
|
||||
def analyze_images(
|
||||
self,
|
||||
prompt: str,
|
||||
model: ResolvedModel,
|
||||
*,
|
||||
images: Sequence[MultimodalImage],
|
||||
parameters: Mapping[str, Any] | None = None,
|
||||
) -> TextGenerationResult:
|
||||
...
|
||||
|
||||
|
||||
def validate_model_config(model: ResolvedModel) -> None:
|
||||
errors = []
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Mapping
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
import requests
|
||||
|
||||
@@ -8,6 +8,7 @@ from .base import (
|
||||
AiCapabilityError,
|
||||
AiResponseParseError,
|
||||
ImageGenerationResult,
|
||||
MultimodalImage,
|
||||
ResolvedModel,
|
||||
TextGenerationResult,
|
||||
validate_model_config,
|
||||
@@ -18,6 +19,7 @@ from .utils import (
|
||||
API_IMAGES,
|
||||
API_IMAGES_EDITS,
|
||||
extract_image_from_response,
|
||||
extract_raw_text,
|
||||
extract_text_from_response,
|
||||
extract_titles_from_response,
|
||||
image_request_timeout,
|
||||
@@ -166,6 +168,37 @@ class ChatCompletionsProvider(BaseHttpProvider):
|
||||
raise AiResponseParseError("AI response did not contain an image")
|
||||
return ImageGenerationResult(image=image_bytes, model_used=model.model, raw=raw)
|
||||
|
||||
def analyze_images(
|
||||
self,
|
||||
prompt: str,
|
||||
model: ResolvedModel,
|
||||
*,
|
||||
images: Sequence[MultimodalImage],
|
||||
parameters: Mapping[str, Any] | None = None,
|
||||
) -> TextGenerationResult:
|
||||
validate_model_config(model)
|
||||
if not images:
|
||||
raise AiCapabilityError("vision analysis requires at least one image")
|
||||
url = normalize_api_url(model.url, API_CHAT)
|
||||
payload = build_chat_vision_payload(
|
||||
model,
|
||||
prompt,
|
||||
images=images,
|
||||
parameters=parameters,
|
||||
)
|
||||
response = self.session.post(
|
||||
url,
|
||||
headers=self._headers(model, json=True),
|
||||
json=payload,
|
||||
timeout=self._timeout(model, "1K"),
|
||||
)
|
||||
response.raise_for_status()
|
||||
raw = response.json()
|
||||
text = extract_raw_text(raw).strip()
|
||||
if not text:
|
||||
raise AiResponseParseError("AI response did not contain text")
|
||||
return TextGenerationResult(text=text, titles=(), model_used=model.model, raw=raw)
|
||||
|
||||
|
||||
class GeminiProvider(ChatCompletionsProvider):
|
||||
def generate_text(
|
||||
@@ -241,6 +274,37 @@ class GeminiProvider(ChatCompletionsProvider):
|
||||
raise AiResponseParseError("AI response did not contain an image")
|
||||
return ImageGenerationResult(image=image_bytes, model_used=model.model, raw=raw)
|
||||
|
||||
def analyze_images(
|
||||
self,
|
||||
prompt: str,
|
||||
model: ResolvedModel,
|
||||
*,
|
||||
images: Sequence[MultimodalImage],
|
||||
parameters: Mapping[str, Any] | None = None,
|
||||
) -> TextGenerationResult:
|
||||
validate_model_config(model)
|
||||
if not images:
|
||||
raise AiCapabilityError("vision analysis requires at least one image")
|
||||
url = normalize_api_url(model.url, API_GEMINI).replace("{model}", model.model)
|
||||
payload = build_gemini_vision_payload(
|
||||
model,
|
||||
prompt,
|
||||
images=images,
|
||||
parameters=parameters,
|
||||
)
|
||||
response = self.session.post(
|
||||
url,
|
||||
headers=self._headers(model, json=True),
|
||||
json=payload,
|
||||
timeout=self._timeout(model, "1K"),
|
||||
)
|
||||
response.raise_for_status()
|
||||
raw = response.json()
|
||||
text = extract_raw_text(raw).strip()
|
||||
if not text:
|
||||
raise AiResponseParseError("AI response did not contain text")
|
||||
return TextGenerationResult(text=text, titles=(), model_used=model.model, raw=raw)
|
||||
|
||||
|
||||
class ImagesGenerationProvider(BaseHttpProvider):
|
||||
def capabilities(self) -> set[str]:
|
||||
@@ -249,6 +313,9 @@ class ImagesGenerationProvider(BaseHttpProvider):
|
||||
def generate_text(self, *args: Any, **kwargs: Any) -> TextGenerationResult:
|
||||
raise AiCapabilityError("images generation provider cannot generate text")
|
||||
|
||||
def analyze_images(self, *args: Any, **kwargs: Any) -> TextGenerationResult:
|
||||
raise AiCapabilityError("images generation provider cannot analyze images")
|
||||
|
||||
def generate_image(
|
||||
self,
|
||||
prompt: str,
|
||||
@@ -298,6 +365,9 @@ class ImagesEditsProvider(BaseHttpProvider):
|
||||
def generate_text(self, *args: Any, **kwargs: Any) -> TextGenerationResult:
|
||||
raise AiCapabilityError("images edits provider cannot generate text")
|
||||
|
||||
def analyze_images(self, *args: Any, **kwargs: Any) -> TextGenerationResult:
|
||||
raise AiCapabilityError("images edits provider cannot analyze images")
|
||||
|
||||
def generate_image(
|
||||
self,
|
||||
prompt: str,
|
||||
@@ -384,6 +454,32 @@ def build_chat_image_payload(
|
||||
)
|
||||
|
||||
|
||||
def build_chat_vision_payload(
|
||||
model: ResolvedModel,
|
||||
prompt: str,
|
||||
*,
|
||||
images: Sequence[MultimodalImage],
|
||||
parameters: Mapping[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
content: list[dict[str, Any]] = [{"type": "text", "text": prompt}]
|
||||
content.extend(
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": image_bytes_to_data_url(image.data, image.mime_type),
|
||||
},
|
||||
}
|
||||
for image in images
|
||||
)
|
||||
payload: dict[str, Any] = {
|
||||
"model": model.model,
|
||||
"messages": [{"role": "user", "content": content}],
|
||||
"stream": False,
|
||||
}
|
||||
apply_extra_body(payload, model, parameters)
|
||||
return payload
|
||||
|
||||
|
||||
def build_gemini_payload(
|
||||
model: ResolvedModel,
|
||||
prompt: str,
|
||||
@@ -406,6 +502,26 @@ def build_gemini_payload(
|
||||
return payload
|
||||
|
||||
|
||||
def build_gemini_vision_payload(
|
||||
model: ResolvedModel,
|
||||
prompt: str,
|
||||
*,
|
||||
images: Sequence[MultimodalImage],
|
||||
parameters: Mapping[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
parts: list[dict[str, Any]] = [{"text": prompt}]
|
||||
for image in images:
|
||||
data_url = image_bytes_to_data_url(image.data, image.mime_type)
|
||||
mime_type, data = split_data_url(data_url)
|
||||
parts.append({"inlineData": {"mimeType": mime_type, "data": data}})
|
||||
payload: dict[str, Any] = {
|
||||
"contents": [{"parts": parts}],
|
||||
"generationConfig": {"responseModalities": ["TEXT"]},
|
||||
}
|
||||
apply_extra_body(payload, model, parameters)
|
||||
return payload
|
||||
|
||||
|
||||
def apply_extra_body(
|
||||
payload: dict[str, Any],
|
||||
model: ResolvedModel,
|
||||
|
||||
+121
-2
@@ -11,8 +11,18 @@ from apps.ai.admin import AiConfigAuditLogAdmin, AiModelAdmin, ModelAliasAdmin
|
||||
from apps.ai.aliases import AliasNotFoundError, ModelCapabilityError, resolve_alias
|
||||
from apps.ai.importers import import_ai_models_config
|
||||
from apps.ai.models import AiConfigAuditLog, AiModel, ModelAlias
|
||||
from apps.ai.providers import AiCapabilityError, ResolvedModel, get_provider, resolve_api_type
|
||||
from apps.ai.providers.openai_compatible import ChatCompletionsProvider, ImagesEditsProvider
|
||||
from apps.ai.providers import (
|
||||
AiCapabilityError,
|
||||
MultimodalImage,
|
||||
ResolvedModel,
|
||||
get_provider,
|
||||
resolve_api_type,
|
||||
)
|
||||
from apps.ai.providers.openai_compatible import (
|
||||
ChatCompletionsProvider,
|
||||
GeminiProvider,
|
||||
ImagesEditsProvider,
|
||||
)
|
||||
from apps.ai.providers.utils import image_request_timeout, resolution_to_size
|
||||
|
||||
|
||||
@@ -255,6 +265,85 @@ class ChatCompletionsProviderTests(SimpleTestCase):
|
||||
|
||||
self.assertEqual(session.posts[0]["timeout"], (30, 600))
|
||||
|
||||
def test_analyze_images_builds_ordered_payload_and_preserves_full_text(self):
|
||||
session = FakeSession(
|
||||
FakeResponse(
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": "第一张是正面图。\n第二张是细节图。"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
)
|
||||
provider = ChatCompletionsProvider(session=session)
|
||||
model = ResolvedModel(
|
||||
name="Vision text",
|
||||
url="https://api.vectorengine.ai/v1",
|
||||
model="vision-model",
|
||||
api_key="test-key",
|
||||
api_type="chat",
|
||||
)
|
||||
|
||||
result = provider.analyze_images(
|
||||
"比较两张商品图",
|
||||
model,
|
||||
images=(
|
||||
MultimodalImage(b"first-image", "image/jpeg"),
|
||||
MultimodalImage(b"second-image", "image/png"),
|
||||
),
|
||||
parameters={"temperature": 0.2, "messages": []},
|
||||
)
|
||||
|
||||
self.assertEqual(result.text, "第一张是正面图。\n第二张是细节图。")
|
||||
self.assertEqual(result.titles, ())
|
||||
content = session.posts[0]["json"]["messages"][0]["content"]
|
||||
self.assertEqual(content[0], {"type": "text", "text": "比较两张商品图"})
|
||||
self.assertTrue(content[1]["image_url"]["url"].startswith("data:image/jpeg;base64,"))
|
||||
self.assertTrue(content[2]["image_url"]["url"].startswith("data:image/png;base64,"))
|
||||
self.assertEqual(session.posts[0]["json"]["temperature"], 0.2)
|
||||
|
||||
def test_gemini_analyze_images_builds_ordered_inline_data(self):
|
||||
session = FakeSession(
|
||||
FakeResponse(
|
||||
{
|
||||
"candidates": [
|
||||
{"content": {"parts": [{"text": "多图分析结果"}]}}
|
||||
]
|
||||
}
|
||||
)
|
||||
)
|
||||
provider = GeminiProvider(session=session)
|
||||
model = ResolvedModel(
|
||||
name="Gemini vision",
|
||||
url="https://gemini.example.com",
|
||||
model="gemini-vision",
|
||||
api_key="test-key",
|
||||
api_type="gemini",
|
||||
)
|
||||
|
||||
result = provider.analyze_images(
|
||||
"理解这些图片",
|
||||
model,
|
||||
images=(
|
||||
MultimodalImage(b"one", "image/webp"),
|
||||
MultimodalImage(b"two", "image/jpeg"),
|
||||
),
|
||||
)
|
||||
|
||||
self.assertEqual(result.text, "多图分析结果")
|
||||
parts = session.posts[0]["json"]["contents"][0]["parts"]
|
||||
self.assertEqual(parts[0], {"text": "理解这些图片"})
|
||||
self.assertEqual(parts[1]["inlineData"]["mimeType"], "image/webp")
|
||||
self.assertEqual(parts[2]["inlineData"]["mimeType"], "image/jpeg")
|
||||
self.assertEqual(
|
||||
session.posts[0]["json"]["generationConfig"]["responseModalities"],
|
||||
["TEXT"],
|
||||
)
|
||||
|
||||
|
||||
class ImagesEditsProviderTests(SimpleTestCase):
|
||||
def test_generate_image_builds_multipart_request_and_parses_base64(self):
|
||||
@@ -344,6 +433,13 @@ class ImagesEditsProviderTests(SimpleTestCase):
|
||||
with self.assertRaises(AiCapabilityError):
|
||||
provider.generate_text("Generate title", model)
|
||||
|
||||
with self.assertRaises(AiCapabilityError):
|
||||
provider.analyze_images(
|
||||
"Analyze image",
|
||||
model,
|
||||
images=(MultimodalImage(b"source-image"),),
|
||||
)
|
||||
|
||||
|
||||
@override_settings(AI_KEY_ENCRYPTION_KEY=TEST_ENCRYPTION_KEY)
|
||||
class AiModelEncryptionTests(TestCase):
|
||||
@@ -440,6 +536,29 @@ class AliasResolutionTests(TestCase):
|
||||
self.assertEqual(resolved.model, "gpt-image-2")
|
||||
self.assertIn("image", resolved.capabilities)
|
||||
|
||||
def test_resolve_vision_alias_requires_text_and_vision_capabilities(self):
|
||||
ModelAlias.objects.create(
|
||||
operation_type=ModelAlias.OperationType.VISION,
|
||||
alias="vision-standard",
|
||||
ai_model=self.text_model,
|
||||
is_default=True,
|
||||
)
|
||||
|
||||
resolved = resolve_alias(ModelAlias.OperationType.VISION)
|
||||
|
||||
self.assertEqual(resolved.model, "gpt-5.5")
|
||||
self.assertTrue({"text", "vision"}.issubset(resolved.capabilities))
|
||||
|
||||
def test_resolve_vision_alias_rejects_vision_model_without_text(self):
|
||||
ModelAlias.objects.create(
|
||||
operation_type=ModelAlias.OperationType.VISION,
|
||||
alias="bad-vision",
|
||||
ai_model=self.image_model,
|
||||
)
|
||||
|
||||
with self.assertRaises(ModelCapabilityError):
|
||||
resolve_alias(ModelAlias.OperationType.VISION, "bad-vision")
|
||||
|
||||
def test_resolve_alias_rejects_capability_mismatch(self):
|
||||
ModelAlias.objects.create(
|
||||
operation_type=ModelAlias.OperationType.TITLE,
|
||||
|
||||
+153
-18
@@ -6,7 +6,7 @@ import ipaddress
|
||||
import socket
|
||||
from dataclasses import dataclass, field
|
||||
from time import perf_counter
|
||||
from typing import Any, Callable, Mapping
|
||||
from typing import Any, Callable, Mapping, Sequence
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
import requests
|
||||
@@ -19,7 +19,12 @@ from apps.ai.aliases import (
|
||||
REQUIRED_CAPABILITIES,
|
||||
resolve_model_alias,
|
||||
)
|
||||
from apps.ai.providers import AiCapabilityError, AiProviderError, get_provider
|
||||
from apps.ai.providers import (
|
||||
AiCapabilityError,
|
||||
AiProviderError,
|
||||
MultimodalImage,
|
||||
get_provider,
|
||||
)
|
||||
from apps.billing.models import CallRecord, normalize_resolution
|
||||
from apps.billing.pricing import NoPricingRuleError, calculate_points_cost
|
||||
from apps.billing.services import (
|
||||
@@ -68,6 +73,7 @@ class GenerationInput:
|
||||
image_url: str = ""
|
||||
image_base64: str = ""
|
||||
aspect_ratio: str = "1:1"
|
||||
images: tuple[Mapping[str, Any], ...] = field(default_factory=tuple)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -80,6 +86,7 @@ class PreparedGeneration:
|
||||
resolution: str
|
||||
parameters: dict[str, Any]
|
||||
image_input: ImageInput | None
|
||||
image_inputs: tuple[ImageInput, ...]
|
||||
model_alias: Any
|
||||
resolved_model: Any
|
||||
provider: Any
|
||||
@@ -105,6 +112,7 @@ class GenerationResult:
|
||||
call_record: CallRecord
|
||||
titles: tuple[str, ...] = field(default_factory=tuple)
|
||||
image_url: str = ""
|
||||
text: str = ""
|
||||
|
||||
def as_response_data(self) -> dict[str, Any]:
|
||||
common = {
|
||||
@@ -116,6 +124,8 @@ class GenerationResult:
|
||||
}
|
||||
if self.operation_type == CallRecord.OperationType.TITLE:
|
||||
return {"titles": list(self.titles), **common}
|
||||
if self.operation_type == CallRecord.OperationType.VISION:
|
||||
return {"text": self.text, **common}
|
||||
return {"image_url": self.image_url, **common}
|
||||
|
||||
|
||||
@@ -165,6 +175,21 @@ def generate_image_response(
|
||||
return result.as_response_data()
|
||||
|
||||
|
||||
def analyze_images_response(*, user, api_key, request_data: Mapping[str, Any]) -> dict:
|
||||
result = run_synchronous_generation(
|
||||
GenerationInput(
|
||||
user=user,
|
||||
api_key=api_key,
|
||||
operation_type=CallRecord.OperationType.VISION,
|
||||
prompt=request_data["prompt"],
|
||||
alias=request_data.get("model") or None,
|
||||
parameters=dict(request_data.get("parameters") or {}),
|
||||
images=tuple(dict(item) for item in request_data.get("images") or ()),
|
||||
)
|
||||
)
|
||||
return result.as_response_data()
|
||||
|
||||
|
||||
def run_synchronous_generation(
|
||||
generation_input: GenerationInput,
|
||||
*,
|
||||
@@ -180,7 +205,11 @@ def run_synchronous_generation(
|
||||
|
||||
def prepare_generation(generation_input: GenerationInput) -> PreparedGeneration:
|
||||
operation_type = normalize_operation_type(generation_input.operation_type)
|
||||
resolution = normalize_resolution(generation_input.resolution or "1K") or "1K"
|
||||
resolution = (
|
||||
""
|
||||
if operation_type == CallRecord.OperationType.VISION
|
||||
else normalize_resolution(generation_input.resolution or "1K") or "1K"
|
||||
)
|
||||
parameters = dict(generation_input.parameters or {})
|
||||
prompt = str(generation_input.prompt or "")
|
||||
|
||||
@@ -189,12 +218,17 @@ def prepare_generation(generation_input: GenerationInput) -> PreparedGeneration:
|
||||
api_key=generation_input.api_key,
|
||||
prompt=prompt,
|
||||
)
|
||||
image_input = load_image_input(
|
||||
{
|
||||
"image_base64": generation_input.image_base64,
|
||||
"image_url": generation_input.image_url,
|
||||
}
|
||||
)
|
||||
if operation_type == CallRecord.OperationType.VISION:
|
||||
image_input = None
|
||||
image_inputs = load_vision_image_inputs(generation_input.images)
|
||||
else:
|
||||
image_input = load_image_input(
|
||||
{
|
||||
"image_base64": generation_input.image_base64,
|
||||
"image_url": generation_input.image_url,
|
||||
}
|
||||
)
|
||||
image_inputs = ()
|
||||
|
||||
model_alias = resolve_model_alias_or_raise(operation_type, generation_input.alias)
|
||||
resolved_model = resolved_model_or_raise(model_alias)
|
||||
@@ -215,6 +249,7 @@ def prepare_generation(generation_input: GenerationInput) -> PreparedGeneration:
|
||||
resolution=resolution,
|
||||
parameters=parameters,
|
||||
image_input=image_input,
|
||||
image_inputs=image_inputs,
|
||||
model_alias=model_alias,
|
||||
resolved_model=resolved_model,
|
||||
provider=provider,
|
||||
@@ -253,6 +288,8 @@ def execute_precharged_generation(
|
||||
started = perf_counter()
|
||||
if prepared.operation_type == CallRecord.OperationType.TITLE:
|
||||
result = execute_title_generation(precharged, started)
|
||||
elif prepared.operation_type == CallRecord.OperationType.VISION:
|
||||
result = execute_vision_generation(precharged, started)
|
||||
else:
|
||||
result = execute_image_generation(
|
||||
precharged,
|
||||
@@ -358,11 +395,44 @@ def execute_image_generation(
|
||||
)
|
||||
|
||||
|
||||
def execute_vision_generation(
|
||||
precharged: PrechargedGeneration,
|
||||
started: float,
|
||||
) -> GenerationResult:
|
||||
prepared = precharged.prepared
|
||||
generation = prepared.provider.analyze_images(
|
||||
prepared.prompt,
|
||||
prepared.resolved_model,
|
||||
images=tuple(
|
||||
MultimodalImage(data=image.data, mime_type=image.mime_type)
|
||||
for image in prepared.image_inputs
|
||||
),
|
||||
parameters=prepared.parameters,
|
||||
)
|
||||
latency_ms = elapsed_ms(started)
|
||||
text = str(generation.text or "").strip()
|
||||
call_record = mark_call_success(
|
||||
precharged.call_record,
|
||||
result_summary=summarize_text(text),
|
||||
upstream_latency_ms=latency_ms,
|
||||
)
|
||||
return GenerationResult(
|
||||
operation_type=prepared.operation_type,
|
||||
alias=prepared.alias,
|
||||
model_used=generation.model_used,
|
||||
points_cost=precharged.points_cost,
|
||||
points_balance=precharged.points_balance_after_charge,
|
||||
call_record=call_record,
|
||||
text=text,
|
||||
)
|
||||
|
||||
|
||||
def normalize_operation_type(operation_type: str) -> str:
|
||||
normalized = str(operation_type or "").strip()
|
||||
if normalized not in {
|
||||
CallRecord.OperationType.TITLE,
|
||||
CallRecord.OperationType.IMAGE,
|
||||
CallRecord.OperationType.VISION,
|
||||
}:
|
||||
raise ValueError(f"Unsupported generation operation type: {operation_type}")
|
||||
return normalized
|
||||
@@ -392,8 +462,8 @@ def resolve_model_alias_or_raise(operation_type: str, alias: str | None):
|
||||
|
||||
|
||||
def ensure_provider_supports(provider, operation_type: str) -> None:
|
||||
required_capability = REQUIRED_CAPABILITIES[operation_type]
|
||||
if required_capability not in provider.capabilities():
|
||||
required_capabilities = REQUIRED_CAPABILITIES[operation_type]
|
||||
if not required_capabilities.issubset(provider.capabilities()):
|
||||
raise ApiRequestError(
|
||||
"model_not_allowed",
|
||||
"该模型不支持此操作",
|
||||
@@ -471,7 +541,57 @@ def load_image_input(data: Mapping[str, Any]) -> ImageInput | None:
|
||||
return None
|
||||
|
||||
|
||||
def decode_image_input(value: str) -> ImageInput:
|
||||
def load_vision_image_inputs(
|
||||
items: Sequence[Mapping[str, Any]],
|
||||
) -> tuple[ImageInput, ...]:
|
||||
max_images = max(1, int(getattr(settings, "VISION_MAX_IMAGES", 8)))
|
||||
max_image_bytes = max(
|
||||
1,
|
||||
int(getattr(settings, "VISION_MAX_IMAGE_BYTES", 10 * 1024 * 1024)),
|
||||
)
|
||||
max_total_bytes = max(
|
||||
1,
|
||||
int(getattr(settings, "VISION_MAX_TOTAL_BYTES", 32 * 1024 * 1024)),
|
||||
)
|
||||
if not items:
|
||||
raise ApiRequestError("bad_request", "images 至少需要一张图片", status.HTTP_400_BAD_REQUEST)
|
||||
if len(items) > max_images:
|
||||
raise ApiRequestError(
|
||||
"bad_request",
|
||||
f"单次最多上传 {max_images} 张图片",
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
image_inputs = []
|
||||
total_bytes = 0
|
||||
for item in items:
|
||||
raw_base64 = str(item.get("image_base64") or "").strip()
|
||||
image_url = str(item.get("image_url") or "").strip()
|
||||
if bool(raw_base64) == bool(image_url):
|
||||
raise ApiRequestError(
|
||||
"bad_request",
|
||||
"每张图片必须且只能提供 image_url 或 image_base64",
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
image_input = (
|
||||
decode_image_input(raw_base64, max_bytes=max_image_bytes)
|
||||
if raw_base64
|
||||
else download_image_input(image_url, max_bytes=max_image_bytes)
|
||||
)
|
||||
if not image_input.mime_type.lower().startswith("image/"):
|
||||
raise ApiRequestError("bad_request", "图片格式无效", status.HTTP_400_BAD_REQUEST)
|
||||
total_bytes += len(image_input.data)
|
||||
if total_bytes > max_total_bytes:
|
||||
raise ApiRequestError(
|
||||
"bad_request",
|
||||
"图片总大小超过限制",
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
image_inputs.append(image_input)
|
||||
return tuple(image_inputs)
|
||||
|
||||
|
||||
def decode_image_input(value: str, *, max_bytes: int | None = None) -> ImageInput:
|
||||
mime_type = "image/png"
|
||||
encoded = value
|
||||
if value.startswith("data:"):
|
||||
@@ -479,16 +599,20 @@ def decode_image_input(value: str) -> ImageInput:
|
||||
raise ApiRequestError("bad_request", "image_base64 格式无效", status.HTTP_400_BAD_REQUEST)
|
||||
prefix, encoded = value.split(",", 1)
|
||||
mime_type = prefix[len("data:") :].split(";", 1)[0] or mime_type
|
||||
if max_bytes is not None and len(encoded) > ((max_bytes + 2) // 3) * 4:
|
||||
raise ApiRequestError("bad_request", "图片过大", status.HTTP_400_BAD_REQUEST)
|
||||
try:
|
||||
image = base64.b64decode(encoded, validate=True)
|
||||
except (binascii.Error, ValueError) as exc:
|
||||
raise ApiRequestError("bad_request", "image_base64 格式无效", status.HTTP_400_BAD_REQUEST) from exc
|
||||
if not image:
|
||||
raise ApiRequestError("bad_request", "image_base64 不能为空", status.HTTP_400_BAD_REQUEST)
|
||||
if max_bytes is not None and len(image) > max_bytes:
|
||||
raise ApiRequestError("bad_request", "图片过大", status.HTTP_400_BAD_REQUEST)
|
||||
return ImageInput(data=image, mime_type=mime_type, filename=filename_for_mime(mime_type))
|
||||
|
||||
|
||||
def download_image_input(url: str) -> ImageInput:
|
||||
def download_image_input(url: str, *, max_bytes: int | None = None) -> ImageInput:
|
||||
session = requests.Session()
|
||||
session.trust_env = False
|
||||
current_url = validated_image_url(url)
|
||||
@@ -522,7 +646,7 @@ def download_image_input(url: str) -> ImageInput:
|
||||
content_type = response.headers.get("Content-Type", "image/png").split(";", 1)[0].strip().lower()
|
||||
if not content_type.startswith("image/"):
|
||||
raise ApiRequestError("bad_request", "image_url 不是图片资源", status.HTTP_400_BAD_REQUEST)
|
||||
image = read_limited_image_response(response)
|
||||
image = read_limited_image_response(response, max_bytes=max_bytes)
|
||||
if not image:
|
||||
raise ApiRequestError("bad_request", "image_url 图片内容为空", status.HTTP_400_BAD_REQUEST)
|
||||
return ImageInput(
|
||||
@@ -593,12 +717,19 @@ def is_redirect_response(response) -> bool:
|
||||
return 300 <= int(getattr(response, "status_code", 0)) < 400
|
||||
|
||||
|
||||
def read_limited_image_response(response) -> bytes:
|
||||
max_bytes = max(1, int(getattr(settings, "IMAGE_URL_MAX_BYTES", 10 * 1024 * 1024)))
|
||||
def read_limited_image_response(response, *, max_bytes: int | None = None) -> bytes:
|
||||
byte_limit = max(
|
||||
1,
|
||||
int(
|
||||
max_bytes
|
||||
if max_bytes is not None
|
||||
else getattr(settings, "IMAGE_URL_MAX_BYTES", 10 * 1024 * 1024)
|
||||
),
|
||||
)
|
||||
content_length = response.headers.get("Content-Length")
|
||||
if content_length:
|
||||
try:
|
||||
if int(content_length) > max_bytes:
|
||||
if int(content_length) > byte_limit:
|
||||
raise ApiRequestError("bad_request", "image_url 图片过大", status.HTTP_400_BAD_REQUEST)
|
||||
except ValueError:
|
||||
pass
|
||||
@@ -609,7 +740,7 @@ def read_limited_image_response(response) -> bytes:
|
||||
if not chunk:
|
||||
continue
|
||||
total += len(chunk)
|
||||
if total > max_bytes:
|
||||
if total > byte_limit:
|
||||
raise ApiRequestError("bad_request", "image_url 图片过大", status.HTTP_400_BAD_REQUEST)
|
||||
chunks.append(chunk)
|
||||
return b"".join(chunks)
|
||||
@@ -629,5 +760,9 @@ def summarize_titles(titles: list[str], text: str) -> str:
|
||||
return summary[:500]
|
||||
|
||||
|
||||
def summarize_text(text: str) -> str:
|
||||
return str(text or "").strip()[:500]
|
||||
|
||||
|
||||
def elapsed_ms(started: float) -> int:
|
||||
return int((perf_counter() - started) * 1000)
|
||||
|
||||
@@ -50,6 +50,38 @@ class GenerateImageRequestSerializer(serializers.Serializer):
|
||||
parameters = serializers.DictField(required=False, default=dict)
|
||||
|
||||
|
||||
class VisionImageInputSerializer(serializers.Serializer):
|
||||
image_url = serializers.URLField(required=False, allow_blank=True)
|
||||
image_base64 = serializers.CharField(required=False, allow_blank=True)
|
||||
|
||||
def validate(self, attrs):
|
||||
has_url = bool(str(attrs.get("image_url") or "").strip())
|
||||
has_base64 = bool(str(attrs.get("image_base64") or "").strip())
|
||||
if has_url == has_base64:
|
||||
raise serializers.ValidationError(
|
||||
"每张图片必须且只能提供 image_url 或 image_base64。"
|
||||
)
|
||||
return attrs
|
||||
|
||||
|
||||
class AnalyzeImagesRequestSerializer(serializers.Serializer):
|
||||
prompt = serializers.CharField(trim_whitespace=True, allow_blank=False)
|
||||
model = serializers.CharField(
|
||||
required=False,
|
||||
allow_blank=True,
|
||||
trim_whitespace=True,
|
||||
max_length=64,
|
||||
)
|
||||
images = VisionImageInputSerializer(many=True, allow_empty=False)
|
||||
parameters = serializers.DictField(required=False, default=dict)
|
||||
|
||||
def validate_images(self, value):
|
||||
max_images = max(1, int(settings.VISION_MAX_IMAGES))
|
||||
if len(value) > max_images:
|
||||
raise serializers.ValidationError(f"单次最多上传 {max_images} 张图片。")
|
||||
return value
|
||||
|
||||
|
||||
class RechargeCreateRequestSerializer(serializers.Serializer):
|
||||
amount = serializers.DecimalField(
|
||||
max_digits=12,
|
||||
|
||||
+328
-1
@@ -331,6 +331,12 @@ class ModelsCatalogApiTests(TestCase):
|
||||
url="https://provider-secret.example/v1/images/edits",
|
||||
model_sku="secret-sku-image-2",
|
||||
)
|
||||
vision_alias = self.create_alias(
|
||||
alias="vision-standard",
|
||||
operation_type=ModelAlias.OperationType.VISION,
|
||||
capabilities=["text", "vision"],
|
||||
model_sku="secret-sku-vision",
|
||||
)
|
||||
PricingRule.objects.create(
|
||||
operation_type=title_alias.operation_type,
|
||||
alias=title_alias.alias,
|
||||
@@ -349,13 +355,19 @@ class ModelsCatalogApiTests(TestCase):
|
||||
resolution="1k",
|
||||
points_cost=12,
|
||||
)
|
||||
PricingRule.objects.create(
|
||||
operation_type=vision_alias.operation_type,
|
||||
alias=vision_alias.alias,
|
||||
resolution="",
|
||||
points_cost=3,
|
||||
)
|
||||
|
||||
response = self.client.get(self.url, **self.auth_header())
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertNotIn(GenerateRateThrottle, ModelsView.throttle_classes)
|
||||
models = {item["alias"]: item for item in response.data["models"]}
|
||||
self.assertEqual(set(models), {"title-standard", "image-edit"})
|
||||
self.assertEqual(set(models), {"title-standard", "image-edit", "vision-standard"})
|
||||
self.assertEqual(
|
||||
set(models["title-standard"]),
|
||||
{
|
||||
@@ -384,6 +396,14 @@ class ModelsCatalogApiTests(TestCase):
|
||||
{"resolution": "1K", "points_cost": 12},
|
||||
],
|
||||
)
|
||||
self.assertEqual(models["vision-standard"]["operation_type"], "vision")
|
||||
self.assertEqual(models["vision-standard"]["capabilities"], ["text", "vision"])
|
||||
self.assertTrue(models["vision-standard"]["requires_image"])
|
||||
self.assertEqual(models["vision-standard"]["pricing_status"], "priced")
|
||||
self.assertEqual(
|
||||
models["vision-standard"]["prices"],
|
||||
[{"resolution": "default", "points_cost": 3}],
|
||||
)
|
||||
response_body = json.dumps(response.data, ensure_ascii=False)
|
||||
self.assertNotIn("secret-sku", response_body)
|
||||
self.assertNotIn("provider-secret.example", response_body)
|
||||
@@ -1056,8 +1076,10 @@ class FakeGenerationProvider:
|
||||
self._capabilities = set(capabilities or {"text", "image", "vision"})
|
||||
self.text_calls = []
|
||||
self.image_calls = []
|
||||
self.vision_calls = []
|
||||
self.text_error = None
|
||||
self.image_error = None
|
||||
self.vision_error = None
|
||||
|
||||
def capabilities(self):
|
||||
return set(self._capabilities)
|
||||
@@ -1083,6 +1105,17 @@ class FakeGenerationProvider:
|
||||
raw={"b64_json": "SECRET_RAW_SHOULD_NOT_BE_STORED"},
|
||||
)
|
||||
|
||||
def analyze_images(self, prompt, model, **kwargs):
|
||||
self.vision_calls.append({"prompt": prompt, "model": model, **kwargs})
|
||||
if self.vision_error is not None:
|
||||
raise self.vision_error
|
||||
return TextGenerationResult(
|
||||
text="第一张展示商品正面。\n第二张展示商品细节。",
|
||||
titles=(),
|
||||
model_used=model.model,
|
||||
raw={"secret": "SECRET_RAW_SHOULD_NOT_BE_STORED"},
|
||||
)
|
||||
|
||||
|
||||
class FakeImageUrlResponse:
|
||||
def __init__(self, *, status_code=200, headers=None, chunks=()):
|
||||
@@ -1143,14 +1176,26 @@ class GenerateApiTests(TestCase):
|
||||
model=f"gpt-image-{suffix}",
|
||||
capabilities=["image", "vision"],
|
||||
)
|
||||
self.vision_model = self.create_ai_model(
|
||||
name=f"vision-model-{suffix}",
|
||||
model=f"gpt-vision-{suffix}",
|
||||
capabilities=["text", "vision"],
|
||||
)
|
||||
self.title_alias = f"title-standard-{suffix}"
|
||||
self.image_alias = f"image-hd-{suffix}"
|
||||
self.vision_alias = f"vision-standard-{suffix}"
|
||||
ModelAlias.objects.create(
|
||||
operation_type=ModelAlias.OperationType.TITLE,
|
||||
alias=self.title_alias,
|
||||
ai_model=self.title_model,
|
||||
is_default=True,
|
||||
)
|
||||
ModelAlias.objects.create(
|
||||
operation_type=ModelAlias.OperationType.VISION,
|
||||
alias=self.vision_alias,
|
||||
ai_model=self.vision_model,
|
||||
is_default=True,
|
||||
)
|
||||
ModelAlias.objects.create(
|
||||
operation_type=ModelAlias.OperationType.IMAGE,
|
||||
alias=self.image_alias,
|
||||
@@ -1168,6 +1213,11 @@ class GenerateApiTests(TestCase):
|
||||
resolution="1K",
|
||||
points_cost=10,
|
||||
)
|
||||
PricingRule.objects.create(
|
||||
operation_type=CallRecord.OperationType.VISION,
|
||||
alias=self.vision_alias,
|
||||
points_cost=3,
|
||||
)
|
||||
|
||||
def create_ai_model(self, *, name, model, capabilities):
|
||||
ai_model = AiModel(
|
||||
@@ -1314,6 +1364,283 @@ class GenerateApiTests(TestCase):
|
||||
1,
|
||||
)
|
||||
|
||||
def test_analyze_images_supports_single_image_with_explicit_alias(self):
|
||||
encoded = base64.b64encode(b"single-image").decode("ascii")
|
||||
|
||||
response = self.post_with_provider(
|
||||
"/api/v1/analyze/images",
|
||||
{
|
||||
"prompt": "描述这张商品图",
|
||||
"model": self.vision_alias,
|
||||
"images": [{"image_base64": encoded}],
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertEqual(response.data["alias"], self.vision_alias)
|
||||
self.assertEqual(response.data["points_cost"], 3)
|
||||
self.assertEqual(len(self.provider.vision_calls), 1)
|
||||
self.assertEqual(
|
||||
[image.data for image in self.provider.vision_calls[0]["images"]],
|
||||
[b"single-image"],
|
||||
)
|
||||
|
||||
def test_analyze_images_supports_ordered_mixed_sources_and_charges_once(self):
|
||||
first = base64.b64encode(b"first-image").decode("ascii")
|
||||
response_from_url = FakeImageUrlResponse(
|
||||
headers={"Content-Type": "image/jpeg"},
|
||||
chunks=(b"second-", b"image"),
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"apps.api.generation.socket.getaddrinfo",
|
||||
return_value=dns_result("93.184.216.34"),
|
||||
),
|
||||
patch(
|
||||
"apps.api.generation.requests.Session.get",
|
||||
return_value=response_from_url,
|
||||
),
|
||||
):
|
||||
response = self.post_with_provider(
|
||||
"/api/v1/analyze/images",
|
||||
{
|
||||
"prompt": "比较两张商品图",
|
||||
"images": [
|
||||
{"image_base64": f"data:image/png;base64,{first}"},
|
||||
{"image_url": "https://images.example.test/detail.jpg"},
|
||||
],
|
||||
"parameters": {"temperature": 0.2},
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertEqual(response.data["text"], "第一张展示商品正面。\n第二张展示商品细节。")
|
||||
self.assertEqual(response.data["alias"], self.vision_alias)
|
||||
self.assertEqual(response.data["model_used"], self.vision_model.model)
|
||||
self.assertEqual(response.data["points_cost"], 3)
|
||||
self.assertEqual(response.data["points_balance"], 97)
|
||||
self.assertEqual(len(self.provider.vision_calls), 1)
|
||||
images = self.provider.vision_calls[0]["images"]
|
||||
self.assertEqual([image.data for image in images], [b"first-image", b"second-image"])
|
||||
self.assertEqual(
|
||||
[image.mime_type for image in images],
|
||||
["image/png", "image/jpeg"],
|
||||
)
|
||||
|
||||
self.wallet.refresh_from_db()
|
||||
self.assertEqual(self.wallet.points_balance, 97)
|
||||
call = CallRecord.objects.get(pk=response.data["call_id"])
|
||||
self.assertEqual(call.operation_type, CallRecord.OperationType.VISION)
|
||||
self.assertEqual(call.status, CallRecord.Status.SUCCESS)
|
||||
self.assertEqual(call.resolution, "")
|
||||
self.assertEqual(call.result_summary, response.data["text"])
|
||||
self.assertNotIn("first-image", call.result_summary)
|
||||
self.assertNotIn("SECRET_RAW", call.result_summary)
|
||||
self.assertEqual(
|
||||
PointsLedger.objects.filter(
|
||||
ref_call=call,
|
||||
change_type=PointsLedger.ChangeType.CONSUME,
|
||||
).count(),
|
||||
1,
|
||||
)
|
||||
|
||||
def test_analyze_images_requires_nonempty_exclusive_image_sources(self):
|
||||
empty = self.post_with_provider(
|
||||
"/api/v1/analyze/images",
|
||||
{"prompt": "分析图片", "images": []},
|
||||
)
|
||||
both = self.post_with_provider(
|
||||
"/api/v1/analyze/images",
|
||||
{
|
||||
"prompt": "分析图片",
|
||||
"images": [
|
||||
{
|
||||
"image_url": "https://images.example.test/input.jpg",
|
||||
"image_base64": "aW1hZ2U=",
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(empty.status_code, 400)
|
||||
self.assertEqual(empty.data["error"]["code"], "bad_request")
|
||||
self.assertEqual(both.status_code, 400)
|
||||
self.assertEqual(both.data["error"]["code"], "bad_request")
|
||||
self.assertEqual(self.provider.vision_calls, [])
|
||||
self.assert_generation_not_charged()
|
||||
|
||||
@override_settings(VISION_MAX_IMAGES=1)
|
||||
def test_analyze_images_rejects_too_many_images_before_charge(self):
|
||||
encoded = base64.b64encode(b"image").decode("ascii")
|
||||
|
||||
response = self.post_with_provider(
|
||||
"/api/v1/analyze/images",
|
||||
{
|
||||
"prompt": "分析图片",
|
||||
"images": [
|
||||
{"image_base64": encoded},
|
||||
{"image_base64": encoded},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 400)
|
||||
self.assertEqual(response.data["error"]["code"], "bad_request")
|
||||
self.assert_generation_not_charged()
|
||||
|
||||
@override_settings(VISION_MAX_IMAGE_BYTES=3, VISION_MAX_TOTAL_BYTES=10)
|
||||
def test_analyze_images_rejects_oversized_single_image_before_charge(self):
|
||||
encoded = base64.b64encode(b"four").decode("ascii")
|
||||
|
||||
response = self.post_with_provider(
|
||||
"/api/v1/analyze/images",
|
||||
{"prompt": "分析图片", "images": [{"image_base64": encoded}]},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 400)
|
||||
self.assertEqual(response.data["error"]["code"], "bad_request")
|
||||
self.assert_generation_not_charged()
|
||||
|
||||
@override_settings(VISION_MAX_IMAGE_BYTES=10, VISION_MAX_TOTAL_BYTES=5)
|
||||
def test_analyze_images_rejects_oversized_total_before_charge(self):
|
||||
encoded = base64.b64encode(b"abc").decode("ascii")
|
||||
|
||||
response = self.post_with_provider(
|
||||
"/api/v1/analyze/images",
|
||||
{
|
||||
"prompt": "分析图片",
|
||||
"images": [
|
||||
{"image_base64": encoded},
|
||||
{"image_base64": encoded},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 400)
|
||||
self.assertEqual(response.data["error"]["code"], "bad_request")
|
||||
self.assert_generation_not_charged()
|
||||
|
||||
def test_analyze_images_rejects_private_image_url_before_charge(self):
|
||||
with patch(
|
||||
"apps.api.generation.socket.getaddrinfo",
|
||||
return_value=dns_result("127.0.0.1"),
|
||||
):
|
||||
response = self.post_with_provider(
|
||||
"/api/v1/analyze/images",
|
||||
{
|
||||
"prompt": "分析图片",
|
||||
"images": [{"image_url": "http://internal.example.test/input.jpg"}],
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 400)
|
||||
self.assertEqual(response.data["error"]["code"], "bad_request")
|
||||
self.assertEqual(self.provider.vision_calls, [])
|
||||
self.assert_generation_not_charged()
|
||||
|
||||
def test_analyze_images_requires_api_key(self):
|
||||
encoded = base64.b64encode(b"image").decode("ascii")
|
||||
|
||||
response = self.client.post(
|
||||
"/api/v1/analyze/images",
|
||||
{"prompt": "分析图片", "images": [{"image_base64": encoded}]},
|
||||
format="json",
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 401)
|
||||
self.assertEqual(response.data["error"]["code"], "unauthorized")
|
||||
self.assert_generation_not_charged()
|
||||
|
||||
@override_settings(
|
||||
MODERATION_ENABLED=True,
|
||||
MODERATION_PROVIDER="keyword",
|
||||
MODERATION_CACHE_VERSION_KEY="test:api:vision:moderation:version",
|
||||
)
|
||||
def test_analyze_images_blocks_prompt_before_loading_images_or_charge(self):
|
||||
SensitiveWord.objects.create(word="敏感词", category="policy")
|
||||
|
||||
with (
|
||||
patch("apps.api.generation.decode_image_input") as decode_image,
|
||||
patch("apps.api.generation.download_image_input") as download_image,
|
||||
):
|
||||
response = self.post_with_provider(
|
||||
"/api/v1/analyze/images",
|
||||
{
|
||||
"prompt": "分析敏-感词图片",
|
||||
"images": [
|
||||
{"image_url": "https://images.example.test/input.jpg"}
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 400)
|
||||
self.assertEqual(response.data["error"]["code"], "content_blocked")
|
||||
decode_image.assert_not_called()
|
||||
download_image.assert_not_called()
|
||||
self.assert_generation_not_charged()
|
||||
|
||||
def test_analyze_images_rejects_model_or_provider_without_text_vision(self):
|
||||
bad_alias = f"vision-without-text-{uuid.uuid4().hex[:8]}"
|
||||
ModelAlias.objects.create(
|
||||
operation_type=ModelAlias.OperationType.VISION,
|
||||
alias=bad_alias,
|
||||
ai_model=self.image_model,
|
||||
)
|
||||
encoded = base64.b64encode(b"image").decode("ascii")
|
||||
payload = {
|
||||
"prompt": "分析图片",
|
||||
"model": bad_alias,
|
||||
"images": [{"image_base64": encoded}],
|
||||
}
|
||||
|
||||
model_rejected = self.post_with_provider("/api/v1/analyze/images", payload)
|
||||
provider_rejected = self.post_with_provider(
|
||||
"/api/v1/analyze/images",
|
||||
{**payload, "model": self.vision_alias},
|
||||
provider=FakeGenerationProvider(capabilities={"vision"}),
|
||||
)
|
||||
|
||||
self.assertEqual(model_rejected.status_code, 400)
|
||||
self.assertEqual(model_rejected.data["error"]["code"], "model_not_allowed")
|
||||
self.assertEqual(provider_rejected.status_code, 400)
|
||||
self.assertEqual(provider_rejected.data["error"]["code"], "model_not_allowed")
|
||||
self.assert_generation_not_charged()
|
||||
|
||||
def test_analyze_images_upstream_failure_refunds_once(self):
|
||||
encoded = base64.b64encode(b"image").decode("ascii")
|
||||
self.provider.vision_error = requests.Timeout("vision timeout")
|
||||
|
||||
response = self.post_with_provider(
|
||||
"/api/v1/analyze/images",
|
||||
{
|
||||
"prompt": "分析图片",
|
||||
"model": self.vision_alias,
|
||||
"images": [{"image_base64": encoded}],
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 502)
|
||||
self.assertEqual(response.data["error"]["code"], "upstream_timeout")
|
||||
self.wallet.refresh_from_db()
|
||||
self.assertEqual(self.wallet.points_balance, 100)
|
||||
call = CallRecord.objects.get(operation_type=CallRecord.OperationType.VISION)
|
||||
self.assertEqual(call.status, CallRecord.Status.FAILED)
|
||||
self.assertEqual(
|
||||
PointsLedger.objects.filter(
|
||||
ref_call=call,
|
||||
change_type=PointsLedger.ChangeType.CONSUME,
|
||||
).count(),
|
||||
1,
|
||||
)
|
||||
self.assertEqual(
|
||||
PointsLedger.objects.filter(
|
||||
ref_call=call,
|
||||
change_type=PointsLedger.ChangeType.REFUND,
|
||||
).count(),
|
||||
1,
|
||||
)
|
||||
|
||||
def test_generate_image_stores_file_returns_url_and_does_not_store_raw_base64(self):
|
||||
encoded = base64.b64encode(b"input-image").decode("ascii")
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ from django.urls import path
|
||||
|
||||
from .views import (
|
||||
AlipayRechargeCallbackView,
|
||||
AnalyzeImagesView,
|
||||
BalanceView,
|
||||
ClientLatestReleaseView,
|
||||
GenerateImageTaskDetailView,
|
||||
@@ -23,6 +24,7 @@ urlpatterns = [
|
||||
name="api-client-release-latest",
|
||||
),
|
||||
path("v1/generate/title", GenerateTitleView.as_view(), name="api-generate-title"),
|
||||
path("v1/analyze/images", AnalyzeImagesView.as_view(), name="api-analyze-images"),
|
||||
path("v1/generate/image", GenerateImageView.as_view(), name="api-generate-image"),
|
||||
path(
|
||||
"v1/generate/image/tasks",
|
||||
|
||||
@@ -15,6 +15,7 @@ from apps.api.authentication import ApiKeyAuthentication
|
||||
from apps.api.errors import api_error
|
||||
from apps.api.generation import (
|
||||
ApiRequestError,
|
||||
analyze_images_response,
|
||||
generate_image_response,
|
||||
generate_title_response,
|
||||
)
|
||||
@@ -25,6 +26,7 @@ from apps.api.image_tasks import (
|
||||
)
|
||||
from apps.api.models import ImageGenerationTask
|
||||
from apps.api.serializers import (
|
||||
AnalyzeImagesRequestSerializer,
|
||||
GenerateImageRequestSerializer,
|
||||
GenerateTitleRequestSerializer,
|
||||
RechargeCreateRequestSerializer,
|
||||
@@ -97,6 +99,27 @@ class GenerateTitleView(ExternalApiView):
|
||||
return Response(data, status=status.HTTP_200_OK)
|
||||
|
||||
|
||||
class AnalyzeImagesView(ExternalApiView):
|
||||
throttle_classes = (GenerateRateThrottle,)
|
||||
|
||||
def post(self, request):
|
||||
serializer = AnalyzeImagesRequestSerializer(data=request.data)
|
||||
if not serializer.is_valid():
|
||||
return Response(
|
||||
api_error("bad_request", "参数错误"),
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
try:
|
||||
data = analyze_images_response(
|
||||
user=request.user,
|
||||
api_key=request.auth,
|
||||
request_data=serializer.validated_data,
|
||||
)
|
||||
except ApiRequestError as exc:
|
||||
return Response(exc.as_response_data(), status=exc.http_status)
|
||||
return Response(data, status=status.HTTP_200_OK)
|
||||
|
||||
|
||||
class GenerateImageView(ExternalApiView):
|
||||
throttle_classes = (GenerateRateThrottle,)
|
||||
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
# Generated by Django 5.2.15 on 2026-07-16 03:52
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('billing', '0007_alter_pointsledger_change_type_signupbonusgrant'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='callrecord',
|
||||
name='operation_type',
|
||||
field=models.CharField(choices=[('title', '生成标题'), ('image', '生成图片'), ('vision', '图片理解')], max_length=32, verbose_name='操作类型'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='pricingrule',
|
||||
name='operation_type',
|
||||
field=models.CharField(choices=[('title', '生成标题'), ('image', '生成图片'), ('vision', '图片理解')], max_length=32, verbose_name='操作类型'),
|
||||
),
|
||||
]
|
||||
@@ -14,6 +14,7 @@ class CallRecord(models.Model):
|
||||
class OperationType(models.TextChoices):
|
||||
TITLE = "title", "生成标题"
|
||||
IMAGE = "image", "生成图片"
|
||||
VISION = "vision", "图片理解"
|
||||
|
||||
class Status(models.TextChoices):
|
||||
PENDING = "pending", "待处理"
|
||||
|
||||
@@ -30,6 +30,8 @@
|
||||
生成标题
|
||||
{% elif item.operation_type == "image" %}
|
||||
生成图片
|
||||
{% elif item.operation_type == "vision" %}
|
||||
图片理解
|
||||
{% else %}
|
||||
{{ item.operation_type }}
|
||||
{% endif %}
|
||||
|
||||
+16
-1
@@ -503,6 +503,12 @@ class PortalAccountFlowTests(TestCase):
|
||||
url="https://provider-secret.example/v1/images/edits",
|
||||
model_sku="secret-sku-image-2",
|
||||
)
|
||||
vision_alias = self.create_model_alias(
|
||||
alias="vision-standard",
|
||||
operation_type=ModelAlias.OperationType.VISION,
|
||||
capabilities=["text", "vision"],
|
||||
model_sku="secret-sku-vision",
|
||||
)
|
||||
PricingRule.objects.create(
|
||||
operation_type=title_alias.operation_type,
|
||||
alias=title_alias.alias,
|
||||
@@ -516,12 +522,18 @@ class PortalAccountFlowTests(TestCase):
|
||||
points_cost=12,
|
||||
is_active=False,
|
||||
)
|
||||
PricingRule.objects.create(
|
||||
operation_type=vision_alias.operation_type,
|
||||
alias=vision_alias.alias,
|
||||
resolution="",
|
||||
points_cost=3,
|
||||
)
|
||||
self.client.force_login(user)
|
||||
|
||||
response = self.client.get("/models")
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertEqual(len(response.context["models"]), 2)
|
||||
self.assertEqual(len(response.context["models"]), 3)
|
||||
self.assertContains(response, "可用模型")
|
||||
self.assertContains(response, "title-standard")
|
||||
self.assertContains(response, "生成标题")
|
||||
@@ -529,6 +541,9 @@ class PortalAccountFlowTests(TestCase):
|
||||
self.assertContains(response, "2 点")
|
||||
self.assertContains(response, "image-edit")
|
||||
self.assertContains(response, "生成图片")
|
||||
self.assertContains(response, "vision-standard")
|
||||
self.assertContains(response, "图片理解")
|
||||
self.assertContains(response, "3 点")
|
||||
self.assertContains(response, "需要")
|
||||
self.assertContains(response, "暂未定价")
|
||||
self.assertNotContains(response, "secret-sku")
|
||||
|
||||
@@ -117,6 +117,9 @@ IMAGE_URL_MAX_BYTES = env_int("IMAGE_URL_MAX_BYTES", 10 * 1024 * 1024)
|
||||
IMAGE_URL_MAX_REDIRECTS = env_int("IMAGE_URL_MAX_REDIRECTS", 3)
|
||||
IMAGE_URL_CONNECT_TIMEOUT_SECONDS = env_int("IMAGE_URL_CONNECT_TIMEOUT_SECONDS", 10)
|
||||
IMAGE_URL_READ_TIMEOUT_SECONDS = env_int("IMAGE_URL_READ_TIMEOUT_SECONDS", 60)
|
||||
VISION_MAX_IMAGES = env_int("VISION_MAX_IMAGES", 8)
|
||||
VISION_MAX_IMAGE_BYTES = env_int("VISION_MAX_IMAGE_BYTES", 10 * 1024 * 1024)
|
||||
VISION_MAX_TOTAL_BYTES = env_int("VISION_MAX_TOTAL_BYTES", 32 * 1024 * 1024)
|
||||
PUBLIC_BASE_URL = os.environ.get("PUBLIC_BASE_URL", "").rstrip("/")
|
||||
MEDIA_PUBLIC_BASE_URL = os.environ.get("MEDIA_PUBLIC_BASE_URL", PUBLIC_BASE_URL).rstrip("/")
|
||||
IMAGE_TASK_RETENTION_HOURS = env_int("IMAGE_TASK_RETENTION_HOURS", 24)
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
## 一句话定位
|
||||
|
||||
`cmhub` 是「**自助用户端 + 计费 API + 运营后台**」三合一服务:终端用户在用户端自助注册、扫码充值、管理 API Key,用 API Key 调用把 `cmbot` 的「生成标题」「生成图片」能力封装成的 HTTP 接口,按**点数计费**;运营用 django-admin 管理用户、点数与记录。
|
||||
`cmhub` 是「**自助用户端 + 计费 API + 运营后台**」三合一服务:终端用户在用户端自助注册、扫码充值、管理 API Key,用 API Key 调用「生成标题」「生成图片」「多图理解」等 HTTP 接口,按**点数计费**;运营用 django-admin 管理用户、点数与记录。
|
||||
|
||||
第一版 MVP 做:**用户自助注册登录 → 注册成功赠送 100 点试用点数 → 扫码充值转点数 → 自助生成/删除 API Key → 带 Key 调用按规则算点数 → 点数足够则调上游生成并扣点 / 不足则报错 → 写调用与消费记录 → django-admin 运营后台**。注册赠点也是账本资产,必须经计费层入账并写点数流水。
|
||||
|
||||
@@ -40,6 +40,8 @@
|
||||
|
||||
当前项目处于:**Phase 6 增强任务推进期**。Phase 2 计费核心已完成到 T-204;Phase 3 已完成 T-301 API Key 鉴权、T-302 生成标题 / 图片接口、T-303 余额查询接口、T-304 充值回调、T-305 扫码充值下单 + 轮询与 T-306 对外 API 安全加固;Phase 4 已完成 T-501 注册 / 登录(allauth)、T-502 API Key 自助管理页、T-503 个人中心 / 记录页、T-504 充值页与 T-505 用户端审核优化;Phase 5 已完成 T-401 运营后台完善、T-402 MVP 完整验收与 T-403 部署 / 运行文档;Phase 6 已完成 T-601「可用别名发现」、T-602「django-admin 中文化第 1-3 层」、T-603「django-admin 字段级中文化」、T-604「中文敏感词本地过滤」、T-605「免邮箱验证策略落地」、T-606「公开首页 + 客户端下载入口」、T-607「桌面端最新版本检查接口」、T-608「新用户注册赠送 100 点试用点数」、T-609「桌面端版本检查接口增加强制更新标记」、T-610「首页导入模板下载入口」、T-611「用户端品牌名统一为虾皮圈」、T-612「生图同步接口止血」、T-613「抽生成核心 service」、T-614「生图异步任务化接口」与 T-615「旧同步生图接口遥测 / 弃用口径」。生图慢 / 504 / 客户端超时已拆为 T-612~T-615 并完成:T-612 已做同步接口上游硬截止止血,T-613 已抽共享生成 core,T-614 已新增异步提交 / 轮询接口与 DB worker,T-615 已给旧同步 / 新异步提交路径接入结构化日志并形成旧同步接口退出条件。生产侧仍需补真实支付回调到账闭环;邮件服务仅用于后续密码找回/通知等邮件能力,不阻塞注册登录。
|
||||
|
||||
T-616~T-619 也已完成:生图失败自动重试、客户端版本文件校验元数据、多图理解 `vision` 操作与同步接口均已落地。生产启用多图理解前仍需在 admin 配置视觉模型、默认别名和计费规则。
|
||||
|
||||
优先路径:
|
||||
|
||||
1. Phase 0:Django 骨架可运行、**自定义 User 模型在首次迁移前定好**、django-admin 可登录;T-004 审核修补项已完成。
|
||||
@@ -48,7 +50,7 @@
|
||||
4. Phase 3:对外 API 与充值 —— T-301 Key 鉴权、T-302 生成接口、T-303 余额查询、T-304 充值回调、T-305 扫码下单与轮询、T-306 安全加固已完成。
|
||||
5. Phase 4:用户端(Django 模板 SSR)—— T-501 注册登录、T-502 API Key 管理、T-503 个人中心 / 记录页、T-504 充值页与 T-505 用户端审核优化已完成。
|
||||
6. Phase 5:后台与发布 —— T-401 运营后台完善、T-402 完整验收 MVP、T-403 部署 / 运行文档已完成;计划内 MVP 任务已收尾。
|
||||
7. Phase 6:增强(MVP 后)—— T-601~T-618 已完成,覆盖可用别名发现、后台中文化、敏感词过滤、免邮箱验证、公开首页与客户端发布、注册赠点、用户端品牌、生图异步化 / 遥测 / 自动重试和客户端发布校验元数据;T-619「多张图片理解并返回文字」已登记为下一项 `TODO`,将新增独立 `vision` 操作、能力别名和同步多图理解接口,当前尚未实现。
|
||||
7. Phase 6:增强(MVP 后)—— T-601~T-619 已完成,覆盖可用别名发现、后台中文化、敏感词过滤、免邮箱验证、公开首页与客户端发布、注册赠点、用户端品牌、生图异步化 / 遥测 / 自动重试、客户端发布校验元数据和同步多图理解;当前没有已登记的下一编号任务。
|
||||
|
||||
## 领取任务规则
|
||||
|
||||
@@ -69,7 +71,7 @@
|
||||
MVP 只做:
|
||||
|
||||
- **用户端(Django 模板 SSR + Bootstrap)**:自助注册/登录、注册成功一次性赠送 100 点试用点数、扫码充值、查看充值记录/充值总额/剩余点数/消费记录、自助生成与删除 API Key。
|
||||
- 对外生成接口:`POST /api/v1/generate/title`(同步返回标题)、`POST /api/v1/generate/image`(旧同步生图,保留兼容)、`POST /api/v1/generate/image/tasks` + `GET /api/v1/generate/image/tasks/{task_id}`(异步生图提交 / 轮询)。
|
||||
- 对外生成 / 理解接口:`POST /api/v1/generate/title`(同步返回标题)、`POST /api/v1/generate/image`(旧同步生图,保留兼容)、`POST /api/v1/generate/image/tasks` + `GET /api/v1/generate/image/tasks/{task_id}`(异步生图提交 / 轮询)、`POST /api/v1/analyze/images`(同步多图理解并返回文字)。
|
||||
- API Key 鉴权,识别所属注册用户(API 只认 Key,不认 Web session)。
|
||||
- 点数计费:按「操作类型 + 能力别名(+ 可选分辨率)」查计费规则得点数;本地余额原子扣减;不足报错。
|
||||
- 支付充值回调:支付系统服务端回调 → 验签 + 幂等 → 按汇率把金额转点数入账。
|
||||
|
||||
@@ -31,6 +31,7 @@
|
||||
| 个人中心 | 查看剩余点数、充值总额、充值记录、点数使用(消费)记录 | P0 |
|
||||
| 生成标题 API | 带 API Key + prompt(+可选商品图)调用,拿到标题文字 | P0 |
|
||||
| 生成图片 API | 带 API Key + prompt + 图 + 模型/分辨率调用;旧同步接口可直接返回图片 URL,新版客户端可用异步提交 / 轮询拿结果 | P0 |
|
||||
| 多图理解 API | 带 API Key + prompt + 一张或多张图片调用,按图片顺序理解、比较或提取信息并返回完整文字 | P1 |
|
||||
| 点数计费与扣减 | 按「操作类型+能力别名+分辨率」算点数,余额足够则扣点放行,不足则报错 | P0 |
|
||||
| 余额查询 API | 查询当前用户点数余额 | P0 |
|
||||
| 调用记录 | 每次调用落库:用户、Key、时间、操作、模型、消耗点数、成功/失败、错误 | P0 |
|
||||
@@ -57,9 +58,10 @@
|
||||
2. 作为已登录用户,我自助生成一把 API Key(明文只显示一次),也能删除不用的 Key。
|
||||
3. 作为接入方,我带着 API Key 调用生成标题接口,传入 prompt,能拿到生成的标题。
|
||||
4. 作为接入方,我可以调用旧同步生成图片接口在同一个响应里拿到图片结果;也可以调用异步图片任务接口先拿到 `task_id`,再轮询直到成功或失败。
|
||||
5. 当我的点数不足以支付本次调用时,系统拒绝调用并返回「点数不足,请先充值」,且不调用上游、不扣点。
|
||||
6. 作为已登录用户,我在个人中心看到剩余点数、充值总额、充值记录和消费(调用)记录。
|
||||
7. 作为运营,我在后台能管理用户与点数(手工调点带原因),配置某操作的点数单价,查看某用户的调用记录和点数流水。
|
||||
5. 作为接入方,我可以提交一张或多张商品图,让支持视觉理解的模型按输入顺序分析并返回文字;一次请求只按对应能力别名扣一次点数。
|
||||
6. 当我的点数不足以支付本次调用时,系统拒绝调用并返回「点数不足,请先充值」,且不调用上游、不扣点。
|
||||
7. 作为已登录用户,我在个人中心看到剩余点数、充值总额、充值记录和消费(调用)记录。
|
||||
8. 作为运营,我在后台能管理用户与点数(手工调点带原因),配置某操作的点数单价,查看某用户的调用记录和点数流水。
|
||||
|
||||
## 五、验收标准(MVP)
|
||||
|
||||
@@ -67,6 +69,7 @@
|
||||
|
||||
- **生成标题 API**:携带有效 API Key + 合法 prompt 调用,返回 200 且响应含标题文本;无效/缺失 Key 返回 401。
|
||||
- **生成图片 API**:旧同步接口携带有效 Key 调用,在超时上限内返回 200 且含图片 URL;异步接口提交后返回 `task_id`,轮询成功后返回稳定图片 URL。上游失败、超时或异步任务僵死时返回明确错误码,且**不最终扣点**(已扣则冲正)。
|
||||
- **多图理解 API**:携带有效 Key、合法 prompt 和 1–8 张图片调用,返回 200 且响应含完整文字;图片顺序保持不变;输入无效、超限、命中敏感词或 URL 不安全时不扣点,上游失败时已扣点数只退一次。
|
||||
- **点数计费与扣减**:调用前按规则算出点数 N;余额 ≥ N 才放行并精确扣 N;余额 < N 返回点数不足错误码且不调上游。并发同时多次调用,最终扣点总额正确,**不出现超扣或扣成负数**。
|
||||
- **余额查询 API**:返回的余额等于该账号点数流水累加结果。
|
||||
- **充值转点数**:模拟一次支付系统回调(含订单号、金额、签名),验签通过后按汇率加点;**同一订单号重复回调只入账一次**(幂等);验签失败不入账。
|
||||
|
||||
@@ -41,6 +41,8 @@ T-301 已实现 `ApiKeyAuthentication` 与 `ExternalApiView`:外部 API 使用
|
||||
|
||||
T-302 已实现 `/api/v1/generate/title` 与 `/api/v1/generate/image`:API 层只做鉴权、参数校验和编排;别名解析、Provider 选择、计费计算、预扣、成功确认、失败退点分别调用 `apps.ai` / `apps.billing` 既有模块。T-613 已把生成链路抽为 `apps.api.generation` 的核心阶段:`prepare_generation()` 负责审核、图片输入、别名、Provider 与计费准备;`precharge_generation()` 只调用 billing 预扣;`execute_precharged_generation()` 复用已预扣 `CallRecord` 调上游并成功确认或失败退点,供旧同步接口和后续异步 worker 共用。图片结果 MVP 先用本地 `default_storage` 保存到 `MEDIA_ROOT/generated/images/...` 并返回 `image_url`;核心阶段通过 URL 构建器生成外部 URL,不依赖 DRF `Request`;`CallRecord` 只写 URL / 摘要,不保存 provider `raw` 或 base64。
|
||||
|
||||
T-619 已在同一生成核心增加 `vision` 分支和 `/api/v1/analyze/images`:API 接收有序的 `images[]`,每项二选一提供 `image_url` / `image_base64`;prompt 审核通过后才下载或解码图片,随后校验同时具备 `text + vision` 的模型与 Provider、按 `vision + alias` 默认价格预扣一次、同步调用 Chat Completions / Gemini 多模态接口并返回完整文字。输入图片只在请求内存中使用,不落库;`CallRecord` 只保存最多 500 字结果摘要。
|
||||
|
||||
T-614 已实现 `/api/v1/generate/image/tasks` 与 `/api/v1/generate/image/tasks/{task_id}`:提交接口同步审核 prompt、解析图片输入和预扣点,创建 `ImageGenerationTask(status=queued)` 后立即返回公开 UUID `task_id`;后台 worker 通过 `select_for_update(skip_locked)` 抢任务,复用 T-613 `execute_precharged_generation()` 对已预扣 `CallRecord` 调上游、保存结果、成功确认或失败退点。任务表记录 worker 租约、心跳和尝试次数;reaper 识别僵尸 `running` 任务后默认判失败并幂等退点,不默认重排队。异步 worker 没有 DRF request,结果 URL 由 `MEDIA_PUBLIC_BASE_URL` / `PUBLIC_BASE_URL` 生成。当前实现中 worker 会基于任务快照再次运行 `prepare_generation()`,因此会复审 prompt、重解析别名 / Provider / 定价;账务仍使用已预扣 `CallRecord.points_cost`,不会重复扣点。这个取舍偏安全(排队期间敏感词库更新后仍能拦截并退款),但如果未来队列积压明显,应单独实现“提交时模型配置快照”,避免执行时别名映射变化导致按旧价预扣、按新模型执行。
|
||||
|
||||
T-616 起异步生图 worker 对临时性上游失败增加自动重试:`upstream_timeout` / `upstream_error` 在未达到最大次数前回到 `queued` 并设置 `next_attempt_at`,`CallRecord` 保持 `pending` 且不退点;不可重试错误或最后一次失败才置 `failed` 并幂等退款。默认 `IMAGE_TASK_MAX_RETRIES=2`,因此 `attempt_count` 最多为 3。
|
||||
@@ -57,6 +59,8 @@ T-504 已实现用户端 `/recharge` 页面:GET 展示当前余额、充值表
|
||||
|
||||
T-306 已实现对外 API 安全加固:`download_image_input()` 在请求前校验 `image_url` 协议与解析后的 IP,只允许公网 `http` / `https`,拒绝私有、回环、链路本地、保留、组播、未指定地址;重定向由服务端手动跟随并逐跳重新校验,响应按 `IMAGE_URL_MAX_BYTES` 流式限长读取。`REST_FRAMEWORK` 全局默认认证为空、默认权限为 `IsAuthenticated`,外部 API 和用户端 session API 必须显式声明认证类;生成接口挂 `GenerateRateThrottle`,认证失败挂 IP 限流;充值下单通过 `RECHARGE_MAX_AMOUNT_CNY` 控制单笔上限。T-607/T-609/T-617 版本检查接口已显式使用 `AllowAny` / 空认证,且只返回公开发布元数据、强制更新标记和文件大小字节数。
|
||||
|
||||
T-619 多图理解继续复用上述 URL 下载器和逐跳 SSRF 校验,并额外用 `VISION_MAX_IMAGES`、`VISION_MAX_IMAGE_BYTES`、`VISION_MAX_TOTAL_BYTES` 控制图片数量、单图解码后大小和请求总大小。第一版只审核文字 prompt,不包含图片内容审核。
|
||||
|
||||
**计费层(`apps/billing`)**
|
||||
|
||||
- 计费规则查询:按「操作类型 + 能力别名(+ 可选分辨率)」算出本次点数 N。**按别名定价,不按具体供应商 SKU 定价**,这样后台换底层模型时计费不变。
|
||||
@@ -146,8 +150,8 @@ CREATE TABLE ai_model (
|
||||
-- 能力别名(对外稳定标识 → 当前映射的具体模型)
|
||||
CREATE TABLE model_alias (
|
||||
id BIGINT PRIMARY KEY,
|
||||
alias VARCHAR(64) NOT NULL, -- 如 title-standard / image-hd,对外用
|
||||
operation_type VARCHAR(32) NOT NULL, -- title / image
|
||||
alias VARCHAR(64) NOT NULL, -- 如 title-standard / image-hd / vision-standard,对外用
|
||||
operation_type VARCHAR(32) NOT NULL, -- title / image / vision
|
||||
ai_model_id BIGINT NOT NULL REFERENCES ai_model(id), -- 可随时改指向
|
||||
is_default BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
is_active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
@@ -172,7 +176,7 @@ CREATE TABLE ai_config_audit_log (
|
||||
-- 计费规则(按别名定价,换底层模型不影响计费)
|
||||
CREATE TABLE pricing_rule (
|
||||
id BIGINT PRIMARY KEY,
|
||||
operation_type VARCHAR(32) NOT NULL, -- title / image
|
||||
operation_type VARCHAR(32) NOT NULL, -- title / image / vision
|
||||
alias VARCHAR(64) NOT NULL, -- 能力别名字符串,不绑具体模型
|
||||
resolution VARCHAR(32) NOT NULL DEFAULT '', -- 空字符串表示该别名默认价
|
||||
points_cost BIGINT NOT NULL, -- > 0
|
||||
@@ -307,7 +311,7 @@ CREATE TABLE call_record (
|
||||
id INTEGER PRIMARY KEY,
|
||||
user_id INTEGER NOT NULL REFERENCES "user"(id),
|
||||
api_key_id INTEGER REFERENCES api_key(id), -- 本次调用所用的 Key
|
||||
operation_type TEXT NOT NULL, -- title / image
|
||||
operation_type TEXT NOT NULL, -- title / image / vision
|
||||
alias TEXT, -- 调用方请求的能力别名(对外稳定标识)
|
||||
model_used TEXT, -- 实际服务该次请求的具体模型(解析后,便于排障/对账)
|
||||
resolution TEXT,
|
||||
@@ -389,6 +393,7 @@ CREATE TABLE image_generation_task (
|
||||
- 扣点锁 `user_wallet` 行(`select_for_update()`)或 `F('points_balance') - N` 原子更新 + DB 约束 `points_balance >= 0`(MySQL 的 CHECK 需 ≥8.0.16),**杜绝并发超扣 / 扣成负数**;钱包与 auth `user` 表分离,扣点不与登录/资料更新抢锁。
|
||||
- **先扣后调、失败必退**:保证不会“调用成功但没扣到”或“失败还扣钱”。
|
||||
- 余额不足在调上游**之前**拦截。
|
||||
- `vision` 同步调用在别名解析前先审核 prompt,再按顺序读取 1–8 张图片;输入失败不进入预扣。计费规则使用空 `resolution` 默认价,一次请求固定预扣一次,不随图片数量重复扣费。
|
||||
|
||||
### 4.1.1 调用扣点(异步图片任务)
|
||||
|
||||
|
||||
+1
-1
@@ -98,7 +98,7 @@
|
||||
|
||||
| T-617 | 桌面端版本检查接口增加文件大小字段 | T-607, T-609 | 在 `GET /api/v1/client/releases/latest?platform=windows` 的当前版本响应中,为 `release` 对象新增 `size_bytes` 字段,用于桌面端下载后比对文件大小。**接口兼容**:只新增字段,不删除既有 `version` / `download_url` / `sha256` / `release_notes` / `force_update` / `published_at`;无当前版本或无下载地址时仍返回 `release:null`,不返回顶层 `size_bytes`。**数据来源**:`DownloadRelease` 新增 `size_bytes` 可空正整数字段,单位字节,django-admin 可填写并在列表展示;老数据为空时接口返回 `null`,客户端只在值为正整数时做大小校验。**安全**:接口仍公开匿名只读,不读取用户、不扣点、不暴露后台 ID、本地文件路径或内部状态。**文档**:同步 `api.md`、`routes.md`、`04-architecture.md`、`02-requirements.md`、`current-state.md`。**测试**:覆盖有值返回整数、未配置返回 `null`、未发布响应不返回顶层字段、响应字段白名单更新、admin 字段可见;`makemigrations --check` / `check` / 目标测试通过并在 `../progress.md` 留证据 | DONE |
|
||||
| T-618 | 客户端发布版本后台必填文件校验元数据 | T-617 | 把 django-admin 里 `DownloadRelease` 的 `sha256` 和 `size_bytes` 改为必填,避免运营发布客户端安装包时漏填校验元数据,导致桌面端无法完整校验下载文件。**范围**:本任务只要求 admin 后台保存时必填;为兼容历史数据和现有 API 合约,第一步不直接把数据库字段改成 `NOT NULL`,`size_bytes` 仍允许旧记录为空,API 暂保持可返回 `null`;若后续要数据库级强约束,需先单独回填全量历史发布记录再做迁移。**实现**:`DownloadReleaseAdmin` 已挂专用 `ModelForm`,将 `sha256`、`size_bytes` 设为 required,并保留 `sha256` 64 位十六进制校验、`size_bytes >= 1` 校验;admin 新增 / 编辑保存时未填会显示字段级错误,不写库。**生产数据**:上线后必须补齐当前 `windows` 发布版本的 `size_bytes`,再验证 `/api/v1/client/releases/latest?platform=windows` 返回正整数 `release.size_bytes`;已有 `sha256` 继续保留并核对。**兼容性**:不删除既有 API 字段,不改变无当前版本时的 `release:null`;非 admin 的历史数据、迁移和只读接口不应因空值直接 500。**文档**:同步 `api.md`、`04-architecture.md`、`routes.md`、`current-state.md` 和 `progress.md` 的发布校验口径。**测试**:已覆盖 admin 表单缺 `sha256` / 缺 `size_bytes` 拒绝保存、合法 `sha256 + size_bytes` 可保存、API 仍能读取历史空值记录、当前版本补齐后接口返回正整数;`check` / `makemigrations --check` / 目标测试通过并在 `../progress.md` 留证据 | DONE |
|
||||
| T-619 | 多张图片理解并返回文字 | T-613, T-601, T-306 | 新增独立的多模态图片理解能力,不复用“生成标题”语义。**操作与别名**:在 `ModelAlias.OperationType`、`CallRecord.OperationType` 和别名解析中新增 `vision`;建议首个数据库别名为 `vision-standard`,但不得在接口中暴露具体供应商模型名。`vision` 别名指向的 `AiModel` 及 Provider 必须同时具备 `vision` 与文字输出能力,图片生成 / 编辑专用 Provider 不得误接。**新增接口**:`POST /api/v1/analyze/images`,继承 `ExternalApiView`、使用 API Key 鉴权和生成限流,同步返回 `{text, alias, model_used, points_cost, points_balance, call_id}`;不改变 `/api/v1/generate/title`、`/api/v1/generate/image` 及异步生图接口。**请求结构**:`prompt` 必填,`model` 可选,`images` 为有序列表且至少 1 张;每项必须且只能提供一个 `image_url` 或 `image_base64`,允许 URL/base64 混合。新增 `VISION_MAX_IMAGES=8`、`VISION_MAX_IMAGE_BYTES=10485760`、`VISION_MAX_TOTAL_BYTES=33554432` 三个可配置上限,分别控制单次图片数量、单图解码后字节数和总字节数;超限、空列表、格式错误或同项双来源返回 `400 bad_request`,不扣点、不调上游。**安全时序**:serializer 后先按 T-604 审核 prompt,再下载 URL / 解码 base64;每个 URL 必须复用 T-306 的协议、公网地址、重定向逐跳校验、超时和响应大小保护,不另写弱化下载器。第一版只审核文字 prompt,不声称已做图片内容审核。**Provider**:复用 `apps/ai/providers` 现有 HTTP 适配层,增加向 Chat Completions / Gemini 发送多张图片并保序的兼容能力;不得复制第二套上游调用实现。现有单图标题 / 生图调用签名与行为保持兼容。**计费与留痕**:复用 T-613 共享生成 core 与 billing 的预扣 / 成功确认 / 幂等退点;第一版按 `operation_type=vision + alias` 的默认 `PricingRule` 对一次请求固定扣点,不按图片张数重复扣费,图片数量上限用于控制成本;上游失败必须退点。`CallRecord` 记录 `vision`、别名、实际模型、点数、耗时和结果摘要,不保存输入图片、base64、provider raw 或完整上游响应。**目录与运营配置**:`GET /api/v1/models` 和 portal 可用模型页按 T-601 既有语义列出 active、能力匹配的 `vision` 别名并显示 `requires_image=true`;有定价时返回 `priced`,缺定价时仍返回 `unpriced`,不得改变现有目录兼容行为。代码部署 / migrate 后由运营在 admin 配置支持视觉理解的 `AiModel`、`vision-standard` 默认别名和计费规则,不通过数据迁移写入真实上游配置或密钥。**范围边界**:本任务只做同步文字结果,不做流式输出、异步 vision task、OCR 专用接口、结构化 JSON schema、图片内容审核,也不为 OCR / 商品识别 / 图片对比各建别名;这些用途先由 prompt 表达,只有底层模型或价格确实不同时再增加别名。**文档**:实现时同步 `02-requirements.md`、`04-architecture.md`、`api.md`、`routes.md`、`env.md`、`current-state.md` 和 `progress.md`。**测试**:覆盖单图 / 多图 / 混合来源及顺序、默认 / 指定别名、模型与 Provider 能力拒绝、图片数量与大小限制、SSRF、敏感词先拦截、成功仅扣一次、上游失败仅退一次、调用记录不保存图片 / raw、模型目录安全字段、旧标题 / 生图接口回归;`makemigrations` / `migrate` / `check` / 目标测试 / `init` 通过并在 `../progress.md` 留证据 | TODO |
|
||||
| T-619 | 多张图片理解并返回文字 | T-613, T-601, T-306 | 新增独立的多模态图片理解能力,不复用“生成标题”语义。**操作与别名**:在 `ModelAlias.OperationType`、`CallRecord.OperationType` 和别名解析中新增 `vision`;建议首个数据库别名为 `vision-standard`,但不得在接口中暴露具体供应商模型名。`vision` 别名指向的 `AiModel` 及 Provider 必须同时具备 `vision` 与文字输出能力,图片生成 / 编辑专用 Provider 不得误接。**新增接口**:`POST /api/v1/analyze/images`,继承 `ExternalApiView`、使用 API Key 鉴权和生成限流,同步返回 `{text, alias, model_used, points_cost, points_balance, call_id}`;不改变 `/api/v1/generate/title`、`/api/v1/generate/image` 及异步生图接口。**请求结构**:`prompt` 必填,`model` 可选,`images` 为有序列表且至少 1 张;每项必须且只能提供一个 `image_url` 或 `image_base64`,允许 URL/base64 混合。新增 `VISION_MAX_IMAGES=8`、`VISION_MAX_IMAGE_BYTES=10485760`、`VISION_MAX_TOTAL_BYTES=33554432` 三个可配置上限,分别控制单次图片数量、单图解码后字节数和总字节数;超限、空列表、格式错误或同项双来源返回 `400 bad_request`,不扣点、不调上游。**安全时序**:serializer 后先按 T-604 审核 prompt,再下载 URL / 解码 base64;每个 URL 必须复用 T-306 的协议、公网地址、重定向逐跳校验、超时和响应大小保护,不另写弱化下载器。第一版只审核文字 prompt,不声称已做图片内容审核。**Provider**:复用 `apps/ai/providers` 现有 HTTP 适配层,增加向 Chat Completions / Gemini 发送多张图片并保序的兼容能力;不得复制第二套上游调用实现。现有单图标题 / 生图调用签名与行为保持兼容。**计费与留痕**:复用 T-613 共享生成 core 与 billing 的预扣 / 成功确认 / 幂等退点;第一版按 `operation_type=vision + alias` 的默认 `PricingRule` 对一次请求固定扣点,不按图片张数重复扣费,图片数量上限用于控制成本;上游失败必须退点。`CallRecord` 记录 `vision`、别名、实际模型、点数、耗时和结果摘要,不保存输入图片、base64、provider raw 或完整上游响应。**目录与运营配置**:`GET /api/v1/models` 和 portal 可用模型页按 T-601 既有语义列出 active、能力匹配的 `vision` 别名并显示 `requires_image=true`;有定价时返回 `priced`,缺定价时仍返回 `unpriced`,不得改变现有目录兼容行为。代码部署 / migrate 后由运营在 admin 配置支持视觉理解的 `AiModel`、`vision-standard` 默认别名和计费规则,不通过数据迁移写入真实上游配置或密钥。**范围边界**:本任务只做同步文字结果,不做流式输出、异步 vision task、OCR 专用接口、结构化 JSON schema、图片内容审核,也不为 OCR / 商品识别 / 图片对比各建别名;这些用途先由 prompt 表达,只有底层模型或价格确实不同时再增加别名。**文档**:实现时同步 `02-requirements.md`、`04-architecture.md`、`api.md`、`routes.md`、`env.md`、`current-state.md` 和 `progress.md`。**测试**:覆盖单图 / 多图 / 混合来源及顺序、默认 / 指定别名、模型与 Provider 能力拒绝、图片数量与大小限制、SSRF、敏感词先拦截、成功仅扣一次、上游失败仅退一次、调用记录不保存图片 / raw、模型目录安全字段、旧标题 / 生图接口回归;`makemigrations` / `migrate` / `check` / 目标测试 / `init` 通过并在 `../progress.md` 留证据 | DONE |
|
||||
|
||||
## 里程碑
|
||||
|
||||
|
||||
+56
-1
@@ -23,6 +23,8 @@ T-301 已实现对外 API 鉴权基线:`apps.api.authentication.ApiKeyAuthenti
|
||||
|
||||
T-302 已实现生成接口基线:`POST /api/v1/generate/title` 与 `POST /api/v1/generate/image` 已接入 API Key 鉴权、别名解析、计费规则、预扣点、Provider 调用、成功确认和失败退点;图片结果当前以本地 `MEDIA_ROOT` 保存并返回 `image_url`,后续可替换为对象存储。T-613 起内部实现已抽为生成核心 service,旧同步接口仍保持原字段、状态码和错误语义。T-614 起新增异步生图任务接口,提交阶段同步审核 prompt 和预扣点,worker 复用同一套已预扣执行 / 成功确认 / 失败退点阶段,不复制第二套资金逻辑。
|
||||
|
||||
T-619 已实现 `POST /api/v1/analyze/images`:使用独立 `vision` 操作和能力别名,同步接收一张或多张有序图片并返回完整文字。该接口复用 T-613 的准备 / 预扣 / 执行 / 成功确认 / 失败退点核心,不改变标题和生图接口。
|
||||
|
||||
T-614 已实现生图异步任务化:`POST /api/v1/generate/image/tasks` 返回 `202` 与公开 UUID `task_id`,`GET /api/v1/generate/image/tasks/{task_id}` 轮询任务状态和结果。任务表 `ImageGenerationTask` 关联 `user`、`api_key` 与已预扣 `CallRecord`,支持 `Idempotency-Key` 去重、payload 冲突 `409 idempotency_conflict`、租约 / 心跳 / reaper 处理僵尸 running 任务并幂等退点。异步结果 URL 由 `MEDIA_PUBLIC_BASE_URL` 或 `PUBLIC_BASE_URL` 生成,不透传上游临时链接。
|
||||
|
||||
T-615 已实现生图接口用量遥测:旧同步 `POST /api/v1/generate/image` 与新异步提交 `POST /api/v1/generate/image/tasks` 都会写 `cmhub.api.generation_usage` 结构化日志,事件名 `generation_route_usage`。调用方可选带 `X-Client-Version` 请求头,便于按客户端版本观察旧同步路迁移进度;该请求头不参与鉴权、计费或幂等判断。日志只记录 route/user/key 前缀/客户端版本/别名/状态/耗时/错误码等白名单字段,不记录 API Key 明文、prompt、图片 base64 或 provider raw。
|
||||
@@ -171,6 +173,45 @@ GET /api/v1/client/releases/latest?platform=windows
|
||||
|
||||
要点:别名必须映射到声明 `text` 能力的模型;映射到图片模型时返回 `model_not_allowed`;别名无计费规则返回 `no_pricing_rule`。T-604 后 prompt 命中本地敏感词时返回 `content_blocked`,不扣点、不写调用记录、不调上游。若传 `image_url`,服务端只会在 prompt 审核通过后下载公网 `http` / `https` 图片,并在扣点前拒绝内网、回环、链路本地、元数据地址、跳转到内网的地址和超过大小上限的响应;被拒绝时返回 `bad_request` 且不预扣点。
|
||||
|
||||
### `POST /api/v1/analyze/images`
|
||||
|
||||
理解一张或多张图片并同步返回文字。请求:
|
||||
|
||||
```json
|
||||
{
|
||||
"prompt": "比较这些商品图,说明款式、材质和细节差异",
|
||||
"model": "vision-standard",
|
||||
"images": [
|
||||
{"image_base64": "data:image/jpeg;base64,..."},
|
||||
{"image_url": "https://example.com/detail.png"}
|
||||
],
|
||||
"parameters": {"temperature": 0.2}
|
||||
}
|
||||
```
|
||||
|
||||
`images` 必填且保持顺序,每项必须且只能提供一个 `image_url` 或 `image_base64`,允许两种来源混合。默认限制由 `VISION_MAX_IMAGES=8`、`VISION_MAX_IMAGE_BYTES=10485760`、`VISION_MAX_TOTAL_BYTES=33554432` 控制;超限、空列表、格式错误或同项双来源返回 `400 bad_request`,不扣点、不调上游。
|
||||
|
||||
成功响应:
|
||||
|
||||
```json
|
||||
{
|
||||
"text": "第一张是商品正面图,第二张展示了领口和面料细节。",
|
||||
"alias": "vision-standard",
|
||||
"model_used": "provider-model-for-debugging",
|
||||
"points_cost": 3,
|
||||
"points_balance": 95,
|
||||
"call_id": 12346
|
||||
}
|
||||
```
|
||||
|
||||
要点:
|
||||
|
||||
- `model` 是 `vision` 操作的能力别名;缺省时使用后台配置的默认别名。别名指向的 `AiModel` 与 Provider 必须同时具备 `text` 和 `vision` 能力,否则返回 `model_not_allowed`。
|
||||
- prompt 审核先于图片下载 / 解码;命中敏感词返回 `content_blocked`。每个 URL 复用现有公网地址、逐跳重定向、超时和 SSRF 防护。第一版不包含图片内容审核。
|
||||
- 第一版按 `PricingRule(operation_type=vision, alias, resolution="")` 对一次请求固定扣点,不按图片数量重复扣费。输入校验失败不预扣;上游失败按现有账务流程幂等退点。
|
||||
- Chat Completions / Gemini Provider 按请求顺序发送多张图片;返回 `text` 保留上游完整文字,不执行标题拆分清洗。
|
||||
- 输入图片只在本次同步请求内存中使用,不保存到数据库或 media;`CallRecord` 只保存最多 500 字结果摘要,不保存 base64、provider raw 或完整上游响应。
|
||||
|
||||
### `POST /api/v1/generate/image`
|
||||
|
||||
生成图片(同步等待)。请求:
|
||||
@@ -389,6 +430,16 @@ Authorization: Bearer sk_cmhub_xxx
|
||||
"requires_image": true,
|
||||
"pricing_status": "unpriced",
|
||||
"prices": []
|
||||
},
|
||||
{
|
||||
"alias": "vision-standard",
|
||||
"operation_type": "vision",
|
||||
"capabilities": ["text", "vision"],
|
||||
"requires_image": true,
|
||||
"pricing_status": "priced",
|
||||
"prices": [
|
||||
{"resolution": "default", "points_cost": 3}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -529,7 +580,7 @@ query_and_apply_recharge_payment(order_no: str, query_func) -> RechargeResult
|
||||
resolve_alias(operation_type: str, alias: str | None) -> ResolvedModel
|
||||
```
|
||||
|
||||
T-102 后,别名解析已接入数据库表 `AiModel` / `ModelAlias`:每次调用读取当前 active 记录,缺省 `alias=None` 时取该 `operation_type` 的默认别名;标题要求 `text` 能力,图片要求 `image` 能力。密钥以 `AiModel.api_key_encrypted` 存储,使用 Fernet 解密后进入 `ResolvedModel`。
|
||||
T-102/T-619 后,别名解析已接入数据库表 `AiModel` / `ModelAlias`:每次调用读取当前 active 记录,缺省 `alias=None` 时取该 `operation_type` 的默认别名;标题要求 `text`,图片生成要求 `image`,图片理解要求同时具备 `text + vision`。密钥以 `AiModel.api_key_encrypted` 存储,使用 Fernet 解密后进入 `ResolvedModel`。
|
||||
|
||||
从 `cmbot` 形状导入配置的管理命令:
|
||||
|
||||
@@ -557,12 +608,16 @@ class Provider(Protocol):
|
||||
image_filename: str = "image.png",
|
||||
resolution: str = "1K", aspect_ratio: str = "1:1",
|
||||
parameters: dict | None = None) -> ImageGenerationResult: ...
|
||||
def analyze_images(self, prompt: str, model: ResolvedModel,
|
||||
images: Sequence[MultimodalImage],
|
||||
parameters: dict | None = None) -> TextGenerationResult: ...
|
||||
```
|
||||
|
||||
要点:
|
||||
|
||||
- `ResolvedModel` 来自数据库 AiModel(形状同 `cmbot/config/ai_models.json`,外加 `capabilities`;`api_key` 在库中以 Fernet 加密,使用时解密,不落明文)。
|
||||
- `TextGenerationResult` 包含 `text`、清洗后的 `titles`、`model_used`、`raw`;`ImageGenerationResult` 包含图片 bytes、`model_used`、`raw`。`raw` 仅供本次解析/排障摘要使用,调用记录不得整包保存或打印,避免 base64 大图和上游敏感字段入库;图片落对象存储并返回 URL 属 T-302 之后的 API 编排职责。
|
||||
- `analyze_images()` 接收有序 `MultimodalImage(data, mime_type)` 序列;Chat / Gemini 适配器翻译为各自多模态 payload,并从原始响应提取完整文字。图片生成 / 编辑专用 Provider 必须明确拒绝该操作。
|
||||
- 适配器按 `api_type`(`chat`/`gemini`/`images`/`images_edits`/`auto`)从注册表选取,新增供应商 = 新增一个适配器,不改对外接口。
|
||||
- `parameters` 为供应商特有参数的安全子集;适配器负责白名单过滤并把统一入参翻译成各家上游格式,核心字段不可被 `parameters` 或 `extra_body` 覆盖。
|
||||
- 配置热生效:每次调用读当前 AiModel/ModelAlias,后台改动及时反映(或带缓存失效)。
|
||||
|
||||
+11
-3
File diff suppressed because one or more lines are too long
+6
-1
@@ -73,16 +73,21 @@ T-614 起新增异步生图任务接口:提交任务仍同步审核 prompt 和
|
||||
|
||||
| 变量 | 必填 | 示例 | 说明 |
|
||||
| --- | --- | --- | --- |
|
||||
| `API_GENERATE_THROTTLE_RATE` | 否 | `60/min` | 生成标题 / 图片接口按 API Key 或用户限流,DRF throttle rate 格式 |
|
||||
| `API_GENERATE_THROTTLE_RATE` | 否 | `60/min` | 生成标题 / 图片 / 多图理解接口按 API Key 或用户限流,DRF throttle rate 格式 |
|
||||
| `API_AUTH_FAILURE_THROTTLE_RATE` | 否 | `30/min` | 缺失、畸形或无效 API Key 的认证失败按 IP 限流 |
|
||||
| `IMAGE_URL_MAX_BYTES` | 否 | `10485760` | `image_url` 服务端下载的最大响应字节数,默认 10 MiB |
|
||||
| `IMAGE_URL_MAX_REDIRECTS` | 否 | `3` | `image_url` 手动跟随重定向次数上限;每跳都会重新校验目标地址 |
|
||||
| `IMAGE_URL_CONNECT_TIMEOUT_SECONDS` | 否 | `10` | `image_url` 下载连接超时秒数 |
|
||||
| `IMAGE_URL_READ_TIMEOUT_SECONDS` | 否 | `60` | `image_url` 下载读取超时秒数 |
|
||||
| `VISION_MAX_IMAGES` | 否 | `8` | 多图理解接口单次最多接受的图片数量,最小按 1 处理 |
|
||||
| `VISION_MAX_IMAGE_BYTES` | 否 | `10485760` | 多图理解接口每张图片解码或下载后的最大字节数,默认 10 MiB |
|
||||
| `VISION_MAX_TOTAL_BYTES` | 否 | `33554432` | 多图理解接口单次所有图片的总字节数上限,默认 32 MiB |
|
||||
| `RECHARGE_MAX_AMOUNT_CNY` | 否 | `100000.00` | 用户端单笔充值金额上限,超过则拒绝创建订单 |
|
||||
|
||||
`image_url` 只允许 `http` / `https`,服务端会在请求前解析域名,拒绝私有网段、回环、链路本地、保留地址、组播、未指定地址;重定向后的目标地址也会重复执行同样校验。内网图片不应通过 `image_url` 传入,调用方应改用 `image_base64`。
|
||||
|
||||
多图理解接口会先审核 prompt,再按 `images` 顺序下载或解码图片;单图同时受 `VISION_MAX_IMAGE_BYTES` 限制,URL 下载不会绕过现有 SSRF、重定向和超时检查。三个 `VISION_*` 限制应按上游模型输入上限与 Web worker 内存共同校准。
|
||||
|
||||
## 六、内容安全 / 本地敏感词配置
|
||||
|
||||
| 变量 | 必填 | 示例 | 说明 |
|
||||
|
||||
@@ -25,6 +25,7 @@ T-606 已落地 `/` 公开首页:匿名访问返回 200,不再重定向到 `
|
||||
| 路由 | 方法 | 职责 | 鉴权 |
|
||||
| --- | --- | --- | --- |
|
||||
| `/api/v1/generate/title` | POST | 生成标题 | API Key |
|
||||
| `/api/v1/analyze/images` | POST | 理解一张或多张图片并同步返回文字 | API Key |
|
||||
| `/api/v1/generate/image` | POST | 生成图片(同步) | API Key |
|
||||
| `/api/v1/generate/image/tasks` | POST | 提交异步图片生成任务,返回 `task_id` | API Key |
|
||||
| `/api/v1/generate/image/tasks/{task_id}` | GET | 轮询异步图片生成任务状态和结果 | API Key |
|
||||
@@ -39,6 +40,8 @@ T-606 已落地 `/` 公开首页:匿名访问返回 200,不再重定向到 `
|
||||
T-607/T-609/T-617 已落地 `/api/v1/client/releases/latest`:公开匿名可访问,不需要 API Key,不读取用户账本,只返回当前 `DownloadRelease` 的版本、下载 URL、SHA256、发布说明、强制更新标记、文件大小字节数和发布时间;无当前版本返回 `release:null`。
|
||||
T-614 已落地 `/api/v1/generate/image/tasks` 与 `/api/v1/generate/image/tasks/{task_id}`:新版桌面端可先提交异步生图任务,再轮询状态;旧 `/api/v1/generate/image` 同步接口继续保留。异步查询只允许同一用户访问自己的任务,跨用户按不存在处理。
|
||||
|
||||
T-619 已落地 `/api/v1/analyze/images`:使用独立 `vision` 操作和能力别名,同步接收有序多图并返回文字;不复用标题接口语义,也不改变现有标题 / 生图路由。
|
||||
|
||||
## 运营后台(django-admin,`/admin/`)
|
||||
|
||||
后台用 Django Session 登录,按模型注册 Admin:
|
||||
|
||||
+37
@@ -1913,3 +1913,40 @@
|
||||
- 多图上游调用必须扩展既有 Provider 和 T-613 共享生成 core;图片 URL 必须复用现有 SSRF 防护,失败复用计费层幂等退点。
|
||||
- 本任务不包含流式、异步 vision task、OCR 专用接口或结构化 JSON 输出;OCR、商品识别和图片对比先通过 prompt 表达。
|
||||
- 验证:`./init.ps1` 通过,Python 3.12.3,依赖已满足,`manage.py check` 0 issues;任务登记完成后继续执行文档 diff 检查。
|
||||
|
||||
## 2026-07-16 开工:T-619 多张图片理解并返回文字
|
||||
|
||||
- 状态:DOING。
|
||||
- 基线:执行 `./init.ps1` 通过,Python 3.12.3,依赖已满足,`manage.py check` 0 issues。
|
||||
- 范围:按 `docs/06-tasks.md` 新增独立 `vision` 操作、多图理解同步接口、Provider 多图支持、固定单次计费、目录展示和输入资源限制;不改旧标题、生图和异步生图契约。
|
||||
- 工作区说明:开工时存在用户侧 `.gitignore` 修改、已跟踪测试脚本 / 提示词删除和临时截图,均不属于 T-619,不恢复、不覆盖、不纳入本任务。
|
||||
|
||||
## 2026-07-16 实施:T-619 多张图片理解并返回文字
|
||||
|
||||
- 状态:DONE。
|
||||
- 代码变更:
|
||||
- `ModelAlias.OperationType`、`CallRecord.OperationType` 与 `PricingRule` 可选操作新增 `vision`,生成并应用 `ai.0005` / `billing.0008` 迁移。
|
||||
- `apps.ai.aliases` 把能力要求统一为集合;`vision` 模型和 Provider 必须同时支持 `text`、`vision`,图片生成 / 编辑专用 Provider 不可误接。
|
||||
- Provider 协议新增 `analyze_images()` 与 `MultimodalImage`;Chat Completions、Gemini 按请求顺序发送多张图片并解析完整文字结果。
|
||||
- 新增 `POST /api/v1/analyze/images`,支持有序 URL/Base64 混合图片;prompt 审核先于图片读取,URL 复用现有 SSRF 防护;数量、单图和总大小受三个 `VISION_*` 环境变量限制。
|
||||
- 多图理解复用 T-613 生成核心和 billing 预扣 / 成功确认 / 幂等退款;每次请求按 `vision + alias` 默认价固定扣一次,调用记录只保留最多 500 字文字摘要,不保存输入图片或 provider raw。
|
||||
- `GET /api/v1/models` 与 portal 可用模型页支持展示能力匹配的 active `vision` 别名,并返回 / 显示 `requires_image=true` 与计费状态。
|
||||
- 文档变更:同步 `README.md`、`docs/00-ai-start-here.md`、`02-requirements.md`、`04-architecture.md`、`06-tasks.md`、`api.md`、`current-state.md`、`env.md`、`routes.md` 与 `.env.example`;T-619 标记为 DONE。
|
||||
- 验证:
|
||||
- `./init.ps1` 开工和收尾均通过;收尾为 Python 3.12.3、依赖已满足、Django system check 0 issues。
|
||||
- `py -3.12 -m py_compile ...`:T-619 相关 Python 文件通过。
|
||||
- `py -3.12 manage.py migrate --noinput`:通过,已应用 `ai.0005` 与 `billing.0008`。
|
||||
- `py -3.12 manage.py check`:通过,0 issues;`py -3.12 manage.py makemigrations --check --dry-run`:通过,No changes detected。
|
||||
- Provider 专项:10 tests OK;T-619 API / 别名 / 目录专项:13 tests OK;旧标题 / 生图关键回归:5 tests OK。
|
||||
- `apps.ai.tests + ModelsCatalogApiTests + GenerateApiTests + portal 模型页` 扩展回归:84 tests OK。
|
||||
- `apps.users.tests + apps.billing.tests + apps.moderation.tests` 分组回归:46 tests OK。
|
||||
- 完整单命令回归运行 604 秒后达到执行器超时,未取得最终结果;随后 `apps.api.tests` 全量 88 条测试重试两次,均在测试库初始化阶段因远程 MySQL `43.128.3.240` 握手超时中断,未进入断言。同期 TCP 3306 探测可达,按环境稳定性问题记录,不把该两次运行记为通过。
|
||||
- `git diff --check`:通过,仅 Windows CRLF 提示。
|
||||
- 生产启用提醒:部署代码并执行 migrate 后,需在 admin 配置同时具有 `text`、`vision` 能力的 active `AiModel`、`vision-standard` active 默认别名和 `operation_type=vision` 的默认 `PricingRule`;本任务不写入真实模型、URL、密钥或价格。
|
||||
- 工作区说明:用户侧 `.gitignore` 修改、`xiaxiuxiu.py` / `图生图提示词.txt` 删除及 `.tmp-t609-layout.png` 未跟踪文件保持原状,未作为 T-619 内容处理。
|
||||
- 数据库恢复后的补充验证:
|
||||
- 新增 `GenerateApiTests.test_analyze_images_supports_single_image_with_explicit_alias`,补齐单图 + 指定别名成功矩阵;单独运行 1 test OK。
|
||||
- API 鉴权 / 余额 / 模型目录 / 客户端版本分组:28 tests OK。
|
||||
- API 充值回调 / 下单状态分组:13 tests OK。
|
||||
- `apps.portal.tests` 全量:39 tests OK。
|
||||
- 结合此前 `GenerateApiTests` 整类 47 tests OK 与新增单测,当前 API 89 条测试均有通过证据;所有应用已通过拆分方式完成完整回归。完整单命令超时仍作为远程 MySQL / 执行器稳定性现象保留,不再构成测试覆盖缺口。
|
||||
|
||||
Reference in New Issue
Block a user