feat: add multi-image vision analysis

apps/ai/providers/__init__.py

@@ -4,6 +4,7 @@ from .base import (
     AiProviderError,
     AiResponseParseError,
     ImageGenerationResult,
+    MultimodalImage,
     Provider,
     ResolvedModel,
     TextGenerationResult,
@@ -16,6 +17,7 @@ __all__ = [
     "AiProviderError",
     "AiResponseParseError",
     "ImageGenerationResult",
+    "MultimodalImage",
     "Provider",
     "ResolvedModel",
     "TextGenerationResult",

apps/ai/providers/base.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
-from typing import Any, Mapping, Protocol
+from typing import Any, Mapping, Protocol, Sequence
 
 
 class AiProviderError(RuntimeError):
@@ -78,6 +78,12 @@ class ImageGenerationResult:
     raw: Mapping[str, Any]
 
 
+@dataclass(frozen=True)
+class MultimodalImage:
+    data: bytes
+    mime_type: str = "image/png"
+
+
 class Provider(Protocol):
     def capabilities(self) -> set[str]:
         ...
@@ -108,6 +114,16 @@ class Provider(Protocol):
     ) -> ImageGenerationResult:
         ...
 
+    def analyze_images(
+        self,
+        prompt: str,
+        model: ResolvedModel,
+        *,
+        images: Sequence[MultimodalImage],
+        parameters: Mapping[str, Any] | None = None,
+    ) -> TextGenerationResult:
+        ...
+
 
 def validate_model_config(model: ResolvedModel) -> None:
     errors = []

apps/ai/providers/openai_compatible.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Any, Mapping
+from typing import Any, Mapping, Sequence
 
 import requests
 
@@ -8,6 +8,7 @@ from .base import (
     AiCapabilityError,
     AiResponseParseError,
     ImageGenerationResult,
+    MultimodalImage,
     ResolvedModel,
     TextGenerationResult,
     validate_model_config,
@@ -18,6 +19,7 @@ from .utils import (
     API_IMAGES,
     API_IMAGES_EDITS,
     extract_image_from_response,
+    extract_raw_text,
     extract_text_from_response,
     extract_titles_from_response,
     image_request_timeout,
@@ -166,6 +168,37 @@ class ChatCompletionsProvider(BaseHttpProvider):
             raise AiResponseParseError("AI response did not contain an image")
         return ImageGenerationResult(image=image_bytes, model_used=model.model, raw=raw)
 
+    def analyze_images(
+        self,
+        prompt: str,
+        model: ResolvedModel,
+        *,
+        images: Sequence[MultimodalImage],
+        parameters: Mapping[str, Any] | None = None,
+    ) -> TextGenerationResult:
+        validate_model_config(model)
+        if not images:
+            raise AiCapabilityError("vision analysis requires at least one image")
+        url = normalize_api_url(model.url, API_CHAT)
+        payload = build_chat_vision_payload(
+            model,
+            prompt,
+            images=images,
+            parameters=parameters,
+        )
+        response = self.session.post(
+            url,
+            headers=self._headers(model, json=True),
+            json=payload,
+            timeout=self._timeout(model, "1K"),
+        )
+        response.raise_for_status()
+        raw = response.json()
+        text = extract_raw_text(raw).strip()
+        if not text:
+            raise AiResponseParseError("AI response did not contain text")
+        return TextGenerationResult(text=text, titles=(), model_used=model.model, raw=raw)
+
 
 class GeminiProvider(ChatCompletionsProvider):
     def generate_text(
@@ -241,6 +274,37 @@ class GeminiProvider(ChatCompletionsProvider):
             raise AiResponseParseError("AI response did not contain an image")
         return ImageGenerationResult(image=image_bytes, model_used=model.model, raw=raw)
 
+    def analyze_images(
+        self,
+        prompt: str,
+        model: ResolvedModel,
+        *,
+        images: Sequence[MultimodalImage],
+        parameters: Mapping[str, Any] | None = None,
+    ) -> TextGenerationResult:
+        validate_model_config(model)
+        if not images:
+            raise AiCapabilityError("vision analysis requires at least one image")
+        url = normalize_api_url(model.url, API_GEMINI).replace("{model}", model.model)
+        payload = build_gemini_vision_payload(
+            model,
+            prompt,
+            images=images,
+            parameters=parameters,
+        )
+        response = self.session.post(
+            url,
+            headers=self._headers(model, json=True),
+            json=payload,
+            timeout=self._timeout(model, "1K"),
+        )
+        response.raise_for_status()
+        raw = response.json()
+        text = extract_raw_text(raw).strip()
+        if not text:
+            raise AiResponseParseError("AI response did not contain text")
+        return TextGenerationResult(text=text, titles=(), model_used=model.model, raw=raw)
+
 
 class ImagesGenerationProvider(BaseHttpProvider):
     def capabilities(self) -> set[str]:
@@ -249,6 +313,9 @@ class ImagesGenerationProvider(BaseHttpProvider):
     def generate_text(self, *args: Any, **kwargs: Any) -> TextGenerationResult:
         raise AiCapabilityError("images generation provider cannot generate text")
 
+    def analyze_images(self, *args: Any, **kwargs: Any) -> TextGenerationResult:
+        raise AiCapabilityError("images generation provider cannot analyze images")
+
     def generate_image(
         self,
         prompt: str,
@@ -298,6 +365,9 @@ class ImagesEditsProvider(BaseHttpProvider):
     def generate_text(self, *args: Any, **kwargs: Any) -> TextGenerationResult:
         raise AiCapabilityError("images edits provider cannot generate text")
 
+    def analyze_images(self, *args: Any, **kwargs: Any) -> TextGenerationResult:
+        raise AiCapabilityError("images edits provider cannot analyze images")
+
     def generate_image(
         self,
         prompt: str,
@@ -384,6 +454,32 @@ def build_chat_image_payload(
     )
 
 
+def build_chat_vision_payload(
+    model: ResolvedModel,
+    prompt: str,
+    *,
+    images: Sequence[MultimodalImage],
+    parameters: Mapping[str, Any] | None = None,
+) -> dict[str, Any]:
+    content: list[dict[str, Any]] = [{"type": "text", "text": prompt}]
+    content.extend(
+        {
+            "type": "image_url",
+            "image_url": {
+                "url": image_bytes_to_data_url(image.data, image.mime_type),
+            },
+        }
+        for image in images
+    )
+    payload: dict[str, Any] = {
+        "model": model.model,
+        "messages": [{"role": "user", "content": content}],
+        "stream": False,
+    }
+    apply_extra_body(payload, model, parameters)
+    return payload
+
+
 def build_gemini_payload(
     model: ResolvedModel,
     prompt: str,
@@ -406,6 +502,26 @@ def build_gemini_payload(
     return payload
 
 
+def build_gemini_vision_payload(
+    model: ResolvedModel,
+    prompt: str,
+    *,
+    images: Sequence[MultimodalImage],
+    parameters: Mapping[str, Any] | None = None,
+) -> dict[str, Any]:
+    parts: list[dict[str, Any]] = [{"text": prompt}]
+    for image in images:
+        data_url = image_bytes_to_data_url(image.data, image.mime_type)
+        mime_type, data = split_data_url(data_url)
+        parts.append({"inlineData": {"mimeType": mime_type, "data": data}})
+    payload: dict[str, Any] = {
+        "contents": [{"parts": parts}],
+        "generationConfig": {"responseModalities": ["TEXT"]},
+    }
+    apply_extra_body(payload, model, parameters)
+    return payload
+
+
 def apply_extra_body(
     payload: dict[str, Any],
     model: ResolvedModel,