feat: retry async image tasks
This commit is contained in:
QiuSW
@@ -41,6 +41,8 @@ IMAGE_TASK_RETENTION_HOURS=24
|
||||
GENERATED_IMAGE_RETENTION_HOURS=72
|
||||
IMAGE_TASK_REAPER_INTERVAL_SECONDS=60
|
||||
IMAGE_TASK_LEASE_SECONDS=600
|
||||
IMAGE_TASK_MAX_RETRIES=2
|
||||
IMAGE_TASK_RETRY_BACKOFF_SECONDS=10,30
|
||||
|
||||
# API safety
|
||||
API_GENERATE_THROTTLE_RATE=60/min
|
||||
|
||||
+3
-1
@@ -10,11 +10,12 @@ class ImageGenerationTaskAdmin(admin.ModelAdmin):
|
||||
"user",
|
||||
"status",
|
||||
"attempt_count",
|
||||
"next_attempt_at",
|
||||
"points_balance_after_charge",
|
||||
"created_at",
|
||||
"finished_at",
|
||||
)
|
||||
list_filter = ("status", "created_at", "finished_at")
|
||||
list_filter = ("status", "created_at", "next_attempt_at", "finished_at")
|
||||
search_fields = (
|
||||
"task_id",
|
||||
"user__username",
|
||||
@@ -37,6 +38,7 @@ class ImageGenerationTaskAdmin(admin.ModelAdmin):
|
||||
"started_at",
|
||||
"finished_at",
|
||||
"expires_at",
|
||||
"next_attempt_at",
|
||||
"locked_at",
|
||||
"lease_expires_at",
|
||||
"heartbeat_at",
|
||||
|
||||
@@ -246,6 +246,7 @@ def execute_precharged_generation(
|
||||
precharged: PrechargedGeneration,
|
||||
*,
|
||||
image_url_builder: ImageUrlBuilder | None = None,
|
||||
refund_on_failure: bool = True,
|
||||
) -> GenerationResult:
|
||||
prepared = precharged.prepared
|
||||
try:
|
||||
@@ -259,6 +260,7 @@ def execute_precharged_generation(
|
||||
image_url_builder=image_url_builder,
|
||||
)
|
||||
except AiCapabilityError as exc:
|
||||
if refund_on_failure:
|
||||
refund_call_points(
|
||||
precharged.call_record,
|
||||
error_message=str(exc),
|
||||
@@ -270,6 +272,7 @@ def execute_precharged_generation(
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
) from exc
|
||||
except Exception as exc:
|
||||
if refund_on_failure:
|
||||
refund_call_points(
|
||||
precharged.call_record,
|
||||
error_message=str(exc),
|
||||
|
||||
+149
-10
@@ -31,6 +31,7 @@ from .models import ImageGenerationTask
|
||||
IDEMPOTENCY_KEY_MAX_LENGTH = 128
|
||||
TASK_FAILURE_MESSAGE = "图片生成失败，已退回点数"
|
||||
TASK_TIMEOUT_MESSAGE = "图片生成任务超时，已退回点数"
|
||||
RETRYABLE_TASK_ERROR_CODES = {"upstream_timeout", "upstream_error"}
|
||||
|
||||
|
||||
def create_image_generation_task(
|
||||
@@ -207,6 +208,10 @@ def claim_next_image_task(worker_id: str | None = None) -> ImageGenerationTask |
|
||||
queryset = (
|
||||
ImageGenerationTask.objects.select_related("user", "api_key", "call_record")
|
||||
.filter(status=ImageGenerationTask.Status.QUEUED)
|
||||
.filter(
|
||||
models_q("next_attempt_at__isnull", True)
|
||||
| models_q("next_attempt_at__lte", now)
|
||||
)
|
||||
.order_by("created_at", "id")
|
||||
)
|
||||
if connection.features.has_select_for_update_skip_locked:
|
||||
@@ -224,6 +229,7 @@ def claim_next_image_task(worker_id: str | None = None) -> ImageGenerationTask |
|
||||
task.lease_expires_at = lease_expires_at
|
||||
task.heartbeat_at = now
|
||||
task.started_at = task.started_at or now
|
||||
task.next_attempt_at = None
|
||||
task.attempt_count += 1
|
||||
task.save(
|
||||
update_fields=(
|
||||
@@ -233,6 +239,7 @@ def claim_next_image_task(worker_id: str | None = None) -> ImageGenerationTask |
|
||||
"lease_expires_at",
|
||||
"heartbeat_at",
|
||||
"started_at",
|
||||
"next_attempt_at",
|
||||
"attempt_count",
|
||||
"updated_at",
|
||||
)
|
||||
@@ -261,24 +268,56 @@ def run_image_generation_task(
|
||||
|
||||
try:
|
||||
precharged = precharged_generation_for_task(task)
|
||||
result = execute_precharged_generation(
|
||||
precharged,
|
||||
image_url_builder=media_public_url_builder,
|
||||
)
|
||||
except ApiRequestError as exc:
|
||||
refund_task_call(task, exc.message)
|
||||
return mark_task_failed_if_running(task.pk, exc.code, exc.message)
|
||||
|
||||
try:
|
||||
result = execute_precharged_generation(
|
||||
precharged,
|
||||
image_url_builder=media_public_url_builder,
|
||||
refund_on_failure=False,
|
||||
)
|
||||
except ApiRequestError as exc:
|
||||
return handle_task_generation_failure(
|
||||
task,
|
||||
error_code=exc.code,
|
||||
error_message=exc.message,
|
||||
retryable=is_retryable_task_error(exc.code),
|
||||
)
|
||||
except Exception as exc:
|
||||
refund_task_call(task, str(exc))
|
||||
return mark_task_failed_if_running(
|
||||
task.pk,
|
||||
"upstream_error",
|
||||
TASK_FAILURE_MESSAGE,
|
||||
return handle_task_generation_failure(
|
||||
task,
|
||||
error_code="upstream_error",
|
||||
error_message=str(exc) or TASK_FAILURE_MESSAGE,
|
||||
retryable=True,
|
||||
)
|
||||
|
||||
return mark_task_succeeded_if_running(task.pk, result.image_url)
|
||||
|
||||
|
||||
def handle_task_generation_failure(
|
||||
task: ImageGenerationTask,
|
||||
*,
|
||||
error_code: str,
|
||||
error_message: str,
|
||||
retryable: bool,
|
||||
) -> ImageGenerationTask:
|
||||
normalized_code = str(error_code or "upstream_error")
|
||||
normalized_message = str(error_message or TASK_FAILURE_MESSAGE)
|
||||
if retryable and task.attempt_count < image_task_max_attempts():
|
||||
return mark_task_retry_if_running(
|
||||
task.pk,
|
||||
normalized_code,
|
||||
retry_error_message(normalized_code, normalized_message),
|
||||
next_attempt_at=timezone.now()
|
||||
+ timedelta(seconds=image_task_retry_backoff_seconds(task.attempt_count)),
|
||||
)
|
||||
|
||||
refund_task_call(task, normalized_message)
|
||||
return mark_task_failed_if_running(task.pk, normalized_code, normalized_message)
|
||||
|
||||
|
||||
def precharged_generation_for_task(task: ImageGenerationTask) -> PrechargedGeneration:
|
||||
payload = dict(task.request_payload or {})
|
||||
prepared = prepare_generation(
|
||||
@@ -344,6 +383,7 @@ def mark_task_succeeded_if_running(
|
||||
task.result_url = str(result_url or "")
|
||||
task.error_code = ""
|
||||
task.error_message = ""
|
||||
task.next_attempt_at = None
|
||||
task.finished_at = now
|
||||
task.heartbeat_at = now
|
||||
task.save(
|
||||
@@ -352,6 +392,7 @@ def mark_task_succeeded_if_running(
|
||||
"result_url",
|
||||
"error_code",
|
||||
"error_message",
|
||||
"next_attempt_at",
|
||||
"finished_at",
|
||||
"heartbeat_at",
|
||||
"updated_at",
|
||||
@@ -360,6 +401,41 @@ def mark_task_succeeded_if_running(
|
||||
return task
|
||||
|
||||
|
||||
def mark_task_retry_if_running(
|
||||
task_pk: int,
|
||||
error_code: str,
|
||||
error_message: str,
|
||||
*,
|
||||
next_attempt_at,
|
||||
) -> ImageGenerationTask:
|
||||
with transaction.atomic():
|
||||
task = ImageGenerationTask.objects.select_for_update().get(pk=task_pk)
|
||||
if task.status != ImageGenerationTask.Status.RUNNING:
|
||||
return task
|
||||
task.status = ImageGenerationTask.Status.QUEUED
|
||||
task.error_code = str(error_code or "upstream_error")
|
||||
task.error_message = str(error_message or TASK_FAILURE_MESSAGE)
|
||||
task.next_attempt_at = next_attempt_at
|
||||
task.worker_id = ""
|
||||
task.locked_at = None
|
||||
task.lease_expires_at = None
|
||||
task.heartbeat_at = None
|
||||
task.save(
|
||||
update_fields=(
|
||||
"status",
|
||||
"error_code",
|
||||
"error_message",
|
||||
"next_attempt_at",
|
||||
"worker_id",
|
||||
"locked_at",
|
||||
"lease_expires_at",
|
||||
"heartbeat_at",
|
||||
"updated_at",
|
||||
)
|
||||
)
|
||||
return task
|
||||
|
||||
|
||||
def mark_task_failed_if_running(
|
||||
task_pk: int,
|
||||
error_code: str,
|
||||
@@ -373,6 +449,7 @@ def mark_task_failed_if_running(
|
||||
task.status = ImageGenerationTask.Status.FAILED
|
||||
task.error_code = str(error_code or "upstream_error")
|
||||
task.error_message = str(error_message or TASK_FAILURE_MESSAGE)
|
||||
task.next_attempt_at = None
|
||||
task.finished_at = now
|
||||
task.heartbeat_at = now
|
||||
task.save(
|
||||
@@ -380,6 +457,7 @@ def mark_task_failed_if_running(
|
||||
"status",
|
||||
"error_code",
|
||||
"error_message",
|
||||
"next_attempt_at",
|
||||
"finished_at",
|
||||
"heartbeat_at",
|
||||
"updated_at",
|
||||
@@ -422,6 +500,7 @@ def reap_stale_image_task(task_pk: int, now) -> bool:
|
||||
task.result_url = task.call_record.result_ref
|
||||
task.error_code = ""
|
||||
task.error_message = ""
|
||||
task.next_attempt_at = None
|
||||
task.finished_at = now
|
||||
task.save(
|
||||
update_fields=(
|
||||
@@ -429,6 +508,7 @@ def reap_stale_image_task(task_pk: int, now) -> bool:
|
||||
"result_url",
|
||||
"error_code",
|
||||
"error_message",
|
||||
"next_attempt_at",
|
||||
"finished_at",
|
||||
"updated_at",
|
||||
)
|
||||
@@ -446,14 +526,24 @@ def reap_stale_image_task(task_pk: int, now) -> bool:
|
||||
if task.call_record.status == CallRecord.Status.SUCCESS:
|
||||
task.status = ImageGenerationTask.Status.SUCCEEDED
|
||||
task.result_url = task.call_record.result_ref
|
||||
task.next_attempt_at = None
|
||||
task.finished_at = now
|
||||
task.save(update_fields=("status", "result_url", "finished_at", "updated_at"))
|
||||
task.save(
|
||||
update_fields=(
|
||||
"status",
|
||||
"result_url",
|
||||
"next_attempt_at",
|
||||
"finished_at",
|
||||
"updated_at",
|
||||
)
|
||||
)
|
||||
return True
|
||||
raise
|
||||
|
||||
task.status = ImageGenerationTask.Status.FAILED
|
||||
task.error_code = "task_timeout"
|
||||
task.error_message = TASK_TIMEOUT_MESSAGE
|
||||
task.next_attempt_at = None
|
||||
task.finished_at = now
|
||||
task.heartbeat_at = now
|
||||
task.save(
|
||||
@@ -461,6 +551,7 @@ def reap_stale_image_task(task_pk: int, now) -> bool:
|
||||
"status",
|
||||
"error_code",
|
||||
"error_message",
|
||||
"next_attempt_at",
|
||||
"finished_at",
|
||||
"heartbeat_at",
|
||||
"updated_at",
|
||||
@@ -510,6 +601,9 @@ def task_submit_response(task: ImageGenerationTask) -> dict[str, Any]:
|
||||
"call_id": call_record.id,
|
||||
"points_cost": call_record.points_cost,
|
||||
"points_balance": task.points_balance_after_charge,
|
||||
"attempt_count": task.attempt_count,
|
||||
"max_attempts": image_task_max_attempts(),
|
||||
"next_attempt_at": task.next_attempt_at.isoformat() if task.next_attempt_at else None,
|
||||
"created_at": task.created_at.isoformat(),
|
||||
"expires_at": task.expires_at.isoformat() if task.expires_at else None,
|
||||
}
|
||||
@@ -522,6 +616,9 @@ def task_detail_response(task: ImageGenerationTask) -> dict[str, Any]:
|
||||
"status": task.status,
|
||||
"call_id": call_record.id,
|
||||
"points_cost": call_record.points_cost,
|
||||
"attempt_count": task.attempt_count,
|
||||
"max_attempts": image_task_max_attempts(),
|
||||
"next_attempt_at": task.next_attempt_at.isoformat() if task.next_attempt_at else None,
|
||||
"created_at": task.created_at.isoformat(),
|
||||
"updated_at": task.updated_at.isoformat(),
|
||||
"expires_at": task.expires_at.isoformat() if task.expires_at else None,
|
||||
@@ -553,6 +650,48 @@ def image_task_lease_seconds() -> int:
|
||||
return max(1, int(getattr(settings, "IMAGE_TASK_LEASE_SECONDS", 600)))
|
||||
|
||||
|
||||
def image_task_max_retries() -> int:
|
||||
return max(0, int(getattr(settings, "IMAGE_TASK_MAX_RETRIES", 2)))
|
||||
|
||||
|
||||
def image_task_max_attempts() -> int:
|
||||
return 1 + image_task_max_retries()
|
||||
|
||||
|
||||
def image_task_retry_backoff_seconds(attempt_count: int) -> int:
|
||||
values = image_task_retry_backoff_values()
|
||||
if not values:
|
||||
return 0
|
||||
index = max(0, int(attempt_count or 1) - 1)
|
||||
return values[min(index, len(values) - 1)]
|
||||
|
||||
|
||||
def image_task_retry_backoff_values() -> list[int]:
|
||||
raw = str(getattr(settings, "IMAGE_TASK_RETRY_BACKOFF_SECONDS", "10,30") or "")
|
||||
values: list[int] = []
|
||||
for part in raw.split(","):
|
||||
item = part.strip()
|
||||
if not item:
|
||||
continue
|
||||
try:
|
||||
values.append(max(0, int(item)))
|
||||
except ValueError:
|
||||
continue
|
||||
return values
|
||||
|
||||
|
||||
def is_retryable_task_error(error_code: str) -> bool:
|
||||
return str(error_code or "") in RETRYABLE_TASK_ERROR_CODES
|
||||
|
||||
|
||||
def retry_error_message(error_code: str, fallback: str) -> str:
|
||||
if error_code == "upstream_timeout":
|
||||
return "上游 AI 调用超时，稍后自动重试"
|
||||
if error_code == "upstream_error":
|
||||
return "上游 AI 调用失败，稍后自动重试"
|
||||
return fallback
|
||||
|
||||
|
||||
def normalize_worker_id(worker_id: str | None) -> str:
|
||||
normalized = str(worker_id or "").strip()
|
||||
if normalized:
|
||||
|
||||
@@ -4,8 +4,12 @@ import uuid
|
||||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from apps.api.image_tasks import (
|
||||
image_task_max_attempts,
|
||||
reap_stale_image_tasks,
|
||||
run_one_image_task,
|
||||
)
|
||||
from apps.api.models import ImageGenerationTask
|
||||
from apps.api.image_tasks import reap_stale_image_tasks, run_one_image_task
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
@@ -70,13 +74,25 @@ class Command(BaseCommand):
|
||||
def format_task_log_line(task: ImageGenerationTask, *, started_at: float) -> str:
|
||||
duration_ms = max(0, int((time.monotonic() - started_at) * 1000))
|
||||
alias = str((task.request_payload or {}).get("model") or "")
|
||||
retrying = bool(
|
||||
task.status == ImageGenerationTask.Status.QUEUED
|
||||
and task.error_code
|
||||
and task.next_attempt_at
|
||||
)
|
||||
fields = {
|
||||
"event": "image_task_processed",
|
||||
"task_id": str(task.task_id),
|
||||
"status": task.status,
|
||||
"alias": alias,
|
||||
"attempt": str(task.attempt_count),
|
||||
"max_attempts": str(image_task_max_attempts()),
|
||||
"retrying": str(retrying).lower(),
|
||||
"next_attempt_at": task.next_attempt_at.isoformat() if task.next_attempt_at else "",
|
||||
"duration_ms": str(duration_ms),
|
||||
}
|
||||
if task.status in {ImageGenerationTask.Status.FAILED, ImageGenerationTask.Status.EXPIRED}:
|
||||
if retrying or task.status in {
|
||||
ImageGenerationTask.Status.FAILED,
|
||||
ImageGenerationTask.Status.EXPIRED,
|
||||
}:
|
||||
fields["error_code"] = task.error_code or "upstream_error"
|
||||
return " ".join(f"{key}={value}" for key, value in fields.items())
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
# Generated by Django 5.2.15 on 2026-07-09 06:28
|
||||
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('api', '0001_initial'),
|
||||
('billing', '0007_alter_pointsledger_change_type_signupbonusgrant'),
|
||||
('users', '0005_alter_apikey_created_at_alter_apikey_key_hash_and_more'),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='imagegenerationtask',
|
||||
name='next_attempt_at',
|
||||
field=models.DateTimeField(blank=True, null=True, verbose_name='下次重试时间'),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name='imagegenerationtask',
|
||||
index=models.Index(fields=['status', 'next_attempt_at'], name='image_gener_status_e2e2f3_idx'),
|
||||
),
|
||||
]
|
||||
@@ -62,6 +62,7 @@ class ImageGenerationTask(models.Model):
|
||||
started_at = models.DateTimeField("开始时间", null=True, blank=True)
|
||||
finished_at = models.DateTimeField("完成时间", null=True, blank=True)
|
||||
expires_at = models.DateTimeField("任务元数据过期时间", null=True, blank=True)
|
||||
next_attempt_at = models.DateTimeField("下次重试时间", null=True, blank=True)
|
||||
locked_at = models.DateTimeField("锁定时间", null=True, blank=True)
|
||||
lease_expires_at = models.DateTimeField("租约过期时间", null=True, blank=True)
|
||||
heartbeat_at = models.DateTimeField("心跳时间", null=True, blank=True)
|
||||
@@ -89,6 +90,7 @@ class ImageGenerationTask(models.Model):
|
||||
models.Index(fields=("user", "created_at")),
|
||||
models.Index(fields=("api_key", "created_at")),
|
||||
models.Index(fields=("status", "created_at")),
|
||||
models.Index(fields=("status", "next_attempt_at")),
|
||||
models.Index(fields=("status", "lease_expires_at")),
|
||||
models.Index(fields=("worker_id", "status")),
|
||||
models.Index(fields=("expires_at",)),
|
||||
|
||||
@@ -1507,6 +1507,9 @@ class GenerateApiTests(TestCase):
|
||||
self.assertEqual(response.status_code, 202)
|
||||
self.assertEqual(response.data["status"], ImageGenerationTask.Status.QUEUED)
|
||||
self.assertEqual(response.data["points_balance"], 90)
|
||||
self.assertEqual(response.data["attempt_count"], 0)
|
||||
self.assertEqual(response.data["max_attempts"], 3)
|
||||
self.assertIsNone(response.data["next_attempt_at"])
|
||||
self.assertEqual(self.provider.image_calls, [])
|
||||
|
||||
with patch("apps.api.generation.get_provider", return_value=self.provider):
|
||||
@@ -1529,6 +1532,9 @@ class GenerateApiTests(TestCase):
|
||||
|
||||
self.assertEqual(poll.status_code, 200)
|
||||
self.assertEqual(poll.data["status"], ImageGenerationTask.Status.SUCCEEDED)
|
||||
self.assertEqual(poll.data["attempt_count"], 1)
|
||||
self.assertEqual(poll.data["max_attempts"], 3)
|
||||
self.assertIsNone(poll.data["next_attempt_at"])
|
||||
self.assertEqual(poll.data["result"]["image_url"], task.result_url)
|
||||
self.assertEqual(repeat.data["result"]["image_url"], task.result_url)
|
||||
|
||||
@@ -1552,6 +1558,7 @@ class GenerateApiTests(TestCase):
|
||||
self.assertEqual(denied.status_code, 404)
|
||||
self.assertEqual(denied.data["error"]["code"], "task_not_found")
|
||||
|
||||
@override_settings(IMAGE_TASK_MAX_RETRIES=0)
|
||||
def test_async_image_worker_failure_refunds_precharged_points(self):
|
||||
self.provider.image_error = requests.Timeout("image upstream deadline exceeded")
|
||||
response = self.post_with_provider(
|
||||
@@ -1586,7 +1593,199 @@ class GenerateApiTests(TestCase):
|
||||
)
|
||||
self.assertEqual(poll.data["status"], ImageGenerationTask.Status.FAILED)
|
||||
self.assertEqual(poll.data["error"]["code"], "upstream_timeout")
|
||||
self.assertEqual(poll.data["attempt_count"], 1)
|
||||
self.assertEqual(poll.data["max_attempts"], 1)
|
||||
self.assertIsNone(poll.data["next_attempt_at"])
|
||||
|
||||
@override_settings(IMAGE_TASK_RETRY_BACKOFF_SECONDS="60,120")
|
||||
def test_async_image_retryable_timeout_requeues_without_refund_and_respects_backoff(self):
|
||||
self.provider.image_error = requests.Timeout("image upstream deadline exceeded")
|
||||
response = self.post_with_provider(
|
||||
"/api/v1/generate/image/tasks",
|
||||
{"prompt": "生成图片", "model": self.image_alias, "resolution": "1K"},
|
||||
)
|
||||
|
||||
with patch("apps.api.generation.get_provider", return_value=self.provider):
|
||||
task = run_image_generation_task(
|
||||
claim_next_image_task("worker-retry"),
|
||||
worker_id="worker-retry",
|
||||
)
|
||||
|
||||
self.assertEqual(task.status, ImageGenerationTask.Status.QUEUED)
|
||||
self.assertEqual(task.attempt_count, 1)
|
||||
self.assertEqual(task.error_code, "upstream_timeout")
|
||||
self.assertEqual(task.error_message, "上游 AI 调用超时，稍后自动重试")
|
||||
self.assertIsNotNone(task.next_attempt_at)
|
||||
self.assertGreater(task.next_attempt_at, timezone.now())
|
||||
self.assertIsNone(claim_next_image_task("worker-too-soon"))
|
||||
self.wallet.refresh_from_db()
|
||||
self.assertEqual(self.wallet.points_balance, 90)
|
||||
|
||||
call = CallRecord.objects.get(pk=response.data["call_id"])
|
||||
self.assertEqual(call.status, CallRecord.Status.PENDING)
|
||||
self.assertEqual(
|
||||
PointsLedger.objects.filter(
|
||||
ref_call=call,
|
||||
change_type=PointsLedger.ChangeType.REFUND,
|
||||
).count(),
|
||||
0,
|
||||
)
|
||||
|
||||
poll = self.client.get(
|
||||
f"/api/v1/generate/image/tasks/{response.data['task_id']}",
|
||||
**self.auth_header(),
|
||||
)
|
||||
self.assertEqual(poll.status_code, 200)
|
||||
self.assertEqual(poll.data["status"], ImageGenerationTask.Status.QUEUED)
|
||||
self.assertEqual(poll.data["attempt_count"], 1)
|
||||
self.assertEqual(poll.data["max_attempts"], 3)
|
||||
self.assertIsNotNone(poll.data["next_attempt_at"])
|
||||
|
||||
@override_settings(
|
||||
MEDIA_PUBLIC_BASE_URL="https://cm.example.test",
|
||||
IMAGE_TASK_RETRY_BACKOFF_SECONDS="0,0",
|
||||
)
|
||||
def test_async_image_retryable_timeouts_then_success_charges_once(self):
|
||||
response = self.post_with_provider(
|
||||
"/api/v1/generate/image/tasks",
|
||||
{"prompt": "生成图片", "model": self.image_alias, "resolution": "1K"},
|
||||
)
|
||||
|
||||
with patch("apps.api.generation.get_provider", return_value=self.provider):
|
||||
self.provider.image_error = requests.Timeout("first timeout")
|
||||
first = run_image_generation_task(
|
||||
claim_next_image_task("worker-retry-1"),
|
||||
worker_id="worker-retry-1",
|
||||
)
|
||||
ImageGenerationTask.objects.filter(pk=first.pk).update(
|
||||
next_attempt_at=timezone.now() - timedelta(seconds=1)
|
||||
)
|
||||
|
||||
self.provider.image_error = requests.Timeout("second timeout")
|
||||
second = run_image_generation_task(
|
||||
claim_next_image_task("worker-retry-2"),
|
||||
worker_id="worker-retry-2",
|
||||
)
|
||||
ImageGenerationTask.objects.filter(pk=second.pk).update(
|
||||
next_attempt_at=timezone.now() - timedelta(seconds=1)
|
||||
)
|
||||
|
||||
self.provider.image_error = None
|
||||
succeeded = run_image_generation_task(
|
||||
claim_next_image_task("worker-retry-3"),
|
||||
worker_id="worker-retry-3",
|
||||
)
|
||||
|
||||
self.assertEqual(first.status, ImageGenerationTask.Status.QUEUED)
|
||||
self.assertEqual(second.status, ImageGenerationTask.Status.QUEUED)
|
||||
self.assertEqual(succeeded.status, ImageGenerationTask.Status.SUCCEEDED)
|
||||
self.assertEqual(succeeded.attempt_count, 3)
|
||||
self.assertTrue(succeeded.result_url.startswith("https://cm.example.test/media/"))
|
||||
self.assertEqual(len(self.provider.image_calls), 3)
|
||||
self.wallet.refresh_from_db()
|
||||
self.assertEqual(self.wallet.points_balance, 90)
|
||||
|
||||
call = CallRecord.objects.get(pk=response.data["call_id"])
|
||||
self.assertEqual(call.status, CallRecord.Status.SUCCESS)
|
||||
self.assertEqual(
|
||||
PointsLedger.objects.filter(
|
||||
ref_call=call,
|
||||
change_type=PointsLedger.ChangeType.CONSUME,
|
||||
).count(),
|
||||
1,
|
||||
)
|
||||
self.assertEqual(
|
||||
PointsLedger.objects.filter(
|
||||
ref_call=call,
|
||||
change_type=PointsLedger.ChangeType.REFUND,
|
||||
).count(),
|
||||
0,
|
||||
)
|
||||
|
||||
@override_settings(IMAGE_TASK_RETRY_BACKOFF_SECONDS="0,0")
|
||||
def test_async_image_retryable_timeouts_final_failure_refunds_once(self):
|
||||
self.provider.image_error = requests.Timeout("image upstream deadline exceeded")
|
||||
response = self.post_with_provider(
|
||||
"/api/v1/generate/image/tasks",
|
||||
{"prompt": "生成图片", "model": self.image_alias, "resolution": "1K"},
|
||||
)
|
||||
|
||||
with patch("apps.api.generation.get_provider", return_value=self.provider):
|
||||
first = run_image_generation_task(
|
||||
claim_next_image_task("worker-final-1"),
|
||||
worker_id="worker-final-1",
|
||||
)
|
||||
ImageGenerationTask.objects.filter(pk=first.pk).update(
|
||||
next_attempt_at=timezone.now() - timedelta(seconds=1)
|
||||
)
|
||||
second = run_image_generation_task(
|
||||
claim_next_image_task("worker-final-2"),
|
||||
worker_id="worker-final-2",
|
||||
)
|
||||
ImageGenerationTask.objects.filter(pk=second.pk).update(
|
||||
next_attempt_at=timezone.now() - timedelta(seconds=1)
|
||||
)
|
||||
failed = run_image_generation_task(
|
||||
claim_next_image_task("worker-final-3"),
|
||||
worker_id="worker-final-3",
|
||||
)
|
||||
|
||||
self.assertEqual(failed.status, ImageGenerationTask.Status.FAILED)
|
||||
self.assertEqual(failed.error_code, "upstream_timeout")
|
||||
self.assertEqual(failed.attempt_count, 3)
|
||||
self.assertIsNone(failed.next_attempt_at)
|
||||
self.assertEqual(len(self.provider.image_calls), 3)
|
||||
self.wallet.refresh_from_db()
|
||||
self.assertEqual(self.wallet.points_balance, 100)
|
||||
|
||||
call = CallRecord.objects.get(pk=response.data["call_id"])
|
||||
self.assertEqual(call.status, CallRecord.Status.FAILED)
|
||||
self.assertEqual(
|
||||
PointsLedger.objects.filter(
|
||||
ref_call=call,
|
||||
change_type=PointsLedger.ChangeType.CONSUME,
|
||||
).count(),
|
||||
1,
|
||||
)
|
||||
self.assertEqual(
|
||||
PointsLedger.objects.filter(
|
||||
ref_call=call,
|
||||
change_type=PointsLedger.ChangeType.REFUND,
|
||||
).count(),
|
||||
1,
|
||||
)
|
||||
|
||||
def test_async_image_non_retryable_provider_error_fails_immediately_and_refunds(self):
|
||||
self.provider.image_error = AiCapabilityError("input image is required")
|
||||
response = self.post_with_provider(
|
||||
"/api/v1/generate/image/tasks",
|
||||
{"prompt": "生成图片", "model": self.image_alias, "resolution": "1K"},
|
||||
)
|
||||
|
||||
with patch("apps.api.generation.get_provider", return_value=self.provider):
|
||||
failed = run_image_generation_task(
|
||||
claim_next_image_task("worker-no-retry"),
|
||||
worker_id="worker-no-retry",
|
||||
)
|
||||
|
||||
self.assertEqual(failed.status, ImageGenerationTask.Status.FAILED)
|
||||
self.assertEqual(failed.error_code, "bad_request")
|
||||
self.assertEqual(failed.attempt_count, 1)
|
||||
self.assertIsNone(failed.next_attempt_at)
|
||||
self.assertEqual(len(self.provider.image_calls), 1)
|
||||
self.wallet.refresh_from_db()
|
||||
self.assertEqual(self.wallet.points_balance, 100)
|
||||
call = CallRecord.objects.get(pk=response.data["call_id"])
|
||||
self.assertEqual(call.status, CallRecord.Status.FAILED)
|
||||
self.assertEqual(
|
||||
PointsLedger.objects.filter(
|
||||
ref_call=call,
|
||||
change_type=PointsLedger.ChangeType.REFUND,
|
||||
).count(),
|
||||
1,
|
||||
)
|
||||
|
||||
@override_settings(IMAGE_TASK_MAX_RETRIES=0)
|
||||
def test_run_image_tasks_logs_failed_task_alias_error_and_duration(self):
|
||||
self.provider.image_error = requests.Timeout("image upstream deadline exceeded")
|
||||
response = self.post_with_provider(
|
||||
@@ -1611,10 +1810,47 @@ class GenerateApiTests(TestCase):
|
||||
self.assertIn(f"task_id={task.task_id}", output)
|
||||
self.assertIn(f"alias={self.image_alias}", output)
|
||||
self.assertIn("status=failed", output)
|
||||
self.assertIn("attempt=1", output)
|
||||
self.assertIn("max_attempts=1", output)
|
||||
self.assertIn("retrying=false", output)
|
||||
self.assertIn("error_code=upstream_timeout", output)
|
||||
self.assertRegex(output, r"duration_ms=\d+")
|
||||
self.assertNotIn("生成图片", output)
|
||||
|
||||
@override_settings(IMAGE_TASK_RETRY_BACKOFF_SECONDS="60,120")
|
||||
def test_run_image_tasks_logs_retrying_task_attempt_fields_without_sensitive_data(self):
|
||||
self.provider.image_error = requests.Timeout("image upstream deadline exceeded")
|
||||
response = self.post_with_provider(
|
||||
"/api/v1/generate/image/tasks",
|
||||
{"prompt": "生成图片", "model": self.image_alias, "resolution": "1K"},
|
||||
)
|
||||
out = io.StringIO()
|
||||
|
||||
with patch("apps.api.generation.get_provider", return_value=self.provider):
|
||||
call_command(
|
||||
"run_image_tasks",
|
||||
"--once",
|
||||
"--worker-id",
|
||||
"worker-log-retry",
|
||||
stdout=out,
|
||||
)
|
||||
|
||||
task = ImageGenerationTask.objects.get(task_id=response.data["task_id"])
|
||||
output = out.getvalue()
|
||||
self.assertEqual(task.status, ImageGenerationTask.Status.QUEUED)
|
||||
self.assertIn("event=image_task_processed", output)
|
||||
self.assertIn(f"task_id={task.task_id}", output)
|
||||
self.assertIn(f"alias={self.image_alias}", output)
|
||||
self.assertIn("status=queued", output)
|
||||
self.assertIn("attempt=1", output)
|
||||
self.assertIn("max_attempts=3", output)
|
||||
self.assertIn("retrying=true", output)
|
||||
self.assertIn("next_attempt_at=", output)
|
||||
self.assertIn("error_code=upstream_timeout", output)
|
||||
self.assertRegex(output, r"duration_ms=\d+")
|
||||
self.assertNotIn("生成图片", output)
|
||||
self.assertNotIn(self.raw_key, output)
|
||||
|
||||
def test_async_image_reaper_fails_stale_running_task_and_refunds(self):
|
||||
response = self.post_with_provider(
|
||||
"/api/v1/generate/image/tasks",
|
||||
|
||||
@@ -123,6 +123,11 @@ IMAGE_TASK_RETENTION_HOURS = env_int("IMAGE_TASK_RETENTION_HOURS", 24)
|
||||
GENERATED_IMAGE_RETENTION_HOURS = env_int("GENERATED_IMAGE_RETENTION_HOURS", 72)
|
||||
IMAGE_TASK_REAPER_INTERVAL_SECONDS = env_int("IMAGE_TASK_REAPER_INTERVAL_SECONDS", 60)
|
||||
IMAGE_TASK_LEASE_SECONDS = env_int("IMAGE_TASK_LEASE_SECONDS", 600)
|
||||
IMAGE_TASK_MAX_RETRIES = env_int("IMAGE_TASK_MAX_RETRIES", 2)
|
||||
IMAGE_TASK_RETRY_BACKOFF_SECONDS = os.environ.get(
|
||||
"IMAGE_TASK_RETRY_BACKOFF_SECONDS",
|
||||
"10,30",
|
||||
)
|
||||
RECHARGE_MAX_AMOUNT_CNY = env_decimal("RECHARGE_MAX_AMOUNT_CNY", "100000.00")
|
||||
MODERATION_ENABLED = env_bool("MODERATION_ENABLED", False)
|
||||
MODERATION_PROVIDER = os.environ.get("MODERATION_PROVIDER", "keyword").strip().lower()
|
||||
|
||||
+10
-4
@@ -43,6 +43,8 @@ T-302 已实现 `/api/v1/generate/title` 与 `/api/v1/generate/image`：API 层
|
||||
|
||||
T-614 已实现 `/api/v1/generate/image/tasks` 与 `/api/v1/generate/image/tasks/{task_id}`：提交接口同步审核 prompt、解析图片输入和预扣点，创建 `ImageGenerationTask(status=queued)` 后立即返回公开 UUID `task_id`；后台 worker 通过 `select_for_update(skip_locked)` 抢任务，复用 T-613 `execute_precharged_generation()` 对已预扣 `CallRecord` 调上游、保存结果、成功确认或失败退点。任务表记录 worker 租约、心跳和尝试次数；reaper 识别僵尸 `running` 任务后默认判失败并幂等退点，不默认重排队。异步 worker 没有 DRF request，结果 URL 由 `MEDIA_PUBLIC_BASE_URL` / `PUBLIC_BASE_URL` 生成。当前实现中 worker 会基于任务快照再次运行 `prepare_generation()`，因此会复审 prompt、重解析别名 / Provider / 定价；账务仍使用已预扣 `CallRecord.points_cost`，不会重复扣点。这个取舍偏安全（排队期间敏感词库更新后仍能拦截并退款），但如果未来队列积压明显，应单独实现“提交时模型配置快照”，避免执行时别名映射变化导致按旧价预扣、按新模型执行。
|
||||
|
||||
T-616 起异步生图 worker 对临时性上游失败增加自动重试：`upstream_timeout` / `upstream_error` 在未达到最大次数前回到 `queued` 并设置 `next_attempt_at`，`CallRecord` 保持 `pending` 且不退点；不可重试错误或最后一次失败才置 `failed` 并幂等退款。默认 `IMAGE_TASK_MAX_RETRIES=2`，因此 `attempt_count` 最多为 3。
|
||||
|
||||
T-615 已给旧同步生图接口和新异步提交接口接入 `cmhub.api.generation_usage` 结构化日志，事件名为 `generation_route_usage`。日志字段只包含 `route_type(sync/async)`、`api_key_id`、`api_key_prefix`、`user_id`、`client_version`、`alias`、`status`、`latency_ms`、`error_code`、`http_status` 等白名单信息；不得记录 API Key 明文、prompt 全文、`image_base64`、provider raw 或上游密钥。第一版用日志查询完成用量观察，不新增报表表结构；如后续要在 admin 做统计报表，需单独评估数据量、索引和保留周期。
|
||||
|
||||
T-303 已实现 `/api/v1/balance`：外部 API 继续只认 API Key，API 层调用 `apps.billing.services.get_balance_snapshot()` 读取当前钱包余额；测试覆盖余额响应与流水累加一致的场景。
|
||||
@@ -339,6 +341,7 @@ CREATE TABLE image_generation_task (
|
||||
started_at DATETIME(6),
|
||||
finished_at DATETIME(6),
|
||||
expires_at DATETIME(6),
|
||||
next_attempt_at DATETIME(6),
|
||||
locked_at DATETIME(6),
|
||||
lease_expires_at DATETIME(6),
|
||||
heartbeat_at DATETIME(6),
|
||||
@@ -353,13 +356,13 @@ CREATE TABLE image_generation_task (
|
||||
需要说明：
|
||||
|
||||
- 主键自增；`api_key.key_hash`、`pricing_rule(operation_type, alias, resolution)`、`recharge_order.order_no`、`user.username` 唯一。
|
||||
- 重要索引：`call_record(user_id, created_at)`、`points_ledger(user_id, created_at)`、`recharge_order(order_no)`、`api_key(key_hash)`、`signup_bonus_grant(user_id)`、`image_generation_task(status, created_at)`、`image_generation_task(status, lease_expires_at)`、`image_generation_task(user_id, created_at)`。
|
||||
- 重要索引：`call_record(user_id, created_at)`、`points_ledger(user_id, created_at)`、`recharge_order(order_no)`、`api_key(key_hash)`、`signup_bonus_grant(user_id)`、`image_generation_task(status, created_at)`、`image_generation_task(status, next_attempt_at)`、`image_generation_task(status, lease_expires_at)`、`image_generation_task(user_id, created_at)`。
|
||||
- 不软删除业务流水；用户/账号可标记 `disabled` 而非物理删；API Key 用 `revoked` 状态而非物理删。
|
||||
- 服务端生成字段：`api_key.key_hash`/`key_prefix`、`points_balance`、`balance_after`、各 `created_at` / `updated_at`。
|
||||
- `payment_user_id`、`payment_txn_no` 为对账预留，字段先建。
|
||||
- `recharge_order.exchange_rate` 与 `points_granted` 在下单时写入，状态为 `pending` 时也必须有值；支付回调金额必须与订单金额一致，入账时不得按新的汇率重算。
|
||||
- `call_record.status` 状态机为 `pending -> success / failed`。上游失败退点后仍保持 `failed`，退款流水通过 `points_ledger(change_type=refund, ref_call_id=call_record.id)` 关联，不单独增加 `refunded` 状态，避免调用结果与账务动作混在一个字段里。
|
||||
- T-201 已落地 `UserWallet` / `ApiKey` 于 `apps.users`，`PointsLedger` / `CallRecord` 于 `apps.billing`；T-203 已落地扣点/退点服务；T-304 已落地 `RechargeOrder`、回调幂等入账服务和 `points_ledger(ref_order_id, change_type)` 复合唯一约束，`ref_order_id` 当前仍为数值引用 `RechargeOrder.id`；T-305 已落地 `create_recharge_order()`，负责创建 pending 订单、锁定汇率/点数并回填二维码票据；T-401 已落地 `adjust_wallet_points()`，手工调整点数必须带原因并写 `adjust` 流水，后台钱包余额字段只读；T-608 已新增 `signup_bonus` 流水类型和 MySQL 兼容的注册赠点幂等标记 `SignupBonusGrant(user UNIQUE)`，并把 allauth 自助注册路径改为调用 `grant_signup_bonus()` 发放 100 点；T-614 已在 `apps.api` 落地 `ImageGenerationTask` 与 `api.0001_initial` 迁移，使用 MySQL 兼容的 `(api_key, idempotency_key_hash)` 唯一约束处理幂等键，不使用条件唯一约束；T-502 已把 API Key 自助管理接到 `ApiKey.create_for_user()`，删除动作写为 `revoked` 状态而非物理删除；T-503/T-608 已把个人中心和记录页接到只读查询，余额用 `get_balance_snapshot()`，充值总额按 paid `RechargeOrder` 汇总，充值 / 注册赠点 / 消费 / 退款按 `PointsLedger` 汇总；T-504 已把 `/recharge` 页面接到 `create_recharge_order()` 与 `/api/v1/recharge/status`。
|
||||
- `call_record.status` 状态机为 `pending -> success / failed`。异步任务临时性失败等待重试时仍保持 `pending`；最终上游失败退点后才变为 `failed`，退款流水通过 `points_ledger(change_type=refund, ref_call_id=call_record.id)` 关联，不单独增加 `refunded` 状态，避免调用结果与账务动作混在一个字段里。
|
||||
- T-201 已落地 `UserWallet` / `ApiKey` 于 `apps.users`，`PointsLedger` / `CallRecord` 于 `apps.billing`；T-203 已落地扣点/退点服务；T-304 已落地 `RechargeOrder`、回调幂等入账服务和 `points_ledger(ref_order_id, change_type)` 复合唯一约束，`ref_order_id` 当前仍为数值引用 `RechargeOrder.id`；T-305 已落地 `create_recharge_order()`，负责创建 pending 订单、锁定汇率/点数并回填二维码票据；T-401 已落地 `adjust_wallet_points()`，手工调整点数必须带原因并写 `adjust` 流水，后台钱包余额字段只读；T-608 已新增 `signup_bonus` 流水类型和 MySQL 兼容的注册赠点幂等标记 `SignupBonusGrant(user UNIQUE)`，并把 allauth 自助注册路径改为调用 `grant_signup_bonus()` 发放 100 点；T-614 已在 `apps.api` 落地 `ImageGenerationTask` 与 `api.0001_initial` 迁移，使用 MySQL 兼容的 `(api_key, idempotency_key_hash)` 唯一约束处理幂等键，不使用条件唯一约束；T-616 已在 `ImageGenerationTask` 增加 `next_attempt_at` 与 `(status, next_attempt_at)` 索引，用于 worker 避免立即反复抢占等待重试的任务；T-502 已把 API Key 自助管理接到 `ApiKey.create_for_user()`，删除动作写为 `revoked` 状态而非物理删除；T-503/T-608 已把个人中心和记录页接到只读查询，余额用 `get_balance_snapshot()`，充值总额按 paid `RechargeOrder` 汇总，充值 / 注册赠点 / 消费 / 退款按 `PointsLedger` 汇总；T-504 已把 `/recharge` 页面接到 `create_recharge_order()` 与 `/api/v1/recharge/status`。
|
||||
|
||||
## 四、计费时序（核心，务必照此实现）
|
||||
|
||||
@@ -398,7 +401,8 @@ CREATE TABLE image_generation_task (
|
||||
7. worker 抢 queued → running，写 worker_id / locked_at / lease_expires_at / heartbeat_at / attempt_count。
|
||||
8. worker 复用已预扣 CallRecord 调上游：
|
||||
成功 → mark_call_success，任务置 succeeded，写稳定 result_url。
|
||||
失败 / 超时 → refund_call_points 幂等退点，任务置 failed。
|
||||
可重试失败且 attempt_count < max_attempts → 任务回 queued，写 next_attempt_at，CallRecord 保持 pending，不退点。
|
||||
不可重试失败或最终失败 → refund_call_points 幂等退点，任务置 failed。
|
||||
9. reaper 扫描 lease/heartbeat 过期的 running：
|
||||
默认置 failed + refund_call_points；不默认重排队。
|
||||
若 call_record 已 success，则只把任务补成 succeeded，不退款。
|
||||
@@ -409,6 +413,7 @@ CREATE TABLE image_generation_task (
|
||||
- 提交即预扣，避免余额只够 1 张却排入大量任务；余额不足仍返回 `402 insufficient_points`。
|
||||
- `Idempotency-Key` 按当前 API Key 去重，同 key 同 payload 返回同一任务且不重复扣点；同 key 不同 payload 返回 `409 idempotency_conflict`。
|
||||
- 桌面端主链路推荐传 `image_base64`，submit 阶段只做解码和落盘，通常是毫秒级；`image_url` 输入会在 submit 阶段下载并可能阻塞，属于边缘路径，换来的是任务自包含和 worker 不再访问调用方外部 URL。
|
||||
- 默认只重试 `upstream_timeout` / `upstream_error` 这类临时性上游失败；敏感词、余额不足、未定价、别名 / 模型不可用、参数错误和 Provider 配置错误不重试。
|
||||
- worker 是至少一次执行模型，但账务终态和结果终态必须 exactly-once：不得重复扣/退，不得覆盖已成功结果，也不得把 reaper 已失败退款的任务改回成功。
|
||||
- 查询接口只按 `task_id` + 当前 API Key 所属用户查任务，防止 IDOR；成功任务重复查询返回同一 `result_url`。
|
||||
|
||||
@@ -469,6 +474,7 @@ CREATE TABLE image_generation_task (
|
||||
| 充值重复入账 | 回调可能重发 | `order_no` 唯一 + 状态机幂等 |
|
||||
| 回调伪造 | 不验签会被刷点 | 强制验签，验签失败不入账并留痕 |
|
||||
| 图片生成耗时长 | 同步等待几十秒到分钟级；后台可换上慢模型拖垮 worker | T-612 后生图 Provider 读取超时取 `min(AiModel.timeout_seconds 或分辨率默认值, AI_IMAGE_UPSTREAM_DEADLINE_SECONDS)`，撞硬截止走失败退点；T-614 已提供异步提交 / 轮询路径，新客户端优先使用异步任务，旧同步接口保留兼容 |
|
||||
| 异步生图临时性上游失败 | 上游偶发 `Read timed out`、502/503/504 直接失败会降低批量任务成功率 | T-616 起异步 worker 对 `upstream_timeout` / `upstream_error` 默认最多重试 2 次，重试等待用 `next_attempt_at` 防止立即打爆上游；最终失败才退款 |
|
||||
| 异步任务僵死 / 迟到 worker | worker 抢到任务后进程崩溃会留下 running；迟到 worker 可能在 reaper 退款后返回成功 | `ImageGenerationTask` 记录租约和心跳；reaper 按 `lease_expires_at` / `heartbeat_at` 幂等失败退款；成功/失败终态写入前重新锁任务，禁止覆盖已成功或已退款失败的任务 |
|
||||
| 异步执行时配置漂移 | worker 当前会复跑审核、别名解析和定价；账务用已预扣点数，不重复扣费，但排队期间后台改别名可能导致按提交时价格、执行时模型运行 | 当前接受该取舍并视为短队列安全优先；队列积压或多模型价差扩大后，单独实现提交时 `ai_model_id` / `model_used` 快照，worker 只复审 prompt、不重选模型 |
|
||||
| 上游错误分类 | 区分参数错误与上游故障 | AI 层抛分类异常；上游故障退点 |
|
||||
|
||||
+1
-1
File diff suppressed because one or more lines are too long
+31
-1
@@ -242,6 +242,9 @@ Content-Type: application/json
|
||||
"call_id": 12346,
|
||||
"points_cost": 10,
|
||||
"points_balance": 88,
|
||||
"attempt_count": 0,
|
||||
"max_attempts": 3,
|
||||
"next_attempt_at": null,
|
||||
"created_at": "2026-07-08T21:30:00+08:00",
|
||||
"expires_at": "2026-07-09T21:30:00+08:00"
|
||||
}
|
||||
@@ -256,6 +259,7 @@ Content-Type: application/json
|
||||
- 桌面端主链路推荐传 `image_base64`。该路径在 submit 阶段只做解码和输入文件落盘，不发生外部网络请求，提交请求应保持短耗时。
|
||||
- `image_url` 仍按同步接口的 SSRF 与大小规则处理，并在 submit 阶段下载成输入文件引用；失败不扣点。这个路径可能因远程图片下载变慢而让 submit 阻塞，适合作为边缘兼容能力，不建议桌面端批量生图主流程使用。
|
||||
- worker 执行时会复审 prompt 并重解析当前别名 / Provider 配置，但账务使用提交阶段已预扣的 `CallRecord.points_cost`，不会重复扣点。若排队时间较长且后台切换别名，可能出现按提交时价格预扣、按执行时模型运行；未来如需强一致模型选择，应单独实现提交时模型配置快照。
|
||||
- T-616 起异步 worker 对临时性上游失败自动重试，默认最多 3 次上游调用（第 1 次执行 + 2 次重试）。提交阶段仍只预扣一次；重试等待期间任务状态回到 `queued`，点数暂不退回，最终失败才退款。
|
||||
|
||||
### `GET /api/v1/generate/image/tasks/{task_id}`
|
||||
|
||||
@@ -274,12 +278,32 @@ Authorization: Bearer sk_cmhub_xxx
|
||||
"status": "running",
|
||||
"call_id": 12346,
|
||||
"points_cost": 10,
|
||||
"attempt_count": 1,
|
||||
"max_attempts": 3,
|
||||
"next_attempt_at": null,
|
||||
"created_at": "2026-07-08T21:30:00+08:00",
|
||||
"updated_at": "2026-07-08T21:30:05+08:00",
|
||||
"expires_at": "2026-07-09T21:30:00+08:00"
|
||||
}
|
||||
```
|
||||
|
||||
临时性上游失败等待重试时仍返回 `queued`：
|
||||
|
||||
```json
|
||||
{
|
||||
"task_id": "2bff8217-47a9-44f1-9bd9-82a375e79dc9",
|
||||
"status": "queued",
|
||||
"call_id": 12346,
|
||||
"points_cost": 10,
|
||||
"attempt_count": 1,
|
||||
"max_attempts": 3,
|
||||
"next_attempt_at": "2026-07-08T21:30:20+08:00",
|
||||
"created_at": "2026-07-08T21:30:00+08:00",
|
||||
"updated_at": "2026-07-08T21:30:10+08:00",
|
||||
"expires_at": "2026-07-09T21:30:00+08:00"
|
||||
}
|
||||
```
|
||||
|
||||
成功：
|
||||
|
||||
```json
|
||||
@@ -288,6 +312,9 @@ Authorization: Bearer sk_cmhub_xxx
|
||||
"status": "succeeded",
|
||||
"call_id": 12346,
|
||||
"points_cost": 10,
|
||||
"attempt_count": 3,
|
||||
"max_attempts": 3,
|
||||
"next_attempt_at": null,
|
||||
"created_at": "2026-07-08T21:30:00+08:00",
|
||||
"updated_at": "2026-07-08T21:31:40+08:00",
|
||||
"expires_at": "2026-07-09T21:30:00+08:00",
|
||||
@@ -305,6 +332,9 @@ Authorization: Bearer sk_cmhub_xxx
|
||||
"status": "failed",
|
||||
"call_id": 12346,
|
||||
"points_cost": 10,
|
||||
"attempt_count": 3,
|
||||
"max_attempts": 3,
|
||||
"next_attempt_at": null,
|
||||
"created_at": "2026-07-08T21:30:00+08:00",
|
||||
"updated_at": "2026-07-08T21:33:00+08:00",
|
||||
"expires_at": "2026-07-09T21:30:00+08:00",
|
||||
@@ -315,7 +345,7 @@ Authorization: Bearer sk_cmhub_xxx
|
||||
}
|
||||
```
|
||||
|
||||
要点：查询只允许任务所属 `user` 与当前 API Key 所属 `user` 一致，跨用户返回 `404 task_not_found`；`succeeded` 重复查询必须返回同一个 `image_url`；`failed` 表示已退款或未扣款，不需要客户端再请求退款。任务元数据默认保留 `IMAGE_TASK_RETENTION_HOURS`（默认 24h），结果图片保留窗口按 `GENERATED_IMAGE_RETENTION_HOURS`（默认 72h）管理；本任务暂不暴露 cancel 路由。
|
||||
要点：查询只允许任务所属 `user` 与当前 API Key 所属 `user` 一致，跨用户返回 `404 task_not_found`；`succeeded` 重复查询必须返回同一个 `image_url`；`queued` 且 `next_attempt_at` 非空表示正在等待自动重试，客户端继续轮询即可；`failed` 表示最终失败且已退款或未扣款，不需要客户端再请求退款。任务元数据默认保留 `IMAGE_TASK_RETENTION_HOURS`（默认 24h），结果图片保留窗口按 `GENERATED_IMAGE_RETENTION_HOURS`（默认 72h）管理；本任务暂不暴露 cancel 路由。
|
||||
|
||||
### `GET /api/v1/balance`
|
||||
|
||||
|
||||
+10
-10
File diff suppressed because one or more lines are too long
+6
-2
@@ -120,6 +120,8 @@ IMAGE_TASK_RETENTION_HOURS=24
|
||||
GENERATED_IMAGE_RETENTION_HOURS=72
|
||||
IMAGE_TASK_REAPER_INTERVAL_SECONDS=60
|
||||
IMAGE_TASK_LEASE_SECONDS=600
|
||||
IMAGE_TASK_MAX_RETRIES=2
|
||||
IMAGE_TASK_RETRY_BACKOFF_SECONDS=10,30
|
||||
|
||||
DJANGO_CACHE_BACKEND=django.core.cache.backends.db.DatabaseCache
|
||||
DJANGO_CACHE_LOCATION=cmhub_cache
|
||||
@@ -279,9 +281,11 @@ python3.12 manage.py run_image_tasks \
|
||||
--sleep-seconds 1
|
||||
```
|
||||
|
||||
建议单独托管为 `cmhub-image-worker.service`。该 worker 从数据库 `image_generation_task` 表抢 `queued` 任务，使用 MySQL `select_for_update(skip_locked)` 标记 `running`，执行成功后写 `succeeded` 和稳定 `result_url`；失败或上游超时会调用计费层退点并写 `failed`。worker 循环会按 `IMAGE_TASK_REAPER_INTERVAL_SECONDS` 扫描租约或心跳过期的 `running` 任务，默认判失败并幂等退点，不默认重排队。
|
||||
建议单独托管为 `cmhub-image-worker.service`。该 worker 从数据库 `image_generation_task` 表抢 `queued` 且 `next_attempt_at` 已到达的任务，使用 MySQL `select_for_update(skip_locked)` 标记 `running`，执行成功后写 `succeeded` 和稳定 `result_url`。T-616 起，`upstream_timeout` / `upstream_error` 这类临时性上游失败会先回到 `queued` 并写 `next_attempt_at`，默认最多重试 2 次；重试期间 `CallRecord` 仍为 `pending`，不退点。不可重试错误或最终失败才调用计费层幂等退点并写 `failed`。worker 循环会按 `IMAGE_TASK_REAPER_INTERVAL_SECONDS` 扫描租约或心跳过期的 `running` 任务，默认判失败并幂等退点，不默认重排队。
|
||||
|
||||
worker 每处理一个任务会向 stdout 输出一行结构化日志，形如 `event=image_task_processed task_id=... status=failed alias=... duration_ms=... error_code=upstream_timeout`。失败日志必须用于区分「还在 queued 未提交给上游」和「已 running 但上游超时 / 失败」；日志不得包含 prompt、`image_base64`、provider raw 或密钥。
|
||||
worker 每处理一个任务会向 stdout 输出一行结构化日志，形如 `event=image_task_processed task_id=... status=queued alias=image-hd attempt=1 max_attempts=3 retrying=true next_attempt_at=2026-07-09T12:00:10+08:00 duration_ms=220015 error_code=upstream_timeout`。失败日志必须用于区分「还在 queued 未提交给上游」「queued 等待重试」和「已最终 failed」；日志不得包含 prompt、`image_base64`、provider raw 或密钥。
|
||||
|
||||
生产默认 `IMAGE_TASK_MAX_RETRIES=2`、`IMAGE_TASK_RETRY_BACKOFF_SECONDS=10,30`，表示第 1 次正常执行失败后最多再重试 2 次。若 `AI_IMAGE_UPSTREAM_DEADLINE_SECONDS=220`，单任务最坏耗时约 `220 * 3 + 10 + 30 = 700` 秒；Nginx / Gunicorn 的异步提交与轮询仍是短请求，但桌面端轮询总超时和任务保留窗口必须覆盖这个最坏耗时。
|
||||
|
||||
多 worker 可以并行运行同一命令，只要 `--worker-id` 不同即可；MySQL 8.4 会通过 `select_for_update(skip_locked)` 避免重复抢同一任务。生产扩容应按 2、4、8、16 逐级观察 `queued` 长度、`duration_ms`、`error_code`、MySQL 连接数、VPS CPU/内存/磁盘写入和上游失败率。不要直接扩到 100 个 worker：这会同时放大 MySQL 连接、上游请求、图片下载和本地写文件压力；如果上游已经频繁 `upstream_timeout`，100 并发通常只会把失败更快放大。
|
||||
|
||||
|
||||
+3
-1
@@ -58,6 +58,8 @@
|
||||
| `GENERATED_IMAGE_RETENTION_HOURS` | 否 | `72` | 生成图片文件保留窗口口径，默认 72 小时；实际清理任务后续单独实现时按此值执行 |
|
||||
| `IMAGE_TASK_REAPER_INTERVAL_SECONDS` | 否 | `60` | 异步生图 worker 循环中扫描僵尸 running 任务的间隔秒数 |
|
||||
| `IMAGE_TASK_LEASE_SECONDS` | 否 | `600` | 异步生图任务租约秒数；worker 认领任务后写 `lease_expires_at` / `heartbeat_at`，超时由 reaper 判失败并退点 |
|
||||
| `IMAGE_TASK_MAX_RETRIES` | 否 | `2` | T-616 异步生图临时性上游失败最大重试次数；默认 2 表示最多 3 次上游调用 |
|
||||
| `IMAGE_TASK_RETRY_BACKOFF_SECONDS` | 否 | `10,30` | T-616 异步生图重试退避秒数列表；第 1 次失败等 10 秒，第 2 次失败等 30 秒，列表不足时复用最后一个值 |
|
||||
|
||||
`AI_KEY_ENCRYPTION_KEY` 必须是 `cryptography.fernet.Fernet.generate_key()` 生成的 base64 字符串，可用 `py -3.12 -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"` 生成。
|
||||
|
||||
@@ -65,7 +67,7 @@
|
||||
|
||||
AI 上游连接超时由 `AiModel.connect_timeout_seconds` 控制。文本读取超时由 `AiModel.timeout_seconds` 控制；当读取超时为 `0` 时，Provider 按分辨率使用内置默认值。生图读取超时在此基础上再套 `AI_IMAGE_UPSTREAM_DEADLINE_SECONDS` 硬截止，避免慢 / 卡死图片上游长期占用生成池线程。
|
||||
|
||||
T-614 起新增异步生图任务接口：提交任务仍同步审核 prompt 和预扣点；worker 成功后返回 cmhub 托管媒体 URL。生产必须配置 `MEDIA_PUBLIC_BASE_URL` 或 `PUBLIC_BASE_URL` 为 HTTPS 域名，否则 worker 只能返回相对 `/media/...` URL，不利于桌面端直接下载。
|
||||
T-614 起新增异步生图任务接口：提交任务仍同步审核 prompt 和预扣点；worker 成功后返回 cmhub 托管媒体 URL。生产必须配置 `MEDIA_PUBLIC_BASE_URL` 或 `PUBLIC_BASE_URL` 为 HTTPS 域名，否则 worker 只能返回相对 `/media/...` URL，不利于桌面端直接下载。T-616 起临时性上游失败会按 `IMAGE_TASK_MAX_RETRIES` 与 `IMAGE_TASK_RETRY_BACKOFF_SECONDS` 自动重试；若 `AI_IMAGE_UPSTREAM_DEADLINE_SECONDS=220` 且默认重试 2 次，单个任务最坏耗时约为 `220 * 3 + 10 + 30 = 700` 秒，桌面端轮询总超时必须覆盖该窗口。
|
||||
|
||||
## 五、对外 API 安全配置
|
||||
|
||||
|
||||
+35
@@ -1805,3 +1805,38 @@
|
||||
- 新增 `next_attempt_at` 与 `IMAGE_TASK_MAX_RETRIES` / `IMAGE_TASK_RETRY_BACKOFF_SECONDS` 等配置；worker 只抢到达重试时间的 queued 任务。
|
||||
- 轮询响应可兼容性新增 `attempt_count`、`max_attempts`、`next_attempt_at`；日志新增 attempt / retrying / next_attempt_at 字段。
|
||||
- 验证：仅文档更新；提交前执行 `git diff --check`。
|
||||
|
||||
## 2026-07-09 实施：T-616 生图失败自动重试 2 次
|
||||
|
||||
- 状态：DONE。
|
||||
- 背景：线上异步生图已出现 `api.vectorengine.ai` 偶发 `Read timed out`。原逻辑第一次 `upstream_timeout` 就把任务置 `failed` 并退点，批量生图成功率受上游短暂波动影响较大。
|
||||
- 代码变更：
|
||||
- `apps/api/models.py` / `apps/api/migrations/0002_imagegenerationtask_next_attempt_at_and_more.py`：给 `ImageGenerationTask` 增加 `next_attempt_at`，并新增 `(status, next_attempt_at)` 索引，worker 只抢到达重试时间的 queued 任务。
|
||||
- `config/settings.py` / `.env.example`：新增 `IMAGE_TASK_MAX_RETRIES=2`、`IMAGE_TASK_RETRY_BACKOFF_SECONDS=10,30`。
|
||||
- `apps/api/generation.py`：`execute_precharged_generation()` 增加 `refund_on_failure` 参数；旧同步接口默认行为不变，异步 worker 可在上游失败时先不退款，由任务状态机决定重试或最终退款。
|
||||
- `apps/api/image_tasks.py`：只对 `upstream_timeout` / `upstream_error` 自动重试；前两次临时性失败回到 `queued`、写 `next_attempt_at`、保留 `CallRecord.pending`、不退点；不可重试错误或最终失败才幂等退款。轮询响应新增 `attempt_count`、`max_attempts`、`next_attempt_at`。
|
||||
- `apps/api/management/commands/run_image_tasks.py`：worker 日志增加 `attempt`、`max_attempts`、`retrying`、`next_attempt_at`，重试中 / 失败 / 过期任务输出 `error_code`。
|
||||
- `apps/api/admin.py`：后台图片任务列表 / 过滤 / 只读字段展示 `next_attempt_at`。
|
||||
- `apps/api/tests.py`：新增 T-616 目标测试，覆盖首次超时回队列不退点、两次超时后三次成功只扣一次、连续三次超时最终只退一次、非重试错误立即退款、worker 重试日志字段不泄露敏感信息；旧失败退款测试用 `IMAGE_TASK_MAX_RETRIES=0` 保留原验收。
|
||||
- 文档变更：
|
||||
- `docs/api.md`：异步提交 / 轮询响应示例新增 `attempt_count`、`max_attempts`、`next_attempt_at`，说明 `queued + next_attempt_at` 为等待自动重试。
|
||||
- `docs/04-architecture.md`：同步 `next_attempt_at` schema、索引、异步计费状态机和临时性失败重试语义。
|
||||
- `docs/env.md` / `docs/deployment.md`：同步新增环境变量、最坏耗时估算和 worker 日志排障口径。
|
||||
- `docs/current-state.md`：同步 T-616 已完成、最新验证和下一步。
|
||||
- `docs/06-tasks.md`：T-616 标记为 DONE。
|
||||
- 验证：
|
||||
- `py -3.12 -m py_compile apps\api\generation.py apps\api\image_tasks.py apps\api\management\commands\run_image_tasks.py apps\api\models.py apps\api\admin.py apps\api\tests.py config\settings.py`：通过。
|
||||
- `py -3.12 manage.py makemigrations api`：生成 `api.0002_imagegenerationtask_next_attempt_at_and_more`。
|
||||
- `py -3.12 manage.py migrate api --noinput`：通过，已应用 `api.0001_initial` 与 `api.0002_imagegenerationtask_next_attempt_at_and_more` 到当前开发库。
|
||||
- `py -3.12 manage.py check`：通过，0 issues。
|
||||
- `py -3.12 manage.py makemigrations --check --dry-run`：通过，No changes detected。
|
||||
- `py -3.12 manage.py test apps.api.tests.GenerateApiTests.test_async_image_retryable_timeout_requeues_without_refund_and_respects_backoff apps.api.tests.GenerateApiTests.test_async_image_retryable_timeouts_then_success_charges_once apps.api.tests.GenerateApiTests.test_async_image_retryable_timeouts_final_failure_refunds_once apps.api.tests.GenerateApiTests.test_async_image_non_retryable_provider_error_fails_immediately_and_refunds apps.api.tests.GenerateApiTests.test_run_image_tasks_logs_retrying_task_attempt_fields_without_sensitive_data --keepdb --noinput --verbosity 2`：通过，5 tests OK。
|
||||
- `py -3.12 manage.py test apps.api.tests.GenerateApiTests.test_async_image_submit_usage_telemetry_logs_safe_success_event apps.api.tests.GenerateApiTests.test_async_image_submit_usage_telemetry_logs_error_code_without_sensitive_data apps.api.tests.GenerateApiTests.test_async_image_blocked_prompt_creates_no_task_or_charge apps.api.tests.GenerateApiTests.test_async_image_insufficient_points_returns_402_without_task apps.api.tests.GenerateApiTests.test_async_image_idempotency_reuses_task_and_rejects_conflict apps.api.tests.GenerateApiTests.test_async_image_worker_success_and_poll_are_idempotent apps.api.tests.GenerateApiTests.test_async_image_poll_rejects_cross_user_access apps.api.tests.GenerateApiTests.test_async_image_worker_failure_refunds_precharged_points apps.api.tests.GenerateApiTests.test_run_image_tasks_logs_failed_task_alias_error_and_duration apps.api.tests.GenerateApiTests.test_async_image_reaper_fails_stale_running_task_and_refunds apps.api.tests.GenerateApiTests.test_async_image_duplicate_worker_does_not_double_charge_or_refund apps.api.tests.GenerateApiTests.test_async_image_late_worker_after_reaper_cannot_flip_failed_task --keepdb --noinput --verbosity 2`：通过，12 tests OK。
|
||||
- `py -3.12 manage.py test apps.api.tests.GenerateApiTests --keepdb --noinput --verbosity 2`：通过，37 tests OK。
|
||||
- `.\init.ps1`：通过（Python 3.12.3，依赖已满足，`manage.py check` 0 issues，打印启动命令）。
|
||||
- `git diff --check`：通过，仅 Windows CRLF 提示。
|
||||
- 测试环境现象：
|
||||
- 测试期仍保留 allauth `account.EmailAddress` 条件唯一约束在 MySQL 上不可创建的既有 `models.W036` 警告。
|
||||
- 决策：
|
||||
- T-616 第一版不重试敏感词、余额不足、未定价、模型 / 别名不可用、参数错误和 Provider 能力错误；这些属于确定性失败，立即失败并退款或在提交阶段不扣点。
|
||||
- 重试期间不展示 `error` 给轮询客户端，客户端只需看到 `queued + next_attempt_at` 后继续轮询；最终 `failed` 才表示已退款。
|
||||
|
||||
Reference in New Issue
Block a user