feat: retry async image tasks

apps/api/management/commands/run_image_tasks.py

@@ -4,8 +4,12 @@ import uuid
 from django.conf import settings
 from django.core.management.base import BaseCommand
 
+from apps.api.image_tasks import (
+    image_task_max_attempts,
+    reap_stale_image_tasks,
+    run_one_image_task,
+)
 from apps.api.models import ImageGenerationTask
-from apps.api.image_tasks import reap_stale_image_tasks, run_one_image_task
 
 
 class Command(BaseCommand):
@@ -70,13 +74,25 @@ class Command(BaseCommand):
 def format_task_log_line(task: ImageGenerationTask, *, started_at: float) -> str:
     duration_ms = max(0, int((time.monotonic() - started_at) * 1000))
     alias = str((task.request_payload or {}).get("model") or "")
+    retrying = bool(
+        task.status == ImageGenerationTask.Status.QUEUED
+        and task.error_code
+        and task.next_attempt_at
+    )
     fields = {
         "event": "image_task_processed",
         "task_id": str(task.task_id),
         "status": task.status,
         "alias": alias,
+        "attempt": str(task.attempt_count),
+        "max_attempts": str(image_task_max_attempts()),
+        "retrying": str(retrying).lower(),
+        "next_attempt_at": task.next_attempt_at.isoformat() if task.next_attempt_at else "",
         "duration_ms": str(duration_ms),
     }
-    if task.status in {ImageGenerationTask.Status.FAILED, ImageGenerationTask.Status.EXPIRED}:
+    if retrying or task.status in {
+        ImageGenerationTask.Status.FAILED,
+        ImageGenerationTask.Status.EXPIRED,
+    }:
         fields["error_code"] = task.error_code or "upstream_error"
     return " ".join(f"{key}={value}" for key, value in fields.items())