feat: 执行并提交 PDD 采集任务 (#32)

2026-08-07 17:38:29 +08:00
parent 34b4b54b63
commit 2d1fdbea3f
17 changed files with 2156 additions and 108 deletions
@@ -8,10 +8,16 @@ import sqlite3
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple, Union
+from uuid import uuid4

 from .db import initialize_database, open_database
 from .task_models import (
    NewClaimedTask,
+    OutboxEventRecord,
+    OutboxEventType,
+    OutboxStatus,
+    RunStatus,
+    StartedTaskRun,
    TaskDetail,
    TaskFilters,
    TaskStatus,
@@ -145,6 +151,353 @@ class TaskRepository:
            connection.close()
        return self._to_detail(row) if row is not None else None

+    def recover_interrupted_work(self) -> None:
+        """恢复上次异常退出留下的可重试状态。"""
+
+        now = utc_now_iso()
+        connection = open_database(self._db_path)
+        try:
+            with connection:
+                connection.execute(
+                    "UPDATE outbox_events SET status = 'pending', updated_at = ?"
+                    " WHERE status = 'sending'",
+                    (now,),
+                )
+                connection.execute(
+                    "UPDATE pdd_tasks SET status = 'retry_wait',"
+                    " current_step = 'interrupted', retry_count = retry_count + 1,"
+                    " last_error_code = 'CLIENT_INTERRUPTED',"
+                    " last_error_message = '客户端上次执行期间退出', updated_at = ?"
+                    " WHERE status = 'running' AND task_type = 'collect'",
+                    (now,),
+                )
+                connection.execute(
+                    "UPDATE task_runs SET run_status = 'failed',"
+                    " error_code = 'CLIENT_INTERRUPTED',"
+                    " error_message = '客户端上次执行期间退出',"
+                    " finished_at = ?, updated_at = ?"
+                    " WHERE run_status = 'running' AND irreversible_action_at IS NULL"
+                    " AND task_id IN (SELECT id FROM pdd_tasks WHERE task_type = 'collect')",
+                    (now, now),
+                )
+        finally:
+            connection.close()
+
+    def next_collect_task(self) -> Optional[TaskDetail]:
+        """返回最早的本地待执行采集任务。"""
+
+        connection = open_database(self._db_path)
+        try:
+            row = connection.execute(
+                "SELECT * FROM pdd_tasks"
+                " WHERE task_type = 'collect' AND status IN ('claimed', 'retry_wait')"
+                " ORDER BY received_at ASC, id ASC LIMIT 1"
+            ).fetchone()
+        finally:
+            connection.close()
+        return self._to_detail(row) if row is not None else None
+
+    def start_collect_run(
+        self, remote_task_id: str, device_address: str
+    ) -> StartedTaskRun:
+        """原子地把待执行任务改为执行中，并创建一次运行记录。"""
+
+        now = utc_now_iso()
+        attempt_id = str(uuid4())
+        connection = open_database(self._db_path)
+        try:
+            with connection:
+                row = connection.execute(
+                    "SELECT * FROM pdd_tasks WHERE remote_task_id = ?",
+                    (remote_task_id,),
+                ).fetchone()
+                if row is None:
+                    raise ValueError(f"任务 {remote_task_id} 不存在")
+                if row["task_type"] != TaskType.COLLECT.value:
+                    raise ValueError("当前只能执行采集任务")
+                if row["status"] not in {
+                    TaskStatus.CLAIMED.value,
+                    TaskStatus.RETRY_WAIT.value,
+                }:
+                    raise ValueError(f"任务状态 {row['status']} 不能开始采集")
+                attempt_no = int(
+                    connection.execute(
+                        "SELECT COALESCE(MAX(attempt_no), 0) + 1"
+                        " FROM task_runs WHERE task_id = ?",
+                        (row["id"],),
+                    ).fetchone()[0]
+                )
+                connection.execute(
+                    "UPDATE pdd_tasks SET status = 'running',"
+                    " current_step = 'collecting', started_at = COALESCE(started_at, ?),"
+                    " last_error_code = NULL, last_error_message = NULL, updated_at = ?"
+                    " WHERE id = ?",
+                    (now, now, row["id"]),
+                )
+                connection.execute(
+                    "INSERT INTO task_runs (task_id, attempt_id, attempt_no,"
+                    " device_address, run_status, current_step, started_at,"
+                    " created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                    (
+                        row["id"], attempt_id, attempt_no, device_address,
+                        RunStatus.RUNNING.value, "collecting", now, now, now,
+                    ),
+                )
+            task = self.get_task(remote_task_id)
+            assert task is not None
+            return StartedTaskRun(task, attempt_id, attempt_no)
+        finally:
+            connection.close()
+
+    def save_collect_result(
+        self,
+        remote_task_id: str,
+        attempt_id: str,
+        pdd_data: Dict[str, object],
+    ) -> OutboxEventRecord:
+        """在一个事务中保存采集结果并创建待提交事件。"""
+
+        now = utc_now_iso()
+        connection = open_database(self._db_path)
+        try:
+            with connection:
+                task = connection.execute(
+                    "SELECT id, version FROM pdd_tasks WHERE remote_task_id = ?",
+                    (remote_task_id,),
+                ).fetchone()
+                if task is None:
+                    raise ValueError(f"任务 {remote_task_id} 不存在")
+                payload = {
+                    "task_version": task["version"],
+                    "attempt_id": attempt_id,
+                    "result_type": "collect",
+                    "completed_at": now,
+                    "pdd_data": pdd_data,
+                }
+                idempotency_key = f"{remote_task_id}:{attempt_id}:result-v1"
+                connection.execute(
+                    "UPDATE pdd_tasks SET status = 'result_pending',"
+                    " current_step = 'submit_result', pdd_data = ?, goods_id = ?,"
+                    " title = ?, price_cent = ?, finished_at = ?, updated_at = ?"
+                    " WHERE id = ?",
+                    (
+                        json.dumps(pdd_data, ensure_ascii=False),
+                        pdd_data.get("goods_id"), pdd_data.get("title"),
+                        self._summary_price(pdd_data), now, now, task["id"],
+                    ),
+                )
+                connection.execute(
+                    "UPDATE task_runs SET run_status = 'succeeded',"
+                    " current_step = 'submit_result', finished_at = ?, updated_at = ?"
+                    " WHERE attempt_id = ?",
+                    (now, now, attempt_id),
+                )
+                cursor = connection.execute(
+                    "INSERT INTO outbox_events (task_id, event_type, idempotency_key,"
+                    " payload_json, status, created_at, updated_at)"
+                    " VALUES (?, 'collect_result', ?, ?, 'pending', ?, ?)",
+                    (
+                        task["id"], idempotency_key,
+                        json.dumps(payload, ensure_ascii=False), now, now,
+                    ),
+                )
+                event_id = int(cursor.lastrowid)
+            event = self.get_outbox_event(event_id)
+            assert event is not None
+            return event
+        finally:
+            connection.close()
+
+    def save_collect_failure(
+        self,
+        remote_task_id: str,
+        attempt_id: str,
+        status: TaskStatus,
+        error_code: str,
+        error_message: str,
+        retryable: bool,
+        diagnostics: Optional[Dict[str, object]] = None,
+    ) -> OutboxEventRecord:
+        """保存结构化失败，并可靠排队提交 Admin。"""
+
+        if status not in {
+            TaskStatus.RETRY_WAIT, TaskStatus.MANUAL_REVIEW,
+            TaskStatus.FAILED, TaskStatus.CANCELLED,
+        }:
+            raise ValueError("失败状态无效")
+        now = utc_now_iso()
+        connection = open_database(self._db_path)
+        try:
+            with connection:
+                task = connection.execute(
+                    "SELECT id, version FROM pdd_tasks WHERE remote_task_id = ?",
+                    (remote_task_id,),
+                ).fetchone()
+                if task is None:
+                    raise ValueError(f"任务 {remote_task_id} 不存在")
+                payload = {
+                    "task_version": task["version"],
+                    "attempt_id": attempt_id,
+                    "status": status.value,
+                    "error": {
+                        "code": error_code,
+                        "message": error_message,
+                        "retryable": retryable,
+                        "step": "collecting",
+                    },
+                    "diagnostics": diagnostics or {"artifacts": []},
+                    "reported_at": now,
+                }
+                idempotency_key = f"{remote_task_id}:{attempt_id}:failure-v1"
+                connection.execute(
+                    "UPDATE pdd_tasks SET status = ?, current_step = 'failed',"
+                    " retry_count = retry_count + ?, last_error_code = ?,"
+                    " last_error_message = ?, finished_at = ?, updated_at = ?"
+                    " WHERE id = ?",
+                    (
+                        status.value, 1 if status is TaskStatus.RETRY_WAIT else 0,
+                        error_code, error_message, now, now, task["id"],
+                    ),
+                )
+                run_status = {
+                    TaskStatus.CANCELLED: RunStatus.CANCELLED,
+                    TaskStatus.MANUAL_REVIEW: RunStatus.MANUAL_REVIEW,
+                }.get(status, RunStatus.FAILED)
+                connection.execute(
+                    "UPDATE task_runs SET run_status = ?, current_step = 'failed',"
+                    " error_code = ?, error_message = ?, finished_at = ?, updated_at = ?"
+                    " WHERE attempt_id = ?",
+                    (run_status.value, error_code, error_message, now, now, attempt_id),
+                )
+                cursor = connection.execute(
+                    "INSERT INTO outbox_events (task_id, event_type, idempotency_key,"
+                    " payload_json, status, created_at, updated_at)"
+                    " VALUES (?, 'task_failure', ?, ?, 'pending', ?, ?)",
+                    (
+                        task["id"], idempotency_key,
+                        json.dumps(payload, ensure_ascii=False), now, now,
+                    ),
+                )
+                event_id = int(cursor.lastrowid)
+            event = self.get_outbox_event(event_id)
+            assert event is not None
+            return event
+        finally:
+            connection.close()
+
+    def next_pending_outbox(self) -> Optional[OutboxEventRecord]:
+        connection = open_database(self._db_path)
+        try:
+            row = connection.execute(
+                "SELECT o.* FROM outbox_events o JOIN pdd_tasks t ON t.id = o.task_id"
+                " WHERE o.status = 'pending'"
+                " AND (o.next_retry_at IS NULL OR o.next_retry_at <= ?)"
+                " ORDER BY o.id ASC LIMIT 1",
+                (utc_now_iso(),),
+            ).fetchone()
+        finally:
+            connection.close()
+        return self._to_outbox(row) if row is not None else None
+
+    def get_outbox_event(self, event_id: int) -> Optional[OutboxEventRecord]:
+        connection = open_database(self._db_path)
+        try:
+            row = connection.execute(
+                "SELECT * FROM outbox_events WHERE id = ?", (event_id,)
+            ).fetchone()
+        finally:
+            connection.close()
+        return self._to_outbox(row) if row is not None else None
+
+    def outbox_task_id(self, event_id: int) -> str:
+        connection = open_database(self._db_path)
+        try:
+            row = connection.execute(
+                "SELECT t.remote_task_id FROM outbox_events o"
+                " JOIN pdd_tasks t ON t.id = o.task_id WHERE o.id = ?",
+                (event_id,),
+            ).fetchone()
+        finally:
+            connection.close()
+        if row is None:
+            raise ValueError(f"Outbox {event_id} 不存在")
+        return str(row[0])
+
+    def mark_outbox_sending(self, event_id: int) -> None:
+        self._update_outbox(event_id, "sending", None)
+
+    def mark_outbox_retry(self, event_id: int, message: str) -> None:
+        self._update_outbox(event_id, "pending", message, increment=True)
+
+    def mark_outbox_failed(self, event_id: int, message: str) -> None:
+        self._update_outbox(event_id, "failed", message, increment=True)
+
+    def mark_outbox_sent(self, event_id: int) -> None:
+        now = utc_now_iso()
+        connection = open_database(self._db_path)
+        try:
+            with connection:
+                row = connection.execute(
+                    "SELECT task_id, event_type FROM outbox_events WHERE id = ?",
+                    (event_id,),
+                ).fetchone()
+                if row is None:
+                    raise ValueError(f"Outbox {event_id} 不存在")
+                connection.execute(
+                    "UPDATE outbox_events SET status = 'sent', sent_at = ?,"
+                    " updated_at = ?, attempt_count = attempt_count + 1"
+                    " WHERE id = ?",
+                    (now, now, event_id),
+                )
+                if row["event_type"] == OutboxEventType.COLLECT_RESULT.value:
+                    connection.execute(
+                        "UPDATE pdd_tasks SET status = 'succeeded',"
+                        " current_step = 'completed', updated_at = ? WHERE id = ?",
+                        (now, row["task_id"]),
+                    )
+        finally:
+            connection.close()
+
+    def _update_outbox(
+        self, event_id: int, status: str, message: Optional[str], increment: bool = False
+    ) -> None:
+        connection = open_database(self._db_path)
+        try:
+            with connection:
+                cursor = connection.execute(
+                    "UPDATE outbox_events SET status = ?, last_error = ?, updated_at = ?,"
+                    f" attempt_count = attempt_count + {1 if increment else 0} WHERE id = ?",
+                    (status, message, utc_now_iso(), event_id),
+                )
+                if cursor.rowcount != 1:
+                    raise ValueError(f"Outbox {event_id} 不存在")
+        finally:
+            connection.close()
+
+    @staticmethod
+    def _summary_price(pdd_data: Dict[str, object]) -> Optional[int]:
+        skus = pdd_data.get("skus")
+        if not isinstance(skus, list):
+            return None
+        prices = [
+            item.get("price_cent") for item in skus
+            if isinstance(item, dict) and isinstance(item.get("price_cent"), int)
+        ]
+        return min(prices) if prices else None
+
+    @staticmethod
+    def _to_outbox(row: sqlite3.Row) -> OutboxEventRecord:
+        return OutboxEventRecord(
+            id=row["id"], task_id=row["task_id"],
+            event_type=OutboxEventType(row["event_type"]),
+            idempotency_key=row["idempotency_key"],
+            payload_json=TaskRepository._load_json_object(row["payload_json"]),
+            status=OutboxStatus(row["status"]), attempt_count=row["attempt_count"],
+            next_retry_at=row["next_retry_at"], last_error=row["last_error"],
+            created_at=row["created_at"], updated_at=row["updated_at"],
+            sent_at=row["sent_at"],
+        )
+
    @staticmethod
    def _validate_page(limit: int, offset: int) -> None:
        if not 1 <= limit <= MAX_PAGE_SIZE: