diff --git a/.gitignore b/.gitignore index 699cd0b..66f6aaa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ # ── 凭证 / 业务数据 / 本地产物（绝不提交）── # AI 模型清单 ai_models.json（含密钥） -config/ +config/ai_models.json # 应用配置（路径/模型选择/参数） config.json # SQLite（账号/任务/结果，含密码） @@ -9,6 +9,10 @@ cmshopee.db chrome_user_data_dir/ # 采集的旧封面 / AI 生成的新封面 images/ +# 本地测试图片 / 临时启动脚本 +1_TY030.jpg +cmshopee_t001_cover.jpg +dev.bat # ── Python ── __pycache__/ diff --git a/app/appconfig.py b/app/appconfig.py new file mode 100644 index 0000000..5aeca1f --- /dev/null +++ b/app/appconfig.py @@ -0,0 +1,165 @@ +"""Application-level configuration for cmshopee. + +This module owns `config.json`, which stores local app settings and AI role/ +generation parameters. AI provider definitions and API keys are intentionally +kept out of this file; T-005 will own `config/ai_models.json`. +""" + +import copy +import json +import os + + +CONFIG_PATH = "config.json" + +DEFAULT_CONFIG = { + "chrome_path": r"C:\Program Files\Google\Chrome\Application\chrome.exe", + "user_data_root": "chrome_user_data_dir", + "image_dir": "images", + "db_path": "cmshopee.db", + "default_debug_port": 9222, + "debug_port_range": [9222, 9260], + "cdp_ready_timeout": 60, + "ai": { + "default_text_model": "GPT-5.5 文本", + "default_image_model": "Nano Banana 2", + "title_concurrency": 4, + "image_concurrency": 4, + "retry": 2, + "jpg_quality": 90, + "resolution": "1k", + "resolution_timeouts": { + "512": 180, + "1k": 240, + "2k": 360, + "4k": 600, + }, + }, +} + +SECRET_FIELD_NAMES = {"api_key", "apikey", "key", "token", "password"} + + +class ConfigError(RuntimeError): + """Raised when app configuration is missing or malformed.""" + + +def default_config() -> dict: + """Return a new copy of the default config.""" + + return copy.deepcopy(DEFAULT_CONFIG) + + +def _deep_merge(defaults, loaded): + if not isinstance(defaults, dict): + return copy.deepcopy(loaded) if loaded is not None else copy.deepcopy(defaults) + merged = copy.deepcopy(defaults) + if not isinstance(loaded, dict): + return merged + for key, value in loaded.items(): + if isinstance(merged.get(key), dict) and isinstance(value, dict): + merged[key] = _deep_merge(merged[key], value) + else: + merged[key] = copy.deepcopy(value) + return merged + + +def _assert_no_secrets(config): + def visit(value, path): + if isinstance(value, dict): + for key, child in value.items(): + lowered = str(key).lower() + if lowered in SECRET_FIELD_NAMES or lowered.endswith("_key"): + raise ConfigError( + f"config.json 不允许保存敏感字段: {'.'.join(path + [str(key)])}" + ) + visit(child, path + [str(key)]) + elif isinstance(value, list): + for index, child in enumerate(value): + visit(child, path + [str(index)]) + + visit(config, []) + + +def save_config(config, path=CONFIG_PATH) -> dict: + """Persist config to JSON and return the normalized config.""" + + normalized = _deep_merge(DEFAULT_CONFIG, config) + _assert_no_secrets(normalized) + directory = os.path.dirname(os.path.abspath(path)) + if directory: + os.makedirs(directory, exist_ok=True) + with open(path, "w", encoding="utf-8") as fh: + json.dump(normalized, fh, ensure_ascii=False, indent=2) + fh.write("\n") + return normalized + + +def load_config(path=CONFIG_PATH) -> dict: + """Load config, writing defaults first if the file does not exist.""" + + if not os.path.exists(path): + return save_config(default_config(), path=path) + with open(path, "r", encoding="utf-8") as fh: + try: + loaded = json.load(fh) + except json.JSONDecodeError as exc: + raise ConfigError(f"配置文件不是有效 JSON: {path}") from exc + normalized = _deep_merge(DEFAULT_CONFIG, loaded) + _assert_no_secrets(normalized) + return normalized + + +def update_config(updates, path=CONFIG_PATH) -> dict: + """Merge updates into the persisted config.""" + + config = load_config(path) + return save_config(_deep_merge(config, updates), path=path) + + +def _config_or_load(config): + return load_config() if config is None else config + + +def chrome_path(config=None) -> str: + return _config_or_load(config).get("chrome_path", "") + + +def user_data_root(config=None) -> str: + return _config_or_load(config).get("user_data_root", "chrome_user_data_dir") + + +def image_dir(config=None) -> str: + return _config_or_load(config).get("image_dir", "images") + + +def db_path(config=None) -> str: + return _config_or_load(config).get("db_path", "cmshopee.db") + + +def default_debug_port(config=None) -> int: + return int(_config_or_load(config).get("default_debug_port", 9222)) + + +def debug_port_range(config=None) -> tuple: + values = _config_or_load(config).get("debug_port_range", [9222, 9260]) + if not isinstance(values, list) or len(values) != 2: + raise ConfigError("debug_port_range 必须是 [start, end]") + return int(values[0]), int(values[1]) + + +def cdp_ready_timeout(config=None) -> int: + return int(_config_or_load(config).get("cdp_ready_timeout", 60)) + + +def ai_config(config=None) -> dict: + return copy.deepcopy(_config_or_load(config).get("ai", DEFAULT_CONFIG["ai"])) + + +def response_timeout(config=None) -> int: + ai = ai_config(config) + resolution = str(ai.get("resolution", DEFAULT_CONFIG["ai"]["resolution"])) + timeouts = ai.get("resolution_timeouts", {}) + if resolution not in timeouts: + raise ConfigError(f"未配置分辨率 {resolution} 的返回超时") + return int(timeouts[resolution]) diff --git a/app/db.py b/app/db.py new file mode 100644 index 0000000..83d03ec --- /dev/null +++ b/app/db.py @@ -0,0 +1,564 @@ +"""SQLite persistence for accounts, import batches, and tasks.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import sqlite3 +import uuid +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime +from typing import Iterable, Optional + +from . import appconfig + + +DEFAULT_BUSY_TIMEOUT_MS = 5000 +VALID_BATCH_FIELDS = {"source_files_json", "status", "note"} +VALID_ACCOUNT_FIELDS = { + "account_name", + "region_host", + "slug", + "user_data_dir", + "debug_port", + "password", + "note", + "last_login_at", +} +PHASE_ATTEMPT_FIELDS = { + "collect": "collect_attempts", + "collected": "collect_attempts", + "generate": "generate_attempts", + "generated": "generate_attempts", + "apply": "apply_attempts", + "applied": "apply_attempts", + "update": "apply_attempts", + "updated": "apply_attempts", +} + + +class DbError(RuntimeError): + """Raised when SQLite persistence cannot complete an operation.""" + + +@dataclass(frozen=True) +class Batch: + id: str + source_files_json: str + status: str + note: Optional[str] + created_at: str + updated_at: str + + @property + def source_files(self) -> list[str]: + return json.loads(self.source_files_json) + + +@dataclass(frozen=True) +class Account: + id: int + account_name: str + alias: str + region_host: str + slug: str + user_data_dir: str + debug_port: int + password: Optional[str] + note: Optional[str] + created_at: str + updated_at: str + last_login_at: Optional[str] + + +@dataclass(frozen=True) +class Task: + id: int + batch_id: str + source_file: str + source_file_abs: str + source_sheet: str + source_row: int + row_key: str + account_name: Optional[str] + alias: str + item_id: str + old_title: Optional[str] + old_cover_path: Optional[str] + new_title: Optional[str] + new_cover_path: Optional[str] + committed: int + stage: str + status: str + last_error: Optional[str] + collect_attempts: int + generate_attempts: int + apply_attempts: int + imported_at: str + collected_at: Optional[str] + generated_at: Optional[str] + applied_at: Optional[str] + updated_at: str + + +SCHEMA_SQL = """ +CREATE TABLE IF NOT EXISTS batches ( + id TEXT PRIMARY KEY, + source_files_json TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active', + note TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS accounts ( + id INTEGER PRIMARY KEY, + account_name TEXT NOT NULL, + alias TEXT UNIQUE NOT NULL, + region_host TEXT NOT NULL, + slug TEXT UNIQUE NOT NULL, + user_data_dir TEXT NOT NULL, + debug_port INTEGER NOT NULL, + password TEXT, + note TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + last_login_at TEXT +); + +CREATE TABLE IF NOT EXISTS tasks ( + id INTEGER PRIMARY KEY, + batch_id TEXT NOT NULL REFERENCES batches(id), + source_file TEXT NOT NULL, + source_file_abs TEXT NOT NULL, + source_sheet TEXT NOT NULL, + source_row INTEGER NOT NULL, + row_key TEXT NOT NULL UNIQUE, + account_name TEXT, + alias TEXT NOT NULL, + item_id TEXT NOT NULL, + old_title TEXT, + old_cover_path TEXT, + new_title TEXT, + new_cover_path TEXT, + committed INTEGER NOT NULL DEFAULT 0, + stage TEXT NOT NULL DEFAULT 'imported', + status TEXT NOT NULL DEFAULT 'pending', + last_error TEXT, + collect_attempts INTEGER NOT NULL DEFAULT 0, + generate_attempts INTEGER NOT NULL DEFAULT 0, + apply_attempts INTEGER NOT NULL DEFAULT 0, + imported_at TEXT NOT NULL, + collected_at TEXT, + generated_at TEXT, + applied_at TEXT, + updated_at TEXT NOT NULL, + UNIQUE(batch_id, source_file_abs, source_sheet, source_row) +); + +CREATE INDEX IF NOT EXISTS idx_tasks_batch_stage_status + ON tasks(batch_id, stage, status); +CREATE INDEX IF NOT EXISTS idx_tasks_alias ON tasks(alias); +CREATE INDEX IF NOT EXISTS idx_tasks_item ON tasks(item_id); +""" + + +def _now() -> str: + return datetime.now().isoformat(timespec="seconds") + + +def _db_path(path=None) -> str: + return path or appconfig.db_path() + + +def connect(path=None) -> sqlite3.Connection: + """Open a SQLite connection with the project concurrency pragmas.""" + + db_path = _db_path(path) + if db_path != ":memory:": + directory = os.path.dirname(os.path.abspath(db_path)) + if directory: + os.makedirs(directory, exist_ok=True) + conn = sqlite3.connect(db_path, timeout=DEFAULT_BUSY_TIMEOUT_MS / 1000) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys=ON") + conn.execute("PRAGMA journal_mode=WAL") + conn.execute(f"PRAGMA busy_timeout={DEFAULT_BUSY_TIMEOUT_MS}") + conn.execute("PRAGMA synchronous=NORMAL") + return conn + + +@contextmanager +def _connection(conn=None, path=None): + owned = conn is None + database = connect(path) if owned else conn + try: + yield database + finally: + if owned: + database.close() + + +def _as_dataclass(row, cls): + return None if row is None else cls(**dict(row)) + + +def _fetch_one(conn, sql, params, cls): + return _as_dataclass(conn.execute(sql, params).fetchone(), cls) + + +def _fetch_all(conn, sql, params, cls): + return [_as_dataclass(row, cls) for row in conn.execute(sql, params).fetchall()] + + +def _validate_fields(fields, allowed): + unknown = sorted(set(fields) - allowed) + if unknown: + raise DbError(f"不支持更新字段: {', '.join(unknown)}") + + +def _slug_from_alias(alias: str) -> str: + base = re.sub(r"[^a-z0-9_]+", "_", alias.lower()).strip("_") + suffix = hashlib.sha1(alias.encode("utf-8")).hexdigest()[:8] + return f"{base}_{suffix}" if base else f"account_{suffix}" + + +def _row_key(batch_id, source_file_abs, source_sheet, source_row) -> str: + return f"{batch_id}:{source_file_abs}:{source_sheet}:{source_row}" + + +def _attempt_field(phase: str) -> str: + key = str(phase).lower() + if key not in PHASE_ATTEMPT_FIELDS: + raise DbError(f"未知阶段: {phase}") + return PHASE_ATTEMPT_FIELDS[key] + + +def init_db(path=None, conn=None) -> None: + """Create tables and indexes if they do not already exist.""" + + with _connection(conn, path) as database: + with database: + database.executescript(SCHEMA_SQL) + + +def create_batch(file_paths: Iterable[str], note=None, path=None, conn=None) -> str: + batch_id = datetime.now().strftime("%Y%m%d_%H%M%S_") + uuid.uuid4().hex[:8] + files = [os.path.abspath(file_path) for file_path in file_paths] + now = _now() + with _connection(conn, path) as database: + with database: + database.execute( + """ + INSERT INTO batches + (id, source_files_json, status, note, created_at, updated_at) + VALUES (?, ?, 'active', ?, ?, ?) + """, + (batch_id, json.dumps(files, ensure_ascii=False), note, now, now), + ) + return batch_id + + +def get_batch(batch_id, path=None, conn=None): + with _connection(conn, path) as database: + return _fetch_one( + database, + "SELECT * FROM batches WHERE id = ?", + (batch_id,), + Batch, + ) + + +def list_batches(status=None, path=None, conn=None): + sql = "SELECT * FROM batches" + params = [] + if status is not None: + sql += " WHERE status = ?" + params.append(status) + sql += " ORDER BY created_at DESC, id DESC" + with _connection(conn, path) as database: + return _fetch_all(database, sql, params, Batch) + + +def update_batch(batch_id, path=None, conn=None, **fields) -> None: + _validate_fields(fields, VALID_BATCH_FIELDS) + if not fields: + return + fields["updated_at"] = _now() + assignments = ", ".join(f"{field} = ?" for field in fields) + params = list(fields.values()) + [batch_id] + with _connection(conn, path) as database: + with database: + database.execute( + f"UPDATE batches SET {assignments} WHERE id = ?", + params, + ) + + +def list_accounts(path=None, conn=None): + with _connection(conn, path) as database: + return _fetch_all( + database, + "SELECT * FROM accounts ORDER BY alias", + (), + Account, + ) + + +def get_account_by_alias(alias, path=None, conn=None): + with _connection(conn, path) as database: + return _fetch_one( + database, + "SELECT * FROM accounts WHERE alias = ?", + (alias,), + Account, + ) + + +def add_account( + account_name, + alias, + region_host, + debug_port, + password=None, + note=None, + slug=None, + user_data_dir=None, + path=None, + conn=None, +): + slug = slug or _slug_from_alias(alias) + user_data_dir = user_data_dir or os.path.join("chrome_user_data_dir", slug) + now = _now() + with _connection(conn, path) as database: + try: + with database: + database.execute( + """ + INSERT INTO accounts + (account_name, alias, region_host, slug, user_data_dir, + debug_port, password, note, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + account_name, + alias, + region_host, + slug, + user_data_dir, + int(debug_port), + password, + note, + now, + now, + ), + ) + except sqlite3.IntegrityError as exc: + raise DbError(f"账号别名或 slug 已存在: {alias}") from exc + return get_account_by_alias(alias, conn=database) + + +def update_account(alias, path=None, conn=None, **fields) -> None: + _validate_fields(fields, VALID_ACCOUNT_FIELDS) + if not fields: + return + fields["updated_at"] = _now() + assignments = ", ".join(f"{field} = ?" for field in fields) + params = list(fields.values()) + [alias] + with _connection(conn, path) as database: + with database: + database.execute( + f"UPDATE accounts SET {assignments} WHERE alias = ?", + params, + ) + + +def delete_account(alias, path=None, conn=None) -> None: + with _connection(conn, path) as database: + with database: + database.execute("DELETE FROM accounts WHERE alias = ?", (alias,)) + + +def insert_tasks(batch_id, rows, path=None, conn=None) -> int: + now = _now() + values = [] + for row in rows: + source_file_abs = os.path.abspath(row["source_file_abs"]) + source_sheet = row["source_sheet"] + source_row = int(row["source_row"]) + values.append( + ( + batch_id, + row.get("source_file") or source_file_abs, + source_file_abs, + source_sheet, + source_row, + row.get("row_key") or _row_key(batch_id, source_file_abs, source_sheet, source_row), + row.get("account_name"), + row["alias"], + str(row["item_id"]), + now, + now, + ) + ) + with _connection(conn, path) as database: + try: + with database: + database.executemany( + """ + INSERT INTO tasks + (batch_id, source_file, source_file_abs, source_sheet, + source_row, row_key, account_name, alias, item_id, + imported_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + values, + ) + except sqlite3.IntegrityError as exc: + raise DbError("任务行重复或批次不存在") from exc + return len(values) + + +def list_tasks(batch_id=None, stage=None, status=None, alias=None, path=None, conn=None): + clauses = [] + params = [] + filters = { + "batch_id": batch_id, + "stage": stage, + "status": status, + "alias": alias, + } + for field, value in filters.items(): + if value is not None: + clauses.append(f"{field} = ?") + params.append(value) + sql = "SELECT * FROM tasks" + if clauses: + sql += " WHERE " + " AND ".join(clauses) + sql += " ORDER BY id" + with _connection(conn, path) as database: + return _fetch_all(database, sql, params, Task) + + +def mark_running(task_id, phase, path=None, conn=None) -> None: + _attempt_field(phase) + with _connection(conn, path) as database: + with database: + database.execute( + """ + UPDATE tasks + SET status = 'running', last_error = NULL, updated_at = ? + WHERE id = ? + """, + (_now(), int(task_id)), + ) + + +def mark_failed(task_id, phase, error, path=None, conn=None) -> None: + attempt_field = _attempt_field(phase) + with _connection(conn, path) as database: + with database: + database.execute( + f""" + UPDATE tasks + SET status = 'failed', + last_error = ?, + {attempt_field} = {attempt_field} + 1, + updated_at = ? + WHERE id = ? + """, + (str(error), _now(), int(task_id)), + ) + + +def mark_skipped(task_id, reason, path=None, conn=None) -> None: + with _connection(conn, path) as database: + with database: + database.execute( + """ + UPDATE tasks + SET status = 'skipped', last_error = ?, updated_at = ? + WHERE id = ? + """, + (str(reason), _now(), int(task_id)), + ) + + +def set_collected(task_id, old_title, old_cover_path, path=None, conn=None) -> None: + now = _now() + with _connection(conn, path) as database: + with database: + database.execute( + """ + UPDATE tasks + SET old_title = ?, + old_cover_path = ?, + stage = 'collected', + status = 'success', + last_error = NULL, + collect_attempts = collect_attempts + 1, + collected_at = ?, + updated_at = ? + WHERE id = ? + """, + (old_title, old_cover_path, now, now, int(task_id)), + ) + + +def set_generated(task_id, new_title, new_cover_path, path=None, conn=None) -> None: + now = _now() + with _connection(conn, path) as database: + with database: + database.execute( + """ + UPDATE tasks + SET new_title = ?, + new_cover_path = ?, + stage = 'generated', + status = 'success', + last_error = NULL, + generate_attempts = generate_attempts + 1, + generated_at = ?, + updated_at = ? + WHERE id = ? + """, + (new_title, new_cover_path, now, now, int(task_id)), + ) + + +def set_applied(task_id, committed, error=None, path=None, conn=None) -> None: + now = _now() + success = bool(committed) and error is None + with _connection(conn, path) as database: + with database: + if success: + database.execute( + """ + UPDATE tasks + SET committed = 1, + stage = 'applied', + status = 'success', + last_error = NULL, + apply_attempts = apply_attempts + 1, + applied_at = ?, + updated_at = ? + WHERE id = ? + """, + (now, now, int(task_id)), + ) + else: + database.execute( + """ + UPDATE tasks + SET committed = 0, + status = 'failed', + last_error = ?, + apply_attempts = apply_attempts + 1, + updated_at = ? + WHERE id = ? + """, + (str(error or "未提交更新"), now, int(task_id)), + ) diff --git a/docs/06-tasks.md b/docs/06-tasks.md index ba350b9..1dfba56 100644 --- a/docs/06-tasks.md +++ b/docs/06-tasks.md @@ -25,9 +25,9 @@ | --- | --- | --- | --- | --- | | T-000 | 正式代码包结构：创建 `app/`、迁入 `cdp.py` 为 `app/cdp.py`、新增 `app/__init__.py`、`app/__main__.py`、根入口 `main.py`、最小 `app/gui.py` 占位入口，并修正 prototypes 导入 | - | `python -m compileall app main.py` 通过；`python -m app`/`python main.py` 可进入入口（本机 `python` 不符合版本时用 `py -3 -m app`）；GUI 未完成时给明确提示并退出；`prototypes/demo.py` 可从项目根导入 `app.cdp` | DONE | | T-001 | `app/editor.py`：改标题/换封面/点更新/登录检测/**采集(读旧标题+旧封面下载)**/apply_task，复用 `app/cdp.py` | T-000 | 函数可调用，在测试商品跑通；与 `prototypes/demo.py` 行为一致 | DONE | -| T-002 | `app/appconfig.py` + `config.json`（含 image_dir、ai 选择/参数段、端口等默认值；不含 AI Key） | T-000 | 读写正常；不存在则写默认；AI Key 留给 `config/ai_models.json`/T-501 | TODO | -| T-003 | `app/db.py` + SQLite 建表（batches/accounts/tasks，含 Excel 行定位、状态、时间戳、重试字段） | T-000 | `init_db` 幂等；`connect` 设置 WAL/busy_timeout/foreign_keys；账号/批次/任务/各 set_* 可用；schema 同架构 5.2 | TODO | -| T-004 | `.gitignore`：排除 `config.json`、`config/ai_models.json`、`cmshopee.db`、`chrome_user_data_dir/`、`images/` | T-002, T-003 | 配置、密钥、凭证、业务数据、图片不被提交 | TODO | +| T-002 | `app/appconfig.py` + `config.json`（含 image_dir、ai 选择/参数段、端口等默认值；不含 AI Key） | T-000 | 读写正常；不存在则写默认；AI Key 留给 `config/ai_models.json`/T-501 | DONE | +| T-003 | `app/db.py` + SQLite 建表（batches/accounts/tasks，含 Excel 行定位、状态、时间戳、重试字段） | T-000 | `init_db` 幂等；`connect` 设置 WAL/busy_timeout/foreign_keys；账号/批次/任务/各 set_* 可用；schema 同架构 5.2 | DONE | +| T-004 | `.gitignore`：排除 `config.json`、`config/ai_models.json`、`cmshopee.db`、`chrome_user_data_dir/`、`images/` | T-002, T-003 | 配置、密钥、凭证、业务数据、图片不被提交 | DONE | | T-005 | AI 模型清单后端：`config/ai_models.json` 读写 + category 过滤 + 测试连接 | T-002 | 本地明文 api_key；UI/API 打码显示；日志脱敏；至少 text/image 各一个；`get_model` 返回调用所需字段 | TODO | | T-006 | 单元测试基座：`tests/` + appconfig/db/excel/prompts 最小测试 | T-002, T-003 | `python -m unittest discover -s tests` 可跑；不依赖真实 Shopee/AI；临时文件在测试目录清理 | TODO | diff --git a/docs/api.md b/docs/api.md index 5d594f8..19f375f 100644 --- a/docs/api.md +++ b/docs/api.md @@ -9,22 +9,34 @@ - 凭证：登录态在 user-data-dir；密码、AI Key 本地明文存于 config/DB；`config.json`、`config/ai_models.json`、`cmshopee.db`、`chrome_user_data_dir/`、`images/` 必须 gitignore；UI 打码显示，不出现在日志/导出。 - 失败处理：抛带中文说明的异常或返回状态字段；GUI 负责提示，不静默吞错。 -## appconfig 模块（`app/appconfig.py`，待建） +## appconfig 模块（`app/appconfig.py`，已建） 读写 `config.json`（schema 见 [架构 5.1](04-architecture.md)）。 ```python +class ConfigError(RuntimeError): ... +default_config() -> dict load_config(path="config.json") -> dict # 不存在则写默认 -chrome_path() -> str -user_data_root() -> str -image_dir() -> str -db_path() -> str -ai_config() -> dict # default_text_model/default_image_model/ +save_config(config, path="config.json") -> dict +update_config(updates, path="config.json") -> dict +chrome_path(config=None) -> str +user_data_root(config=None) -> str +image_dir(config=None) -> str +db_path(config=None) -> str +default_debug_port(config=None) -> int +debug_port_range(config=None) -> tuple # (start, end) +cdp_ready_timeout(config=None) -> int +ai_config(config=None) -> dict # default_text_model/default_image_model/ # title_concurrency/image_concurrency/retry/jpg_quality/ # resolution/resolution_timeouts -response_timeout() -> int # = resolution_timeouts[resolution]（返回超时，随分辨率） +response_timeout(config=None) -> int # = resolution_timeouts[resolution]（返回超时，随分辨率） +``` -# AI 模型清单 config/ai_models.json（含本地明文密钥；CRUD 由 ⑤ 设置） +`config.json` 不保存 AI Key；写入 `api_key` / `*_key` / `token` / `password` 等敏感字段时抛 `ConfigError`。AI Key 留给 `config/ai_models.json`。 + +AI 模型清单（`config/ai_models.json`，含本地明文密钥；CRUD 由 ⑤ 设置，T-005 待建）： + +```python list_ai_models(category=None) -> list[dict] # category=text/image 过滤；含 connect_timeout_seconds 等 add_ai_model(model) -> None # name 唯一校验 update_ai_model(name, **fields) -> None @@ -33,32 +45,34 @@ test_ai_model(name) -> dict # 「测试连接」：用 key/url get_model(name) -> dict # 返回模型定义，含 api_key（调用方不得写日志） ``` -## db 模块（`app/db.py`，待建） +## db 模块（`app/db.py`，已建） SQLite 读写，表见 [架构 5.2](04-architecture.md)。 ```python +class DbError(RuntimeError): ... +Batch / Account / Task # dataclass，字段同 SQLite schema connect(path=None) -> sqlite3.Connection # 设置 foreign_keys/WAL/busy_timeout/synchronous/row_factory -init_db(path) -create_batch(file_paths, note=None) -> str -get_batch(batch_id) -> Batch|None -list_batches(status=None) -> list[Batch] -update_batch(batch_id, **fields) -> None +init_db(path=None) +create_batch(file_paths, note=None, path=None) -> str +get_batch(batch_id, path=None) -> Batch|None +list_batches(status=None, path=None) -> list[Batch] +update_batch(batch_id, **fields) -> None # 支持 status/note/source_files_json # 账号 -list_accounts() -> list[Account] -get_account_by_alias(alias) -> Account|None +list_accounts(path=None) -> list[Account] +get_account_by_alias(alias, path=None) -> Account|None add_account(account_name, alias, region_host, debug_port, password=None, note=None) -> Account -update_account(alias, **fields) -> None +update_account(alias, **fields) -> None # 支持账号展示字段、端口、密码、note、last_login_at delete_account(alias) -> None # 任务 / 各阶段结果 -insert_tasks(batch_id, rows) -> int # 写输入列；rows 含 source_file_abs/source_sheet/source_row/row_key -list_tasks(batch_id=None, stage=None, status=None, alias=None) -> list[Task] +insert_tasks(batch_id, rows, path=None) -> int # 写输入列；rows 含 source_file_abs/source_sheet/source_row/row_key +list_tasks(batch_id=None, stage=None, status=None, alias=None, path=None) -> list[Task] mark_running(task_id, phase) -> None -mark_failed(task_id, phase, error) -> None -mark_skipped(task_id, reason) -> None -set_collected(task_id, old_title, old_cover_path) -> None # stage=collected,status=success, attempts+1 -set_generated(task_id, new_title, new_cover_path) -> None # stage=generated,status=success, attempts+1 -set_applied(task_id, committed, error=None) -> None # stage=applied,status=success/failed, attempts+1 +mark_failed(task_id, phase, error) -> None # status=failed，stage 不前进，对应 attempts+1 +mark_skipped(task_id, reason) -> None # status=skipped，stage 不前进 +set_collected(task_id, old_title, old_cover_path) -> None # stage=collected,status=success,collect_attempts+1 +set_generated(task_id, new_title, new_cover_path) -> None # stage=generated,status=success,generate_attempts+1 +set_applied(task_id, committed, error=None) -> None # 成功 stage=applied；失败 status=failed 且 stage 不前进 ``` `stage` 表示最后成功业务阶段（imported→collected→generated→applied）；`status` 表示当前处理结果（pending/running/success/failed/skipped/cancelled）。任意步失败写 `last_error` 且 `status=failed`，`stage` 不前进。无 confirmed 阶段。 diff --git a/docs/current-state.md b/docs/current-state.md index cb99b42..0e04a5f 100644 --- a/docs/current-state.md +++ b/docs/current-state.md @@ -5,12 +5,12 @@ ## 当前快照 -- 日期：2026-06-26 -- 阶段：V0 单账号 CDP 流程已验证；V1 已完成 T-000 正式代码包结构与 T-001 `app/editor.py` 模块化，下一步开始应用配置。 +- 日期：2026-06-27 +- 阶段：V0 单账号 CDP 流程已验证；V1 已完成 T-000 正式代码包结构、T-001 `app/editor.py` 模块化、T-002 `app/appconfig.py` 应用配置、T-003 SQLite 持久化地基、T-004 本地数据忽略规则。 - 技术栈：Python 3.10+，自研 CDP（websocket-client + requests），SQLite（sqlite3）+ `config.json` + openpyxl + AI（服务商待定），GUI PySide6 5 Tab（已定）。 -- 生产代码：已建立 `app/` 包 + 根入口 `main.py`；`app/cdp.py` 为已验证 CDP 底座；`app/editor.py` 已封装标题/封面/采集/更新按钮能力；`app/gui.py` 目前是入口占位，完整 PySide6 主窗口待 T-104。 +- 生产代码：已建立 `app/` 包 + 根入口 `main.py`；`app/cdp.py` 为已验证 CDP 底座；`app/editor.py` 已封装标题/封面/采集/更新按钮能力；`app/appconfig.py` 已实现 `config.json` 默认值、读写、更新、AI 参数与端口配置读取；`app/db.py` 已实现 SQLite schema、连接 PRAGMA、批次/账号/任务与阶段写库函数；`app/gui.py` 目前是入口占位，完整 PySide6 主窗口待 T-104。 - 测试：当前以 `compileall` + 测试商品手动 CDP 验证为主；`tests/` 与 `python -m unittest discover -s tests` 由 T-006 建立，T-006 完成前不把缺少 `tests/` 视为验证失败。 -- 数据：无 `config.json`、`config/ai_models.json`、`cmshopee.db`、`chrome_user_data_dir/`、`images/`（待 Phase 0/1 建立，均须 gitignore）。 +- 数据：`config.json`、`config/ai_models.json`、`cmshopee.db`、`chrome_user_data_dir/`、`images/` 已由 `.gitignore` 排除；`app/appconfig.py` 首次读取缺失的 `config.json` 时会在本地写默认配置，`app/db.py` 调用 `init_db()` 时会在本地创建 SQLite DB。 ## 既定设计要点（文档已定） @@ -29,13 +29,15 @@ | --- | --- | --- | | `docs/` | 已有 | 本 harness coding 文档集合 | | `app/cdp.py` | 已有 | CDP 底座：连接/找 tab/开 tab/执行 JS/拖拽 | -| `prototypes/` | 已有 | 已验证原型/探查脚本（demo/set_title/set_cover/get_title/cookies/inspect_images/grab/1.py），逻辑待并入 `app/editor.py` 后清理；见 `prototypes/README.md` | +| `prototypes/` | 已有 | 已验证原型/探查脚本（demo/set_title/set_cover/get_title/cookies/inspect_images/grab/1.py），保留作人工回归与探查参考；见 `prototypes/README.md` | | `chrome-remote-debug-lan.md` | 已有 | WSL→Windows CDP 转发排查记录 | | `app/__init__.py` / `app/__main__.py` / `main.py` | 已有 | 正式包与启动入口；`python main.py` / `python -m app` 可运行占位入口 | | `app/gui.py` | 已有 | GUI 占位入口；完整 PySide6 主窗口待 T-104 | | `app/editor.py` | 已有 | T-001 产出：登录检测、打开商品页、读/写标题、读/下载封面、上传拖封面、更新按钮、apply_task | -| `app/appconfig.py` / `app/db.py` / `app/excel.py` / `app/config.py` / `app/chrome.py` / `app/workers.py` | 待建 | Phase 0-3 产出 | -| `config.json` / `config/ai_models.json` / `cmshopee.db` / `chrome_user_data_dir/` / `images/` | 待建 | 含配置、密钥、业务、登录态、图片，须 gitignore | +| `app/appconfig.py` | 已有 | T-002 产出：`config.json` 默认值、读写、更新、路径/端口/AI 参数读取；拒绝敏感字段写入 | +| `app/db.py` | 已有 | T-003 产出：batches/accounts/tasks schema；WAL/busy_timeout/foreign_keys；账号/批次/任务与 set_* 阶段写库 | +| `app/excel.py` / `app/config.py` / `app/chrome.py` / `app/workers.py` | 待建 | Phase 0-3 产出 | +| `config.json` / `config/ai_models.json` / `cmshopee.db` / `chrome_user_data_dir/` / `images/` | 本地待建，已忽略 | 含配置、密钥、业务、登录态、图片，不提交版本库 | ## 已验证能力（单账号） @@ -49,9 +51,9 @@ 任务状态以 [`06-tasks.md`](06-tasks.md) 为准，历史记录见 [`../progress.md`](../progress.md)。 -- 已完成：T-000（正式代码包结构）、T-001（`app/editor.py` 模块化）。 +- 已完成：T-000（正式代码包结构）、T-001（`app/editor.py` 模块化）、T-002（`app/appconfig.py` + `config.json`）、T-003（`app/db.py` + SQLite 建表）、T-004（本地数据 gitignore）。 - 正在进行：无。 -- 下一个可领取任务：**T-002（`app/appconfig.py` + `config.json`）**。虽然 T-003 也依赖满足，但任务领取规则固定为“取 `06-tasks.md` 中第一个 TODO 且依赖均 DONE 的任务”，因此当前不能任选。 +- 下一个可领取任务：**T-005（AI 模型清单后端）**。 ## 当前可运行内容 @@ -59,6 +61,15 @@ # 语法检查（T-000 后；本机优先用 py -3） py -3 -m compileall app main.py +# appconfig 默认配置读写（写入临时目录） +py -3 -c "import os,tempfile; from app import appconfig; d=tempfile.TemporaryDirectory(dir='.'); print(appconfig.load_config(os.path.join(d.name,'config.json'))['image_dir'])" + +# db 临时库初始化与 PRAGMA 检查 +py -3 -c "import os,tempfile; from app import db; d=tempfile.TemporaryDirectory(dir='.'); p=os.path.join(d.name,'cmshopee.db'); db.init_db(p); c=db.connect(p); print(c.execute('PRAGMA foreign_keys').fetchone()[0], c.execute('PRAGMA journal_mode').fetchone()[0], c.execute('PRAGMA busy_timeout').fetchone()[0]); c.close()" + +# gitignore 核心本地数据检查 +git check-ignore -v -- config.json config/ai_models.json cmshopee.db chrome_user_data_dir/ images/ + # 当前入口占位 python main.py py -3 -m app diff --git a/progress.md b/progress.md index 2264dff..270b390 100644 --- a/progress.md +++ b/progress.md @@ -249,3 +249,30 @@ - 下一步：按任务看板领取 T-002。 + +## 【2026-06-27】T-002 appconfig 应用配置 + +- 状态：DONE +- 变更：新增 `app/appconfig.py`，实现 `config.json` 默认配置、缺失时写默认、读取/保存/合并更新、路径/端口/CDP 等待时间/AI 参数读取、按分辨率派生返回超时；同步 `docs/06-tasks.md`、`docs/current-state.md`、`docs/api.md`。 +- 配置边界：`config.json` 只保存应用路径、端口、AI 角色选择与生成参数；`api_key` / `*_key` / `token` / `password` 等敏感字段写入时抛 `ConfigError`，AI Key 仍留给 T-005 的 `config/ai_models.json`。 +- 验证：`py -3 -m compileall app main.py` 通过；临时 `config.json` 的 `load_config` / `update_config` / `response_timeout` 验证通过；尝试写入 `api_key` 返回 `ConfigError: config.json 不允许保存敏感字段: api_key`。 +- 注意：`python -m unittest discover -s tests` 仍等 T-006 建立测试基座后再运行。 +- 下一步：按任务看板领取 T-003。 + +## 【2026-06-27】T-003 SQLite 持久化地基 + +- 状态：DONE +- 变更：新增 `app/db.py`，按架构 5.2 建立 `batches`、`accounts`、`tasks` 三张表和索引；实现 `connect/init_db`、批次、账号、任务导入、任务筛选，以及 `mark_running/mark_failed/mark_skipped/set_collected/set_generated/set_applied`；同步 `docs/06-tasks.md`、`docs/current-state.md`、`docs/api.md`。 +- 连接规则：`connect()` 设置 `foreign_keys=ON`、`journal_mode=WAL`、`busy_timeout=5000`、`synchronous=NORMAL`、`row_factory=sqlite3.Row`；每个函数默认短事务写入并提交，worker 后续可按线程自建 connection。 +- 阶段规则：失败和跳过只更新 `status/last_error/attempts`，`stage` 保持最后成功阶段；成功采集/生成/应用分别推进到 `collected/generated/applied`。 +- 验证：`py -3 -m compileall app main.py` 通过；临时 SQLite 文件中重复 `init_db` 通过；PRAGMA 输出 `(1, 'wal', 5000)`；批次/账号/任务写入、失败状态、采集/生成/应用阶段推进验证通过。 +- 注意：`python -m unittest discover -s tests` 仍等 T-006 建立测试基座后再运行。 +- 下一步：按任务看板领取 T-004。 + +## 【2026-06-27】T-004 本地数据 gitignore + +- 状态：DONE +- 变更：校准 `.gitignore`，明确排除 `config.json`、`config/ai_models.json`、`cmshopee.db`、`chrome_user_data_dir/`、`images/`；保留本地测试图片与临时启动脚本忽略项；同步 `docs/06-tasks.md`、`docs/current-state.md`。 +- 决策：将原来的 `config/` 整目录忽略收窄为 `config/ai_models.json`，与文档安全红线保持一致，避免后续非敏感模板或说明文件被误隐藏。 +- 验证：`git check-ignore -v -- config.json config/ai_models.json cmshopee.db chrome_user_data_dir/ images/` 五个路径均命中仓库 `.gitignore`；命令同时提示全局 ignore 文件权限不可读，但不影响仓库规则生效。 +- 下一步：按任务看板领取 T-005。