feat: 建立商品目录接入基础 (#132)

2026-08-11 10:00:07 +08:00
parent b1a78f3e2f
commit 107ee5ce85
13 changed files with 631 additions and 7 deletions
@@ -19,7 +19,7 @@ import (
 	"cmautobuy/admin/spec"
 )

-const mysqlSchemaVersion = 6
+const mysqlSchemaVersion = 7

 // OpenMySQL 打开生产 MySQL 8 数据库。错误信息绝不包含完整 DSN 或密码。
 func OpenMySQL(cfg config.DatabaseConfig) (*sql.DB, error) {
@@ -495,10 +495,98 @@ func MigrateMySQL(db *sql.DB) error {
 		if _, err := db.Exec(`INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)`, 6, time.Now().UTC().Format(time.RFC3339Nano)); err != nil {
 			return fmt.Errorf("记录 MySQL schema v6 失败: %w", err)
 		}
+		current = 6
+	}
+	if current < 7 {
+		if err := migrateMySQLV7(db); err != nil {
+			return fmt.Errorf("执行 MySQL schema v7 失败: %w", err)
+		}
+		if err := checkMySQLV7Shape(db); err != nil {
+			return fmt.Errorf("MySQL schema v7 自检失败，未记录版本: %w", err)
+		}
+		if _, err := db.Exec(`INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)`, 7, time.Now().UTC().Format(time.RFC3339Nano)); err != nil {
+			return fmt.Errorf("记录 MySQL schema v7 失败: %w", err)
+		}
 	}
 	return CheckMySQLSchema(db)
 }

+const mysqlSchemaV7CatalogRuns = `CREATE TABLE IF NOT EXISTS catalog_import_runs (
+	source VARCHAR(64) COLLATE utf8mb4_bin NOT NULL,
+	batch_id VARCHAR(191) COLLATE utf8mb4_bin NOT NULL,
+	request_hash VARCHAR(64) COLLATE utf8mb4_bin NOT NULL,
+	status VARCHAR(16) COLLATE utf8mb4_bin NOT NULL DEFAULT 'processing',
+	request_count INT UNSIGNED NOT NULL DEFAULT 1,
+	conflict_count INT UNSIGNED NOT NULL DEFAULT 0,
+	observed_at VARCHAR(35) NULL,
+	last_request_at VARCHAR(35) NOT NULL,
+	last_conflict_at VARCHAR(35) NULL,
+	shopee_created INT UNSIGNED NOT NULL DEFAULT 0,
+	shopee_updated INT UNSIGNED NOT NULL DEFAULT 0,
+	sku_created INT UNSIGNED NOT NULL DEFAULT 0,
+	sku_updated INT UNSIGNED NOT NULL DEFAULT 0,
+	pdd_created INT UNSIGNED NOT NULL DEFAULT 0,
+	pdd_updated INT UNSIGNED NOT NULL DEFAULT 0,
+	association_created INT UNSIGNED NOT NULL DEFAULT 0,
+	association_unchanged INT UNSIGNED NOT NULL DEFAULT 0,
+	failure_count INT UNSIGNED NOT NULL DEFAULT 0,
+	error_summary VARCHAR(1000) NULL,
+	response_body LONGTEXT NULL,
+	created_at VARCHAR(35) NOT NULL,
+	finished_at VARCHAR(35) NULL,
+	PRIMARY KEY (source, batch_id),
+	KEY idx_catalog_runs_list (created_at DESC, source, batch_id),
+	KEY idx_catalog_runs_status (status, created_at DESC),
+	CONSTRAINT chk_catalog_runs_status CHECK (status IN ('processing','succeeded','failed'))
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`
+
+// migrateMySQLV7 建立第三方商品目录批次基础。所有 DDL 均逐项检查，支持隐式提交后重放。
+func migrateMySQLV7(db *sql.DB) error {
+	columns := []struct {
+		table, name, ddl string
+	}{
+		{"shopee_products", "source_observed_at", `ALTER TABLE shopee_products ADD COLUMN source_observed_at VARCHAR(35) NULL AFTER source`},
+		{"shopee_skus", "source_observed_at", `ALTER TABLE shopee_skus ADD COLUMN source_observed_at VARCHAR(35) NULL AFTER is_manual`},
+		{"pdd_products", "source", `ALTER TABLE pdd_products ADD COLUMN source VARCHAR(16) COLLATE utf8mb4_bin NULL AFTER collect_status`},
+		{"pdd_products", "source_observed_at", `ALTER TABLE pdd_products ADD COLUMN source_observed_at VARCHAR(35) NULL AFTER source`},
+	}
+	for _, column := range columns {
+		exists, err := mysqlColumnExists(db, column.table, column.name)
+		if err != nil {
+			return err
+		}
+		if !exists {
+			if _, err := db.Exec(column.ddl); err != nil {
+				return fmt.Errorf("增加 %s.%s 失败: %w", column.table, column.name, err)
+			}
+		}
+	}
+
+	var clause string
+	err := db.QueryRow(`SELECT cc.check_clause FROM information_schema.table_constraints tc
+		JOIN information_schema.check_constraints cc ON cc.constraint_schema=tc.constraint_schema AND cc.constraint_name=tc.constraint_name
+		WHERE tc.constraint_schema=DATABASE() AND tc.table_name='shopee_products'
+		  AND tc.constraint_name='chk_shopee_products_source' AND tc.constraint_type='CHECK'`).Scan(&clause)
+	if err != nil && err != sql.ErrNoRows {
+		return fmt.Errorf("读取蝦皮来源约束失败: %w", err)
+	}
+	normalized := strings.NewReplacer("`", "", " ", "", "(", "", ")", "", "_utf8mb4", "", `\`, "").Replace(strings.ToLower(clause))
+	if !strings.Contains(normalized, "'api'") {
+		if err == nil {
+			if _, dropErr := db.Exec(`ALTER TABLE shopee_products DROP CHECK chk_shopee_products_source`); dropErr != nil {
+				return fmt.Errorf("移除旧蝦皮来源约束失败: %w", dropErr)
+			}
+		}
+		if _, addErr := db.Exec(`ALTER TABLE shopee_products ADD CONSTRAINT chk_shopee_products_source CHECK (source IN ('report','syb','api'))`); addErr != nil {
+			return fmt.Errorf("扩展蝦皮来源约束失败: %w", addErr)
+		}
+	}
+	if _, err := db.Exec(mysqlSchemaV7CatalogRuns); err != nil {
+		return fmt.Errorf("建立商品目录导入记录表失败: %w", err)
+	}
+	return nil
+}
+
 // migrateMySQLV6 给任务补充创建人。存量任务保持 NULL，明确标记为历史任务；
 // 新代码创建任务时写入当前网页登录用户。DDL 逐项检查，支持中断后重放。
 func migrateMySQLV6(db *sql.DB) error {
@@ -843,6 +931,7 @@ func CheckMySQLSchema(db *sql.DB) error {
 		"tasks", "clients", "idempotency_keys", "task_claims", "syb_session", "syb_sync_state",
 		"users", "web_sessions", "client_user_assignments", "syb_sync_runs", "admin_initialization_lock",
 		"spec_mapping_decisions",
+		"catalog_import_runs",
 	}
 	if err := checkMySQLSchema(db, mysqlRequiredTables); err != nil {
 		return err
@@ -856,7 +945,71 @@ func CheckMySQLSchema(db *sql.DB) error {
 	if err := checkMySQLV5Shape(db); err != nil {
 		return err
 	}
-	return checkMySQLV6Shape(db)
+	if err := checkMySQLV6Shape(db); err != nil {
+		return err
+	}
+	return checkMySQLV7Shape(db)
+}
+
+func checkMySQLV7Shape(db *sql.DB) error {
+	for _, column := range []struct{ table, name string }{
+		{"shopee_products", "source_observed_at"},
+		{"shopee_skus", "source_observed_at"},
+		{"pdd_products", "source_observed_at"},
+	} {
+		if err := checkMySQLVarcharColumn(db, column.table, column.name, 35, true, "utf8mb4_0900_ai_ci", ""); err != nil {
+			return err
+		}
+		if err := checkMySQLNullDefault(db, column.table, column.name); err != nil {
+			return err
+		}
+	}
+	if err := checkMySQLVarcharColumn(db, "pdd_products", "source", 16, true, "utf8mb4_bin", ""); err != nil {
+		return err
+	}
+	if err := checkMySQLNullDefault(db, "pdd_products", "source"); err != nil {
+		return err
+	}
+	if err := checkMySQLSchema(db, []string{"catalog_import_runs"}); err != nil {
+		return err
+	}
+	for _, column := range []struct {
+		name      string
+		length    int64
+		nullable  bool
+		collation string
+		def       string
+	}{
+		{"source", 64, false, "utf8mb4_bin", ""},
+		{"batch_id", 191, false, "utf8mb4_bin", ""},
+		{"request_hash", 64, false, "utf8mb4_bin", ""},
+		{"status", 16, false, "utf8mb4_bin", "processing"},
+	} {
+		if err := checkMySQLVarcharColumn(db, "catalog_import_runs", column.name, column.length, column.nullable, column.collation, column.def); err != nil {
+			return err
+		}
+	}
+	var clause string
+	if err := db.QueryRow(`SELECT cc.check_clause FROM information_schema.table_constraints tc
+		JOIN information_schema.check_constraints cc ON cc.constraint_schema=tc.constraint_schema AND cc.constraint_name=tc.constraint_name
+		WHERE tc.constraint_schema=DATABASE() AND tc.table_name='catalog_import_runs'
+		  AND tc.constraint_name='chk_catalog_runs_status' AND tc.constraint_type='CHECK'`).Scan(&clause); err != nil {
+		return fmt.Errorf("商品目录批次状态约束缺失: %w", err)
+	}
+	normalized := strings.NewReplacer("`", "", " ", "", "(", "", ")", "", "_utf8mb4", "", `\`, "").Replace(strings.ToLower(clause))
+	for _, value := range []string{"'processing'", "'succeeded'", "'failed'"} {
+		if !strings.Contains(normalized, value) {
+			return fmt.Errorf("商品目录批次状态约束不正确")
+		}
+	}
+	var sourceClause string
+	if err := db.QueryRow(`SELECT cc.check_clause FROM information_schema.table_constraints tc
+		JOIN information_schema.check_constraints cc ON cc.constraint_schema=tc.constraint_schema AND cc.constraint_name=tc.constraint_name
+		WHERE tc.constraint_schema=DATABASE() AND tc.table_name='shopee_products'
+		  AND tc.constraint_name='chk_shopee_products_source' AND tc.constraint_type='CHECK'`).Scan(&sourceClause); err != nil || !strings.Contains(strings.ToLower(sourceClause), "api") {
+		return fmt.Errorf("蝦皮商品来源约束未允许 api")
+	}
+	return nil
 }

 func checkMySQLV6Shape(db *sql.DB) error {