feat: 支持无SKU目录导入策略 (#141)

2026-08-11 11:03:18 +08:00
parent 6386d76488
commit 4259be0e55
17 changed files with 766 additions and 73 deletions
@@ -19,7 +19,7 @@ import (
 	"cmautobuy/admin/spec"
 )

-const mysqlSchemaVersion = 7
+const mysqlSchemaVersion = 8

 // OpenMySQL 打开生产 MySQL 8 数据库。错误信息绝不包含完整 DSN 或密码。
 func OpenMySQL(cfg config.DatabaseConfig) (*sql.DB, error) {
@@ -507,10 +507,115 @@ func MigrateMySQL(db *sql.DB) error {
 		if _, err := db.Exec(`INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)`, 7, time.Now().UTC().Format(time.RFC3339Nano)); err != nil {
 			return fmt.Errorf("记录 MySQL schema v7 失败: %w", err)
 		}
+		current = 7
+	}
+	if current < 8 {
+		if err := migrateMySQLV8(db); err != nil {
+			return fmt.Errorf("执行 MySQL schema v8 失败: %w", err)
+		}
+		if err := checkMySQLV8Shape(db); err != nil {
+			return fmt.Errorf("MySQL schema v8 自检失败，未记录版本: %w", err)
+		}
+		if _, err := db.Exec(`INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)`, 8, time.Now().UTC().Format(time.RFC3339Nano)); err != nil {
+			return fmt.Errorf("记录 MySQL schema v8 失败: %w", err)
+		}
 	}
 	return CheckMySQLSchema(db)
 }

+// migrateMySQLV8 保留 sku_id 作为内部主键，并增加可空的真实蝦皮 SKU ID。
+// 每个 DDL 都先检查形状，支持 MySQL 隐式提交后的中断重放。
+func migrateMySQLV8(db *sql.DB) error {
+	columns := []struct{ table, name, ddl string }{
+		{"shopee_skus", "shopee_sku_id", `ALTER TABLE shopee_skus ADD COLUMN shopee_sku_id VARCHAR(191) COLLATE utf8mb4_bin NULL AFTER sku_id`},
+		{"shopee_skus", "spec_key", `ALTER TABLE shopee_skus ADD COLUMN spec_key VARCHAR(191) COLLATE utf8mb4_bin NULL AFTER spec_raw`},
+		{"shopee_skus", "source", `ALTER TABLE shopee_skus ADD COLUMN source VARCHAR(64) COLLATE utf8mb4_bin NULL AFTER is_manual`},
+		{"shopee_skus", "field_sources", `ALTER TABLE shopee_skus ADD COLUMN field_sources JSON NULL AFTER source`},
+		{"shopee_skus", "field_observed_at", `ALTER TABLE shopee_skus ADD COLUMN field_observed_at JSON NULL AFTER field_sources`},
+		{"catalog_import_runs", "update_policy", `ALTER TABLE catalog_import_runs ADD COLUMN update_policy VARCHAR(32) COLLATE utf8mb4_bin NOT NULL DEFAULT 'fill_missing' AFTER observed_at`},
+		{"catalog_import_runs", "sku_filled", `ALTER TABLE catalog_import_runs ADD COLUMN sku_filled INT UNSIGNED NOT NULL DEFAULT 0 AFTER sku_updated`},
+		{"catalog_import_runs", "sku_same_source_updated", `ALTER TABLE catalog_import_runs ADD COLUMN sku_same_source_updated INT UNSIGNED NOT NULL DEFAULT 0 AFTER sku_filled`},
+		{"catalog_import_runs", "sku_skipped", `ALTER TABLE catalog_import_runs ADD COLUMN sku_skipped INT UNSIGNED NOT NULL DEFAULT 0 AFTER sku_same_source_updated`},
+		{"catalog_import_runs", "sku_manual_skipped", `ALTER TABLE catalog_import_runs ADD COLUMN sku_manual_skipped INT UNSIGNED NOT NULL DEFAULT 0 AFTER sku_skipped`},
+		{"catalog_import_runs", "sku_stale_skipped", `ALTER TABLE catalog_import_runs ADD COLUMN sku_stale_skipped INT UNSIGNED NOT NULL DEFAULT 0 AFTER sku_manual_skipped`},
+	}
+	for _, column := range columns {
+		exists, err := mysqlColumnExists(db, column.table, column.name)
+		if err != nil {
+			return err
+		}
+		if !exists {
+			if _, err := db.Exec(column.ddl); err != nil {
+				return fmt.Errorf("增加 %s.%s 失败: %w", column.table, column.name, err)
+			}
+		}
+	}
+	if _, err := db.Exec(`UPDATE shopee_skus SET shopee_sku_id=sku_id WHERE shopee_sku_id IS NULL AND sku_id NOT LIKE 'catalog:%'`); err != nil {
+		return fmt.Errorf("回填真实蝦皮 SKU ID 失败: %w", err)
+	}
+	if _, err := db.Exec(`UPDATE shopee_skus SET source='report' WHERE source IS NULL OR source=''`); err != nil {
+		return fmt.Errorf("回填 SKU 来源失败: %w", err)
+	}
+	if _, err := db.Exec(`UPDATE shopee_skus SET field_sources=JSON_OBJECT('color',IF(color IS NULL OR color='','',COALESCE(source,'report')),'size',IF(size IS NULL OR size='','',COALESCE(source,'report')),'advice',IF(advice IS NULL OR advice='','',COALESCE(source,'report')),'sku_code',IF(sku_code IS NULL OR sku_code='','',COALESCE(source,'report'))) WHERE field_sources IS NULL`); err != nil {
+		return fmt.Errorf("回填 SKU 字段来源失败: %w", err)
+	}
+	if _, err := db.Exec(`UPDATE shopee_skus SET field_observed_at=JSON_OBJECT('color',IF(color IS NULL OR color='','',COALESCE(source_observed_at,updated_at)),'size',IF(size IS NULL OR size='','',COALESCE(source_observed_at,updated_at)),'advice',IF(advice IS NULL OR advice='','',COALESCE(source_observed_at,updated_at)),'sku_code',IF(sku_code IS NULL OR sku_code='','',COALESCE(source_observed_at,updated_at))) WHERE field_observed_at IS NULL`); err != nil {
+		return fmt.Errorf("回填 SKU 字段观测时间失败: %w", err)
+	}
+	rows, err := db.Query(`SELECT sku_id,spec_raw FROM shopee_skus WHERE spec_key IS NULL OR spec_key=''`)
+	if err != nil {
+		return fmt.Errorf("读取待回填规格键失败: %w", err)
+	}
+	type pendingKey struct{ id, raw string }
+	var pending []pendingKey
+	for rows.Next() {
+		var row pendingKey
+		if err := rows.Scan(&row.id, &row.raw); err != nil {
+			rows.Close()
+			return err
+		}
+		pending = append(pending, row)
+	}
+	if err := rows.Close(); err != nil {
+		return err
+	}
+	for _, row := range pending {
+		key, keyErr := spec.SpecKey(row.raw)
+		if keyErr != nil {
+			return fmt.Errorf("SKU 内部记录 %s 无法生成规格键: %w", row.id, keyErr)
+		}
+		if len([]rune(key)) > 191 {
+			return fmt.Errorf("SKU 内部记录 %s 的规格键超过 191 个字符", row.id)
+		}
+		if _, err := db.Exec(`UPDATE shopee_skus SET spec_key=? WHERE sku_id=?`, key, row.id); err != nil {
+			return err
+		}
+	}
+	var duplicate string
+	err = db.QueryRow(`SELECT CONCAT(goods_id,' / ',spec_key) FROM shopee_skus GROUP BY goods_id,spec_key HAVING COUNT(*)>1 LIMIT 1`).Scan(&duplicate)
+	if err != nil && err != sql.ErrNoRows {
+		return err
+	}
+	if err == nil {
+		return fmt.Errorf("存在同商品重复规格 %s，请人工确认真实 SKU 后再迁移", duplicate)
+	}
+	for _, index := range []struct{ name, ddl string }{
+		{"uq_shopee_skus_external", `ALTER TABLE shopee_skus ADD UNIQUE KEY uq_shopee_skus_external (shopee_sku_id)`},
+		{"uq_shopee_skus_spec", `ALTER TABLE shopee_skus ADD UNIQUE KEY uq_shopee_skus_spec (goods_id,spec_key)`},
+	} {
+		exists, err := mysqlIndexExists(db, "shopee_skus", index.name)
+		if err != nil {
+			return err
+		}
+		if !exists {
+			if _, err := db.Exec(index.ddl); err != nil {
+				return fmt.Errorf("增加索引 %s 失败: %w", index.name, err)
+			}
+		}
+	}
+	return nil
+}
+
 const mysqlSchemaV7CatalogRuns = `CREATE TABLE IF NOT EXISTS catalog_import_runs (
 	source VARCHAR(64) COLLATE utf8mb4_bin NOT NULL,
 	batch_id VARCHAR(191) COLLATE utf8mb4_bin NOT NULL,
@@ -948,7 +1053,49 @@ func CheckMySQLSchema(db *sql.DB) error {
 	if err := checkMySQLV6Shape(db); err != nil {
 		return err
 	}
-	return checkMySQLV7Shape(db)
+	if err := checkMySQLV7Shape(db); err != nil {
+		return err
+	}
+	return checkMySQLV8Shape(db)
+}
+
+func checkMySQLV8Shape(db *sql.DB) error {
+	for _, column := range []struct {
+		name           string
+		length         int64
+		nullable       bool
+		collation, def string
+	}{
+		{"shopee_sku_id", 191, true, "utf8mb4_bin", ""},
+		{"spec_key", 191, true, "utf8mb4_bin", ""},
+		{"source", 64, true, "utf8mb4_bin", ""},
+	} {
+		if err := checkMySQLVarcharColumn(db, "shopee_skus", column.name, column.length, column.nullable, column.collation, column.def); err != nil {
+			return err
+		}
+	}
+	for _, name := range []string{"field_sources", "field_observed_at"} {
+		exists, err := mysqlColumnExists(db, "shopee_skus", name)
+		if err != nil || !exists {
+			return fmt.Errorf("蝦皮 SKU 字段 %s 缺失", name)
+		}
+	}
+	if err := checkMySQLVarcharColumn(db, "catalog_import_runs", "update_policy", 32, false, "utf8mb4_bin", "fill_missing"); err != nil {
+		return err
+	}
+	for _, name := range []string{"sku_filled", "sku_same_source_updated", "sku_skipped", "sku_manual_skipped", "sku_stale_skipped"} {
+		exists, err := mysqlColumnExists(db, "catalog_import_runs", name)
+		if err != nil || !exists {
+			return fmt.Errorf("商品目录导入记录列 %s 缺失", name)
+		}
+	}
+	for _, name := range []string{"uq_shopee_skus_external", "uq_shopee_skus_spec"} {
+		exists, err := mysqlIndexExists(db, "shopee_skus", name)
+		if err != nil || !exists {
+			return fmt.Errorf("蝦皮 SKU 唯一索引 %s 缺失", name)
+		}
+	}
+	return nil
 }

 func checkMySQLV7Shape(db *sql.DB) error {