diff --git a/admin/handler/integration/auth_test.go b/admin/handler/integration/auth_test.go index d54fb12..131a86e 100644 --- a/admin/handler/integration/auth_test.go +++ b/admin/handler/integration/auth_test.go @@ -57,7 +57,7 @@ func TestCatalogBatchAPI_按批次查询摘要(t *testing.T) { t.Fatal(err) } defer db.Close() - _, err = db.Exec(`CREATE TABLE catalog_import_runs(source TEXT,batch_id TEXT,request_hash TEXT,status TEXT,request_count INTEGER,conflict_count INTEGER,observed_at TEXT,last_request_at TEXT,last_conflict_at TEXT,shopee_created INTEGER DEFAULT 0,shopee_updated INTEGER DEFAULT 0,sku_created INTEGER DEFAULT 0,sku_updated INTEGER DEFAULT 0,pdd_created INTEGER DEFAULT 0,pdd_updated INTEGER DEFAULT 0,association_created INTEGER DEFAULT 0,association_unchanged INTEGER DEFAULT 0,failure_count INTEGER DEFAULT 0,error_summary TEXT,response_body TEXT,created_at TEXT,finished_at TEXT,PRIMARY KEY(source,batch_id))`) + _, err = db.Exec(`CREATE TABLE catalog_import_runs(source TEXT,batch_id TEXT,request_hash TEXT,status TEXT,request_count INTEGER,conflict_count INTEGER,observed_at TEXT,update_policy TEXT DEFAULT 'fill_missing',last_request_at TEXT,last_conflict_at TEXT,shopee_created INTEGER DEFAULT 0,shopee_updated INTEGER DEFAULT 0,sku_created INTEGER DEFAULT 0,sku_updated INTEGER DEFAULT 0,sku_filled INTEGER DEFAULT 0,sku_same_source_updated INTEGER DEFAULT 0,sku_skipped INTEGER DEFAULT 0,sku_manual_skipped INTEGER DEFAULT 0,sku_stale_skipped INTEGER DEFAULT 0,pdd_created INTEGER DEFAULT 0,pdd_updated INTEGER DEFAULT 0,association_created INTEGER DEFAULT 0,association_unchanged INTEGER DEFAULT 0,failure_count INTEGER DEFAULT 0,error_summary TEXT,response_body TEXT,created_at TEXT,finished_at TEXT,PRIMARY KEY(source,batch_id))`) if err != nil { t.Fatal(err) } @@ -75,6 +75,9 @@ func TestCatalogBatchAPI_按批次查询摘要(t *testing.T) { if resp.Code != 200 || !strings.Contains(resp.Body.String(), `"batch_id":"B-1"`) { t.Fatalf("status=%d body=%s", resp.Code, resp.Body.String()) } + if !strings.Contains(resp.Body.String(), `"sku_filled":0`) || !strings.Contains(resp.Body.String(), `"update_policy"`) { + t.Fatal("批次摘要缺少更新策略统计") + } if strings.Contains(resp.Body.String(), "request_hash") { t.Fatal("查询响应不应暴露请求哈希") } diff --git a/admin/handler/integration/catalog_api.go b/admin/handler/integration/catalog_api.go index 92fa8df..d5726c3 100644 --- a/admin/handler/integration/catalog_api.go +++ b/admin/handler/integration/catalog_api.go @@ -37,7 +37,7 @@ func (h *Handler) GetBatch(c *gin.Context) { integrationError(c, 500, "INTERNAL_ERROR", "查询批次失败", true, nil) return } - c.JSON(200, gin.H{"source": run.Source, "batch_id": run.BatchID, "status": run.Status, "request_count": run.RequestCount, "conflict_count": run.ConflictCount, "observed_at": run.ObservedAt, "created_at": run.CreatedAt, "finished_at": run.FinishedAt, "counts": gin.H{"shopee_created": run.ShopeeCreated, "shopee_updated": run.ShopeeUpdated, "sku_created": run.SKUCreated, "sku_updated": run.SKUUpdated, "pdd_created": run.PddCreated, "pdd_updated": run.PddUpdated, "association_created": run.AssociationCreated, "association_unchanged": run.AssociationUnchanged}, "failure_count": run.FailureCount, "error_summary": run.ErrorSummary}) + c.JSON(200, gin.H{"source": run.Source, "batch_id": run.BatchID, "update_policy": run.UpdatePolicy, "status": run.Status, "request_count": run.RequestCount, "conflict_count": run.ConflictCount, "observed_at": run.ObservedAt, "created_at": run.CreatedAt, "finished_at": run.FinishedAt, "counts": gin.H{"shopee_created": run.ShopeeCreated, "shopee_updated": run.ShopeeUpdated, "sku_created": run.SKUCreated, "sku_updated": run.SKUUpdated, "sku_filled": run.SKUFilled, "sku_same_source_updated": run.SKUSameSourceUpdated, "sku_skipped": run.SKUSkipped, "sku_manual_skipped": run.SKUManualSkipped, "sku_stale_skipped": run.SKUStaleSkipped, "pdd_created": run.PddCreated, "pdd_updated": run.PddUpdated, "association_created": run.AssociationCreated, "association_unchanged": run.AssociationUnchanged}, "failure_count": run.FailureCount, "error_summary": run.ErrorSummary}) } func (h *Handler) CreateBatch(c *gin.Context) { diff --git a/admin/main_test.go b/admin/main_test.go index abe38e1..8493501 100644 --- a/admin/main_test.go +++ b/admin/main_test.go @@ -151,6 +151,23 @@ func TestShopeeDetail_区分顺运宝观测与正式SKU(t *testing.T) { } } +func TestCatalogHistory_展示更新策略和分级统计(t *testing.T) { + list, err := os.ReadFile("templates/catalog/list.html") + if err != nil { + t.Fatal(err) + } + detail, err := os.ReadFile("templates/catalog/detail_modal.html") + if err != nil { + t.Fatal(err) + } + content := string(list) + string(detail) + for _, want := range []string{"更新策略", "SKUFilled", "SKUSameSourceUpdated", "SKUManualSkipped", "SKUStaleSkipped"} { + if !strings.Contains(content, want) { + t.Errorf("目录记录缺少 %q", want) + } + } +} + func TestShopeeImportRoute_已移除(t *testing.T) { router, err := newRouter(nil) if err != nil { diff --git a/admin/model/catalog_import.go b/admin/model/catalog_import.go index 6ad784d..073e66b 100644 --- a/admin/model/catalog_import.go +++ b/admin/model/catalog_import.go @@ -24,6 +24,12 @@ type CatalogImportRun struct { ShopeeUpdated int SKUCreated int SKUUpdated int + UpdatePolicy string + SKUFilled int + SKUSameSourceUpdated int + SKUSkipped int + SKUManualSkipped int + SKUStaleSkipped int PddCreated int PddUpdated int AssociationCreated int diff --git a/admin/model/model.go b/admin/model/model.go index a9ed665..b922ec9 100644 --- a/admin/model/model.go +++ b/admin/model/model.go @@ -181,7 +181,8 @@ type ShopeeProduct struct { // **永远原样保留**。Color/Size/Advice 是尽力解析的结果， // 解析失败时留空并把 ParseOK 置 false，不要瞎猜。 type ShopeeSKU struct { - SKUID string + RecordID string // 系统内部记录主键 + SKUID string // 可空的真实蝦皮 SKU ID，不得用内部主键冒充 GoodsID string SpecRaw string Color string diff --git a/admin/repository/catalog_import.go b/admin/repository/catalog_import.go index 3c6c83a..d66b379 100644 --- a/admin/repository/catalog_import.go +++ b/admin/repository/catalog_import.go @@ -2,6 +2,7 @@ package repository import ( "database/sql" + "encoding/json" "errors" "fmt" "strings" @@ -21,9 +22,9 @@ var ( func InsertCatalogImportRun(q Execer, run model.CatalogImportRun) error { _, err := q.Exec(`INSERT INTO catalog_import_runs ( source,batch_id,request_hash,status,request_count,conflict_count,observed_at, - last_request_at,created_at - ) VALUES (?,?,?,?,1,0,NULLIF(?,''),?,?)`, run.Source, run.BatchID, run.RequestHash, - run.Status, run.ObservedAt, run.LastRequestAt, run.CreatedAt) + update_policy,last_request_at,created_at + ) VALUES (?,?,?,?,1,0,NULLIF(?,''),?,?,?)`, run.Source, run.BatchID, run.RequestHash, + run.Status, run.ObservedAt, run.UpdatePolicy, run.LastRequestAt, run.CreatedAt) if err == nil { return nil } @@ -42,13 +43,15 @@ func GetCatalogImportRun(q Execer, source, batchID string) (model.CatalogImportR var run model.CatalogImportRun var observedAt, lastConflictAt, errorSummary, responseBody, finishedAt sql.NullString err := q.QueryRow(`SELECT source,batch_id,request_hash,status,request_count,conflict_count, - observed_at,last_request_at,last_conflict_at,shopee_created,shopee_updated,sku_created, - sku_updated,pdd_created,pdd_updated,association_created,association_unchanged,failure_count, + observed_at,update_policy,last_request_at,last_conflict_at,shopee_created,shopee_updated,sku_created, + sku_updated,sku_filled,sku_same_source_updated,sku_skipped,sku_manual_skipped,sku_stale_skipped, + pdd_created,pdd_updated,association_created,association_unchanged,failure_count, error_summary,response_body,created_at,finished_at FROM catalog_import_runs WHERE source=? AND batch_id=?`, source, batchID).Scan( &run.Source, &run.BatchID, &run.RequestHash, &run.Status, &run.RequestCount, &run.ConflictCount, - &observedAt, &run.LastRequestAt, &lastConflictAt, &run.ShopeeCreated, &run.ShopeeUpdated, - &run.SKUCreated, &run.SKUUpdated, &run.PddCreated, &run.PddUpdated, &run.AssociationCreated, + &observedAt, &run.UpdatePolicy, &run.LastRequestAt, &lastConflictAt, &run.ShopeeCreated, &run.ShopeeUpdated, + &run.SKUCreated, &run.SKUUpdated, &run.SKUFilled, &run.SKUSameSourceUpdated, &run.SKUSkipped, + &run.SKUManualSkipped, &run.SKUStaleSkipped, &run.PddCreated, &run.PddUpdated, &run.AssociationCreated, &run.AssociationUnchanged, &run.FailureCount, &errorSummary, &responseBody, &run.CreatedAt, &finishedAt) if errors.Is(err, sql.ErrNoRows) { return model.CatalogImportRun{}, ErrCatalogImportRunNotFound @@ -147,10 +150,12 @@ func RecordCatalogImportConflict(q Execer, source, batchID, requestedAt string) // CompleteCatalogImportRun 把处理结果摘要和安全的响应 JSON 固化，供幂等重放。 func CompleteCatalogImportRun(q Execer, run model.CatalogImportRun) error { result, err := q.Exec(`UPDATE catalog_import_runs SET status=?,shopee_created=?,shopee_updated=?, - sku_created=?,sku_updated=?,pdd_created=?,pdd_updated=?,association_created=?, + sku_created=?,sku_updated=?,sku_filled=?,sku_same_source_updated=?,sku_skipped=?, + sku_manual_skipped=?,sku_stale_skipped=?,pdd_created=?,pdd_updated=?,association_created=?, association_unchanged=?,failure_count=?,error_summary=NULLIF(?,''),response_body=NULLIF(?,''), finished_at=? WHERE source=? AND batch_id=?`, run.Status, run.ShopeeCreated, run.ShopeeUpdated, - run.SKUCreated, run.SKUUpdated, run.PddCreated, run.PddUpdated, run.AssociationCreated, + run.SKUCreated, run.SKUUpdated, run.SKUFilled, run.SKUSameSourceUpdated, run.SKUSkipped, + run.SKUManualSkipped, run.SKUStaleSkipped, run.PddCreated, run.PddUpdated, run.AssociationCreated, run.AssociationUnchanged, run.FailureCount, run.ErrorSummary, run.ResponseBody, run.FinishedAt, run.Source, run.BatchID) return catalogRunUpdateResult(result, err, "完成商品目录批次") @@ -177,34 +182,194 @@ func UpsertCatalogShopeeProduct(q Execer, goodsID, title, status, mainSKU, obser return false, err == nil, err } -// UpsertCatalogShopeeSKU 拒绝把同一 SKU 静默挪到另一商品，并保留 is_manual。 -func UpsertCatalogShopeeSKU(q Execer, skuID, goodsID, specRaw, color, size, advice string, parseOK bool, skuCode, observedAt, now string) (created, updated bool, err error) { - var oldGoods string - var oldObserved sql.NullString - err = q.QueryRow(`SELECT goods_id,source_observed_at FROM shopee_skus WHERE sku_id=?`, skuID).Scan(&oldGoods, &oldObserved) - parse := 0 - if parseOK { - parse = 1 - } +type CatalogSKUOutcome string + +const ( + CatalogSKUCreated CatalogSKUOutcome = "created" + CatalogSKUFilled CatalogSKUOutcome = "filled" + CatalogSKUSameSourceUpdated CatalogSKUOutcome = "same_source_updated" + CatalogSKUSkipped CatalogSKUOutcome = "skipped" + CatalogSKUManualSkipped CatalogSKUOutcome = "manual_skipped" + CatalogSKUStaleSkipped CatalogSKUOutcome = "stale_skipped" +) + +type CatalogShopeeSKUInput struct { + RecordID, ShopeeSKUID, GoodsID, SpecRaw, SpecKey string + Color, Size, Advice, SKUCode string + ParseOK bool + Source, ObservedAt, Now, UpdatePolicy string +} + +type catalogSKUStored struct { + RecordID, GoodsID, SpecRaw, SpecKey string + ShopeeSKUID, Color, Size, Advice, SKUCode sql.NullString + ParseOK, IsManual int + Source, ObservedAt sql.NullString + FieldSources, FieldObservedAt sql.NullString +} + +func scanCatalogSKU(dbRow *sql.Row) (*catalogSKUStored, error) { + var row catalogSKUStored + err := dbRow.Scan(&row.RecordID, &row.ShopeeSKUID, &row.GoodsID, &row.SpecRaw, &row.SpecKey, &row.Color, &row.Size, &row.Advice, &row.ParseOK, &row.SKUCode, &row.IsManual, &row.Source, &row.ObservedAt, &row.FieldSources, &row.FieldObservedAt) if errors.Is(err, sql.ErrNoRows) { - _, err = q.Exec(`INSERT INTO shopee_skus(sku_id,goods_id,spec_raw,color,size,advice,parse_ok,sku_code, - is_manual,source_observed_at,created_at,updated_at) VALUES(?,?,?,NULLIF(?,''),NULLIF(?,''),NULLIF(?,''),?,NULLIF(?,''),0,?,?,?)`, - skuID, goodsID, specRaw, color, size, advice, parse, skuCode, observedAt, now, now) - return err == nil, false, err + return nil, nil } if err != nil { - return false, false, err + return nil, err } - if oldGoods != goodsID { - return false, false, fmt.Errorf("SKU %s 已属于蝦皮商品 %s", skuID, oldGoods) + return &row, nil +} + +func findCatalogSKUBySpec(q Execer, goodsID, specKey string) (*catalogSKUStored, error) { + return scanCatalogSKU(q.QueryRow(`SELECT sku_id,shopee_sku_id,goods_id,spec_raw,spec_key,color,size,advice,parse_ok,sku_code,is_manual,source,source_observed_at,field_sources,field_observed_at FROM shopee_skus WHERE goods_id=? AND spec_key=?`, goodsID, specKey)) +} + +func findCatalogSKUByExternalID(q Execer, skuID string) (*catalogSKUStored, error) { + return scanCatalogSKU(q.QueryRow(`SELECT sku_id,shopee_sku_id,goods_id,spec_raw,spec_key,color,size,advice,parse_ok,sku_code,is_manual,source,source_observed_at,field_sources,field_observed_at FROM shopee_skus WHERE shopee_sku_id=?`, skuID)) +} + +// UpsertCatalogShopeeSKU 用真实 ID 或商品规格身份定位同一内部记录，人工行永不覆盖。 +func UpsertCatalogShopeeSKU(q Execer, in CatalogShopeeSKUInput) (CatalogSKUOutcome, error) { + bySpec, err := findCatalogSKUBySpec(q, in.GoodsID, in.SpecKey) + if err != nil { + return "", err } - if oldObserved.Valid && oldObserved.String > observedAt { - return false, false, nil + var byID *catalogSKUStored + if in.ShopeeSKUID != "" { + byID, err = findCatalogSKUByExternalID(q, in.ShopeeSKUID) + if err != nil { + return "", err + } } - _, err = q.Exec(`UPDATE shopee_skus SET spec_raw=?,color=NULLIF(?,''),size=NULLIF(?,''),advice=NULLIF(?,''), - parse_ok=?,sku_code=NULLIF(?,''),source_observed_at=?,updated_at=? WHERE sku_id=?`, - specRaw, color, size, advice, parse, skuCode, observedAt, now, skuID) - return false, err == nil, err + if byID != nil && (byID.GoodsID != in.GoodsID || byID.SpecKey != in.SpecKey) { + return "", fmt.Errorf("真实蝦皮 SKU %s 已属于其他商品或规格", in.ShopeeSKUID) + } + if byID != nil && bySpec != nil && byID.RecordID != bySpec.RecordID { + return "", fmt.Errorf("商品 %s 的规格 %s 已对应另一个真实蝦皮 SKU", in.GoodsID, in.SpecRaw) + } + current := byID + if current == nil { + current = bySpec + } + parse := 0 + if in.ParseOK { + parse = 1 + } + if current == nil { + fieldSources, fieldTimes := newCatalogFieldProvenance(in) + sourcesJSON, _ := json.Marshal(fieldSources) + timesJSON, _ := json.Marshal(fieldTimes) + _, err = q.Exec(`INSERT INTO shopee_skus(sku_id,shopee_sku_id,goods_id,spec_raw,spec_key,color,size,advice,parse_ok,sku_code,is_manual,source,field_sources,field_observed_at,source_observed_at,created_at,updated_at) VALUES(?,NULLIF(?,''),?,?,?,NULLIF(?,''),NULLIF(?,''),NULLIF(?,''),?,NULLIF(?,''),0,?,?,?,?,?,?)`, in.RecordID, in.ShopeeSKUID, in.GoodsID, in.SpecRaw, in.SpecKey, in.Color, in.Size, in.Advice, parse, in.SKUCode, in.Source, string(sourcesJSON), string(timesJSON), in.ObservedAt, in.Now, in.Now) + if err != nil { + return "", err + } + return CatalogSKUCreated, nil + } + if current.ShopeeSKUID.Valid && in.ShopeeSKUID != "" && current.ShopeeSKUID.String != in.ShopeeSKUID { + return "", fmt.Errorf("商品 %s 的规格 %s 已对应真实蝦皮 SKU %s", in.GoodsID, in.SpecRaw, current.ShopeeSKUID.String) + } + if in.UpdatePolicy == "insert_only" { + return CatalogSKUSkipped, nil + } + if current.IsManual != 0 { + return CatalogSKUManualSkipped, nil + } + external := current.ShopeeSKUID.String + externalAdded := external == "" && in.ShopeeSKUID != "" + if external == "" { + external = in.ShopeeSKUID + } + if in.UpdatePolicy == "fill_missing" { + color, size, advice, skuCode := current.Color.String, current.Size.String, current.Advice.String, current.SKUCode.String + sources, times := catalogFieldProvenance(current) + changed := external != current.ShopeeSKUID.String + if color == "" && strings.TrimSpace(in.Color) != "" { + color = in.Color + sources["color"], times["color"] = in.Source, in.ObservedAt + changed = true + } + if size == "" && strings.TrimSpace(in.Size) != "" { + size = in.Size + sources["size"], times["size"] = in.Source, in.ObservedAt + changed = true + } + if advice == "" && strings.TrimSpace(in.Advice) != "" { + advice = in.Advice + sources["advice"], times["advice"] = in.Source, in.ObservedAt + changed = true + } + if skuCode == "" && strings.TrimSpace(in.SKUCode) != "" { + skuCode = in.SKUCode + sources["sku_code"], times["sku_code"] = in.Source, in.ObservedAt + changed = true + } + if !changed { + return CatalogSKUSkipped, nil + } + sourcesJSON, _ := json.Marshal(sources) + timesJSON, _ := json.Marshal(times) + _, err = q.Exec(`UPDATE shopee_skus SET shopee_sku_id=NULLIF(?,''),color=NULLIF(?,''),size=NULLIF(?,''),advice=NULLIF(?,''),sku_code=NULLIF(?,''),parse_ok=CASE WHEN parse_ok=1 THEN 1 ELSE ? END,field_sources=?,field_observed_at=?,updated_at=? WHERE sku_id=?`, external, color, size, advice, skuCode, parse, string(sourcesJSON), string(timesJSON), in.Now, current.RecordID) + if err != nil { + return "", err + } + return CatalogSKUFilled, nil + } + sources, times := catalogFieldProvenance(current) + color, size, advice, skuCode := current.Color.String, current.Size.String, current.Advice.String, current.SKUCode.String + changed, stale := false, false + for _, field := range []struct { + name string + incoming string + current *string + }{{"color", in.Color, &color}, {"size", in.Size, &size}, {"advice", in.Advice, &advice}, {"sku_code", in.SKUCode, &skuCode}} { + if sources[field.name] != in.Source { + continue + } + if times[field.name] > in.ObservedAt { + stale = true + continue + } + *field.current = field.incoming + times[field.name] = in.ObservedAt + changed = true + } + if !changed { + if externalAdded { + _, err = q.Exec(`UPDATE shopee_skus SET shopee_sku_id=?,updated_at=? WHERE sku_id=?`, external, in.Now, current.RecordID) + if err != nil { + return "", err + } + return CatalogSKUFilled, nil + } + if stale { + return CatalogSKUStaleSkipped, nil + } + return CatalogSKUSkipped, nil + } + sourcesJSON, _ := json.Marshal(sources) + timesJSON, _ := json.Marshal(times) + _, err = q.Exec(`UPDATE shopee_skus SET shopee_sku_id=NULLIF(?,''),color=NULLIF(?,''),size=NULLIF(?,''),advice=NULLIF(?,''),parse_ok=?,sku_code=NULLIF(?,''),field_sources=?,field_observed_at=?,source_observed_at=?,updated_at=? WHERE sku_id=?`, external, color, size, advice, parse, skuCode, string(sourcesJSON), string(timesJSON), in.ObservedAt, in.Now, current.RecordID) + if err != nil { + return "", err + } + return CatalogSKUSameSourceUpdated, nil +} + +func newCatalogFieldProvenance(in CatalogShopeeSKUInput) (map[string]string, map[string]string) { + sources, times := map[string]string{}, map[string]string{} + for name, value := range map[string]string{"color": in.Color, "size": in.Size, "advice": in.Advice, "sku_code": in.SKUCode} { + if strings.TrimSpace(value) != "" { + sources[name], times[name] = in.Source, in.ObservedAt + } + } + return sources, times +} + +func catalogFieldProvenance(row *catalogSKUStored) (map[string]string, map[string]string) { + sources, times := map[string]string{}, map[string]string{} + _ = json.Unmarshal([]byte(row.FieldSources.String), &sources) + _ = json.Unmarshal([]byte(row.FieldObservedAt.String), ×) + return sources, times } // UpsertCatalogPddProduct 写入结构化 PDD 数据转换后的规范 JSON；空规格不清除既有采集结果。 diff --git a/admin/repository/catalog_import_test.go b/admin/repository/catalog_import_test.go index d5b9355..e21c2a7 100644 --- a/admin/repository/catalog_import_test.go +++ b/admin/repository/catalog_import_test.go @@ -19,9 +19,9 @@ func TestCatalogImportRun_登记查询重放和冲突(t *testing.T) { defer db.Close() if _, err := db.Exec(`CREATE TABLE catalog_import_runs ( source TEXT NOT NULL,batch_id TEXT NOT NULL,request_hash TEXT NOT NULL,status TEXT NOT NULL, - request_count INTEGER NOT NULL,conflict_count INTEGER NOT NULL,observed_at TEXT,last_request_at TEXT NOT NULL, + request_count INTEGER NOT NULL,conflict_count INTEGER NOT NULL,observed_at TEXT,update_policy TEXT NOT NULL DEFAULT 'fill_missing',last_request_at TEXT NOT NULL, last_conflict_at TEXT,shopee_created INTEGER NOT NULL DEFAULT 0,shopee_updated INTEGER NOT NULL DEFAULT 0, - sku_created INTEGER NOT NULL DEFAULT 0,sku_updated INTEGER NOT NULL DEFAULT 0,pdd_created INTEGER NOT NULL DEFAULT 0, + sku_created INTEGER NOT NULL DEFAULT 0,sku_updated INTEGER NOT NULL DEFAULT 0,sku_filled INTEGER NOT NULL DEFAULT 0,sku_same_source_updated INTEGER NOT NULL DEFAULT 0,sku_skipped INTEGER NOT NULL DEFAULT 0,sku_manual_skipped INTEGER NOT NULL DEFAULT 0,sku_stale_skipped INTEGER NOT NULL DEFAULT 0,pdd_created INTEGER NOT NULL DEFAULT 0, pdd_updated INTEGER NOT NULL DEFAULT 0,association_created INTEGER NOT NULL DEFAULT 0, association_unchanged INTEGER NOT NULL DEFAULT 0,failure_count INTEGER NOT NULL DEFAULT 0, error_summary TEXT,response_body TEXT,created_at TEXT NOT NULL,finished_at TEXT, diff --git a/admin/repository/mysql_db.go b/admin/repository/mysql_db.go index bd22cc0..f3ec6be 100644 --- a/admin/repository/mysql_db.go +++ b/admin/repository/mysql_db.go @@ -19,7 +19,7 @@ import ( "cmautobuy/admin/spec" ) -const mysqlSchemaVersion = 7 +const mysqlSchemaVersion = 8 // OpenMySQL 打开生产 MySQL 8 数据库。错误信息绝不包含完整 DSN 或密码。 func OpenMySQL(cfg config.DatabaseConfig) (*sql.DB, error) { @@ -507,10 +507,115 @@ func MigrateMySQL(db *sql.DB) error { if _, err := db.Exec(`INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)`, 7, time.Now().UTC().Format(time.RFC3339Nano)); err != nil { return fmt.Errorf("记录 MySQL schema v7 失败: %w", err) } + current = 7 + } + if current < 8 { + if err := migrateMySQLV8(db); err != nil { + return fmt.Errorf("执行 MySQL schema v8 失败: %w", err) + } + if err := checkMySQLV8Shape(db); err != nil { + return fmt.Errorf("MySQL schema v8 自检失败，未记录版本: %w", err) + } + if _, err := db.Exec(`INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)`, 8, time.Now().UTC().Format(time.RFC3339Nano)); err != nil { + return fmt.Errorf("记录 MySQL schema v8 失败: %w", err) + } } return CheckMySQLSchema(db) } +// migrateMySQLV8 保留 sku_id 作为内部主键，并增加可空的真实蝦皮 SKU ID。 +// 每个 DDL 都先检查形状，支持 MySQL 隐式提交后的中断重放。 +func migrateMySQLV8(db *sql.DB) error { + columns := []struct{ table, name, ddl string }{ + {"shopee_skus", "shopee_sku_id", `ALTER TABLE shopee_skus ADD COLUMN shopee_sku_id VARCHAR(191) COLLATE utf8mb4_bin NULL AFTER sku_id`}, + {"shopee_skus", "spec_key", `ALTER TABLE shopee_skus ADD COLUMN spec_key VARCHAR(191) COLLATE utf8mb4_bin NULL AFTER spec_raw`}, + {"shopee_skus", "source", `ALTER TABLE shopee_skus ADD COLUMN source VARCHAR(64) COLLATE utf8mb4_bin NULL AFTER is_manual`}, + {"shopee_skus", "field_sources", `ALTER TABLE shopee_skus ADD COLUMN field_sources JSON NULL AFTER source`}, + {"shopee_skus", "field_observed_at", `ALTER TABLE shopee_skus ADD COLUMN field_observed_at JSON NULL AFTER field_sources`}, + {"catalog_import_runs", "update_policy", `ALTER TABLE catalog_import_runs ADD COLUMN update_policy VARCHAR(32) COLLATE utf8mb4_bin NOT NULL DEFAULT 'fill_missing' AFTER observed_at`}, + {"catalog_import_runs", "sku_filled", `ALTER TABLE catalog_import_runs ADD COLUMN sku_filled INT UNSIGNED NOT NULL DEFAULT 0 AFTER sku_updated`}, + {"catalog_import_runs", "sku_same_source_updated", `ALTER TABLE catalog_import_runs ADD COLUMN sku_same_source_updated INT UNSIGNED NOT NULL DEFAULT 0 AFTER sku_filled`}, + {"catalog_import_runs", "sku_skipped", `ALTER TABLE catalog_import_runs ADD COLUMN sku_skipped INT UNSIGNED NOT NULL DEFAULT 0 AFTER sku_same_source_updated`}, + {"catalog_import_runs", "sku_manual_skipped", `ALTER TABLE catalog_import_runs ADD COLUMN sku_manual_skipped INT UNSIGNED NOT NULL DEFAULT 0 AFTER sku_skipped`}, + {"catalog_import_runs", "sku_stale_skipped", `ALTER TABLE catalog_import_runs ADD COLUMN sku_stale_skipped INT UNSIGNED NOT NULL DEFAULT 0 AFTER sku_manual_skipped`}, + } + for _, column := range columns { + exists, err := mysqlColumnExists(db, column.table, column.name) + if err != nil { + return err + } + if !exists { + if _, err := db.Exec(column.ddl); err != nil { + return fmt.Errorf("增加 %s.%s 失败: %w", column.table, column.name, err) + } + } + } + if _, err := db.Exec(`UPDATE shopee_skus SET shopee_sku_id=sku_id WHERE shopee_sku_id IS NULL AND sku_id NOT LIKE 'catalog:%'`); err != nil { + return fmt.Errorf("回填真实蝦皮 SKU ID 失败: %w", err) + } + if _, err := db.Exec(`UPDATE shopee_skus SET source='report' WHERE source IS NULL OR source=''`); err != nil { + return fmt.Errorf("回填 SKU 来源失败: %w", err) + } + if _, err := db.Exec(`UPDATE shopee_skus SET field_sources=JSON_OBJECT('color',IF(color IS NULL OR color='','',COALESCE(source,'report')),'size',IF(size IS NULL OR size='','',COALESCE(source,'report')),'advice',IF(advice IS NULL OR advice='','',COALESCE(source,'report')),'sku_code',IF(sku_code IS NULL OR sku_code='','',COALESCE(source,'report'))) WHERE field_sources IS NULL`); err != nil { + return fmt.Errorf("回填 SKU 字段来源失败: %w", err) + } + if _, err := db.Exec(`UPDATE shopee_skus SET field_observed_at=JSON_OBJECT('color',IF(color IS NULL OR color='','',COALESCE(source_observed_at,updated_at)),'size',IF(size IS NULL OR size='','',COALESCE(source_observed_at,updated_at)),'advice',IF(advice IS NULL OR advice='','',COALESCE(source_observed_at,updated_at)),'sku_code',IF(sku_code IS NULL OR sku_code='','',COALESCE(source_observed_at,updated_at))) WHERE field_observed_at IS NULL`); err != nil { + return fmt.Errorf("回填 SKU 字段观测时间失败: %w", err) + } + rows, err := db.Query(`SELECT sku_id,spec_raw FROM shopee_skus WHERE spec_key IS NULL OR spec_key=''`) + if err != nil { + return fmt.Errorf("读取待回填规格键失败: %w", err) + } + type pendingKey struct{ id, raw string } + var pending []pendingKey + for rows.Next() { + var row pendingKey + if err := rows.Scan(&row.id, &row.raw); err != nil { + rows.Close() + return err + } + pending = append(pending, row) + } + if err := rows.Close(); err != nil { + return err + } + for _, row := range pending { + key, keyErr := spec.SpecKey(row.raw) + if keyErr != nil { + return fmt.Errorf("SKU 内部记录 %s 无法生成规格键: %w", row.id, keyErr) + } + if len([]rune(key)) > 191 { + return fmt.Errorf("SKU 内部记录 %s 的规格键超过 191 个字符", row.id) + } + if _, err := db.Exec(`UPDATE shopee_skus SET spec_key=? WHERE sku_id=?`, key, row.id); err != nil { + return err + } + } + var duplicate string + err = db.QueryRow(`SELECT CONCAT(goods_id,' / ',spec_key) FROM shopee_skus GROUP BY goods_id,spec_key HAVING COUNT(*)>1 LIMIT 1`).Scan(&duplicate) + if err != nil && err != sql.ErrNoRows { + return err + } + if err == nil { + return fmt.Errorf("存在同商品重复规格 %s，请人工确认真实 SKU 后再迁移", duplicate) + } + for _, index := range []struct{ name, ddl string }{ + {"uq_shopee_skus_external", `ALTER TABLE shopee_skus ADD UNIQUE KEY uq_shopee_skus_external (shopee_sku_id)`}, + {"uq_shopee_skus_spec", `ALTER TABLE shopee_skus ADD UNIQUE KEY uq_shopee_skus_spec (goods_id,spec_key)`}, + } { + exists, err := mysqlIndexExists(db, "shopee_skus", index.name) + if err != nil { + return err + } + if !exists { + if _, err := db.Exec(index.ddl); err != nil { + return fmt.Errorf("增加索引 %s 失败: %w", index.name, err) + } + } + } + return nil +} + const mysqlSchemaV7CatalogRuns = `CREATE TABLE IF NOT EXISTS catalog_import_runs ( source VARCHAR(64) COLLATE utf8mb4_bin NOT NULL, batch_id VARCHAR(191) COLLATE utf8mb4_bin NOT NULL, @@ -948,7 +1053,49 @@ func CheckMySQLSchema(db *sql.DB) error { if err := checkMySQLV6Shape(db); err != nil { return err } - return checkMySQLV7Shape(db) + if err := checkMySQLV7Shape(db); err != nil { + return err + } + return checkMySQLV8Shape(db) +} + +func checkMySQLV8Shape(db *sql.DB) error { + for _, column := range []struct { + name string + length int64 + nullable bool + collation, def string + }{ + {"shopee_sku_id", 191, true, "utf8mb4_bin", ""}, + {"spec_key", 191, true, "utf8mb4_bin", ""}, + {"source", 64, true, "utf8mb4_bin", ""}, + } { + if err := checkMySQLVarcharColumn(db, "shopee_skus", column.name, column.length, column.nullable, column.collation, column.def); err != nil { + return err + } + } + for _, name := range []string{"field_sources", "field_observed_at"} { + exists, err := mysqlColumnExists(db, "shopee_skus", name) + if err != nil || !exists { + return fmt.Errorf("蝦皮 SKU 字段 %s 缺失", name) + } + } + if err := checkMySQLVarcharColumn(db, "catalog_import_runs", "update_policy", 32, false, "utf8mb4_bin", "fill_missing"); err != nil { + return err + } + for _, name := range []string{"sku_filled", "sku_same_source_updated", "sku_skipped", "sku_manual_skipped", "sku_stale_skipped"} { + exists, err := mysqlColumnExists(db, "catalog_import_runs", name) + if err != nil || !exists { + return fmt.Errorf("商品目录导入记录列 %s 缺失", name) + } + } + for _, name := range []string{"uq_shopee_skus_external", "uq_shopee_skus_spec"} { + exists, err := mysqlIndexExists(db, "shopee_skus", name) + if err != nil || !exists { + return fmt.Errorf("蝦皮 SKU 唯一索引 %s 缺失", name) + } + } + return nil } func checkMySQLV7Shape(db *sql.DB) error { diff --git a/admin/repository/mysql_db_integration_test.go b/admin/repository/mysql_db_integration_test.go index 4c172fc..c7d2397 100644 --- a/admin/repository/mysql_db_integration_test.go +++ b/admin/repository/mysql_db_integration_test.go @@ -382,6 +382,53 @@ func TestMySQLMigrate_V7形状错误不记版本(t *testing.T) { } } +func TestMySQLMigrate_V7升级V8保留内部引用且可重放(t *testing.T) { + db := openMySQLMigrationTestDB(t) + defer db.Close() + cleanMySQLTestSchema(t, db) + defer cleanMySQLTestSchema(t, db) + prepareMySQLV7(t, db) + now := "2026-08-11T00:00:00Z" + mustExec(t, db, `INSERT INTO shopee_products(goods_id,title,source,created_at,updated_at) VALUES('S-1','商品','report',?,?)`, now, now) + mustExec(t, db, `INSERT INTO shopee_skus(sku_id,goods_id,spec_raw,created_at,updated_at) VALUES('REAL-1','S-1',' 黑色, M ',?,?)`, now, now) + // 模拟全部 DDL/回填已提交，但 schema_migrations 尚未记录 v8。 + if err := migrateMySQLV8(db); err != nil { + t.Fatal(err) + } + if err := MigrateMySQL(db); err != nil { + t.Fatal(err) + } + if err := MigrateMySQL(db); err != nil { + t.Fatalf("v8 重跑失败: %v", err) + } + var internalID, externalID, key string + if err := db.QueryRow(`SELECT sku_id,shopee_sku_id,spec_key FROM shopee_skus`).Scan(&internalID, &externalID, &key); err != nil { + t.Fatal(err) + } + if internalID != "REAL-1" || externalID != "REAL-1" || key != "黑色, M" { + t.Fatalf("历史身份未保留：internal=%q external=%q key=%q", internalID, externalID, key) + } +} + +func TestMySQLMigrate_V8重复规格停止且不记版本(t *testing.T) { + db := openMySQLMigrationTestDB(t) + defer db.Close() + cleanMySQLTestSchema(t, db) + defer cleanMySQLTestSchema(t, db) + prepareMySQLV7(t, db) + now := "2026-08-11T00:00:00Z" + mustExec(t, db, `INSERT INTO shopee_products(goods_id,title,source,created_at,updated_at) VALUES('S-1','商品','report',?,?)`, now, now) + mustExec(t, db, `INSERT INTO shopee_skus(sku_id,goods_id,spec_raw,created_at,updated_at) VALUES('A','S-1','黑色,M',?,?),('B','S-1','黑色,M',?,?)`, now, now, now, now) + if err := MigrateMySQL(db); err == nil || !strings.Contains(err.Error(), "重复规格") { + t.Fatalf("应明确阻止冲突迁移：%v", err) + } + var count int + db.QueryRow(`SELECT COUNT(*) FROM schema_migrations WHERE version=8`).Scan(&count) + if count != 0 { + t.Fatal("冲突迁移不得记录 v8") + } +} + func openMySQLMigrationTestDB(t *testing.T) *sql.DB { t.Helper() if os.Getenv("CMAUTOBUY_MYSQL_TEST") != "1" { @@ -436,6 +483,15 @@ func prepareMySQLV6(t *testing.T, db *sql.DB) { VALUES (5,'2026-08-10T00:00:00Z'),(6,'2026-08-10T00:00:00Z')`) } +func prepareMySQLV7(t *testing.T, db *sql.DB) { + t.Helper() + prepareMySQLV6(t, db) + if err := migrateMySQLV7(db); err != nil { + t.Fatal(err) + } + mustExec(t, db, `INSERT INTO schema_migrations(version,applied_at) VALUES (7,'2026-08-11T00:00:00Z')`) +} + func mustExec(t *testing.T, db *sql.DB, query string, args ...any) { t.Helper() if _, err := db.Exec(query, args...); err != nil { diff --git a/admin/repository/shopee.go b/admin/repository/shopee.go index 8e41aff..8d91531 100644 --- a/admin/repository/shopee.go +++ b/admin/repository/shopee.go @@ -2,11 +2,13 @@ package repository import ( "database/sql" + "encoding/json" "errors" "fmt" "strings" "cmautobuy/admin/model" + "cmautobuy/admin/spec" ) // UpsertShopeeProduct 写入或更新一条蝦皮商品汇总行。 @@ -57,22 +59,35 @@ func UpsertShopeeSKU(q Execer, skuID, goodsID, specRaw, color, size, advice stri parseOKInt = 1 } now := model.NowISO() - _, err := q.Exec(` + specKey, err := spec.SpecKey(specRaw) + if err != nil { + return fmt.Errorf("规格原文无效: %w", err) + } + fieldSources, fieldTimes := map[string]string{}, map[string]string{} + for name, value := range map[string]string{"color": color, "size": size, "advice": advice, "sku_code": skuCode} { + if strings.TrimSpace(value) != "" { + fieldSources[name], fieldTimes[name] = "report", now + } + } + fieldSourcesJSON, _ := json.Marshal(fieldSources) + fieldTimesJSON, _ := json.Marshal(fieldTimes) + _, err = q.Exec(` INSERT INTO shopee_skus - (sku_id, goods_id, spec_raw, color, size, advice, parse_ok, sku_code, - is_manual, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?) + (sku_id, shopee_sku_id, goods_id, spec_raw, spec_key, color, size, advice, parse_ok, sku_code, + is_manual, source, field_sources, field_observed_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 'report', ?, ?, ?, ?) ON DUPLICATE KEY UPDATE - goods_id = VALUES(goods_id), - spec_raw = VALUES(spec_raw), - color = VALUES(color), - size = VALUES(size), - advice = VALUES(advice), - parse_ok = VALUES(parse_ok), - sku_code = VALUES(sku_code), - updated_at = VALUES(updated_at)`, - // is_manual 不在上面的 SET 列表里，SQLite 对没提到的列保持原值不变。 - skuID, goodsID, specRaw, color, size, advice, parseOKInt, skuCode, now, now) + goods_id = CASE WHEN is_manual=1 THEN goods_id ELSE VALUES(goods_id) END, + color = CASE WHEN is_manual=1 THEN color ELSE VALUES(color) END, + size = CASE WHEN is_manual=1 THEN size ELSE VALUES(size) END, + advice = CASE WHEN is_manual=1 THEN advice ELSE VALUES(advice) END, + parse_ok = CASE WHEN is_manual=1 THEN parse_ok ELSE VALUES(parse_ok) END, + sku_code = CASE WHEN is_manual=1 THEN sku_code ELSE VALUES(sku_code) END, + field_sources = CASE WHEN is_manual=1 THEN field_sources ELSE VALUES(field_sources) END, + field_observed_at = CASE WHEN is_manual=1 THEN field_observed_at ELSE VALUES(field_observed_at) END, + updated_at = CASE WHEN is_manual=1 THEN updated_at ELSE VALUES(updated_at) END`, + // 人工行的业务字段和更新时间均由 CASE 保留。 + skuID, skuID, goodsID, specRaw, specKey, color, size, advice, parseOKInt, skuCode, string(fieldSourcesJSON), string(fieldTimesJSON), now, now) if err != nil { return fmt.Errorf("写入蝦皮 SKU %s 失败: %w", skuID, err) } @@ -293,7 +308,7 @@ func GetShopeeCollectStatus(q Execer, pddGoodsID string) (status, msg sql.NullSt // 见 #41「弹窗里待补的行必须显示 spec_raw 原文」。 func ListShopeeSKUsByGoodsID(q Execer, goodsID string) ([]model.ShopeeSKU, error) { rows, err := q.Query(` - SELECT sku_id, goods_id, spec_raw, color, size, advice, parse_ok, sku_code, + SELECT sku_id, COALESCE(shopee_sku_id,''), goods_id, spec_raw, color, size, advice, parse_ok, sku_code, is_manual, created_at, updated_at FROM shopee_skus WHERE goods_id = ? @@ -309,7 +324,7 @@ func ListShopeeSKUsByGoodsID(q Execer, goodsID string) ([]model.ShopeeSKU, error var color, size, advice, skuCode sql.NullString var parseOK, isManual int if err := rows.Scan( - &sk.SKUID, &sk.GoodsID, &sk.SpecRaw, &color, &size, &advice, + &sk.RecordID, &sk.SKUID, &sk.GoodsID, &sk.SpecRaw, &color, &size, &advice, &parseOK, &skuCode, &isManual, &sk.CreatedAt, &sk.UpdatedAt, ); err != nil { return nil, fmt.Errorf("读取蝦皮商品 %s 的规格失败: %w", goodsID, err) @@ -331,10 +346,10 @@ func GetShopeeSKUByID(q Execer, skuID string) (*model.ShopeeSKU, error) { var color, size, advice, skuCode sql.NullString var parseOK, isManual int err := q.QueryRow(` - SELECT sku_id, goods_id, spec_raw, color, size, advice, parse_ok, sku_code, + SELECT sku_id, COALESCE(shopee_sku_id,''), goods_id, spec_raw, color, size, advice, parse_ok, sku_code, is_manual, created_at, updated_at FROM shopee_skus WHERE sku_id = ?`, skuID).Scan( - &sk.SKUID, &sk.GoodsID, &sk.SpecRaw, &color, &size, &advice, + &sk.RecordID, &sk.SKUID, &sk.GoodsID, &sk.SpecRaw, &color, &size, &advice, &parseOK, &skuCode, &isManual, &sk.CreatedAt, &sk.UpdatedAt, ) if errors.Is(err, sql.ErrNoRows) { diff --git a/admin/repository/sqlite_to_mysql.go b/admin/repository/sqlite_to_mysql.go index 435312c..9d7270f 100644 --- a/admin/repository/sqlite_to_mysql.go +++ b/admin/repository/sqlite_to_mysql.go @@ -5,11 +5,14 @@ import ( "crypto/sha256" "database/sql" "encoding/binary" + "encoding/json" "fmt" "path/filepath" "sort" "strings" "time" + + "cmautobuy/admin/spec" ) // SQLiteMigrationSummary 只包含表名和数量，可以安全写入终端或工单。 @@ -117,6 +120,9 @@ func MigrateSQLiteToMySQL(source, target *sql.DB, dryRun bool) (*SQLiteMigration return nil, err } } + if err := normalizeMigratedShopeeSKUs(tx); err != nil { + return nil, err + } if err := tx.Commit(); err != nil { return nil, &SQLiteMigrationError{Stage: "无法提交 MySQL 导入事务", Cause: err} } @@ -126,6 +132,52 @@ func MigrateSQLiteToMySQL(source, target *sql.DB, dryRun bool) (*SQLiteMigration return summary, nil } +// normalizeMigratedShopeeSKUs 补齐 MySQL v8 新增、历史 SQLite 没有的规格身份列。 +func normalizeMigratedShopeeSKUs(tx *sql.Tx) error { + rows, err := tx.Query(`SELECT sku_id,spec_raw,color,size,advice,sku_code,updated_at FROM shopee_skus`) + if err != nil { + return &SQLiteMigrationError{Stage: "无法读取待转换 SKU", Table: "shopee_skus", Cause: err} + } + type item struct { + id, raw, updated string + color, size, advice, code sql.NullString + } + var list []item + for rows.Next() { + var row item + if err := rows.Scan(&row.id, &row.raw, &row.color, &row.size, &row.advice, &row.code, &row.updated); err != nil { + rows.Close() + return err + } + list = append(list, row) + } + if err := rows.Close(); err != nil { + return err + } + for _, row := range list { + key, err := spec.SpecKey(row.raw) + if err != nil { + return &SQLiteMigrationError{Stage: "历史 SKU 规格原文无效", Table: "shopee_skus", Cause: err} + } + if len([]rune(key)) > 191 { + return &SQLiteMigrationError{Stage: "历史 SKU 规格键过长", Table: "shopee_skus"} + } + sources, times := map[string]string{}, map[string]string{} + for name, value := range map[string]string{"color": row.color.String, "size": row.size.String, "advice": row.advice.String, "sku_code": row.code.String} { + if value != "" { + sources[name] = "report" + times[name] = row.updated + } + } + sourcesJSON, _ := json.Marshal(sources) + timesJSON, _ := json.Marshal(times) + if _, err := tx.Exec(`UPDATE shopee_skus SET shopee_sku_id=sku_id,spec_key=?,source='report',field_sources=?,field_observed_at=? WHERE sku_id=?`, key, string(sourcesJSON), string(timesJSON), row.id); err != nil { + return &SQLiteMigrationError{Stage: "无法补齐 MySQL v8 SKU 身份", Table: "shopee_skus", Cause: err} + } + } + return nil +} + // VerifySQLiteToMySQL 比较逐表数量和全部迁移列的确定性摘要，不输出业务内容。 func VerifySQLiteToMySQL(source, target *sql.DB) (*SQLiteMigrationSummary, error) { if err := validateLegacySQLite(source); err != nil { diff --git a/admin/service/catalog_import.go b/admin/service/catalog_import.go index d8befde..31d8c36 100644 --- a/admin/service/catalog_import.go +++ b/admin/service/catalog_import.go @@ -11,9 +11,11 @@ import ( "net/http" "strings" "time" + "unicode/utf8" "cmautobuy/admin/model" "cmautobuy/admin/repository" + "cmautobuy/admin/spec" ) const ( @@ -63,6 +65,7 @@ type CatalogBatchRequest struct { SchemaVersion int `json:"schema_version"` BatchID string `json:"batch_id"` ObservedAt string `json:"observed_at"` + UpdatePolicy string `json:"update_policy"` ShopeeProducts []CatalogShopeeProduct `json:"shopee_products"` ShopeeSKUs []CatalogShopeeSKU `json:"shopee_skus"` PddProducts []CatalogPddProduct `json:"pdd_products"` @@ -73,6 +76,11 @@ type CatalogCounts struct { ShopeeUpdated int `json:"shopee_updated"` SKUCreated int `json:"sku_created"` SKUUpdated int `json:"sku_updated"` + SKUFilled int `json:"sku_filled"` + SKUSameSourceUpdated int `json:"sku_same_source_updated"` + SKUSkipped int `json:"sku_skipped"` + SKUManualSkipped int `json:"sku_manual_skipped"` + SKUStaleSkipped int `json:"sku_stale_skipped"` PddCreated int `json:"pdd_created"` PddUpdated int `json:"pdd_updated"` AssociationCreated int `json:"association_created"` @@ -108,13 +116,16 @@ func ValidateCatalogBatch(req CatalogBatchRequest) error { if _, err := time.Parse(time.RFC3339Nano, req.ObservedAt); err != nil { return catalogInvalid("INVALID_OBSERVED_AT", "observed_at 必须是带时区 ISO 8601", nil) } + if _, err := normalizeCatalogUpdatePolicy(req.UpdatePolicy); err != nil { + return err + } if len(req.ShopeeProducts)+len(req.PddProducts) > CatalogMaxProducts { return catalogInvalid("TOO_MANY_PRODUCTS", "每批商品总数不能超过 500", nil) } if len(req.ShopeeSKUs) > CatalogMaxSKUs { return catalogInvalid("TOO_MANY_SKUS", "每批 SKU 不能超过 5000", nil) } - seenProducts, seenSKUs, seenPDD := map[string]bool{}, map[string]bool{}, map[string]bool{} + seenProducts, seenSKUs, seenSpecs, seenPDD := map[string]bool{}, map[string]bool{}, map[string]bool{}, map[string]bool{} for i, p := range req.ShopeeProducts { p.GoodsID = strings.TrimSpace(p.GoodsID) if p.GoodsID == "" || strings.TrimSpace(p.Title) == "" || seenProducts[p.GoodsID] { @@ -123,10 +134,16 @@ func ValidateCatalogBatch(req CatalogBatchRequest) error { seenProducts[p.GoodsID] = true } for i, s := range req.ShopeeSKUs { - if strings.TrimSpace(s.SKUID) == "" || strings.TrimSpace(s.GoodsID) == "" || strings.TrimSpace(s.SpecRaw) == "" || seenSKUs[s.SKUID] { - return catalogInvalid("INVALID_SHOPEE_SKU", "蝦皮 SKU ID、商品 ID、spec_raw 必填且 SKU 不能重复", map[string]any{"index": i, "sku_id": s.SKUID}) + key, keyErr := spec.SpecKey(s.SpecRaw) + externalID := strings.TrimSpace(s.SKUID) + identity := strings.TrimSpace(s.GoodsID) + "\x00" + key + if strings.TrimSpace(s.GoodsID) == "" || keyErr != nil || utf8.RuneCountInString(key) > 191 || seenSpecs[identity] || (externalID != "" && seenSKUs[externalID]) { + return catalogInvalid("INVALID_SHOPEE_SKU", "商品 ID、有效 spec_raw 必填，批内规格身份和非空 SKU ID 不能重复", map[string]any{"index": i, "sku_id": externalID}) + } + seenSpecs[identity] = true + if externalID != "" { + seenSKUs[externalID] = true } - seenSKUs[s.SKUID] = true } for i, p := range req.PddProducts { if strings.TrimSpace(p.GoodsID) == "" || strings.TrimSpace(p.URL) == "" || seenPDD[p.GoodsID] { @@ -153,6 +170,8 @@ func ImportCatalogBatch(db *sql.DB, source string, req CatalogBatchRequest, raw } observed, _ := time.Parse(time.RFC3339Nano, req.ObservedAt) req.ObservedAt = observed.UTC().Format(model.TimeLayout) + policy, _ := normalizeCatalogUpdatePolicy(req.UpdatePolicy) + req.UpdatePolicy = policy h := sha256.Sum256(raw) hash := hex.EncodeToString(h[:]) now := model.NowISO() @@ -160,7 +179,7 @@ func ImportCatalogBatch(db *sql.DB, source string, req CatalogBatchRequest, raw if err != nil { return CatalogBatchResponse{}, err } - run := model.CatalogImportRun{Source: source, BatchID: req.BatchID, RequestHash: hash, Status: model.CatalogImportProcessing, ObservedAt: req.ObservedAt, LastRequestAt: now, CreatedAt: now} + run := model.CatalogImportRun{Source: source, BatchID: req.BatchID, RequestHash: hash, Status: model.CatalogImportProcessing, UpdatePolicy: policy, ObservedAt: req.ObservedAt, LastRequestAt: now, CreatedAt: now} if err = repository.InsertCatalogImportRun(tx, run); err != nil { tx.Rollback() if errors.Is(err, repository.ErrCatalogImportRunExists) { @@ -215,15 +234,29 @@ func ImportCatalogBatch(db *sql.DB, source string, req CatalogBatchRequest, raw } } for _, s := range req.ShopeeSKUs { - c, u, e := repository.UpsertCatalogShopeeSKU(tx, s.SKUID, s.GoodsID, s.SpecRaw, s.Color, s.Size, s.Advice, s.ParseOK, s.SKUCode, req.ObservedAt, now) + key, _ := spec.SpecKey(s.SpecRaw) + internalID := catalogSKURecordID(s.GoodsID, key) + outcome, e := repository.UpsertCatalogShopeeSKU(tx, repository.CatalogShopeeSKUInput{RecordID: internalID, ShopeeSKUID: strings.TrimSpace(s.SKUID), GoodsID: strings.TrimSpace(s.GoodsID), SpecRaw: s.SpecRaw, SpecKey: key, Color: s.Color, Size: s.Size, Advice: s.Advice, ParseOK: s.ParseOK, SKUCode: s.SKUCode, Source: source, ObservedAt: req.ObservedAt, Now: now, UpdatePolicy: policy}) if e != nil { return fail(&CatalogError{Status: 409, Code: "SKU_OWNERSHIP_CONFLICT", Message: e.Error()}) } - if c { + switch outcome { + case repository.CatalogSKUCreated: counts.SKUCreated++ - } - if u { + case repository.CatalogSKUFilled: + counts.SKUFilled++ counts.SKUUpdated++ + case repository.CatalogSKUSameSourceUpdated: + counts.SKUSameSourceUpdated++ + counts.SKUUpdated++ + case repository.CatalogSKUManualSkipped: + counts.SKUManualSkipped++ + counts.SKUSkipped++ + case repository.CatalogSKUStaleSkipped: + counts.SKUStaleSkipped++ + counts.SKUSkipped++ + default: + counts.SKUSkipped++ } } for _, a := range req.Associations { @@ -247,6 +280,11 @@ func ImportCatalogBatch(db *sql.DB, source string, req CatalogBatchRequest, raw run.ShopeeUpdated = counts.ShopeeUpdated run.SKUCreated = counts.SKUCreated run.SKUUpdated = counts.SKUUpdated + run.SKUFilled = counts.SKUFilled + run.SKUSameSourceUpdated = counts.SKUSameSourceUpdated + run.SKUSkipped = counts.SKUSkipped + run.SKUManualSkipped = counts.SKUManualSkipped + run.SKUStaleSkipped = counts.SKUStaleSkipped run.PddCreated = counts.PddCreated run.PddUpdated = counts.PddUpdated run.AssociationCreated = counts.AssociationCreated @@ -260,6 +298,22 @@ func ImportCatalogBatch(db *sql.DB, source string, req CatalogBatchRequest, raw return resp, nil } +func normalizeCatalogUpdatePolicy(raw string) (string, error) { + switch strings.TrimSpace(raw) { + case "", "fill_missing": + return "fill_missing", nil + case "insert_only", "overwrite_same_source": + return strings.TrimSpace(raw), nil + default: + return "", catalogInvalid("INVALID_UPDATE_POLICY", "update_policy 只支持 insert_only、fill_missing、overwrite_same_source", nil) + } +} + +func catalogSKURecordID(goodsID, specKey string) string { + sum := sha256.Sum256([]byte(goodsID + "\x00" + specKey)) + return "catalog:" + hex.EncodeToString(sum[:]) +} + func replayCatalogBatch(db *sql.DB, source, batchID, hash, now string) (CatalogBatchResponse, error) { run, err := repository.GetCatalogImportRun(db, source, batchID) if err != nil { diff --git a/admin/service/catalog_import_test.go b/admin/service/catalog_import_test.go index e51ae10..83884b5 100644 --- a/admin/service/catalog_import_test.go +++ b/admin/service/catalog_import_test.go @@ -20,9 +20,9 @@ func newCatalogTestDB(t *testing.T) *sql.DB { db.SetMaxOpenConns(1) statements := []string{ `CREATE TABLE shopee_products(goods_id TEXT PRIMARY KEY,title TEXT NOT NULL,shopee_status TEXT,main_sku_code TEXT,source TEXT,source_observed_at TEXT,pdd_goods_url TEXT,pdd_goods_id TEXT,created_at TEXT,updated_at TEXT)`, - `CREATE TABLE shopee_skus(sku_id TEXT PRIMARY KEY,goods_id TEXT NOT NULL,spec_raw TEXT,color TEXT,size TEXT,advice TEXT,parse_ok INTEGER,sku_code TEXT,is_manual INTEGER,source_observed_at TEXT,created_at TEXT,updated_at TEXT)`, + `CREATE TABLE shopee_skus(sku_id TEXT PRIMARY KEY,shopee_sku_id TEXT UNIQUE,goods_id TEXT NOT NULL,spec_raw TEXT,spec_key TEXT,color TEXT,size TEXT,advice TEXT,parse_ok INTEGER,sku_code TEXT,is_manual INTEGER,source TEXT,field_sources TEXT,field_observed_at TEXT,source_observed_at TEXT,created_at TEXT,updated_at TEXT,UNIQUE(goods_id,spec_key))`, `CREATE TABLE pdd_products(id INTEGER PRIMARY KEY AUTOINCREMENT,goods_id TEXT UNIQUE,url TEXT,title TEXT,shop_name TEXT,skus_json TEXT,collect_status TEXT,collect_msg TEXT,artifact_ref TEXT,collected_at TEXT,deleted_at TEXT,source TEXT,source_observed_at TEXT,created_at TEXT,updated_at TEXT)`, - `CREATE TABLE catalog_import_runs(source TEXT,batch_id TEXT,request_hash TEXT,status TEXT,request_count INTEGER,conflict_count INTEGER,observed_at TEXT,last_request_at TEXT,last_conflict_at TEXT,shopee_created INTEGER DEFAULT 0,shopee_updated INTEGER DEFAULT 0,sku_created INTEGER DEFAULT 0,sku_updated INTEGER DEFAULT 0,pdd_created INTEGER DEFAULT 0,pdd_updated INTEGER DEFAULT 0,association_created INTEGER DEFAULT 0,association_unchanged INTEGER DEFAULT 0,failure_count INTEGER DEFAULT 0,error_summary TEXT,response_body TEXT,created_at TEXT,finished_at TEXT,PRIMARY KEY(source,batch_id))`, + `CREATE TABLE catalog_import_runs(source TEXT,batch_id TEXT,request_hash TEXT,status TEXT,request_count INTEGER,conflict_count INTEGER,observed_at TEXT,update_policy TEXT DEFAULT 'fill_missing',last_request_at TEXT,last_conflict_at TEXT,shopee_created INTEGER DEFAULT 0,shopee_updated INTEGER DEFAULT 0,sku_created INTEGER DEFAULT 0,sku_updated INTEGER DEFAULT 0,sku_filled INTEGER DEFAULT 0,sku_same_source_updated INTEGER DEFAULT 0,sku_skipped INTEGER DEFAULT 0,sku_manual_skipped INTEGER DEFAULT 0,sku_stale_skipped INTEGER DEFAULT 0,pdd_created INTEGER DEFAULT 0,pdd_updated INTEGER DEFAULT 0,association_created INTEGER DEFAULT 0,association_unchanged INTEGER DEFAULT 0,failure_count INTEGER DEFAULT 0,error_summary TEXT,response_body TEXT,created_at TEXT,finished_at TEXT,PRIMARY KEY(source,batch_id))`, } for _, statement := range statements { if _, err := db.Exec(statement); err != nil { @@ -110,4 +110,153 @@ func TestValidateCatalogBatch_拒绝旧版超限和负金额(t *testing.T) { if err := ValidateCatalogBatch(req); err == nil { t.Fatal("应拒绝负金额") } + req = validCatalogBatch() + req.UpdatePolicy = "force_overwrite" + if err := ValidateCatalogBatch(req); err == nil { + t.Fatal("应拒绝强制覆盖策略") + } +} + +func TestImportCatalogBatch_无真实SKU并在后续补回同一记录(t *testing.T) { + db := newCatalogTestDB(t) + req := validCatalogBatch() + req.BatchID = "missing-id" + req.ShopeeSKUs[0].SKUID = "" + raw, _ := json.Marshal(req) + if got, err := ImportCatalogBatch(db, "script-a", req, raw); err != nil || got.Counts.SKUCreated != 1 { + t.Fatalf("首次导入失败：%+v %v", got, err) + } + var recordID string + if err := db.QueryRow(`SELECT sku_id FROM shopee_skus WHERE goods_id='S-1'`).Scan(&recordID); err != nil || recordID == "" { + t.Fatalf("内部记录缺失：%q %v", recordID, err) + } + req.BatchID = "fill-real-id" + req.ShopeeSKUs[0].SKUID = "REAL-1" + raw, _ = json.Marshal(req) + got, err := ImportCatalogBatch(db, "script-a", req, raw) + if err != nil || got.Counts.SKUFilled != 1 { + t.Fatalf("补真实 ID 失败：%+v %v", got, err) + } + var after, external string + _ = db.QueryRow(`SELECT sku_id,shopee_sku_id FROM shopee_skus WHERE goods_id='S-1'`).Scan(&after, &external) + if after != recordID || external != "REAL-1" { + t.Fatalf("不应新建记录：before=%q after=%q external=%q", recordID, after, external) + } +} + +func TestImportCatalogBatch_三种策略和保护边界(t *testing.T) { + db := newCatalogTestDB(t) + req := validCatalogBatch() + req.ShopeeSKUs[0].Color = "" + raw, _ := json.Marshal(req) + if _, err := ImportCatalogBatch(db, "script-a", req, raw); err != nil { + t.Fatal(err) + } + + fill := validCatalogBatch() + fill.BatchID = "fill" + fill.ShopeeSKUs[0].Color = "黑色" + fill.ShopeeSKUs[0].Size = "L" + raw, _ = json.Marshal(fill) + got, err := ImportCatalogBatch(db, "script-a", fill, raw) + if err != nil || got.Counts.SKUFilled != 1 { + t.Fatalf("补空失败：%+v %v", got, err) + } + var color, size string + _ = db.QueryRow(`SELECT color,size FROM shopee_skus`).Scan(&color, &size) + if color != "黑色" || size != "M" { + t.Fatalf("fill_missing 覆盖了非空字段：%q/%q", color, size) + } + + insertOnly := validCatalogBatch() + insertOnly.BatchID = "insert" + insertOnly.UpdatePolicy = "insert_only" + insertOnly.ShopeeSKUs[0].Color = "白色" + raw, _ = json.Marshal(insertOnly) + got, err = ImportCatalogBatch(db, "script-a", insertOnly, raw) + if err != nil || got.Counts.SKUSkipped != 1 { + t.Fatalf("insert_only 失败：%+v %v", got, err) + } + + overwrite := validCatalogBatch() + overwrite.BatchID = "overwrite" + overwrite.UpdatePolicy = "overwrite_same_source" + overwrite.ObservedAt = "2026-08-12T08:00:00+08:00" + overwrite.ShopeeSKUs[0].Color = "白色" + raw, _ = json.Marshal(overwrite) + got, err = ImportCatalogBatch(db, "script-a", overwrite, raw) + if err != nil || got.Counts.SKUSameSourceUpdated != 1 { + t.Fatalf("同来源覆盖失败：%+v %v", got, err) + } + _ = db.QueryRow(`SELECT color FROM shopee_skus`).Scan(&color) + if color != "白色" { + t.Fatalf("color=%q", color) + } + stale := validCatalogBatch() + stale.BatchID = "stale" + stale.UpdatePolicy = "overwrite_same_source" + stale.ObservedAt = "2026-08-11T12:00:00+08:00" + stale.ShopeeSKUs[0].Color = "旧红色" + raw, _ = json.Marshal(stale) + got, err = ImportCatalogBatch(db, "script-a", stale, raw) + if err != nil || got.Counts.SKUStaleSkipped != 1 { + t.Fatalf("旧观测保护失败：%+v %v", got, err) + } + + cross := validCatalogBatch() + cross.BatchID = "cross-source" + cross.ObservedAt = "2026-08-13T08:00:00+08:00" + cross.ShopeeSKUs[0].Advice = "来源B建议" + raw, _ = json.Marshal(cross) + if _, err := ImportCatalogBatch(db, "script-b", cross, raw); err != nil { + t.Fatal(err) + } + protect := validCatalogBatch() + protect.BatchID = "protect-cross" + protect.UpdatePolicy = "overwrite_same_source" + protect.ObservedAt = "2026-08-14T08:00:00+08:00" + protect.ShopeeSKUs[0].Advice = "来源A不得覆盖" + raw, _ = json.Marshal(protect) + if _, err := ImportCatalogBatch(db, "script-a", protect, raw); err != nil { + t.Fatal(err) + } + var advice string + _ = db.QueryRow(`SELECT advice FROM shopee_skus`).Scan(&advice) + if advice != "来源B建议" { + t.Fatalf("跨来源字段被覆盖：%q", advice) + } + + _, _ = db.Exec(`UPDATE shopee_skus SET is_manual=1`) + manual := validCatalogBatch() + manual.BatchID = "manual" + manual.ObservedAt = "2026-08-13T08:00:00+08:00" + manual.ShopeeSKUs[0].Color = "红色" + raw, _ = json.Marshal(manual) + got, err = ImportCatalogBatch(db, "script-a", manual, raw) + if err != nil || got.Counts.SKUManualSkipped != 1 { + t.Fatalf("人工保护失败：%+v %v", got, err) + } +} + +func TestImportCatalogBatch_同规格不同真实SKU冲突并回滚(t *testing.T) { + db := newCatalogTestDB(t) + req := validCatalogBatch() + raw, _ := json.Marshal(req) + if _, err := ImportCatalogBatch(db, "script-a", req, raw); err != nil { + t.Fatal(err) + } + req.BatchID = "conflict" + req.ShopeeSKUs[0].SKUID = "SKU-2" + req.ShopeeProducts[0].Title = "不应提交" + raw, _ = json.Marshal(req) + _, err := ImportCatalogBatch(db, "script-a", req, raw) + var catalogErr *CatalogError + if !errors.As(err, &catalogErr) || catalogErr.Status != 409 { + t.Fatalf("应返回 409：%v", err) + } + var title string + _ = db.QueryRow(`SELECT title FROM shopee_products WHERE goods_id='S-1'`).Scan(&title) + if title != "蝦皮上衣" { + t.Fatalf("冲突批次未整体回滚：%q", title) + } } diff --git a/admin/templates/catalog/detail_modal.html b/admin/templates/catalog/detail_modal.html index 551f526..bcd6d34 100644 --- a/admin/templates/catalog/detail_modal.html +++ b/admin/templates/catalog/detail_modal.html @@ -1 +1 @@ -{{define "catalog/detail_modal"}}
来源
{{.Run.Source}}
批次号
{{.Run.BatchID}}
状态
{{.Run.StatusText}}
上游观测时间
{{.Run.ObservedAt}}
请求/冲突次数
{{.Run.RequestCount}} / {{.Run.ConflictCount}}
失败数
{{.Run.FailureCount}}
错误码
{{if .Run.ErrorCode}}{{.Run.ErrorCode}}{{else}}—{{end}}
可重试
{{if .Run.Retryable}}是{{else}}否{{end}}
错误摘要
{{if .Run.ErrorSummary}}{{.Run.ErrorSummary}}{{else}}—{{end}}
定位信息
{{if .Run.ErrorDetails}}{{.Run.ErrorDetails}}{{else}}—{{end}}

定位信息只包含数组位置和业务 ID；这里不保存完整请求 JSON 或任何凭据。

{{end}} +{{define "catalog/detail_modal"}}
来源
{{.Run.Source}}
批次号
{{.Run.BatchID}}
更新策略
{{.Run.UpdatePolicy}}
状态
{{.Run.StatusText}}
上游观测时间
{{.Run.ObservedAt}}
SKU 新增/补空/同源更新
{{.Run.SKUCreated}} / {{.Run.SKUFilled}} / {{.Run.SKUSameSourceUpdated}}
SKU 跳过（人工/旧数据）
{{.Run.SKUSkipped}}（{{.Run.SKUManualSkipped}} / {{.Run.SKUStaleSkipped}}）
请求/冲突次数
{{.Run.RequestCount}} / {{.Run.ConflictCount}}
失败数
{{.Run.FailureCount}}
错误码
{{if .Run.ErrorCode}}{{.Run.ErrorCode}}{{else}}—{{end}}
可重试
{{if .Run.Retryable}}是{{else}}否{{end}}
错误摘要
{{if .Run.ErrorSummary}}{{.Run.ErrorSummary}}{{else}}—{{end}}
定位信息
{{if .Run.ErrorDetails}}{{.Run.ErrorDetails}}{{else}}—{{end}}

定位信息只包含数组位置和业务 ID；这里不保存完整请求 JSON 或任何凭据。

{{end}} diff --git a/admin/templates/catalog/list.html b/admin/templates/catalog/list.html index fa80bff..cbd7bed 100644 --- a/admin/templates/catalog/list.html +++ b/admin/templates/catalog/list.html @@ -7,8 +7,8 @@ -
-{{range .Rows}}{{else}}{{end}}
来源批次号状态请求冲突蝦皮 新/更SKU 新/更PDD 新/更关联 新/同首次接收完成耗时
{{.Source}}{{.BatchID}}{{.StatusText}}{{.RequestCount}}{{.ConflictCount}}{{.ShopeeCreated}} / {{.ShopeeUpdated}}{{.SKUCreated}} / {{.SKUUpdated}}{{.PddCreated}} / {{.PddUpdated}}{{.AssociationCreated}} / {{.AssociationUnchanged}}{{.CreatedText}}{{.FinishedText}}{{.DurationText}}
还没有符合条件的导入记录。
+
+{{range .Rows}}{{else}}{{end}}
来源批次号策略状态请求冲突蝦皮 新/更SKU 新/补/同源更/跳过PDD 新/更关联 新/同首次接收完成耗时
{{.Source}}{{.BatchID}}{{.UpdatePolicy}}{{.StatusText}}{{.RequestCount}}{{.ConflictCount}}{{.ShopeeCreated}} / {{.ShopeeUpdated}}{{.SKUCreated}} / {{.SKUFilled}} / {{.SKUSameSourceUpdated}} / {{.SKUSkipped}}{{.PddCreated}} / {{.PddUpdated}}{{.AssociationCreated}} / {{.AssociationUnchanged}}{{.CreatedText}}{{.FinishedText}}{{.DurationText}}
还没有符合条件的导入记录。

双击一行查看批次摘要；系统不会保存或展示 Token 和完整请求体。

diff --git a/docs/admin/03-data-model.md b/docs/admin/03-data-model.md index 5576dcc..7aa5bff 100644 --- a/docs/admin/03-data-model.md +++ b/docs/admin/03-data-model.md @@ -429,6 +429,15 @@ CREATE INDEX idx_syb_orders_list ON syb_orders(updated_at DESC, syb_id DESC); 仍存在于最新 `skus_json`。 - `shopee_sku_id` 只为兼容历史数据保留；顺运宝同步、弹窗、映射和建采购任务均不再读写它。 +MySQL schema v8 将 `shopee_skus.sku_id` 保留为系统内部记录主键（兼容历史引用）， +新增可空且唯一的 `shopee_sku_id` 保存真实外部 ID，并用唯一 +`(goods_id, spec_key)` 标识无真实 ID 的规格。既有行将真实 ID复制到新列；新记录没有 +真实 ID 时只生成内部主键，页面和接口不得把内部值冒充真实 SKU ID。`source` 和 +`source_observed_at` 用于限制同来源更新，`is_manual=1` 始终优先保护。 +`field_sources` 与 `field_observed_at` 是只含颜色、尺码、建议和 SKU 编码的 JSON +来源表；`fill_missing` 补字段时分别登记，`overwrite_same_source` 只更新来源属于本次 +调用方且观测时间不旧的字段，避免不同脚本各补一部分后互相覆盖。 + 蝦皮详情把 `syb_orders` 按 `(shopee_goods_id, spec_key)` 聚合为“顺运宝观测规格”： 订单数按唯一 `syb_id` 行计数，累计数量求和，最近一行提供历史台币售价和图片。 该读模型不写入 `shopee_skus`。只有同一商品下恰好一条正式 SKU 的 `spec_raw` diff --git a/docs/admin/10-商品目录接入接口.md b/docs/admin/10-商品目录接入接口.md index 13d8e12..0b77365 100644 --- a/docs/admin/10-商品目录接入接口.md +++ b/docs/admin/10-商品目录接入接口.md @@ -19,11 +19,12 @@ Token 通过 `CMAUTOBUY_CATALOG_TOKEN` 或未提交的 `config.yaml` 配置。 "schema_version": 1, "batch_id": "source-file-20260811-001", "observed_at": "2026-08-11T10:00:00+08:00", + "update_policy": "fill_missing", "shopee_products": [ {"goods_id": "S-1", "title": "蝦皮上衣", "status": "NORMAL", "main_sku_code": "A01"} ], "shopee_skus": [ - {"sku_id": "SKU-1", "goods_id": "S-1", "spec_raw": "黑色,M", "color": "黑色", "size": "M", "parse_ok": true, "sku_code": "A01-B-M"} + {"sku_id": null, "goods_id": "S-1", "spec_raw": "黑色,M", "color": "黑色", "size": "M", "parse_ok": true, "sku_code": "A01-B-M"} ], "pdd_products": [ { @@ -42,16 +43,25 @@ Token 通过 `CMAUTOBUY_CATALOG_TOKEN` 或未提交的 `config.yaml` 配置。 金额单位固定为人民币分，不接受小数金额。`spec_raw` 必须原样提交。PDD 规格必须用 结构化 `dimensions` 和 `skus`，第三方不能提交数据库内部的 `skus_json`。 +`sku_id` 是可选的真实蝦皮 SKU ID；来源文件没有该值时省略、传 `null` 或空字符串， +不得自行生成。系统用内部主键和 `goods_id + spec_key` 保持记录身份，后续取得真实 ID +会补到原记录，不会重复新增。 + ## 3. 写入与幂等规则 - 整个批次在一个事务中写入：实体、SKU、关联任一步失败就全部回滚。 - `(source, batch_id)` 是幂等键。请求字节相同则重放首次结果；同键不同内容返回 409。 -- 较旧的 `observed_at` 不覆盖较新的接口数据。 +- `update_policy` 可省略，默认 `fill_missing`： + - `insert_only`：只新增，已有规格不修改； + - `fill_missing`：只补颜色、尺码、建议、SKU 编码等空字段； + - `overwrite_same_source`：只有来源相同且 `observed_at` 更新时才覆盖来源字段。 +- 较旧的 `observed_at` 不覆盖较新的接口数据，任何策略都不能覆盖 `is_manual=1`。 +- `spec_raw` 和由它生成的规格身份不被更新；同商品同规格出现不同真实 SKU ID 返回 409。 - 蝦皮 upsert 不覆盖采购员维护的 PDD 链接；SKU 的 `is_manual` 不被接口改写。 - 空关联可以建立、相同关联不重复写；已有不同 PDD 关联返回 409，绝不静默替换。 - 批次缺少某条记录不表示删除，接口没有“全量覆盖”语义。 -成功响应包含批次状态和各类新增、更新、关联计数。相同请求重放时 +成功响应包含批次状态以及 SKU 新增、补空、同来源更新、人工跳过、旧数据跳过等计数。相同请求重放时 `replayed=true`。 ## 4. 错误格式 @@ -70,3 +80,12 @@ Token 通过 `CMAUTOBUY_CATALOG_TOKEN` 或未提交的 `config.yaml` 配置。 调用方应按 `code` 处理，不要解析中文 `message`。校验错误通常是 400/422，业务冲突 是 409，临时服务错误是 500/503。响应和导入记录都不会回显 Token 或完整请求体。 + +## 5. schema v8 发布与回退 + +发布前先备份生产库，并在 MySQL 8.4、库名以 `_test` 结尾的测试库演练 v7→v8。 +迁移只增加列和唯一索引，既有 `sku_id` 继续作为内部主键，因此历史引用不变；若检测 +到同商品重复 `spec_key`，迁移会停止且不记录 v8，必须人工确认，不能自动合并。 + +旧版 Admin 不理解无真实 SKU ID 的新记录。应用回退前应暂停目录接口并确认没有此类 +记录；不要删除 v8 列或把内部主键复制成伪造的外部 SKU ID。