feat: 支持无SKU目录导入策略 (#141)

2026-08-11 11:03:18 +08:00
parent 6386d76488
commit 4259be0e55
17 changed files with 766 additions and 73 deletions
@@ -11,9 +11,11 @@ import (
 	"net/http"
 	"strings"
 	"time"
+	"unicode/utf8"

 	"cmautobuy/admin/model"
 	"cmautobuy/admin/repository"
+	"cmautobuy/admin/spec"
 )

 const (
@@ -63,6 +65,7 @@ type CatalogBatchRequest struct {
 	SchemaVersion  int                    `json:"schema_version"`
 	BatchID        string                 `json:"batch_id"`
 	ObservedAt     string                 `json:"observed_at"`
+	UpdatePolicy   string                 `json:"update_policy"`
 	ShopeeProducts []CatalogShopeeProduct `json:"shopee_products"`
 	ShopeeSKUs     []CatalogShopeeSKU     `json:"shopee_skus"`
 	PddProducts    []CatalogPddProduct    `json:"pdd_products"`
@@ -73,6 +76,11 @@ type CatalogCounts struct {
 	ShopeeUpdated        int `json:"shopee_updated"`
 	SKUCreated           int `json:"sku_created"`
 	SKUUpdated           int `json:"sku_updated"`
+	SKUFilled            int `json:"sku_filled"`
+	SKUSameSourceUpdated int `json:"sku_same_source_updated"`
+	SKUSkipped           int `json:"sku_skipped"`
+	SKUManualSkipped     int `json:"sku_manual_skipped"`
+	SKUStaleSkipped      int `json:"sku_stale_skipped"`
 	PddCreated           int `json:"pdd_created"`
 	PddUpdated           int `json:"pdd_updated"`
 	AssociationCreated   int `json:"association_created"`
@@ -108,13 +116,16 @@ func ValidateCatalogBatch(req CatalogBatchRequest) error {
 	if _, err := time.Parse(time.RFC3339Nano, req.ObservedAt); err != nil {
 		return catalogInvalid("INVALID_OBSERVED_AT", "observed_at 必须是带时区 ISO 8601", nil)
 	}
+	if _, err := normalizeCatalogUpdatePolicy(req.UpdatePolicy); err != nil {
+		return err
+	}
 	if len(req.ShopeeProducts)+len(req.PddProducts) > CatalogMaxProducts {
 		return catalogInvalid("TOO_MANY_PRODUCTS", "每批商品总数不能超过 500", nil)
 	}
 	if len(req.ShopeeSKUs) > CatalogMaxSKUs {
 		return catalogInvalid("TOO_MANY_SKUS", "每批 SKU 不能超过 5000", nil)
 	}
-	seenProducts, seenSKUs, seenPDD := map[string]bool{}, map[string]bool{}, map[string]bool{}
+	seenProducts, seenSKUs, seenSpecs, seenPDD := map[string]bool{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
 	for i, p := range req.ShopeeProducts {
 		p.GoodsID = strings.TrimSpace(p.GoodsID)
 		if p.GoodsID == "" || strings.TrimSpace(p.Title) == "" || seenProducts[p.GoodsID] {
@@ -123,10 +134,16 @@ func ValidateCatalogBatch(req CatalogBatchRequest) error {
 		seenProducts[p.GoodsID] = true
 	}
 	for i, s := range req.ShopeeSKUs {
-		if strings.TrimSpace(s.SKUID) == "" || strings.TrimSpace(s.GoodsID) == "" || strings.TrimSpace(s.SpecRaw) == "" || seenSKUs[s.SKUID] {
-			return catalogInvalid("INVALID_SHOPEE_SKU", "蝦皮 SKU ID、商品 ID、spec_raw 必填且 SKU 不能重复", map[string]any{"index": i, "sku_id": s.SKUID})
+		key, keyErr := spec.SpecKey(s.SpecRaw)
+		externalID := strings.TrimSpace(s.SKUID)
+		identity := strings.TrimSpace(s.GoodsID) + "\x00" + key
+		if strings.TrimSpace(s.GoodsID) == "" || keyErr != nil || utf8.RuneCountInString(key) > 191 || seenSpecs[identity] || (externalID != "" && seenSKUs[externalID]) {
+			return catalogInvalid("INVALID_SHOPEE_SKU", "商品 ID、有效 spec_raw 必填，批内规格身份和非空 SKU ID 不能重复", map[string]any{"index": i, "sku_id": externalID})
+		}
+		seenSpecs[identity] = true
+		if externalID != "" {
+			seenSKUs[externalID] = true
 		}
-		seenSKUs[s.SKUID] = true
 	}
 	for i, p := range req.PddProducts {
 		if strings.TrimSpace(p.GoodsID) == "" || strings.TrimSpace(p.URL) == "" || seenPDD[p.GoodsID] {
@@ -153,6 +170,8 @@ func ImportCatalogBatch(db *sql.DB, source string, req CatalogBatchRequest, raw
 	}
 	observed, _ := time.Parse(time.RFC3339Nano, req.ObservedAt)
 	req.ObservedAt = observed.UTC().Format(model.TimeLayout)
+	policy, _ := normalizeCatalogUpdatePolicy(req.UpdatePolicy)
+	req.UpdatePolicy = policy
 	h := sha256.Sum256(raw)
 	hash := hex.EncodeToString(h[:])
 	now := model.NowISO()
@@ -160,7 +179,7 @@ func ImportCatalogBatch(db *sql.DB, source string, req CatalogBatchRequest, raw
 	if err != nil {
 		return CatalogBatchResponse{}, err
 	}
-	run := model.CatalogImportRun{Source: source, BatchID: req.BatchID, RequestHash: hash, Status: model.CatalogImportProcessing, ObservedAt: req.ObservedAt, LastRequestAt: now, CreatedAt: now}
+	run := model.CatalogImportRun{Source: source, BatchID: req.BatchID, RequestHash: hash, Status: model.CatalogImportProcessing, UpdatePolicy: policy, ObservedAt: req.ObservedAt, LastRequestAt: now, CreatedAt: now}
 	if err = repository.InsertCatalogImportRun(tx, run); err != nil {
 		tx.Rollback()
 		if errors.Is(err, repository.ErrCatalogImportRunExists) {
@@ -215,15 +234,29 @@ func ImportCatalogBatch(db *sql.DB, source string, req CatalogBatchRequest, raw
 		}
 	}
 	for _, s := range req.ShopeeSKUs {
-		c, u, e := repository.UpsertCatalogShopeeSKU(tx, s.SKUID, s.GoodsID, s.SpecRaw, s.Color, s.Size, s.Advice, s.ParseOK, s.SKUCode, req.ObservedAt, now)
+		key, _ := spec.SpecKey(s.SpecRaw)
+		internalID := catalogSKURecordID(s.GoodsID, key)
+		outcome, e := repository.UpsertCatalogShopeeSKU(tx, repository.CatalogShopeeSKUInput{RecordID: internalID, ShopeeSKUID: strings.TrimSpace(s.SKUID), GoodsID: strings.TrimSpace(s.GoodsID), SpecRaw: s.SpecRaw, SpecKey: key, Color: s.Color, Size: s.Size, Advice: s.Advice, ParseOK: s.ParseOK, SKUCode: s.SKUCode, Source: source, ObservedAt: req.ObservedAt, Now: now, UpdatePolicy: policy})
 		if e != nil {
 			return fail(&CatalogError{Status: 409, Code: "SKU_OWNERSHIP_CONFLICT", Message: e.Error()})
 		}
-		if c {
+		switch outcome {
+		case repository.CatalogSKUCreated:
 			counts.SKUCreated++
-		}
-		if u {
+		case repository.CatalogSKUFilled:
+			counts.SKUFilled++
 			counts.SKUUpdated++
+		case repository.CatalogSKUSameSourceUpdated:
+			counts.SKUSameSourceUpdated++
+			counts.SKUUpdated++
+		case repository.CatalogSKUManualSkipped:
+			counts.SKUManualSkipped++
+			counts.SKUSkipped++
+		case repository.CatalogSKUStaleSkipped:
+			counts.SKUStaleSkipped++
+			counts.SKUSkipped++
+		default:
+			counts.SKUSkipped++
 		}
 	}
 	for _, a := range req.Associations {
@@ -247,6 +280,11 @@ func ImportCatalogBatch(db *sql.DB, source string, req CatalogBatchRequest, raw
 	run.ShopeeUpdated = counts.ShopeeUpdated
 	run.SKUCreated = counts.SKUCreated
 	run.SKUUpdated = counts.SKUUpdated
+	run.SKUFilled = counts.SKUFilled
+	run.SKUSameSourceUpdated = counts.SKUSameSourceUpdated
+	run.SKUSkipped = counts.SKUSkipped
+	run.SKUManualSkipped = counts.SKUManualSkipped
+	run.SKUStaleSkipped = counts.SKUStaleSkipped
 	run.PddCreated = counts.PddCreated
 	run.PddUpdated = counts.PddUpdated
 	run.AssociationCreated = counts.AssociationCreated
@@ -260,6 +298,22 @@ func ImportCatalogBatch(db *sql.DB, source string, req CatalogBatchRequest, raw
 	return resp, nil
 }

+func normalizeCatalogUpdatePolicy(raw string) (string, error) {
+	switch strings.TrimSpace(raw) {
+	case "", "fill_missing":
+		return "fill_missing", nil
+	case "insert_only", "overwrite_same_source":
+		return strings.TrimSpace(raw), nil
+	default:
+		return "", catalogInvalid("INVALID_UPDATE_POLICY", "update_policy 只支持 insert_only、fill_missing、overwrite_same_source", nil)
+	}
+}
+
+func catalogSKURecordID(goodsID, specKey string) string {
+	sum := sha256.Sum256([]byte(goodsID + "\x00" + specKey))
+	return "catalog:" + hex.EncodeToString(sum[:])
+}
+
 func replayCatalogBatch(db *sql.DB, source, batchID, hash, now string) (CatalogBatchResponse, error) {
 	run, err := repository.GetCatalogImportRun(db, source, batchID)
 	if err != nil {
@@ -20,9 +20,9 @@ func newCatalogTestDB(t *testing.T) *sql.DB {
 	db.SetMaxOpenConns(1)
 	statements := []string{
 		`CREATE TABLE shopee_products(goods_id TEXT PRIMARY KEY,title TEXT NOT NULL,shopee_status TEXT,main_sku_code TEXT,source TEXT,source_observed_at TEXT,pdd_goods_url TEXT,pdd_goods_id TEXT,created_at TEXT,updated_at TEXT)`,
-		`CREATE TABLE shopee_skus(sku_id TEXT PRIMARY KEY,goods_id TEXT NOT NULL,spec_raw TEXT,color TEXT,size TEXT,advice TEXT,parse_ok INTEGER,sku_code TEXT,is_manual INTEGER,source_observed_at TEXT,created_at TEXT,updated_at TEXT)`,
+		`CREATE TABLE shopee_skus(sku_id TEXT PRIMARY KEY,shopee_sku_id TEXT UNIQUE,goods_id TEXT NOT NULL,spec_raw TEXT,spec_key TEXT,color TEXT,size TEXT,advice TEXT,parse_ok INTEGER,sku_code TEXT,is_manual INTEGER,source TEXT,field_sources TEXT,field_observed_at TEXT,source_observed_at TEXT,created_at TEXT,updated_at TEXT,UNIQUE(goods_id,spec_key))`,
 		`CREATE TABLE pdd_products(id INTEGER PRIMARY KEY AUTOINCREMENT,goods_id TEXT UNIQUE,url TEXT,title TEXT,shop_name TEXT,skus_json TEXT,collect_status TEXT,collect_msg TEXT,artifact_ref TEXT,collected_at TEXT,deleted_at TEXT,source TEXT,source_observed_at TEXT,created_at TEXT,updated_at TEXT)`,
-		`CREATE TABLE catalog_import_runs(source TEXT,batch_id TEXT,request_hash TEXT,status TEXT,request_count INTEGER,conflict_count INTEGER,observed_at TEXT,last_request_at TEXT,last_conflict_at TEXT,shopee_created INTEGER DEFAULT 0,shopee_updated INTEGER DEFAULT 0,sku_created INTEGER DEFAULT 0,sku_updated INTEGER DEFAULT 0,pdd_created INTEGER DEFAULT 0,pdd_updated INTEGER DEFAULT 0,association_created INTEGER DEFAULT 0,association_unchanged INTEGER DEFAULT 0,failure_count INTEGER DEFAULT 0,error_summary TEXT,response_body TEXT,created_at TEXT,finished_at TEXT,PRIMARY KEY(source,batch_id))`,
+		`CREATE TABLE catalog_import_runs(source TEXT,batch_id TEXT,request_hash TEXT,status TEXT,request_count INTEGER,conflict_count INTEGER,observed_at TEXT,update_policy TEXT DEFAULT 'fill_missing',last_request_at TEXT,last_conflict_at TEXT,shopee_created INTEGER DEFAULT 0,shopee_updated INTEGER DEFAULT 0,sku_created INTEGER DEFAULT 0,sku_updated INTEGER DEFAULT 0,sku_filled INTEGER DEFAULT 0,sku_same_source_updated INTEGER DEFAULT 0,sku_skipped INTEGER DEFAULT 0,sku_manual_skipped INTEGER DEFAULT 0,sku_stale_skipped INTEGER DEFAULT 0,pdd_created INTEGER DEFAULT 0,pdd_updated INTEGER DEFAULT 0,association_created INTEGER DEFAULT 0,association_unchanged INTEGER DEFAULT 0,failure_count INTEGER DEFAULT 0,error_summary TEXT,response_body TEXT,created_at TEXT,finished_at TEXT,PRIMARY KEY(source,batch_id))`,
 	}
 	for _, statement := range statements {
 		if _, err := db.Exec(statement); err != nil {
@@ -110,4 +110,153 @@ func TestValidateCatalogBatch_拒绝旧版超限和负金额(t *testing.T) {
 	if err := ValidateCatalogBatch(req); err == nil {
 		t.Fatal("应拒绝负金额")
 	}
+	req = validCatalogBatch()
+	req.UpdatePolicy = "force_overwrite"
+	if err := ValidateCatalogBatch(req); err == nil {
+		t.Fatal("应拒绝强制覆盖策略")
+	}
+}
+
+func TestImportCatalogBatch_无真实SKU并在后续补回同一记录(t *testing.T) {
+	db := newCatalogTestDB(t)
+	req := validCatalogBatch()
+	req.BatchID = "missing-id"
+	req.ShopeeSKUs[0].SKUID = ""
+	raw, _ := json.Marshal(req)
+	if got, err := ImportCatalogBatch(db, "script-a", req, raw); err != nil || got.Counts.SKUCreated != 1 {
+		t.Fatalf("首次导入失败：%+v %v", got, err)
+	}
+	var recordID string
+	if err := db.QueryRow(`SELECT sku_id FROM shopee_skus WHERE goods_id='S-1'`).Scan(&recordID); err != nil || recordID == "" {
+		t.Fatalf("内部记录缺失：%q %v", recordID, err)
+	}
+	req.BatchID = "fill-real-id"
+	req.ShopeeSKUs[0].SKUID = "REAL-1"
+	raw, _ = json.Marshal(req)
+	got, err := ImportCatalogBatch(db, "script-a", req, raw)
+	if err != nil || got.Counts.SKUFilled != 1 {
+		t.Fatalf("补真实 ID 失败：%+v %v", got, err)
+	}
+	var after, external string
+	_ = db.QueryRow(`SELECT sku_id,shopee_sku_id FROM shopee_skus WHERE goods_id='S-1'`).Scan(&after, &external)
+	if after != recordID || external != "REAL-1" {
+		t.Fatalf("不应新建记录：before=%q after=%q external=%q", recordID, after, external)
+	}
+}
+
+func TestImportCatalogBatch_三种策略和保护边界(t *testing.T) {
+	db := newCatalogTestDB(t)
+	req := validCatalogBatch()
+	req.ShopeeSKUs[0].Color = ""
+	raw, _ := json.Marshal(req)
+	if _, err := ImportCatalogBatch(db, "script-a", req, raw); err != nil {
+		t.Fatal(err)
+	}
+
+	fill := validCatalogBatch()
+	fill.BatchID = "fill"
+	fill.ShopeeSKUs[0].Color = "黑色"
+	fill.ShopeeSKUs[0].Size = "L"
+	raw, _ = json.Marshal(fill)
+	got, err := ImportCatalogBatch(db, "script-a", fill, raw)
+	if err != nil || got.Counts.SKUFilled != 1 {
+		t.Fatalf("补空失败：%+v %v", got, err)
+	}
+	var color, size string
+	_ = db.QueryRow(`SELECT color,size FROM shopee_skus`).Scan(&color, &size)
+	if color != "黑色" || size != "M" {
+		t.Fatalf("fill_missing 覆盖了非空字段：%q/%q", color, size)
+	}
+
+	insertOnly := validCatalogBatch()
+	insertOnly.BatchID = "insert"
+	insertOnly.UpdatePolicy = "insert_only"
+	insertOnly.ShopeeSKUs[0].Color = "白色"
+	raw, _ = json.Marshal(insertOnly)
+	got, err = ImportCatalogBatch(db, "script-a", insertOnly, raw)
+	if err != nil || got.Counts.SKUSkipped != 1 {
+		t.Fatalf("insert_only 失败：%+v %v", got, err)
+	}
+
+	overwrite := validCatalogBatch()
+	overwrite.BatchID = "overwrite"
+	overwrite.UpdatePolicy = "overwrite_same_source"
+	overwrite.ObservedAt = "2026-08-12T08:00:00+08:00"
+	overwrite.ShopeeSKUs[0].Color = "白色"
+	raw, _ = json.Marshal(overwrite)
+	got, err = ImportCatalogBatch(db, "script-a", overwrite, raw)
+	if err != nil || got.Counts.SKUSameSourceUpdated != 1 {
+		t.Fatalf("同来源覆盖失败：%+v %v", got, err)
+	}
+	_ = db.QueryRow(`SELECT color FROM shopee_skus`).Scan(&color)
+	if color != "白色" {
+		t.Fatalf("color=%q", color)
+	}
+	stale := validCatalogBatch()
+	stale.BatchID = "stale"
+	stale.UpdatePolicy = "overwrite_same_source"
+	stale.ObservedAt = "2026-08-11T12:00:00+08:00"
+	stale.ShopeeSKUs[0].Color = "旧红色"
+	raw, _ = json.Marshal(stale)
+	got, err = ImportCatalogBatch(db, "script-a", stale, raw)
+	if err != nil || got.Counts.SKUStaleSkipped != 1 {
+		t.Fatalf("旧观测保护失败：%+v %v", got, err)
+	}
+
+	cross := validCatalogBatch()
+	cross.BatchID = "cross-source"
+	cross.ObservedAt = "2026-08-13T08:00:00+08:00"
+	cross.ShopeeSKUs[0].Advice = "来源B建议"
+	raw, _ = json.Marshal(cross)
+	if _, err := ImportCatalogBatch(db, "script-b", cross, raw); err != nil {
+		t.Fatal(err)
+	}
+	protect := validCatalogBatch()
+	protect.BatchID = "protect-cross"
+	protect.UpdatePolicy = "overwrite_same_source"
+	protect.ObservedAt = "2026-08-14T08:00:00+08:00"
+	protect.ShopeeSKUs[0].Advice = "来源A不得覆盖"
+	raw, _ = json.Marshal(protect)
+	if _, err := ImportCatalogBatch(db, "script-a", protect, raw); err != nil {
+		t.Fatal(err)
+	}
+	var advice string
+	_ = db.QueryRow(`SELECT advice FROM shopee_skus`).Scan(&advice)
+	if advice != "来源B建议" {
+		t.Fatalf("跨来源字段被覆盖：%q", advice)
+	}
+
+	_, _ = db.Exec(`UPDATE shopee_skus SET is_manual=1`)
+	manual := validCatalogBatch()
+	manual.BatchID = "manual"
+	manual.ObservedAt = "2026-08-13T08:00:00+08:00"
+	manual.ShopeeSKUs[0].Color = "红色"
+	raw, _ = json.Marshal(manual)
+	got, err = ImportCatalogBatch(db, "script-a", manual, raw)
+	if err != nil || got.Counts.SKUManualSkipped != 1 {
+		t.Fatalf("人工保护失败：%+v %v", got, err)
+	}
+}
+
+func TestImportCatalogBatch_同规格不同真实SKU冲突并回滚(t *testing.T) {
+	db := newCatalogTestDB(t)
+	req := validCatalogBatch()
+	raw, _ := json.Marshal(req)
+	if _, err := ImportCatalogBatch(db, "script-a", req, raw); err != nil {
+		t.Fatal(err)
+	}
+	req.BatchID = "conflict"
+	req.ShopeeSKUs[0].SKUID = "SKU-2"
+	req.ShopeeProducts[0].Title = "不应提交"
+	raw, _ = json.Marshal(req)
+	_, err := ImportCatalogBatch(db, "script-a", req, raw)
+	var catalogErr *CatalogError
+	if !errors.As(err, &catalogErr) || catalogErr.Status != 409 {
+		t.Fatalf("应返回 409：%v", err)
+	}
+	var title string
+	_ = db.QueryRow(`SELECT title FROM shopee_products WHERE goods_id='S-1'`).Scan(&title)
+	if title != "蝦皮上衣" {
+		t.Fatalf("冲突批次未整体回滚：%q", title)
+	}
 }