feat: 支持无SKU目录导入策略 (#141)

2026-08-11 11:03:18 +08:00
parent 6386d76488
commit 4259be0e55
17 changed files with 766 additions and 73 deletions
@@ -2,6 +2,7 @@ package repository

 import (
 	"database/sql"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"strings"
@@ -21,9 +22,9 @@ var (
 func InsertCatalogImportRun(q Execer, run model.CatalogImportRun) error {
 	_, err := q.Exec(`INSERT INTO catalog_import_runs (
 		source,batch_id,request_hash,status,request_count,conflict_count,observed_at,
-		last_request_at,created_at
-	) VALUES (?,?,?,?,1,0,NULLIF(?,''),?,?)`, run.Source, run.BatchID, run.RequestHash,
-		run.Status, run.ObservedAt, run.LastRequestAt, run.CreatedAt)
+		update_policy,last_request_at,created_at
+	) VALUES (?,?,?,?,1,0,NULLIF(?,''),?,?,?)`, run.Source, run.BatchID, run.RequestHash,
+		run.Status, run.ObservedAt, run.UpdatePolicy, run.LastRequestAt, run.CreatedAt)
 	if err == nil {
 		return nil
 	}
@@ -42,13 +43,15 @@ func GetCatalogImportRun(q Execer, source, batchID string) (model.CatalogImportR
 	var run model.CatalogImportRun
 	var observedAt, lastConflictAt, errorSummary, responseBody, finishedAt sql.NullString
 	err := q.QueryRow(`SELECT source,batch_id,request_hash,status,request_count,conflict_count,
-		observed_at,last_request_at,last_conflict_at,shopee_created,shopee_updated,sku_created,
-		sku_updated,pdd_created,pdd_updated,association_created,association_unchanged,failure_count,
+		observed_at,update_policy,last_request_at,last_conflict_at,shopee_created,shopee_updated,sku_created,
+		sku_updated,sku_filled,sku_same_source_updated,sku_skipped,sku_manual_skipped,sku_stale_skipped,
+		pdd_created,pdd_updated,association_created,association_unchanged,failure_count,
 		error_summary,response_body,created_at,finished_at
 		FROM catalog_import_runs WHERE source=? AND batch_id=?`, source, batchID).Scan(
 		&run.Source, &run.BatchID, &run.RequestHash, &run.Status, &run.RequestCount, &run.ConflictCount,
-		&observedAt, &run.LastRequestAt, &lastConflictAt, &run.ShopeeCreated, &run.ShopeeUpdated,
-		&run.SKUCreated, &run.SKUUpdated, &run.PddCreated, &run.PddUpdated, &run.AssociationCreated,
+		&observedAt, &run.UpdatePolicy, &run.LastRequestAt, &lastConflictAt, &run.ShopeeCreated, &run.ShopeeUpdated,
+		&run.SKUCreated, &run.SKUUpdated, &run.SKUFilled, &run.SKUSameSourceUpdated, &run.SKUSkipped,
+		&run.SKUManualSkipped, &run.SKUStaleSkipped, &run.PddCreated, &run.PddUpdated, &run.AssociationCreated,
 		&run.AssociationUnchanged, &run.FailureCount, &errorSummary, &responseBody, &run.CreatedAt, &finishedAt)
 	if errors.Is(err, sql.ErrNoRows) {
 		return model.CatalogImportRun{}, ErrCatalogImportRunNotFound
@@ -147,10 +150,12 @@ func RecordCatalogImportConflict(q Execer, source, batchID, requestedAt string)
 // CompleteCatalogImportRun 把处理结果摘要和安全的响应 JSON 固化，供幂等重放。
 func CompleteCatalogImportRun(q Execer, run model.CatalogImportRun) error {
 	result, err := q.Exec(`UPDATE catalog_import_runs SET status=?,shopee_created=?,shopee_updated=?,
-		sku_created=?,sku_updated=?,pdd_created=?,pdd_updated=?,association_created=?,
+		sku_created=?,sku_updated=?,sku_filled=?,sku_same_source_updated=?,sku_skipped=?,
+		sku_manual_skipped=?,sku_stale_skipped=?,pdd_created=?,pdd_updated=?,association_created=?,
 		association_unchanged=?,failure_count=?,error_summary=NULLIF(?,''),response_body=NULLIF(?,''),
 		finished_at=? WHERE source=? AND batch_id=?`, run.Status, run.ShopeeCreated, run.ShopeeUpdated,
-		run.SKUCreated, run.SKUUpdated, run.PddCreated, run.PddUpdated, run.AssociationCreated,
+		run.SKUCreated, run.SKUUpdated, run.SKUFilled, run.SKUSameSourceUpdated, run.SKUSkipped,
+		run.SKUManualSkipped, run.SKUStaleSkipped, run.PddCreated, run.PddUpdated, run.AssociationCreated,
 		run.AssociationUnchanged, run.FailureCount, run.ErrorSummary, run.ResponseBody,
 		run.FinishedAt, run.Source, run.BatchID)
 	return catalogRunUpdateResult(result, err, "完成商品目录批次")
@@ -177,34 +182,194 @@ func UpsertCatalogShopeeProduct(q Execer, goodsID, title, status, mainSKU, obser
 	return false, err == nil, err
 }

-// UpsertCatalogShopeeSKU 拒绝把同一 SKU 静默挪到另一商品，并保留 is_manual。
-func UpsertCatalogShopeeSKU(q Execer, skuID, goodsID, specRaw, color, size, advice string, parseOK bool, skuCode, observedAt, now string) (created, updated bool, err error) {
-	var oldGoods string
-	var oldObserved sql.NullString
-	err = q.QueryRow(`SELECT goods_id,source_observed_at FROM shopee_skus WHERE sku_id=?`, skuID).Scan(&oldGoods, &oldObserved)
-	parse := 0
-	if parseOK {
-		parse = 1
-	}
+type CatalogSKUOutcome string
+
+const (
+	CatalogSKUCreated           CatalogSKUOutcome = "created"
+	CatalogSKUFilled            CatalogSKUOutcome = "filled"
+	CatalogSKUSameSourceUpdated CatalogSKUOutcome = "same_source_updated"
+	CatalogSKUSkipped           CatalogSKUOutcome = "skipped"
+	CatalogSKUManualSkipped     CatalogSKUOutcome = "manual_skipped"
+	CatalogSKUStaleSkipped      CatalogSKUOutcome = "stale_skipped"
+)
+
+type CatalogShopeeSKUInput struct {
+	RecordID, ShopeeSKUID, GoodsID, SpecRaw, SpecKey string
+	Color, Size, Advice, SKUCode                     string
+	ParseOK                                          bool
+	Source, ObservedAt, Now, UpdatePolicy            string
+}
+
+type catalogSKUStored struct {
+	RecordID, GoodsID, SpecRaw, SpecKey       string
+	ShopeeSKUID, Color, Size, Advice, SKUCode sql.NullString
+	ParseOK, IsManual                         int
+	Source, ObservedAt                        sql.NullString
+	FieldSources, FieldObservedAt             sql.NullString
+}
+
+func scanCatalogSKU(dbRow *sql.Row) (*catalogSKUStored, error) {
+	var row catalogSKUStored
+	err := dbRow.Scan(&row.RecordID, &row.ShopeeSKUID, &row.GoodsID, &row.SpecRaw, &row.SpecKey, &row.Color, &row.Size, &row.Advice, &row.ParseOK, &row.SKUCode, &row.IsManual, &row.Source, &row.ObservedAt, &row.FieldSources, &row.FieldObservedAt)
 	if errors.Is(err, sql.ErrNoRows) {
-		_, err = q.Exec(`INSERT INTO shopee_skus(sku_id,goods_id,spec_raw,color,size,advice,parse_ok,sku_code,
-			is_manual,source_observed_at,created_at,updated_at) VALUES(?,?,?,NULLIF(?,''),NULLIF(?,''),NULLIF(?,''),?,NULLIF(?,''),0,?,?,?)`,
-			skuID, goodsID, specRaw, color, size, advice, parse, skuCode, observedAt, now, now)
-		return err == nil, false, err
+		return nil, nil
 	}
 	if err != nil {
-		return false, false, err
+		return nil, err
 	}
-	if oldGoods != goodsID {
-		return false, false, fmt.Errorf("SKU %s 已属于蝦皮商品 %s", skuID, oldGoods)
+	return &row, nil
+}
+
+func findCatalogSKUBySpec(q Execer, goodsID, specKey string) (*catalogSKUStored, error) {
+	return scanCatalogSKU(q.QueryRow(`SELECT sku_id,shopee_sku_id,goods_id,spec_raw,spec_key,color,size,advice,parse_ok,sku_code,is_manual,source,source_observed_at,field_sources,field_observed_at FROM shopee_skus WHERE goods_id=? AND spec_key=?`, goodsID, specKey))
+}
+
+func findCatalogSKUByExternalID(q Execer, skuID string) (*catalogSKUStored, error) {
+	return scanCatalogSKU(q.QueryRow(`SELECT sku_id,shopee_sku_id,goods_id,spec_raw,spec_key,color,size,advice,parse_ok,sku_code,is_manual,source,source_observed_at,field_sources,field_observed_at FROM shopee_skus WHERE shopee_sku_id=?`, skuID))
+}
+
+// UpsertCatalogShopeeSKU 用真实 ID 或商品规格身份定位同一内部记录，人工行永不覆盖。
+func UpsertCatalogShopeeSKU(q Execer, in CatalogShopeeSKUInput) (CatalogSKUOutcome, error) {
+	bySpec, err := findCatalogSKUBySpec(q, in.GoodsID, in.SpecKey)
+	if err != nil {
+		return "", err
 	}
-	if oldObserved.Valid && oldObserved.String > observedAt {
-		return false, false, nil
+	var byID *catalogSKUStored
+	if in.ShopeeSKUID != "" {
+		byID, err = findCatalogSKUByExternalID(q, in.ShopeeSKUID)
+		if err != nil {
+			return "", err
+		}
 	}
-	_, err = q.Exec(`UPDATE shopee_skus SET spec_raw=?,color=NULLIF(?,''),size=NULLIF(?,''),advice=NULLIF(?,''),
-		parse_ok=?,sku_code=NULLIF(?,''),source_observed_at=?,updated_at=? WHERE sku_id=?`,
-		specRaw, color, size, advice, parse, skuCode, observedAt, now, skuID)
-	return false, err == nil, err
+	if byID != nil && (byID.GoodsID != in.GoodsID || byID.SpecKey != in.SpecKey) {
+		return "", fmt.Errorf("真实蝦皮 SKU %s 已属于其他商品或规格", in.ShopeeSKUID)
+	}
+	if byID != nil && bySpec != nil && byID.RecordID != bySpec.RecordID {
+		return "", fmt.Errorf("商品 %s 的规格 %s 已对应另一个真实蝦皮 SKU", in.GoodsID, in.SpecRaw)
+	}
+	current := byID
+	if current == nil {
+		current = bySpec
+	}
+	parse := 0
+	if in.ParseOK {
+		parse = 1
+	}
+	if current == nil {
+		fieldSources, fieldTimes := newCatalogFieldProvenance(in)
+		sourcesJSON, _ := json.Marshal(fieldSources)
+		timesJSON, _ := json.Marshal(fieldTimes)
+		_, err = q.Exec(`INSERT INTO shopee_skus(sku_id,shopee_sku_id,goods_id,spec_raw,spec_key,color,size,advice,parse_ok,sku_code,is_manual,source,field_sources,field_observed_at,source_observed_at,created_at,updated_at) VALUES(?,NULLIF(?,''),?,?,?,NULLIF(?,''),NULLIF(?,''),NULLIF(?,''),?,NULLIF(?,''),0,?,?,?,?,?,?)`, in.RecordID, in.ShopeeSKUID, in.GoodsID, in.SpecRaw, in.SpecKey, in.Color, in.Size, in.Advice, parse, in.SKUCode, in.Source, string(sourcesJSON), string(timesJSON), in.ObservedAt, in.Now, in.Now)
+		if err != nil {
+			return "", err
+		}
+		return CatalogSKUCreated, nil
+	}
+	if current.ShopeeSKUID.Valid && in.ShopeeSKUID != "" && current.ShopeeSKUID.String != in.ShopeeSKUID {
+		return "", fmt.Errorf("商品 %s 的规格 %s 已对应真实蝦皮 SKU %s", in.GoodsID, in.SpecRaw, current.ShopeeSKUID.String)
+	}
+	if in.UpdatePolicy == "insert_only" {
+		return CatalogSKUSkipped, nil
+	}
+	if current.IsManual != 0 {
+		return CatalogSKUManualSkipped, nil
+	}
+	external := current.ShopeeSKUID.String
+	externalAdded := external == "" && in.ShopeeSKUID != ""
+	if external == "" {
+		external = in.ShopeeSKUID
+	}
+	if in.UpdatePolicy == "fill_missing" {
+		color, size, advice, skuCode := current.Color.String, current.Size.String, current.Advice.String, current.SKUCode.String
+		sources, times := catalogFieldProvenance(current)
+		changed := external != current.ShopeeSKUID.String
+		if color == "" && strings.TrimSpace(in.Color) != "" {
+			color = in.Color
+			sources["color"], times["color"] = in.Source, in.ObservedAt
+			changed = true
+		}
+		if size == "" && strings.TrimSpace(in.Size) != "" {
+			size = in.Size
+			sources["size"], times["size"] = in.Source, in.ObservedAt
+			changed = true
+		}
+		if advice == "" && strings.TrimSpace(in.Advice) != "" {
+			advice = in.Advice
+			sources["advice"], times["advice"] = in.Source, in.ObservedAt
+			changed = true
+		}
+		if skuCode == "" && strings.TrimSpace(in.SKUCode) != "" {
+			skuCode = in.SKUCode
+			sources["sku_code"], times["sku_code"] = in.Source, in.ObservedAt
+			changed = true
+		}
+		if !changed {
+			return CatalogSKUSkipped, nil
+		}
+		sourcesJSON, _ := json.Marshal(sources)
+		timesJSON, _ := json.Marshal(times)
+		_, err = q.Exec(`UPDATE shopee_skus SET shopee_sku_id=NULLIF(?,''),color=NULLIF(?,''),size=NULLIF(?,''),advice=NULLIF(?,''),sku_code=NULLIF(?,''),parse_ok=CASE WHEN parse_ok=1 THEN 1 ELSE ? END,field_sources=?,field_observed_at=?,updated_at=? WHERE sku_id=?`, external, color, size, advice, skuCode, parse, string(sourcesJSON), string(timesJSON), in.Now, current.RecordID)
+		if err != nil {
+			return "", err
+		}
+		return CatalogSKUFilled, nil
+	}
+	sources, times := catalogFieldProvenance(current)
+	color, size, advice, skuCode := current.Color.String, current.Size.String, current.Advice.String, current.SKUCode.String
+	changed, stale := false, false
+	for _, field := range []struct {
+		name     string
+		incoming string
+		current  *string
+	}{{"color", in.Color, &color}, {"size", in.Size, &size}, {"advice", in.Advice, &advice}, {"sku_code", in.SKUCode, &skuCode}} {
+		if sources[field.name] != in.Source {
+			continue
+		}
+		if times[field.name] > in.ObservedAt {
+			stale = true
+			continue
+		}
+		*field.current = field.incoming
+		times[field.name] = in.ObservedAt
+		changed = true
+	}
+	if !changed {
+		if externalAdded {
+			_, err = q.Exec(`UPDATE shopee_skus SET shopee_sku_id=?,updated_at=? WHERE sku_id=?`, external, in.Now, current.RecordID)
+			if err != nil {
+				return "", err
+			}
+			return CatalogSKUFilled, nil
+		}
+		if stale {
+			return CatalogSKUStaleSkipped, nil
+		}
+		return CatalogSKUSkipped, nil
+	}
+	sourcesJSON, _ := json.Marshal(sources)
+	timesJSON, _ := json.Marshal(times)
+	_, err = q.Exec(`UPDATE shopee_skus SET shopee_sku_id=NULLIF(?,''),color=NULLIF(?,''),size=NULLIF(?,''),advice=NULLIF(?,''),parse_ok=?,sku_code=NULLIF(?,''),field_sources=?,field_observed_at=?,source_observed_at=?,updated_at=? WHERE sku_id=?`, external, color, size, advice, parse, skuCode, string(sourcesJSON), string(timesJSON), in.ObservedAt, in.Now, current.RecordID)
+	if err != nil {
+		return "", err
+	}
+	return CatalogSKUSameSourceUpdated, nil
+}
+
+func newCatalogFieldProvenance(in CatalogShopeeSKUInput) (map[string]string, map[string]string) {
+	sources, times := map[string]string{}, map[string]string{}
+	for name, value := range map[string]string{"color": in.Color, "size": in.Size, "advice": in.Advice, "sku_code": in.SKUCode} {
+		if strings.TrimSpace(value) != "" {
+			sources[name], times[name] = in.Source, in.ObservedAt
+		}
+	}
+	return sources, times
+}
+
+func catalogFieldProvenance(row *catalogSKUStored) (map[string]string, map[string]string) {
+	sources, times := map[string]string{}, map[string]string{}
+	_ = json.Unmarshal([]byte(row.FieldSources.String), &sources)
+	_ = json.Unmarshal([]byte(row.FieldObservedAt.String), &times)
+	return sources, times
 }

 // UpsertCatalogPddProduct 写入结构化 PDD 数据转换后的规范 JSON；空规格不清除既有采集结果。