feat: 规范 CSV 预检与安全入库 (#281)

2026-08-20 14:41:01 +08:00
parent b8537fa776
commit fe64d3b3c7
5 changed files with 260 additions and 14 deletions
@@ -18,6 +18,48 @@ var (
 	ErrCatalogImportRunNotFound = errors.New("商品目录批次不存在")
 )

+// CatalogEntityExists 仅用于目录导入预检，不会修改任何记录。
+func CatalogEntityExists(q Execer, table, goodsID string) (bool, error) {
+	var found int
+	var query string
+	switch table {
+	case "shopee_products":
+		query = `SELECT 1 FROM shopee_products WHERE goods_id=? AND deleted_at IS NULL`
+	case "pdd_products":
+		query = `SELECT 1 FROM pdd_products WHERE goods_id=? AND deleted_at IS NULL`
+	default:
+		return false, fmt.Errorf("不支持的目录表: %s", table)
+	}
+	err := q.QueryRow(query, goodsID).Scan(&found)
+	if errors.Is(err, sql.ErrNoRows) {
+		return false, nil
+	}
+	return err == nil, err
+}
+
+// CatalogSKUExists 按与实际写入一致的商品+规格键判断 SKU 是否已存在。
+func CatalogSKUExists(q Execer, goodsID, specKey string) (bool, error) {
+	var found int
+	err := q.QueryRow(`SELECT 1 FROM shopee_skus WHERE goods_id=? AND spec_key=?`, goodsID, specKey).Scan(&found)
+	if errors.Is(err, sql.ErrNoRows) {
+		return false, nil
+	}
+	return err == nil, err
+}
+
+// CatalogAssociationCurrentPDD 返回当前关联；空字符串表示该蝦皮商品没有 PDD 关联。
+func CatalogAssociationCurrentPDD(q Execer, goodsID string) (string, bool, error) {
+	var current sql.NullString
+	err := q.QueryRow(`SELECT pdd_goods_id FROM shopee_products WHERE goods_id=? AND deleted_at IS NULL`, goodsID).Scan(&current)
+	if errors.Is(err, sql.ErrNoRows) {
+		return "", false, nil
+	}
+	if err != nil {
+		return "", false, err
+	}
+	return current.String, true, nil
+}
+
 // InsertCatalogImportRun 先登记 processing 批次；表的复合主键是并发幂等的最终防线。
 func InsertCatalogImportRun(q Execer, run model.CatalogImportRun) error {
 	_, err := q.Exec(`INSERT INTO catalog_import_runs (
@@ -169,13 +211,13 @@ type CatalogProductOutcome struct {

 // UpsertCatalogShopeeProduct 独立保护主图和店铺字段，绝不覆盖人工 PDD 关联。
 func UpsertCatalogShopeeProduct(q Execer, in CatalogShopeeProductInput) (out CatalogProductOutcome, err error) {
-	var oldObserved, oldShopID, imageURL, shopName, imageSource, imageObserved, shopSource, shopObserved sql.NullString
+	var oldObserved, oldShopID, title, status, mainSKU, imageURL, shopName, imageSource, imageObserved, shopSource, shopObserved sql.NullString
 	var imageManual, shopManual int
 	shopID, err := FindShopIDByName(q, in.ShopName)
 	if err != nil {
 		return out, err
 	}
-	err = q.QueryRow(`SELECT source_observed_at,shop_id,image_url,shopee_shop_name,image_source,image_observed_at,image_is_manual,shop_name_source,shop_name_observed_at,shop_name_is_manual FROM shopee_products WHERE goods_id=?`, in.GoodsID).Scan(&oldObserved, &oldShopID, &imageURL, &shopName, &imageSource, &imageObserved, &imageManual, &shopSource, &shopObserved, &shopManual)
+	err = q.QueryRow(`SELECT source_observed_at,shop_id,title,shopee_status,main_sku_code,image_url,shopee_shop_name,image_source,image_observed_at,image_is_manual,shop_name_source,shop_name_observed_at,shop_name_is_manual FROM shopee_products WHERE goods_id=?`, in.GoodsID).Scan(&oldObserved, &oldShopID, &title, &status, &mainSKU, &imageURL, &shopName, &imageSource, &imageObserved, &imageManual, &shopSource, &shopObserved, &shopManual)
 	if errors.Is(err, sql.ErrNoRows) {
 		_, err = q.Exec(`INSERT INTO shopee_products(goods_id,shop_id,title,shopee_status,main_sku_code,image_url,shopee_shop_name,image_source,image_observed_at,image_is_manual,shop_name_source,shop_name_observed_at,shop_name_is_manual,source,source_observed_at,created_at,updated_at) VALUES(?,NULLIF(?,''),?,NULLIF(?,''),NULLIF(?,''),NULLIF(?,''),NULLIF(?,''),CASE WHEN TRIM(?)='' THEN NULL ELSE ? END,CASE WHEN TRIM(?)='' THEN NULL ELSE ? END,0,CASE WHEN TRIM(?)='' THEN NULL ELSE ? END,CASE WHEN TRIM(?)='' THEN NULL ELSE ? END,0,'api',?,?,?)`, in.GoodsID, shopID, in.Title, in.Status, in.MainSKU, in.ImageURL, in.ShopName, in.ImageURL, in.Source, in.ImageURL, in.ObservedAt, in.ShopName, in.Source, in.ShopName, in.ObservedAt, in.ObservedAt, in.Now, in.Now)
 		out.Created = err == nil
@@ -184,7 +226,10 @@ func UpsertCatalogShopeeProduct(q Execer, in CatalogShopeeProductInput) (out Cat
 	if err != nil {
 		return out, err
 	}
-	baseUpdate := !oldObserved.Valid || oldObserved.String <= in.ObservedAt
+	baseUpdate := in.UpdatePolicy == "overwrite_same_source" && (!oldObserved.Valid || oldObserved.String <= in.ObservedAt)
+	fillTitle := in.UpdatePolicy == "fill_missing" && strings.TrimSpace(in.Title) != "" && strings.TrimSpace(title.String) == ""
+	fillStatus := in.UpdatePolicy == "fill_missing" && strings.TrimSpace(in.Status) != "" && strings.TrimSpace(status.String) == ""
+	fillMainSKU := in.UpdatePolicy == "fill_missing" && strings.TrimSpace(in.MainSKU) != "" && strings.TrimSpace(mainSKU.String) == ""
 	newImage, newShop := imageURL.String, shopName.String
 	imageChanged, shopChanged := false, false
 	apply := func(incoming string, current *string, source, observed sql.NullString, manual int) (changed bool) {
@@ -241,10 +286,10 @@ func UpsertCatalogShopeeProduct(q Execer, in CatalogShopeeProductInput) (out Cat
 		resolvedShopID = shopID
 		shopAssociationChanged = true
 	}
-	if !baseUpdate && !imageChanged && !shopChanged && !shopAssociationChanged {
+	if !baseUpdate && !fillTitle && !fillStatus && !fillMainSKU && !imageChanged && !shopChanged && !shopAssociationChanged {
 		return out, nil
 	}
-	_, err = q.Exec(`UPDATE shopee_products SET title=CASE WHEN ? THEN ? ELSE title END,shopee_status=CASE WHEN ? THEN NULLIF(?,'') ELSE shopee_status END,main_sku_code=CASE WHEN ? THEN NULLIF(?,'') ELSE main_sku_code END,source=CASE WHEN ? THEN 'api' ELSE source END,source_observed_at=CASE WHEN ? THEN ? ELSE source_observed_at END,shop_id=CASE WHEN ? THEN NULLIF(?,'') ELSE shop_id END,image_url=NULLIF(?,''),shopee_shop_name=NULLIF(?,''),image_source=CASE WHEN ? THEN ? ELSE image_source END,image_observed_at=CASE WHEN ? THEN ? ELSE image_observed_at END,shop_name_source=CASE WHEN ? THEN ? ELSE shop_name_source END,shop_name_observed_at=CASE WHEN ? THEN ? ELSE shop_name_observed_at END,updated_at=? WHERE goods_id=?`, baseUpdate, in.Title, baseUpdate, in.Status, baseUpdate, in.MainSKU, baseUpdate, baseUpdate, in.ObservedAt, shopAssociationChanged, resolvedShopID, newImage, newShop, imageChanged, in.Source, imageChanged, in.ObservedAt, shopChanged, in.Source, shopChanged, in.ObservedAt, in.Now, in.GoodsID)
+	_, err = q.Exec(`UPDATE shopee_products SET title=CASE WHEN ? OR ? THEN ? ELSE title END,shopee_status=CASE WHEN ? OR ? THEN NULLIF(?,'') ELSE shopee_status END,main_sku_code=CASE WHEN ? OR ? THEN NULLIF(?,'') ELSE main_sku_code END,source=CASE WHEN ? THEN 'api' ELSE source END,source_observed_at=CASE WHEN ? THEN ? ELSE source_observed_at END,shop_id=CASE WHEN ? THEN NULLIF(?,'') ELSE shop_id END,image_url=NULLIF(?,''),shopee_shop_name=NULLIF(?,''),image_source=CASE WHEN ? THEN ? ELSE image_source END,image_observed_at=CASE WHEN ? THEN ? ELSE image_observed_at END,shop_name_source=CASE WHEN ? THEN ? ELSE shop_name_source END,shop_name_observed_at=CASE WHEN ? THEN ? ELSE shop_name_observed_at END,updated_at=? WHERE goods_id=?`, baseUpdate, fillTitle, in.Title, baseUpdate, fillStatus, in.Status, baseUpdate, fillMainSKU, in.MainSKU, baseUpdate, baseUpdate, in.ObservedAt, shopAssociationChanged, resolvedShopID, newImage, newShop, imageChanged, in.Source, imageChanged, in.ObservedAt, shopChanged, in.Source, shopChanged, in.ObservedAt, in.Now, in.GoodsID)
 	out.Updated = err == nil
 	return out, err
 }
@@ -351,8 +396,7 @@ func UpsertCatalogShopeeSKU(q Execer, in CatalogShopeeSKUInput) (CatalogSKUOutco
 		sources, times := catalogFieldProvenance(current)
 		changed := external != current.ShopeeSKUID.String
 		canFill := func(field, current, incoming string) bool {
-			return sources[field] != "manual" && strings.TrimSpace(incoming) != "" &&
-				(current == "" || (sources[field] == "syb" && in.Source != "syb"))
+			return sources[field] != "manual" && strings.TrimSpace(incoming) != "" && current == ""
 		}
 		if canFill("color", color, in.Color) {
 			color = in.Color
@@ -444,7 +488,7 @@ func catalogFieldProvenance(row *catalogSKUStored) (map[string]string, map[strin
 }

 // UpsertCatalogPddProduct 写入结构化 PDD 数据转换后的规范 JSON；空规格不清除既有采集结果。
-func UpsertCatalogPddProduct(q Execer, goodsID, url, title, shopName, skusJSON, observedAt, now string) (created, updated bool, err error) {
+func UpsertCatalogPddProduct(q Execer, goodsID, url, title, shopName, skusJSON, observedAt, now, updatePolicy string) (created, updated bool, err error) {
 	var oldObserved, deletedAt sql.NullString
 	err = q.QueryRow(`SELECT source_observed_at,deleted_at FROM pdd_products WHERE goods_id=?`, goodsID).Scan(&oldObserved, &deletedAt)
 	status := "pending"
@@ -460,13 +504,17 @@ func UpsertCatalogPddProduct(q Execer, goodsID, url, title, shopName, skusJSON,
 	if err != nil {
 		return false, false, err
 	}
-	if oldObserved.Valid && oldObserved.String > observedAt {
+	if updatePolicy == "insert_only" || (oldObserved.Valid && oldObserved.String > observedAt && updatePolicy != "fill_missing") {
 		return false, false, nil
 	}
-	if skusJSON == "" {
-		_, err = q.Exec(`UPDATE pdd_products SET url=?,title=CASE WHEN TRIM(?)='' THEN title ELSE ? END,
-			shop_name=CASE WHEN TRIM(?)='' THEN shop_name ELSE ? END,deleted_at=NULL,source='api',source_observed_at=?,updated_at=? WHERE goods_id=?`,
-			url, title, title, shopName, shopName, observedAt, now, goodsID)
+	if skusJSON == "" || updatePolicy == "fill_missing" {
+		if updatePolicy == "fill_missing" {
+			_, err = q.Exec(`UPDATE pdd_products SET url=CASE WHEN TRIM(COALESCE(url,''))='' THEN ? ELSE url END,title=CASE WHEN TRIM(COALESCE(title,''))='' THEN NULLIF(?,'') ELSE title END,shop_name=CASE WHEN TRIM(COALESCE(shop_name,''))='' THEN NULLIF(?,'') ELSE shop_name END,updated_at=? WHERE goods_id=?`, url, title, shopName, now, goodsID)
+		} else {
+			_, err = q.Exec(`UPDATE pdd_products SET url=?,title=CASE WHEN TRIM(?)='' THEN title ELSE ? END,
+				shop_name=CASE WHEN TRIM(?)='' THEN shop_name ELSE ? END,deleted_at=NULL,source='api',source_observed_at=?,updated_at=? WHERE goods_id=?`,
+				url, title, title, shopName, shopName, observedAt, now, goodsID)
+		}
 	} else {
 		_, err = q.Exec(`UPDATE pdd_products SET url=?,title=NULLIF(?,''),shop_name=NULLIF(?,''),skus_json=?,
 			collect_status='collected',collect_msg=NULL,artifact_ref=NULL,collected_at=?,deleted_at=NULL,
@@ -73,6 +73,8 @@ type CatalogBatchRequest struct {
 	ShopeeSKUs     []CatalogShopeeSKU     `json:"shopee_skus"`
 	PddProducts    []CatalogPddProduct    `json:"pdd_products"`
 	Associations   []CatalogAssociation   `json:"associations"`
+	// DryRun 只执行数据库预检；不得登记导入批次或写入任何业务数据。
+	DryRun         bool                   `json:"dry_run"`
 }
 type CatalogCounts struct {
 	ShopeeCreated                 int `json:"shopee_created"`
@@ -99,6 +101,14 @@ type CatalogBatchResponse struct {
 	Replayed   bool          `json:"replayed"`
 	Counts     CatalogCounts `json:"counts"`
 	FinishedAt string        `json:"finished_at"`
+	Conflicts  []CatalogAssociationConflict `json:"conflicts,omitempty"`
+}
+
+// CatalogAssociationConflict 是预检发现的既有人工关联冲突，不包含请求体或凭据。
+type CatalogAssociationConflict struct {
+	ShopeeGoodsID string `json:"shopee_goods_id"`
+	ExistingPddID string `json:"existing_pdd_goods_id"`
+	IncomingPddID string `json:"incoming_pdd_goods_id"`
 }

 type CatalogError struct {
@@ -177,10 +187,52 @@ func ValidateCatalogBatch(req CatalogBatchRequest) error {
 	return nil
 }

+// PreviewCatalogBatch 按当前数据库做只读预检。它不登记 catalog_import_runs，
+// 也不写业务表；正式执行仍会重新检查，避免预检后人工修改造成覆盖。
+func PreviewCatalogBatch(db *sql.DB, req CatalogBatchRequest) (CatalogBatchResponse, error) {
+	if err := ValidateCatalogBatch(req); err != nil {
+		return CatalogBatchResponse{}, err
+	}
+	tx, err := db.BeginTx(context.Background(), &sql.TxOptions{ReadOnly: true, Isolation: sql.LevelReadCommitted})
+	if err != nil {
+		return CatalogBatchResponse{}, err
+	}
+	defer tx.Rollback()
+	counts := CatalogCounts{}
+	for _, p := range req.ShopeeProducts {
+		exists, err := repository.CatalogEntityExists(tx, "shopee_products", p.GoodsID)
+		if err != nil { return CatalogBatchResponse{}, err }
+		if exists { counts.ShopeeUpdated++ } else { counts.ShopeeCreated++ }
+	}
+	for _, p := range req.PddProducts {
+		exists, err := repository.CatalogEntityExists(tx, "pdd_products", p.GoodsID)
+		if err != nil { return CatalogBatchResponse{}, err }
+		if exists { counts.PddUpdated++ } else { counts.PddCreated++ }
+	}
+	for _, s := range req.ShopeeSKUs {
+		key, _ := spec.SpecKey(s.SpecRaw)
+		exists, err := repository.CatalogSKUExists(tx, s.GoodsID, key)
+		if err != nil { return CatalogBatchResponse{}, err }
+		if exists { counts.SKUSkipped++ } else { counts.SKUCreated++ }
+	}
+	conflicts := make([]CatalogAssociationConflict, 0)
+	for _, a := range req.Associations {
+		current, exists, err := repository.CatalogAssociationCurrentPDD(tx, a.ShopeeGoodsID)
+		if err != nil { return CatalogBatchResponse{}, err }
+		if !exists || current == "" { counts.AssociationCreated++; continue }
+		if current == a.PddGoodsID { counts.AssociationUnchanged++; continue }
+		conflicts = append(conflicts, CatalogAssociationConflict{ShopeeGoodsID: a.ShopeeGoodsID, ExistingPddID: current, IncomingPddID: a.PddGoodsID})
+	}
+	return CatalogBatchResponse{BatchID: req.BatchID, Status: "previewed", Counts: counts, FinishedAt: model.NowISO(), Conflicts: conflicts}, nil
+}
+
 func ImportCatalogBatch(db *sql.DB, source string, req CatalogBatchRequest, raw []byte) (CatalogBatchResponse, error) {
 	if err := ValidateCatalogBatch(req); err != nil {
 		return CatalogBatchResponse{}, err
 	}
+	if req.DryRun {
+		return PreviewCatalogBatch(db, req)
+	}
 	observed, _ := time.Parse(time.RFC3339Nano, req.ObservedAt)
 	req.ObservedAt = observed.UTC().Format(model.TimeLayout)
 	policy, _ := normalizeCatalogUpdatePolicy(req.UpdatePolicy)
@@ -239,7 +291,7 @@ func ImportCatalogBatch(db *sql.DB, source string, req CatalogBatchRequest, raw
 			b, _ := json.Marshal(map[string]any{"schema_version": 1, "goods_id": p.GoodsID, "title": p.Title, "shop_name": p.ShopName, "dimensions": p.Dimensions, "skus": p.SKUs})
 			skus = string(b)
 		}
-		c, u, e := repository.UpsertCatalogPddProduct(tx, p.GoodsID, p.URL, p.Title, p.ShopName, skus, req.ObservedAt, now)
+		c, u, e := repository.UpsertCatalogPddProduct(tx, p.GoodsID, p.URL, p.Title, p.ShopName, skus, req.ObservedAt, now, policy)
 		if e != nil {
 			return fail(e)
 		}
@@ -356,3 +356,44 @@ func TestValidateCatalogBatch_拒绝危险图片URL(t *testing.T) {
 		t.Fatalf("合法图片被拒绝：%v", err)
 	}
 }
+
+func TestPreviewCatalogBatch_只读且报告既有关联冲突(t *testing.T) {
+	db := newCatalogTestDB(t)
+	base := validCatalogBatch()
+	raw, _ := json.Marshal(base)
+	if _, err := ImportCatalogBatch(db, "script-a", base, raw); err != nil {
+		t.Fatal(err)
+	}
+	preview := validCatalogBatch()
+	preview.BatchID = "preview-only"
+	preview.DryRun = true
+	preview.Associations[0].PddGoodsID = "P-2"
+	preview.PddProducts[0].GoodsID = "P-2"
+	preview.PddProducts[0].URL = "https://mobile.yangkeduo.com/goods.html?goods_id=P-2"
+	got, err := ImportCatalogBatch(db, "script-a", preview, []byte("unused"))
+	if err != nil || got.Status != "previewed" || len(got.Conflicts) != 1 {
+		t.Fatalf("预检结果错误：%+v %v", got, err)
+	}
+	var runs, pdd int
+	_ = db.QueryRow(`SELECT COUNT(*) FROM catalog_import_runs WHERE batch_id='preview-only'`).Scan(&runs)
+	_ = db.QueryRow(`SELECT COUNT(*) FROM pdd_products WHERE goods_id='P-2'`).Scan(&pdd)
+	if runs != 0 || pdd != 0 {
+		t.Fatalf("预检写入了数据：runs=%d pdd=%d", runs, pdd)
+	}
+}
+
+func TestImportCatalogBatch_FillMissing不覆盖已有PDD和商品字段(t *testing.T) {
+	db := newCatalogTestDB(t)
+	base := validCatalogBatch()
+	raw, _ := json.Marshal(base)
+	if _, err := ImportCatalogBatch(db, "script-a", base, raw); err != nil { t.Fatal(err) }
+	_, _ = db.Exec(`UPDATE shopee_products SET title='人工标题',main_sku_code='人工货号' WHERE goods_id='S-1'`)
+	_, _ = db.Exec(`UPDATE pdd_products SET url='https://manual.example/P-1',title='已采集标题',shop_name='已采集店铺',collect_status='collected' WHERE goods_id='P-1'`)
+	fill := validCatalogBatch(); fill.BatchID="no-overwrite"; fill.ShopeeProducts[0].Title="来源标题"; fill.ShopeeProducts[0].MainSKUCode="来源货号"; fill.PddProducts[0].URL="https://mobile.yangkeduo.com/goods.html?goods_id=P-1"; fill.PddProducts[0].Title="来源PDD标题"; fill.PddProducts[0].ShopName="来源PDD店铺"
+	raw, _ = json.Marshal(fill)
+	if _, err := ImportCatalogBatch(db, "thirdparty-csv", fill, raw); err != nil { t.Fatal(err) }
+	var title, mainSKU, url, pddTitle, shop, status string
+	_ = db.QueryRow(`SELECT title,main_sku_code FROM shopee_products WHERE goods_id='S-1'`).Scan(&title,&mainSKU)
+	_ = db.QueryRow(`SELECT url,title,shop_name,collect_status FROM pdd_products WHERE goods_id='P-1'`).Scan(&url,&pddTitle,&shop,&status)
+	if title!="人工标题" || mainSKU!="人工货号" || url!="https://manual.example/P-1" || pddTitle!="已采集标题" || shop!="已采集店铺" || status!="collected" { t.Fatalf("fill_missing 覆盖了已有字段：%q/%q/%q/%q/%q/%q",title,mainSKU,url,pddTitle,shop,status) }
+}