feat: 规范 CSV 预检与安全入库 (#281)

2026-08-20 14:41:01 +08:00
parent b8537fa776
commit fe64d3b3c7
5 changed files with 260 additions and 14 deletions
@@ -73,6 +73,8 @@ type CatalogBatchRequest struct {
 	ShopeeSKUs     []CatalogShopeeSKU     `json:"shopee_skus"`
 	PddProducts    []CatalogPddProduct    `json:"pdd_products"`
 	Associations   []CatalogAssociation   `json:"associations"`
+	// DryRun 只执行数据库预检；不得登记导入批次或写入任何业务数据。
+	DryRun         bool                   `json:"dry_run"`
 }
 type CatalogCounts struct {
 	ShopeeCreated                 int `json:"shopee_created"`
@@ -99,6 +101,14 @@ type CatalogBatchResponse struct {
 	Replayed   bool          `json:"replayed"`
 	Counts     CatalogCounts `json:"counts"`
 	FinishedAt string        `json:"finished_at"`
+	Conflicts  []CatalogAssociationConflict `json:"conflicts,omitempty"`
+}
+
+// CatalogAssociationConflict 是预检发现的既有人工关联冲突，不包含请求体或凭据。
+type CatalogAssociationConflict struct {
+	ShopeeGoodsID string `json:"shopee_goods_id"`
+	ExistingPddID string `json:"existing_pdd_goods_id"`
+	IncomingPddID string `json:"incoming_pdd_goods_id"`
 }

 type CatalogError struct {
@@ -177,10 +187,52 @@ func ValidateCatalogBatch(req CatalogBatchRequest) error {
 	return nil
 }

+// PreviewCatalogBatch 按当前数据库做只读预检。它不登记 catalog_import_runs，
+// 也不写业务表；正式执行仍会重新检查，避免预检后人工修改造成覆盖。
+func PreviewCatalogBatch(db *sql.DB, req CatalogBatchRequest) (CatalogBatchResponse, error) {
+	if err := ValidateCatalogBatch(req); err != nil {
+		return CatalogBatchResponse{}, err
+	}
+	tx, err := db.BeginTx(context.Background(), &sql.TxOptions{ReadOnly: true, Isolation: sql.LevelReadCommitted})
+	if err != nil {
+		return CatalogBatchResponse{}, err
+	}
+	defer tx.Rollback()
+	counts := CatalogCounts{}
+	for _, p := range req.ShopeeProducts {
+		exists, err := repository.CatalogEntityExists(tx, "shopee_products", p.GoodsID)
+		if err != nil { return CatalogBatchResponse{}, err }
+		if exists { counts.ShopeeUpdated++ } else { counts.ShopeeCreated++ }
+	}
+	for _, p := range req.PddProducts {
+		exists, err := repository.CatalogEntityExists(tx, "pdd_products", p.GoodsID)
+		if err != nil { return CatalogBatchResponse{}, err }
+		if exists { counts.PddUpdated++ } else { counts.PddCreated++ }
+	}
+	for _, s := range req.ShopeeSKUs {
+		key, _ := spec.SpecKey(s.SpecRaw)
+		exists, err := repository.CatalogSKUExists(tx, s.GoodsID, key)
+		if err != nil { return CatalogBatchResponse{}, err }
+		if exists { counts.SKUSkipped++ } else { counts.SKUCreated++ }
+	}
+	conflicts := make([]CatalogAssociationConflict, 0)
+	for _, a := range req.Associations {
+		current, exists, err := repository.CatalogAssociationCurrentPDD(tx, a.ShopeeGoodsID)
+		if err != nil { return CatalogBatchResponse{}, err }
+		if !exists || current == "" { counts.AssociationCreated++; continue }
+		if current == a.PddGoodsID { counts.AssociationUnchanged++; continue }
+		conflicts = append(conflicts, CatalogAssociationConflict{ShopeeGoodsID: a.ShopeeGoodsID, ExistingPddID: current, IncomingPddID: a.PddGoodsID})
+	}
+	return CatalogBatchResponse{BatchID: req.BatchID, Status: "previewed", Counts: counts, FinishedAt: model.NowISO(), Conflicts: conflicts}, nil
+}
+
 func ImportCatalogBatch(db *sql.DB, source string, req CatalogBatchRequest, raw []byte) (CatalogBatchResponse, error) {
 	if err := ValidateCatalogBatch(req); err != nil {
 		return CatalogBatchResponse{}, err
 	}
+	if req.DryRun {
+		return PreviewCatalogBatch(db, req)
+	}
 	observed, _ := time.Parse(time.RFC3339Nano, req.ObservedAt)
 	req.ObservedAt = observed.UTC().Format(model.TimeLayout)
 	policy, _ := normalizeCatalogUpdatePolicy(req.UpdatePolicy)
@@ -239,7 +291,7 @@ func ImportCatalogBatch(db *sql.DB, source string, req CatalogBatchRequest, raw
 			b, _ := json.Marshal(map[string]any{"schema_version": 1, "goods_id": p.GoodsID, "title": p.Title, "shop_name": p.ShopName, "dimensions": p.Dimensions, "skus": p.SKUs})
 			skus = string(b)
 		}
-		c, u, e := repository.UpsertCatalogPddProduct(tx, p.GoodsID, p.URL, p.Title, p.ShopName, skus, req.ObservedAt, now)
+		c, u, e := repository.UpsertCatalogPddProduct(tx, p.GoodsID, p.URL, p.Title, p.ShopName, skus, req.ObservedAt, now, policy)
 		if e != nil {
 			return fail(e)
 		}
@@ -356,3 +356,44 @@ func TestValidateCatalogBatch_拒绝危险图片URL(t *testing.T) {
 		t.Fatalf("合法图片被拒绝：%v", err)
 	}
 }
+
+func TestPreviewCatalogBatch_只读且报告既有关联冲突(t *testing.T) {
+	db := newCatalogTestDB(t)
+	base := validCatalogBatch()
+	raw, _ := json.Marshal(base)
+	if _, err := ImportCatalogBatch(db, "script-a", base, raw); err != nil {
+		t.Fatal(err)
+	}
+	preview := validCatalogBatch()
+	preview.BatchID = "preview-only"
+	preview.DryRun = true
+	preview.Associations[0].PddGoodsID = "P-2"
+	preview.PddProducts[0].GoodsID = "P-2"
+	preview.PddProducts[0].URL = "https://mobile.yangkeduo.com/goods.html?goods_id=P-2"
+	got, err := ImportCatalogBatch(db, "script-a", preview, []byte("unused"))
+	if err != nil || got.Status != "previewed" || len(got.Conflicts) != 1 {
+		t.Fatalf("预检结果错误：%+v %v", got, err)
+	}
+	var runs, pdd int
+	_ = db.QueryRow(`SELECT COUNT(*) FROM catalog_import_runs WHERE batch_id='preview-only'`).Scan(&runs)
+	_ = db.QueryRow(`SELECT COUNT(*) FROM pdd_products WHERE goods_id='P-2'`).Scan(&pdd)
+	if runs != 0 || pdd != 0 {
+		t.Fatalf("预检写入了数据：runs=%d pdd=%d", runs, pdd)
+	}
+}
+
+func TestImportCatalogBatch_FillMissing不覆盖已有PDD和商品字段(t *testing.T) {
+	db := newCatalogTestDB(t)
+	base := validCatalogBatch()
+	raw, _ := json.Marshal(base)
+	if _, err := ImportCatalogBatch(db, "script-a", base, raw); err != nil { t.Fatal(err) }
+	_, _ = db.Exec(`UPDATE shopee_products SET title='人工标题',main_sku_code='人工货号' WHERE goods_id='S-1'`)
+	_, _ = db.Exec(`UPDATE pdd_products SET url='https://manual.example/P-1',title='已采集标题',shop_name='已采集店铺',collect_status='collected' WHERE goods_id='P-1'`)
+	fill := validCatalogBatch(); fill.BatchID="no-overwrite"; fill.ShopeeProducts[0].Title="来源标题"; fill.ShopeeProducts[0].MainSKUCode="来源货号"; fill.PddProducts[0].URL="https://mobile.yangkeduo.com/goods.html?goods_id=P-1"; fill.PddProducts[0].Title="来源PDD标题"; fill.PddProducts[0].ShopName="来源PDD店铺"
+	raw, _ = json.Marshal(fill)
+	if _, err := ImportCatalogBatch(db, "thirdparty-csv", fill, raw); err != nil { t.Fatal(err) }
+	var title, mainSKU, url, pddTitle, shop, status string
+	_ = db.QueryRow(`SELECT title,main_sku_code FROM shopee_products WHERE goods_id='S-1'`).Scan(&title,&mainSKU)
+	_ = db.QueryRow(`SELECT url,title,shop_name,collect_status FROM pdd_products WHERE goods_id='P-1'`).Scan(&url,&pddTitle,&shop,&status)
+	if title!="人工标题" || mainSKU!="人工货号" || url!="https://manual.example/P-1" || pddTitle!="已采集标题" || shop!="已采集店铺" || status!="collected" { t.Fatalf("fill_missing 覆盖了已有字段：%q/%q/%q/%q/%q/%q",title,mainSKU,url,pddTitle,shop,status) }
+}