Files
cmautobuy/admin/service/catalog_import_test.go
T

400 lines
18 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package service
import (
"database/sql"
"encoding/json"
"errors"
"testing"
_ "modernc.org/sqlite"
"cmautobuy/admin/repository"
)
func newCatalogTestDB(t *testing.T) *sql.DB {
t.Helper()
db, err := sql.Open("sqlite", ":memory:")
if err != nil {
t.Fatal(err)
}
db.SetMaxOpenConns(1)
statements := []string{
`CREATE TABLE shops(shop_id TEXT PRIMARY KEY,display_name TEXT,normalized_name TEXT,enabled INTEGER,created_by_user_id TEXT,created_at TEXT,updated_at TEXT)`,
`CREATE TABLE shop_channel_aliases(alias_id TEXT PRIMARY KEY,shop_id TEXT,channel TEXT,alias_name TEXT,normalized_alias TEXT,enabled INTEGER,created_at TEXT,updated_at TEXT)`,
`CREATE TABLE shopee_products(goods_id TEXT PRIMARY KEY,shop_id TEXT,title TEXT NOT NULL,shopee_status TEXT,main_sku_code TEXT,image_url TEXT,shopee_shop_name TEXT,image_source TEXT,image_observed_at TEXT,image_is_manual INTEGER DEFAULT 0,shop_name_source TEXT,shop_name_observed_at TEXT,shop_name_is_manual INTEGER DEFAULT 0,source TEXT,source_observed_at TEXT,pdd_goods_url TEXT,pdd_goods_id TEXT,deleted_at TEXT,deleted_by_user_id TEXT,created_at TEXT,updated_at TEXT)`,
`CREATE TABLE shopee_skus(sku_id TEXT PRIMARY KEY,shopee_sku_id TEXT UNIQUE,goods_id TEXT NOT NULL,spec_raw TEXT,spec_key TEXT,color TEXT,size TEXT,advice TEXT,parse_ok INTEGER,sku_code TEXT,is_manual INTEGER,source TEXT,field_sources TEXT,field_observed_at TEXT,source_observed_at TEXT,created_at TEXT,updated_at TEXT,UNIQUE(goods_id,spec_key))`,
`CREATE TABLE pdd_products(id INTEGER PRIMARY KEY AUTOINCREMENT,goods_id TEXT UNIQUE,url TEXT,title TEXT,shop_name TEXT,skus_json TEXT,collect_status TEXT,collect_msg TEXT,artifact_ref TEXT,collected_at TEXT,deleted_at TEXT,source TEXT,source_observed_at TEXT,created_at TEXT,updated_at TEXT)`,
`CREATE TABLE catalog_import_runs(source TEXT,batch_id TEXT,request_hash TEXT,status TEXT,request_count INTEGER,conflict_count INTEGER,observed_at TEXT,update_policy TEXT DEFAULT 'fill_missing',last_request_at TEXT,last_conflict_at TEXT,shopee_created INTEGER DEFAULT 0,shopee_updated INTEGER DEFAULT 0,shopee_fields_filled INTEGER DEFAULT 0,shopee_fields_same_source_updated INTEGER DEFAULT 0,shopee_fields_manual_skipped INTEGER DEFAULT 0,shopee_fields_stale_skipped INTEGER DEFAULT 0,sku_created INTEGER DEFAULT 0,sku_updated INTEGER DEFAULT 0,sku_filled INTEGER DEFAULT 0,sku_same_source_updated INTEGER DEFAULT 0,sku_skipped INTEGER DEFAULT 0,sku_manual_skipped INTEGER DEFAULT 0,sku_stale_skipped INTEGER DEFAULT 0,pdd_created INTEGER DEFAULT 0,pdd_updated INTEGER DEFAULT 0,association_created INTEGER DEFAULT 0,association_unchanged INTEGER DEFAULT 0,failure_count INTEGER DEFAULT 0,error_summary TEXT,response_body TEXT,created_at TEXT,finished_at TEXT,PRIMARY KEY(source,batch_id))`,
}
for _, statement := range statements {
if _, err := db.Exec(statement); err != nil {
t.Fatal(err)
}
}
t.Cleanup(func() { db.Close() })
return db
}
func validCatalogBatch() CatalogBatchRequest {
price := int64(1299)
return CatalogBatchRequest{SchemaVersion: 1, BatchID: "batch-001", ObservedAt: "2026-08-11T08:00:00+08:00",
ShopeeProducts: []CatalogShopeeProduct{{GoodsID: "S-1", Title: "蝦皮上衣"}},
ShopeeSKUs: []CatalogShopeeSKU{{SKUID: "SKU-1", GoodsID: "S-1", SpecRaw: "黑色,M", Color: "黑色", Size: "M", ParseOK: true}},
PddProducts: []CatalogPddProduct{{GoodsID: "P-1", URL: "https://mobile.yangkeduo.com/goods.html?goods_id=P-1", Title: "PDD上衣", Dimensions: []CatalogPddDimension{{Key: "color", Name: "颜色"}}, SKUs: []CatalogPddSKU{{Options: map[string]string{"color": "黑色"}, PriceCent: &price, Available: true}}}},
Associations: []CatalogAssociation{{ShopeeGoodsID: "S-1", PddGoodsID: "P-1"}}}
}
func TestImportCatalogBatch_原子写入重放与冲突(t *testing.T) {
db := newCatalogTestDB(t)
req := validCatalogBatch()
raw, _ := json.Marshal(req)
got, err := ImportCatalogBatch(db, "script-a", req, raw)
if err != nil {
t.Fatal(err)
}
if got.Counts.ShopeeCreated != 1 || got.Counts.SKUCreated != 1 || got.Counts.PddCreated != 1 || got.Counts.AssociationCreated != 1 {
t.Fatalf("统计不正确:%+v", got)
}
replay, err := ImportCatalogBatch(db, "script-a", req, raw)
if err != nil || !replay.Replayed {
t.Fatalf("相同批次应重放:resp=%+v err=%v", replay, err)
}
changed := append([]byte{}, raw...)
changed = append(changed, ' ')
_, err = ImportCatalogBatch(db, "script-a", req, changed)
var catalogErr *CatalogError
if !errors.As(err, &catalogErr) || catalogErr.Code != "IDEMPOTENCY_CONFLICT" {
t.Fatalf("不同内容应冲突:%v", err)
}
run, err := repository.GetCatalogImportRun(db, "script-a", req.BatchID)
if err != nil || run.RequestCount != 3 || run.ConflictCount != 1 {
t.Fatalf("批次计数不正确:%+v err=%v", run, err)
}
var pddID string
if err := db.QueryRow(`SELECT pdd_goods_id FROM shopee_products WHERE goods_id='S-1'`).Scan(&pddID); err != nil || pddID != "P-1" {
t.Fatalf("关联未写入:%q %v", pddID, err)
}
}
func TestImportCatalogBatch_关联冲突整批回滚且同请求重放错误(t *testing.T) {
db := newCatalogTestDB(t)
now := "2026-08-11T00:00:00.000000000Z"
_, _ = db.Exec(`INSERT INTO shopee_products(goods_id,title,source,pdd_goods_id,created_at,updated_at) VALUES('S-1','旧标题','report','P-OLD',?,?)`, now, now)
_, _ = db.Exec(`INSERT INTO pdd_products(goods_id,url,collect_status,created_at,updated_at) VALUES('P-OLD','old','pending',?,?)`, now, now)
req := validCatalogBatch()
raw, _ := json.Marshal(req)
_, err := ImportCatalogBatch(db, "script-a", req, raw)
var first *CatalogError
if !errors.As(err, &first) || first.Code != "ASSOCIATION_CONFLICT" {
t.Fatalf("应返回关联冲突:%v", err)
}
var title string
_ = db.QueryRow(`SELECT title FROM shopee_products WHERE goods_id='S-1'`).Scan(&title)
if title != "旧标题" {
t.Fatalf("失败批次必须整体回滚,title=%q", title)
}
_, err = ImportCatalogBatch(db, "script-a", req, raw)
var replay *CatalogError
if !errors.As(err, &replay) || replay.Code != first.Code || replay.Status != first.Status {
t.Fatalf("失败重放应保持原错误:%+v", err)
}
}
func TestValidateCatalogBatch_拒绝旧版超限和负金额(t *testing.T) {
req := validCatalogBatch()
req.SchemaVersion = 2
if err := ValidateCatalogBatch(req); err == nil {
t.Fatal("应拒绝未知 schema")
}
req = validCatalogBatch()
negative := int64(-1)
req.PddProducts[0].SKUs[0].PriceCent = &negative
if err := ValidateCatalogBatch(req); err == nil {
t.Fatal("应拒绝负金额")
}
req = validCatalogBatch()
req.UpdatePolicy = "force_overwrite"
if err := ValidateCatalogBatch(req); err == nil {
t.Fatal("应拒绝强制覆盖策略")
}
}
func TestImportCatalogBatch_无真实SKU并在后续补回同一记录(t *testing.T) {
db := newCatalogTestDB(t)
req := validCatalogBatch()
req.BatchID = "missing-id"
req.ShopeeSKUs[0].SKUID = ""
raw, _ := json.Marshal(req)
if got, err := ImportCatalogBatch(db, "script-a", req, raw); err != nil || got.Counts.SKUCreated != 1 {
t.Fatalf("首次导入失败:%+v %v", got, err)
}
var recordID string
if err := db.QueryRow(`SELECT sku_id FROM shopee_skus WHERE goods_id='S-1'`).Scan(&recordID); err != nil || recordID == "" {
t.Fatalf("内部记录缺失:%q %v", recordID, err)
}
req.BatchID = "fill-real-id"
req.ShopeeSKUs[0].SKUID = "REAL-1"
raw, _ = json.Marshal(req)
got, err := ImportCatalogBatch(db, "script-a", req, raw)
if err != nil || got.Counts.SKUFilled != 1 {
t.Fatalf("补真实 ID 失败:%+v %v", got, err)
}
var after, external string
_ = db.QueryRow(`SELECT sku_id,shopee_sku_id FROM shopee_skus WHERE goods_id='S-1'`).Scan(&after, &external)
if after != recordID || external != "REAL-1" {
t.Fatalf("不应新建记录:before=%q after=%q external=%q", recordID, after, external)
}
}
func TestImportCatalogBatch_三种策略和保护边界(t *testing.T) {
db := newCatalogTestDB(t)
req := validCatalogBatch()
req.ShopeeSKUs[0].Color = ""
raw, _ := json.Marshal(req)
if _, err := ImportCatalogBatch(db, "script-a", req, raw); err != nil {
t.Fatal(err)
}
fill := validCatalogBatch()
fill.BatchID = "fill"
fill.ShopeeSKUs[0].Color = "黑色"
fill.ShopeeSKUs[0].Size = "L"
raw, _ = json.Marshal(fill)
got, err := ImportCatalogBatch(db, "script-a", fill, raw)
if err != nil || got.Counts.SKUFilled != 1 {
t.Fatalf("补空失败:%+v %v", got, err)
}
var color, size string
_ = db.QueryRow(`SELECT color,size FROM shopee_skus`).Scan(&color, &size)
if color != "黑色" || size != "M" {
t.Fatalf("fill_missing 覆盖了非空字段:%q/%q", color, size)
}
insertOnly := validCatalogBatch()
insertOnly.BatchID = "insert"
insertOnly.UpdatePolicy = "insert_only"
insertOnly.ShopeeSKUs[0].Color = "白色"
raw, _ = json.Marshal(insertOnly)
got, err = ImportCatalogBatch(db, "script-a", insertOnly, raw)
if err != nil || got.Counts.SKUSkipped != 1 {
t.Fatalf("insert_only 失败:%+v %v", got, err)
}
overwrite := validCatalogBatch()
overwrite.BatchID = "overwrite"
overwrite.UpdatePolicy = "overwrite_same_source"
overwrite.ObservedAt = "2026-08-12T08:00:00+08:00"
overwrite.ShopeeSKUs[0].Color = "白色"
raw, _ = json.Marshal(overwrite)
got, err = ImportCatalogBatch(db, "script-a", overwrite, raw)
if err != nil || got.Counts.SKUSameSourceUpdated != 1 {
t.Fatalf("同来源覆盖失败:%+v %v", got, err)
}
_ = db.QueryRow(`SELECT color FROM shopee_skus`).Scan(&color)
if color != "白色" {
t.Fatalf("color=%q", color)
}
stale := validCatalogBatch()
stale.BatchID = "stale"
stale.UpdatePolicy = "overwrite_same_source"
stale.ObservedAt = "2026-08-11T12:00:00+08:00"
stale.ShopeeSKUs[0].Color = "旧红色"
raw, _ = json.Marshal(stale)
got, err = ImportCatalogBatch(db, "script-a", stale, raw)
if err != nil || got.Counts.SKUStaleSkipped != 1 {
t.Fatalf("旧观测保护失败:%+v %v", got, err)
}
cross := validCatalogBatch()
cross.BatchID = "cross-source"
cross.ObservedAt = "2026-08-13T08:00:00+08:00"
cross.ShopeeSKUs[0].Advice = "来源B建议"
raw, _ = json.Marshal(cross)
if _, err := ImportCatalogBatch(db, "script-b", cross, raw); err != nil {
t.Fatal(err)
}
protect := validCatalogBatch()
protect.BatchID = "protect-cross"
protect.UpdatePolicy = "overwrite_same_source"
protect.ObservedAt = "2026-08-14T08:00:00+08:00"
protect.ShopeeSKUs[0].Advice = "来源A不得覆盖"
raw, _ = json.Marshal(protect)
if _, err := ImportCatalogBatch(db, "script-a", protect, raw); err != nil {
t.Fatal(err)
}
var advice string
_ = db.QueryRow(`SELECT advice FROM shopee_skus`).Scan(&advice)
if advice != "来源B建议" {
t.Fatalf("跨来源字段被覆盖:%q", advice)
}
_, _ = db.Exec(`UPDATE shopee_skus SET is_manual=1`)
manual := validCatalogBatch()
manual.BatchID = "manual"
manual.ObservedAt = "2026-08-13T08:00:00+08:00"
manual.ShopeeSKUs[0].Color = "红色"
raw, _ = json.Marshal(manual)
got, err = ImportCatalogBatch(db, "script-a", manual, raw)
if err != nil || got.Counts.SKUManualSkipped != 1 {
t.Fatalf("人工保护失败:%+v %v", got, err)
}
}
func TestImportCatalogBatch_同规格不同真实SKU冲突并回滚(t *testing.T) {
db := newCatalogTestDB(t)
req := validCatalogBatch()
raw, _ := json.Marshal(req)
if _, err := ImportCatalogBatch(db, "script-a", req, raw); err != nil {
t.Fatal(err)
}
req.BatchID = "conflict"
req.ShopeeSKUs[0].SKUID = "SKU-2"
req.ShopeeProducts[0].Title = "不应提交"
raw, _ = json.Marshal(req)
_, err := ImportCatalogBatch(db, "script-a", req, raw)
var catalogErr *CatalogError
if !errors.As(err, &catalogErr) || catalogErr.Status != 409 {
t.Fatalf("应返回 409:%v", err)
}
var title string
_ = db.QueryRow(`SELECT title FROM shopee_products WHERE goods_id='S-1'`).Scan(&title)
if title != "蝦皮上衣" {
t.Fatalf("冲突批次未整体回滚:%q", title)
}
}
func TestImportCatalogBatch_蝦皮图片店铺分级更新和人工保护(t *testing.T) {
db := newCatalogTestDB(t)
base := validCatalogBatch()
raw, _ := json.Marshal(base)
if _, err := ImportCatalogBatch(db, "script-a", base, raw); err != nil {
t.Fatal(err)
}
fill := validCatalogBatch()
fill.BatchID = "product-fill"
fill.ShopeeProducts[0].ImageURL = "https://img.example.com/a.jpg"
fill.ShopeeProducts[0].ShopName = "蝦皮店铺A"
raw, _ = json.Marshal(fill)
got, err := ImportCatalogBatch(db, "script-a", fill, raw)
if err != nil || got.Counts.ShopeeFieldsFilled != 2 {
t.Fatalf("补空失败:%+v %v", got, err)
}
keep := validCatalogBatch()
keep.BatchID = "product-keep"
keep.ShopeeProducts[0].ImageURL = "https://img.example.com/b.jpg"
keep.ShopeeProducts[0].ShopName = "店铺B"
raw, _ = json.Marshal(keep)
got, err = ImportCatalogBatch(db, "script-b", keep, raw)
if err != nil || got.Counts.ShopeeFieldsFilled != 0 {
t.Fatalf("默认策略不应覆盖:%+v %v", got, err)
}
cross := keep
cross.BatchID = "product-cross"
cross.UpdatePolicy = "overwrite_same_source"
cross.ObservedAt = "2026-08-12T07:00:00+08:00"
raw, _ = json.Marshal(cross)
got, err = ImportCatalogBatch(db, "script-b", cross, raw)
if err != nil || got.Counts.ShopeeFieldsSameSourceUpdated != 0 {
t.Fatalf("跨来源不应覆盖:%+v %v", got, err)
}
insertOnly := keep
insertOnly.BatchID = "product-insert-only"
insertOnly.UpdatePolicy = "insert_only"
raw, _ = json.Marshal(insertOnly)
got, err = ImportCatalogBatch(db, "script-a", insertOnly, raw)
if err != nil || got.Counts.ShopeeFieldsFilled != 0 || got.Counts.ShopeeFieldsSameSourceUpdated != 0 {
t.Fatalf("insert_only 不应修改已有商品:%+v %v", got, err)
}
overwrite := keep
overwrite.BatchID = "product-overwrite"
overwrite.UpdatePolicy = "overwrite_same_source"
overwrite.ObservedAt = "2026-08-12T08:00:00+08:00"
raw, _ = json.Marshal(overwrite)
got, err = ImportCatalogBatch(db, "script-a", overwrite, raw)
if err != nil || got.Counts.ShopeeFieldsSameSourceUpdated != 2 {
t.Fatalf("同来源更新失败:%+v %v", got, err)
}
var image, shop string
_ = db.QueryRow(`SELECT image_url,shopee_shop_name FROM shopee_products WHERE goods_id='S-1'`).Scan(&image, &shop)
if image != "https://img.example.com/b.jpg" || shop != "店铺B" {
t.Fatalf("字段未更新:%q %q", image, shop)
}
stale := overwrite
stale.BatchID = "product-stale"
stale.ObservedAt = "2026-08-11T09:00:00+08:00"
stale.ShopeeProducts[0].ImageURL = "https://img.example.com/old.jpg"
stale.ShopeeProducts[0].ShopName = "旧店铺"
raw, _ = json.Marshal(stale)
got, err = ImportCatalogBatch(db, "script-a", stale, raw)
if err != nil || got.Counts.ShopeeFieldsStaleSkipped != 2 {
t.Fatalf("旧观测保护失败:%+v %v", got, err)
}
_, _ = db.Exec(`UPDATE shopee_products SET image_is_manual=1,shop_name_is_manual=1 WHERE goods_id='S-1'`)
manual := overwrite
manual.BatchID = "product-manual"
manual.ObservedAt = "2026-08-13T08:00:00+08:00"
manual.ShopeeProducts[0].ImageURL = "https://img.example.com/c.jpg"
manual.ShopeeProducts[0].ShopName = "店铺C"
raw, _ = json.Marshal(manual)
got, err = ImportCatalogBatch(db, "script-a", manual, raw)
if err != nil || got.Counts.ShopeeFieldsManualSkipped != 2 {
t.Fatalf("人工保护失败:%+v %v", got, err)
}
}
func TestValidateCatalogBatch_拒绝危险图片URL(t *testing.T) {
for _, rawURL := range []string{"file:///tmp/a.jpg", "data:image/png;base64,AA", "https://user:pass@example.com/a.jpg", "https://example.com/a.jpg?access_token=secret", "javascript:alert(1)"} {
req := validCatalogBatch()
req.ShopeeProducts[0].ImageURL = rawURL
if err := ValidateCatalogBatch(req); err == nil {
t.Errorf("应拒绝 %q", rawURL)
}
}
req := validCatalogBatch()
req.ShopeeProducts[0].ImageURL = "https://img.example.com/a.jpg?v=1"
if err := ValidateCatalogBatch(req); err != nil {
t.Fatalf("合法图片被拒绝:%v", err)
}
}
func TestPreviewCatalogBatch_只读且报告既有关联冲突(t *testing.T) {
db := newCatalogTestDB(t)
base := validCatalogBatch()
raw, _ := json.Marshal(base)
if _, err := ImportCatalogBatch(db, "script-a", base, raw); err != nil {
t.Fatal(err)
}
preview := validCatalogBatch()
preview.BatchID = "preview-only"
preview.DryRun = true
preview.Associations[0].PddGoodsID = "P-2"
preview.PddProducts[0].GoodsID = "P-2"
preview.PddProducts[0].URL = "https://mobile.yangkeduo.com/goods.html?goods_id=P-2"
got, err := ImportCatalogBatch(db, "script-a", preview, []byte("unused"))
if err != nil || got.Status != "previewed" || len(got.Conflicts) != 1 {
t.Fatalf("预检结果错误:%+v %v", got, err)
}
var runs, pdd int
_ = db.QueryRow(`SELECT COUNT(*) FROM catalog_import_runs WHERE batch_id='preview-only'`).Scan(&runs)
_ = db.QueryRow(`SELECT COUNT(*) FROM pdd_products WHERE goods_id='P-2'`).Scan(&pdd)
if runs != 0 || pdd != 0 {
t.Fatalf("预检写入了数据:runs=%d pdd=%d", runs, pdd)
}
}
func TestImportCatalogBatch_FillMissing不覆盖已有PDD和商品字段(t *testing.T) {
db := newCatalogTestDB(t)
base := validCatalogBatch()
raw, _ := json.Marshal(base)
if _, err := ImportCatalogBatch(db, "script-a", base, raw); err != nil { t.Fatal(err) }
_, _ = db.Exec(`UPDATE shopee_products SET title='人工标题',main_sku_code='人工货号' WHERE goods_id='S-1'`)
_, _ = db.Exec(`UPDATE pdd_products SET url='https://manual.example/P-1',title='已采集标题',shop_name='已采集店铺',collect_status='collected' WHERE goods_id='P-1'`)
fill := validCatalogBatch(); fill.BatchID="no-overwrite"; fill.ShopeeProducts[0].Title="来源标题"; fill.ShopeeProducts[0].MainSKUCode="来源货号"; fill.PddProducts[0].URL="https://mobile.yangkeduo.com/goods.html?goods_id=P-1"; fill.PddProducts[0].Title="来源PDD标题"; fill.PddProducts[0].ShopName="来源PDD店铺"
raw, _ = json.Marshal(fill)
if _, err := ImportCatalogBatch(db, "thirdparty-csv", fill, raw); err != nil { t.Fatal(err) }
var title, mainSKU, url, pddTitle, shop, status string
_ = db.QueryRow(`SELECT title,main_sku_code FROM shopee_products WHERE goods_id='S-1'`).Scan(&title,&mainSKU)
_ = db.QueryRow(`SELECT url,title,shop_name,collect_status FROM pdd_products WHERE goods_id='P-1'`).Scan(&url,&pddTitle,&shop,&status)
if title!="人工标题" || mainSKU!="人工货号" || url!="https://manual.example/P-1" || pddTitle!="已采集标题" || shop!="已采集店铺" || status!="collected" { t.Fatalf("fill_missing 覆盖了已有字段:%q/%q/%q/%q/%q/%q",title,mainSKU,url,pddTitle,shop,status) }
}