chengma/cmautobuy
2
0
Fork 0
Code Issues 89 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 规范 CSV 预检与安全入库 (#281)

Browse Source
This commit is contained in:
chengma
2026-08-20 14:41:01 +08:00
parent b8537fa776
commit fe64d3b3c7
5 changed files with 260 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
admin/repository/catalog_import.go
+61 -13
View File
@@ -18,6 +18,48 @@ var (
ErrCatalogImportRunNotFound = errors.New("商品目录批次不存在")
)
// CatalogEntityExists 仅用于目录导入预检，不会修改任何记录。
func CatalogEntityExists(q Execer, table, goodsID string) (bool, error) {
var found int
var query string
switch table {
case "shopee_products":
query = `SELECT 1 FROM shopee_products WHERE goods_id=? AND deleted_at IS NULL`
case "pdd_products":
query = `SELECT 1 FROM pdd_products WHERE goods_id=? AND deleted_at IS NULL`
default:
return false, fmt.Errorf("不支持的目录表: %s", table)
}
err := q.QueryRow(query, goodsID).Scan(&found)
if errors.Is(err, sql.ErrNoRows) {
return false, nil
}
return err == nil, err
}
// CatalogSKUExists 按与实际写入一致的商品+规格键判断 SKU 是否已存在。
func CatalogSKUExists(q Execer, goodsID, specKey string) (bool, error) {
var found int
err := q.QueryRow(`SELECT 1 FROM shopee_skus WHERE goods_id=? AND spec_key=?`, goodsID, specKey).Scan(&found)
if errors.Is(err, sql.ErrNoRows) {
return false, nil
}
return err == nil, err
}
// CatalogAssociationCurrentPDD 返回当前关联；空字符串表示该蝦皮商品没有 PDD 关联。
func CatalogAssociationCurrentPDD(q Execer, goodsID string) (string, bool, error) {
var current sql.NullString
err := q.QueryRow(`SELECT pdd_goods_id FROM shopee_products WHERE goods_id=? AND deleted_at IS NULL`, goodsID).Scan(&current)
if errors.Is(err, sql.ErrNoRows) {
return "", false, nil
}
if err != nil {
return "", false, err
}
return current.String, true, nil
}
// InsertCatalogImportRun 先登记 processing 批次；表的复合主键是并发幂等的最终防线。
func InsertCatalogImportRun(q Execer, run model.CatalogImportRun) error {
_, err := q.Exec(`INSERT INTO catalog_import_runs (
@@ -169,13 +211,13 @@ type CatalogProductOutcome struct {
// UpsertCatalogShopeeProduct 独立保护主图和店铺字段，绝不覆盖人工 PDD 关联。
func UpsertCatalogShopeeProduct(q Execer, in CatalogShopeeProductInput) (out CatalogProductOutcome, err error) {
var oldObserved, oldShopID, imageURL, shopName, imageSource, imageObserved, shopSource, shopObserved sql.NullString
var oldObserved, oldShopID, title, status, mainSKU, imageURL, shopName, imageSource, imageObserved, shopSource, shopObserved sql.NullString
var imageManual, shopManual int
shopID, err := FindShopIDByName(q, in.ShopName)
if err != nil {
return out, err
}
err = q.QueryRow(`SELECT source_observed_at,shop_id,image_url,shopee_shop_name,image_source,image_observed_at,image_is_manual,shop_name_source,shop_name_observed_at,shop_name_is_manual FROM shopee_products WHERE goods_id=?`, in.GoodsID).Scan(&oldObserved, &oldShopID, &imageURL, &shopName, &imageSource, &imageObserved, &imageManual, &shopSource, &shopObserved, &shopManual)
err = q.QueryRow(`SELECT source_observed_at,shop_id,title,shopee_status,main_sku_code,image_url,shopee_shop_name,image_source,image_observed_at,image_is_manual,shop_name_source,shop_name_observed_at,shop_name_is_manual FROM shopee_products WHERE goods_id=?`, in.GoodsID).Scan(&oldObserved, &oldShopID, &title, &status, &mainSKU, &imageURL, &shopName, &imageSource, &imageObserved, &imageManual, &shopSource, &shopObserved, &shopManual)
if errors.Is(err, sql.ErrNoRows) {
_, err = q.Exec(`INSERT INTO shopee_products(goods_id,shop_id,title,shopee_status,main_sku_code,image_url,shopee_shop_name,image_source,image_observed_at,image_is_manual,shop_name_source,shop_name_observed_at,shop_name_is_manual,source,source_observed_at,created_at,updated_at) VALUES(?,NULLIF(?,''),?,NULLIF(?,''),NULLIF(?,''),NULLIF(?,''),NULLIF(?,''),CASE WHEN TRIM(?)='' THEN NULL ELSE ? END,CASE WHEN TRIM(?)='' THEN NULL ELSE ? END,0,CASE WHEN TRIM(?)='' THEN NULL ELSE ? END,CASE WHEN TRIM(?)='' THEN NULL ELSE ? END,0,'api',?,?,?)`, in.GoodsID, shopID, in.Title, in.Status, in.MainSKU, in.ImageURL, in.ShopName, in.ImageURL, in.Source, in.ImageURL, in.ObservedAt, in.ShopName, in.Source, in.ShopName, in.ObservedAt, in.ObservedAt, in.Now, in.Now)
out.Created = err == nil
@@ -184,7 +226,10 @@ func UpsertCatalogShopeeProduct(q Execer, in CatalogShopeeProductInput) (out Cat
if err != nil {
return out, err
}
baseUpdate := !oldObserved.Valid || oldObserved.String <= in.ObservedAt
baseUpdate := in.UpdatePolicy == "overwrite_same_source" && (!oldObserved.Valid || oldObserved.String <= in.ObservedAt)
fillTitle := in.UpdatePolicy == "fill_missing" && strings.TrimSpace(in.Title) != "" && strings.TrimSpace(title.String) == ""
fillStatus := in.UpdatePolicy == "fill_missing" && strings.TrimSpace(in.Status) != "" && strings.TrimSpace(status.String) == ""
fillMainSKU := in.UpdatePolicy == "fill_missing" && strings.TrimSpace(in.MainSKU) != "" && strings.TrimSpace(mainSKU.String) == ""
newImage, newShop := imageURL.String, shopName.String
imageChanged, shopChanged := false, false
apply := func(incoming string, current *string, source, observed sql.NullString, manual int) (changed bool) {
@@ -241,10 +286,10 @@ func UpsertCatalogShopeeProduct(q Execer, in CatalogShopeeProductInput) (out Cat
resolvedShopID = shopID
shopAssociationChanged = true
}
if !baseUpdate && !imageChanged && !shopChanged && !shopAssociationChanged {
if !baseUpdate && !fillTitle && !fillStatus && !fillMainSKU && !imageChanged && !shopChanged && !shopAssociationChanged {
return out, nil
}
_, err = q.Exec(`UPDATE shopee_products SET title=CASE WHEN ? THEN ? ELSE title END,shopee_status=CASE WHEN ? THEN NULLIF(?,'') ELSE shopee_status END,main_sku_code=CASE WHEN ? THEN NULLIF(?,'') ELSE main_sku_code END,source=CASE WHEN ? THEN 'api' ELSE source END,source_observed_at=CASE WHEN ? THEN ? ELSE source_observed_at END,shop_id=CASE WHEN ? THEN NULLIF(?,'') ELSE shop_id END,image_url=NULLIF(?,''),shopee_shop_name=NULLIF(?,''),image_source=CASE WHEN ? THEN ? ELSE image_source END,image_observed_at=CASE WHEN ? THEN ? ELSE image_observed_at END,shop_name_source=CASE WHEN ? THEN ? ELSE shop_name_source END,shop_name_observed_at=CASE WHEN ? THEN ? ELSE shop_name_observed_at END,updated_at=? WHERE goods_id=?`, baseUpdate, in.Title, baseUpdate, in.Status, baseUpdate, in.MainSKU, baseUpdate, baseUpdate, in.ObservedAt, shopAssociationChanged, resolvedShopID, newImage, newShop, imageChanged, in.Source, imageChanged, in.ObservedAt, shopChanged, in.Source, shopChanged, in.ObservedAt, in.Now, in.GoodsID)
_, err = q.Exec(`UPDATE shopee_products SET title=CASE WHEN ? OR ? THEN ? ELSE title END,shopee_status=CASE WHEN ? OR ? THEN NULLIF(?,'') ELSE shopee_status END,main_sku_code=CASE WHEN ? OR ? THEN NULLIF(?,'') ELSE main_sku_code END,source=CASE WHEN ? THEN 'api' ELSE source END,source_observed_at=CASE WHEN ? THEN ? ELSE source_observed_at END,shop_id=CASE WHEN ? THEN NULLIF(?,'') ELSE shop_id END,image_url=NULLIF(?,''),shopee_shop_name=NULLIF(?,''),image_source=CASE WHEN ? THEN ? ELSE image_source END,image_observed_at=CASE WHEN ? THEN ? ELSE image_observed_at END,shop_name_source=CASE WHEN ? THEN ? ELSE shop_name_source END,shop_name_observed_at=CASE WHEN ? THEN ? ELSE shop_name_observed_at END,updated_at=? WHERE goods_id=?`, baseUpdate, fillTitle, in.Title, baseUpdate, fillStatus, in.Status, baseUpdate, fillMainSKU, in.MainSKU, baseUpdate, baseUpdate, in.ObservedAt, shopAssociationChanged, resolvedShopID, newImage, newShop, imageChanged, in.Source, imageChanged, in.ObservedAt, shopChanged, in.Source, shopChanged, in.ObservedAt, in.Now, in.GoodsID)
out.Updated = err == nil
return out, err
}
@@ -351,8 +396,7 @@ func UpsertCatalogShopeeSKU(q Execer, in CatalogShopeeSKUInput) (CatalogSKUOutco
sources, times := catalogFieldProvenance(current)
changed := external != current.ShopeeSKUID.String
canFill := func(field, current, incoming string) bool {
return sources[field] != "manual" && strings.TrimSpace(incoming) != "" &&
(current == "" || (sources[field] == "syb" && in.Source != "syb"))
return sources[field] != "manual" && strings.TrimSpace(incoming) != "" && current == ""
}
if canFill("color", color, in.Color) {
color = in.Color
@@ -444,7 +488,7 @@ func catalogFieldProvenance(row *catalogSKUStored) (map[string]string, map[strin
}
// UpsertCatalogPddProduct 写入结构化 PDD 数据转换后的规范 JSON；空规格不清除既有采集结果。
func UpsertCatalogPddProduct(q Execer, goodsID, url, title, shopName, skusJSON, observedAt, now string) (created, updated bool, err error) {
func UpsertCatalogPddProduct(q Execer, goodsID, url, title, shopName, skusJSON, observedAt, now, updatePolicy string) (created, updated bool, err error) {
var oldObserved, deletedAt sql.NullString
err = q.QueryRow(`SELECT source_observed_at,deleted_at FROM pdd_products WHERE goods_id=?`, goodsID).Scan(&oldObserved, &deletedAt)
status := "pending"
@@ -460,13 +504,17 @@ func UpsertCatalogPddProduct(q Execer, goodsID, url, title, shopName, skusJSON,
if err != nil {
return false, false, err
}
if oldObserved.Valid && oldObserved.String > observedAt {
if updatePolicy == "insert_only" || (oldObserved.Valid && oldObserved.String > observedAt && updatePolicy != "fill_missing") {
return false, false, nil
}
if skusJSON == "" {
_, err = q.Exec(`UPDATE pdd_products SET url=?,title=CASE WHEN TRIM(?)='' THEN title ELSE ? END,
shop_name=CASE WHEN TRIM(?)='' THEN shop_name ELSE ? END,deleted_at=NULL,source='api',source_observed_at=?,updated_at=? WHERE goods_id=?`,
url, title, title, shopName, shopName, observedAt, now, goodsID)
if skusJSON == "" || updatePolicy == "fill_missing" {
if updatePolicy == "fill_missing" {
_, err = q.Exec(`UPDATE pdd_products SET url=CASE WHEN TRIM(COALESCE(url,''))='' THEN ? ELSE url END,title=CASE WHEN TRIM(COALESCE(title,''))='' THEN NULLIF(?,'') ELSE title END,shop_name=CASE WHEN TRIM(COALESCE(shop_name,''))='' THEN NULLIF(?,'') ELSE shop_name END,updated_at=? WHERE goods_id=?`, url, title, shopName, now, goodsID)
} else {
_, err = q.Exec(`UPDATE pdd_products SET url=?,title=CASE WHEN TRIM(?)='' THEN title ELSE ? END,
shop_name=CASE WHEN TRIM(?)='' THEN shop_name ELSE ? END,deleted_at=NULL,source='api',source_observed_at=?,updated_at=? WHERE goods_id=?`,
url, title, title, shopName, shopName, observedAt, now, goodsID)
}
} else {
_, err = q.Exec(`UPDATE pdd_products SET url=?,title=NULLIF(?,''),shop_name=NULLIF(?,''),skus_json=?,
collect_status='collected',collect_msg=NULL,artifact_ref=NULL,collected_at=?,deleted_at=NULL,