feat: 完善 SYB 蝦皮主数据与软删除 (#203 #204 #205)

This commit is contained in:
chengma
2026-08-13 10:39:21 +08:00
parent 3af9bf245a
commit ad1ccc2c60
23 changed files with 864 additions and 90 deletions
+20 -7
View File
@@ -191,6 +191,16 @@ func UpsertCatalogShopeeProduct(q Execer, in CatalogShopeeProductInput) (out Cat
out.FieldsManualSkipped++
return false
}
// 商品目录是权威来源,可以替换顺运宝为了预览而填入的低优先级值。
// 这不是任意跨来源覆盖:只有明确标记为 syb 的值享受升级规则。
if source.Valid && source.String == "syb" {
if *current != incoming {
*current = incoming
out.FieldsFilled++
return true
}
return false
}
if in.UpdatePolicy == "fill_missing" {
if *current == "" {
*current = incoming
@@ -324,22 +334,25 @@ func UpsertCatalogShopeeSKU(q Execer, in CatalogShopeeSKUInput) (CatalogSKUOutco
color, size, advice, skuCode := current.Color.String, current.Size.String, current.Advice.String, current.SKUCode.String
sources, times := catalogFieldProvenance(current)
changed := external != current.ShopeeSKUID.String
if color == "" && strings.TrimSpace(in.Color) != "" {
canFill := func(field, current, incoming string) bool {
return strings.TrimSpace(incoming) != "" && (current == "" || (sources[field] == "syb" && in.Source != "syb"))
}
if canFill("color", color, in.Color) {
color = in.Color
sources["color"], times["color"] = in.Source, in.ObservedAt
changed = true
}
if size == "" && strings.TrimSpace(in.Size) != "" {
if canFill("size", size, in.Size) {
size = in.Size
sources["size"], times["size"] = in.Source, in.ObservedAt
changed = true
}
if advice == "" && strings.TrimSpace(in.Advice) != "" {
if canFill("advice", advice, in.Advice) {
advice = in.Advice
sources["advice"], times["advice"] = in.Source, in.ObservedAt
changed = true
}
if skuCode == "" && strings.TrimSpace(in.SKUCode) != "" {
if canFill("sku_code", skuCode, in.SKUCode) {
skuCode = in.SKUCode
sources["sku_code"], times["sku_code"] = in.Source, in.ObservedAt
changed = true
@@ -349,7 +362,7 @@ func UpsertCatalogShopeeSKU(q Execer, in CatalogShopeeSKUInput) (CatalogSKUOutco
}
sourcesJSON, _ := json.Marshal(sources)
timesJSON, _ := json.Marshal(times)
_, err = q.Exec(`UPDATE shopee_skus SET shopee_sku_id=NULLIF(?,''),color=NULLIF(?,''),size=NULLIF(?,''),advice=NULLIF(?,''),sku_code=NULLIF(?,''),parse_ok=CASE WHEN parse_ok=1 THEN 1 ELSE ? END,field_sources=?,field_observed_at=?,updated_at=? WHERE sku_id=?`, external, color, size, advice, skuCode, parse, string(sourcesJSON), string(timesJSON), in.Now, current.RecordID)
_, err = q.Exec(`UPDATE shopee_skus SET shopee_sku_id=NULLIF(?,''),color=NULLIF(?,''),size=NULLIF(?,''),advice=NULLIF(?,''),sku_code=NULLIF(?,''),parse_ok=CASE WHEN parse_ok=1 THEN 1 ELSE ? END,field_sources=?,field_observed_at=?,source_observed_at=CASE WHEN source='syb' AND ?<>'syb' THEN ? ELSE source_observed_at END,source=CASE WHEN source='syb' AND ?<>'syb' THEN ? ELSE source END,updated_at=? WHERE sku_id=?`, external, color, size, advice, skuCode, parse, string(sourcesJSON), string(timesJSON), in.Source, in.ObservedAt, in.Source, in.Source, in.Now, current.RecordID)
if err != nil {
return "", err
}
@@ -448,7 +461,7 @@ func UpsertCatalogPddProduct(q Execer, goodsID, url, title, shopName, skusJSON,
// ApplyCatalogAssociation 只允许空关联建立或相同关联重放。
func ApplyCatalogAssociation(q Execer, shopeeGoodsID, pddGoodsID, now string) (created, unchanged bool, err error) {
var current sql.NullString
if err = q.QueryRow(`SELECT pdd_goods_id FROM shopee_products WHERE goods_id=?`, shopeeGoodsID).Scan(&current); err != nil {
if err = q.QueryRow(`SELECT pdd_goods_id FROM shopee_products WHERE goods_id=? AND deleted_at IS NULL`, shopeeGoodsID).Scan(&current); err != nil {
return
}
var url string
@@ -461,7 +474,7 @@ func ApplyCatalogAssociation(q Execer, shopeeGoodsID, pddGoodsID, now string) (c
}
return false, false, fmt.Errorf("蝦皮商品 %s 已关联 PDD 商品 %s", shopeeGoodsID, current.String)
}
_, err = q.Exec(`UPDATE shopee_products SET pdd_goods_id=?,pdd_goods_url=?,updated_at=? WHERE goods_id=?`, pddGoodsID, url, now, shopeeGoodsID)
_, err = q.Exec(`UPDATE shopee_products SET pdd_goods_id=?,pdd_goods_url=?,updated_at=? WHERE goods_id=? AND deleted_at IS NULL`, pddGoodsID, url, now, shopeeGoodsID)
return err == nil, false, err
}
+133 -2
View File
@@ -20,7 +20,7 @@ import (
"cmautobuy/admin/spec"
)
const mysqlSchemaVersion = 14
const mysqlSchemaVersion = 17
// OpenMySQL 打开生产 MySQL 8 数据库。错误信息绝不包含完整 DSN 或密码。
func OpenMySQL(cfg config.DatabaseConfig) (*sql.DB, error) {
@@ -592,10 +592,89 @@ func MigrateMySQL(db *sql.DB) error {
if _, err := db.Exec(`INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)`, 14, time.Now().UTC().Format(time.RFC3339Nano)); err != nil {
return fmt.Errorf("记录 MySQL schema v14 失败: %w", err)
}
current = 14
}
if current < 15 {
if err := migrateMySQLV15(db); err != nil {
return fmt.Errorf("执行 MySQL schema v15 失败: %w", err)
}
if err := checkMySQLV15Shape(db); err != nil {
return fmt.Errorf("MySQL schema v15 自检失败,未记录版本: %w", err)
}
if _, err := db.Exec(`INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)`, 15, time.Now().UTC().Format(time.RFC3339Nano)); err != nil {
return fmt.Errorf("记录 MySQL schema v15 失败: %w", err)
}
current = 15
}
if current < 16 {
if err := migrateMySQLV16(db); err != nil {
return fmt.Errorf("执行 MySQL schema v16 失败: %w", err)
}
if err := checkMySQLV16Shape(db); err != nil {
return fmt.Errorf("MySQL schema v16 自检失败,未记录版本: %w", err)
}
if _, err := db.Exec(`INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)`, 16, time.Now().UTC().Format(time.RFC3339Nano)); err != nil {
return fmt.Errorf("记录 MySQL schema v16 失败: %w", err)
}
current = 16
}
if current < 17 {
if err := migrateMySQLV17(db); err != nil {
return fmt.Errorf("执行 MySQL schema v17 失败: %w", err)
}
if err := checkMySQLV17Shape(db); err != nil {
return fmt.Errorf("MySQL schema v17 自检失败,未记录版本: %w", err)
}
if _, err := db.Exec(`INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)`, 17, time.Now().UTC().Format(time.RFC3339Nano)); err != nil {
return fmt.Errorf("记录 MySQL schema v17 失败: %w", err)
}
}
return CheckMySQLSchema(db)
}
// migrateMySQLV15 以低优先级补全历史 SYB 商品的店铺和图片。
func migrateMySQLV15(db *sql.DB) error {
return backfillSybProductMetadata(db)
}
// migrateMySQLV16 为历史 SYB 数据补建格式明确的低优先级蝦皮 SKU。
func migrateMySQLV16(db *sql.DB) error {
return backfillSybShopeeSKUs(db)
}
// migrateMySQLV17 为蝦皮商品增加可恢复的软删除标记。
func migrateMySQLV17(db *sql.DB) error {
for _, column := range []struct{ name, ddl string }{
{"deleted_at", `ALTER TABLE shopee_products ADD COLUMN deleted_at VARCHAR(35) NULL AFTER pdd_goods_id`},
{"deleted_by_user_id", `ALTER TABLE shopee_products ADD COLUMN deleted_by_user_id VARCHAR(191) COLLATE utf8mb4_bin NULL AFTER deleted_at`},
} {
exists, err := mysqlColumnExists(db, "shopee_products", column.name)
if err != nil {
return err
}
if !exists {
if _, err := db.Exec(column.ddl); err != nil {
return fmt.Errorf("增加 shopee_products.%s 失败: %w", column.name, err)
}
}
}
if exists, err := mysqlIndexExists(db, "shopee_products", "idx_shopee_products_deleted"); err != nil {
return err
} else if !exists {
if _, err := db.Exec(`ALTER TABLE shopee_products ADD INDEX idx_shopee_products_deleted (deleted_at,updated_at,goods_id)`); err != nil {
return fmt.Errorf("增加蝦皮商品删除状态索引失败: %w", err)
}
}
if exists, err := mysqlConstraintExists(db, "shopee_products", "fk_shopee_products_deleted_by"); err != nil {
return err
} else if !exists {
if _, err := db.Exec(`ALTER TABLE shopee_products ADD CONSTRAINT fk_shopee_products_deleted_by FOREIGN KEY (deleted_by_user_id) REFERENCES users(user_id) ON DELETE SET NULL`); err != nil {
return fmt.Errorf("增加蝦皮商品删除人外键失败: %w", err)
}
}
return nil
}
// migrateMySQLV14 增加顺运宝同步店铺准入表和审计统计。
func migrateMySQLV14(db *sql.DB) error {
if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS syb_allowed_shops (
@@ -1579,7 +1658,59 @@ func CheckMySQLSchema(db *sql.DB) error {
if err := checkMySQLV13Shape(db); err != nil {
return err
}
return checkMySQLV14Shape(db)
if err := checkMySQLV14Shape(db); err != nil {
return err
}
if err := checkMySQLV15Shape(db); err != nil {
return err
}
if err := checkMySQLV16Shape(db); err != nil {
return err
}
return checkMySQLV17Shape(db)
}
func checkMySQLV15Shape(db *sql.DB) error {
rows, err := db.Query(`SELECT image_source,image_observed_at,image_is_manual,
shop_name_source,shop_name_observed_at,shop_name_is_manual
FROM shopee_products LIMIT 0`)
if err != nil {
return fmt.Errorf("蝦皮商品店铺与图片来源字段缺失: %w", err)
}
return rows.Close()
}
func checkMySQLV16Shape(db *sql.DB) error {
rows, err := db.Query(`SELECT spec_key,color,size,advice,parse_ok,source,field_sources,field_observed_at FROM shopee_skus LIMIT 0`)
if err != nil {
return fmt.Errorf("蝦皮 SKU 解析与来源字段缺失: %w", err)
}
return rows.Close()
}
func checkMySQLV17Shape(db *sql.DB) error {
if err := checkMySQLVarcharColumn(db, "shopee_products", "deleted_at", 35, true, "utf8mb4_0900_ai_ci", ""); err != nil {
return err
}
if err := checkMySQLVarcharColumn(db, "shopee_products", "deleted_by_user_id", 191, true, "utf8mb4_bin", ""); err != nil {
return err
}
for _, item := range []struct{ kind, name string }{{"index", "idx_shopee_products_deleted"}, {"constraint", "fk_shopee_products_deleted_by"}} {
var exists bool
var err error
if item.kind == "index" {
exists, err = mysqlIndexExists(db, "shopee_products", item.name)
} else {
exists, err = mysqlConstraintExists(db, "shopee_products", item.name)
}
if err != nil {
return err
}
if !exists {
return fmt.Errorf("蝦皮商品软删除%s %s 缺失", item.kind, item.name)
}
}
return nil
}
func checkMySQLV14Shape(db *sql.DB) error {
+45 -5
View File
@@ -560,7 +560,7 @@ func TestMySQLMigrate_V11回填顺运宝店铺且可重放(t *testing.T) {
if err := MigrateMySQL(db); err != nil {
t.Fatal(err)
}
mustExec(t, db, `DELETE FROM schema_migrations WHERE version=11`)
mustExec(t, db, `DELETE FROM schema_migrations WHERE version>=11`)
mustExec(t, db, `ALTER TABLE syb_orders DROP COLUMN shop_name`)
now := model.NowISO()
mustExec(t, db, `INSERT INTO syb_orders
@@ -595,7 +595,7 @@ func TestMySQLMigrate_V11升级V12并修复孤儿采集中状态(t *testing.T) {
t.Fatal(err)
}
// 模拟已在 v11 的生产库:去掉 v12 版本及其附加表,保留全部 v1-v11 结构。
mustExec(t, db, `DELETE FROM schema_migrations WHERE version=12`)
mustExec(t, db, `DELETE FROM schema_migrations WHERE version>=12`)
mustExec(t, db, `DROP TABLE task_syb_sources`)
now := model.NowISO()
mustExec(t, db, `INSERT INTO pdd_products(goods_id,url,collect_status,created_at,updated_at) VALUES
@@ -634,7 +634,7 @@ func TestMySQLMigrate_V12形状错误不记录版本(t *testing.T) {
if err := MigrateMySQL(db); err != nil {
t.Fatal(err)
}
mustExec(t, db, `DELETE FROM schema_migrations WHERE version=12`)
mustExec(t, db, `DELETE FROM schema_migrations WHERE version>=12`)
mustExec(t, db, `DROP TABLE task_syb_sources`)
mustExec(t, db, `CREATE TABLE task_syb_sources (
task_id VARCHAR(191) COLLATE utf8mb4_bin NOT NULL,
@@ -660,7 +660,7 @@ func TestMySQLMigrate_V12升级V13迁移任务主键和关联(t *testing.T) {
t.Fatal(err)
}
// 模拟生产 v12:移除 v13 表、版本和新增的级联更新外键。
mustExec(t, db, `DELETE FROM schema_migrations WHERE version=13`)
mustExec(t, db, `DELETE FROM schema_migrations WHERE version>=13`)
mustExec(t, db, `DROP TABLE task_sequences`)
mustExec(t, db, `ALTER TABLE task_claims DROP FOREIGN KEY fk_task_claims_task`)
mustExec(t, db, `ALTER TABLE task_syb_sources DROP FOREIGN KEY fk_task_syb_sources_task`)
@@ -814,7 +814,7 @@ func TestMySQLMigrate_V13升级V14并可重放(t *testing.T) {
mustExec(t, db, `ALTER TABLE syb_sync_runs DROP COLUMN shop_skipped_count`)
mustExec(t, db, `ALTER TABLE syb_sync_runs DROP COLUMN accepted_stock_count`)
mustExec(t, db, `DROP TABLE syb_allowed_shops`)
mustExec(t, db, `DELETE FROM schema_migrations WHERE version=14`)
mustExec(t, db, `DELETE FROM schema_migrations WHERE version>=14`)
mustExec(t, db, `INSERT INTO users(user_id,username,password_hash,role,status,password_changed_at,created_at,updated_at)
VALUES('V14-USER','v14-user','x','admin','active','2026-08-12T00:00:00Z','2026-08-12T00:00:00Z','2026-08-12T00:00:00Z')`)
mustExec(t, db, `INSERT INTO syb_sync_runs(run_id,user_id,date_from,date_to,status,stock_count,started_at,finished_at)
@@ -835,6 +835,46 @@ func TestMySQLMigrate_V13升级V14并可重放(t *testing.T) {
}
}
func TestMySQLMigrate_V14升级V17回填元数据规格并增加软删除(t *testing.T) {
db := openMySQLMigrationTestDB(t)
defer db.Close()
cleanMySQLTestSchema(t, db)
defer cleanMySQLTestSchema(t, db)
if err := MigrateMySQL(db); err != nil {
t.Fatal(err)
}
// 模拟真实 v14:移除后三版新增结构和版本记录,保留历史业务数据。
mustExec(t, db, `ALTER TABLE shopee_products DROP FOREIGN KEY fk_shopee_products_deleted_by`)
mustExec(t, db, `ALTER TABLE shopee_products DROP INDEX idx_shopee_products_deleted`)
mustExec(t, db, `ALTER TABLE shopee_products DROP COLUMN deleted_by_user_id`)
mustExec(t, db, `ALTER TABLE shopee_products DROP COLUMN deleted_at`)
mustExec(t, db, `DELETE FROM schema_migrations WHERE version>=15`)
now := "2026-08-13T01:00:00Z"
mustExec(t, db, `INSERT INTO shopee_products(goods_id,title,source,created_at,updated_at)
VALUES('S-V15','历史商品','syb',?,?)`, now, now)
mustExec(t, db, `INSERT INTO syb_orders(syb_id,order_no,shop_name,title,product_spec,spec_key,shopee_goods_id,quantity,image_url,syb_data,created_at,updated_at)
VALUES('SO-V15','O-V15','历史店铺','历史商品','白色,L【建議50-60公斤】','白色,L【建議50-60公斤】','S-V15',1,'https://example.com/history.jpg','{}',?,?)`, now, now)
if err := MigrateMySQL(db); err != nil {
t.Fatalf("v14 升级 v17 失败: %v", err)
}
if err := MigrateMySQL(db); err != nil {
t.Fatalf("v17 重放失败: %v", err)
}
var shop, image string
if err := db.QueryRow(`SELECT shopee_shop_name,image_url FROM shopee_products WHERE goods_id='S-V15'`).Scan(&shop, &image); err != nil || shop != "历史店铺" || image != "https://example.com/history.jpg" {
t.Fatalf("v15 元数据回填错误: shop=%q image=%q err=%v", shop, image, err)
}
var color, size, advice, source string
if err := db.QueryRow(`SELECT color,size,advice,source FROM shopee_skus WHERE goods_id='S-V15'`).Scan(&color, &size, &advice, &source); err != nil || color != "白色" || size != "L" || advice != "50-60公斤" || source != "syb" {
t.Fatalf("v16 规格回填错误: color=%q size=%q advice=%q source=%q err=%v", color, size, advice, source, err)
}
if err := checkMySQLV17Shape(db); err != nil {
t.Fatalf("v17 软删除结构错误: %v", err)
}
}
func openMySQLMigrationTestDB(t *testing.T) *sql.DB {
t.Helper()
if os.Getenv("CMAUTOBUY_MYSQL_TEST") != "1" {
+79 -15
View File
@@ -136,6 +136,7 @@ type ShopeeFilter struct {
Status string
Shop string // has / missing / 空(全部)
Image string // has / missing / 空(全部)
Deleted bool // true 只看软删除数据;false 只看正常数据
}
// shopeeFilterClause 把关键字、状态、店铺和图片筛选拼成 WHERE 子句,供 ListShopeeProducts
@@ -146,7 +147,10 @@ type ShopeeFilter struct {
// SKU 时 JOIN 会出重复行,DISTINCT 又会让外层 LIMIT/OFFSET 的行为难推理
// (见工单 #43)。
func shopeeFilterClause(filter ShopeeFilter) (string, []any) {
var clauses []string
clauses := []string{"sp.deleted_at IS NULL"}
if filter.Deleted {
clauses[0] = "sp.deleted_at IS NOT NULL"
}
var args []any
if kw := strings.TrimSpace(filter.Keyword); kw != "" {
@@ -183,9 +187,6 @@ func shopeeFilterClause(filter ShopeeFilter) (string, []any) {
clauses = append(clauses, `(sp.image_url IS NULL OR TRIM(sp.image_url) = '')`)
}
if len(clauses) == 0 {
return "", args
}
return " WHERE " + strings.Join(clauses, " AND "), args
}
@@ -200,7 +201,7 @@ func ListShopeeProducts(q Execer, filter ShopeeFilter, limit, offset int) ([]Sho
where, args := shopeeFilterClause(filter)
sqlText := `
SELECT sp.goods_id, sp.title, sp.shopee_status, sp.main_sku_code, sp.image_url,sp.shopee_shop_name,sp.image_source,sp.image_observed_at,sp.image_is_manual,sp.shop_name_source,sp.shop_name_observed_at,sp.shop_name_is_manual,sp.source,
sp.pdd_goods_url, sp.pdd_goods_id, sp.created_at, sp.updated_at,
sp.pdd_goods_url, sp.pdd_goods_id,sp.deleted_at,sp.deleted_by_user_id, sp.created_at, sp.updated_at,
COUNT(DISTINCT CASE WHEN sk.parse_ok = 1 THEN sk.color END) AS color_count,
COUNT(DISTINCT CASE WHEN sk.parse_ok = 1 THEN sk.size END) AS size_count,
COUNT(sk.sku_id) AS sku_count,
@@ -222,11 +223,11 @@ func ListShopeeProducts(q Execer, filter ShopeeFilter, limit, offset int) ([]Sho
var list []ShopeeProductRow
for rows.Next() {
var r ShopeeProductRow
var title, shopeeStatus, mainSKUCode, imageURL, shopName, imageSource, imageObserved, shopSource, shopObserved, source, pddGoodsURL, pddGoodsID sql.NullString
var title, shopeeStatus, mainSKUCode, imageURL, shopName, imageSource, imageObserved, shopSource, shopObserved, source, pddGoodsURL, pddGoodsID, deletedAt, deletedBy sql.NullString
var imageManual, shopManual int
if err := rows.Scan(
&r.GoodsID, &title, &shopeeStatus, &mainSKUCode, &imageURL, &shopName, &imageSource, &imageObserved, &imageManual, &shopSource, &shopObserved, &shopManual, &source,
&pddGoodsURL, &pddGoodsID, &r.CreatedAt, &r.UpdatedAt,
&pddGoodsURL, &pddGoodsID, &deletedAt, &deletedBy, &r.CreatedAt, &r.UpdatedAt,
&r.ColorCount, &r.SizeCount, &r.SKUCount, &r.PendingCount,
&r.CollectStatus, &r.CollectMsg,
); err != nil {
@@ -246,6 +247,8 @@ func ListShopeeProducts(q Execer, filter ShopeeFilter, limit, offset int) ([]Sho
r.Source = source.String
r.PddGoodsURL = pddGoodsURL.String
r.PddGoodsID = pddGoodsID.String
r.DeletedAt = deletedAt.String
r.DeletedByUserID = deletedBy.String
list = append(list, r)
}
return list, rows.Err()
@@ -270,14 +273,14 @@ func CountShopeeProductsFiltered(q Execer, filter ShopeeFilter) (int, error) {
// 弹窗组装商品信息时用。查不到返回 (nil, nil)。
func GetShopeeProductByGoodsID(q Execer, goodsID string) (*model.ShopeeProduct, error) {
var p model.ShopeeProduct
var title, shopeeStatus, mainSKUCode, imageURL, shopName, imageSource, imageObserved, shopSource, shopObserved, source, pddGoodsURL, pddGoodsID sql.NullString
var title, shopeeStatus, mainSKUCode, imageURL, shopName, imageSource, imageObserved, shopSource, shopObserved, source, pddGoodsURL, pddGoodsID, deletedAt, deletedBy sql.NullString
var imageManual, shopManual int
err := q.QueryRow(`
SELECT goods_id, title, shopee_status, main_sku_code,image_url,shopee_shop_name,image_source,image_observed_at,image_is_manual,shop_name_source,shop_name_observed_at,shop_name_is_manual, source,
pdd_goods_url, pdd_goods_id, created_at, updated_at
FROM shopee_products WHERE goods_id = ?`, goodsID).Scan(
pdd_goods_url, pdd_goods_id,deleted_at,deleted_by_user_id, created_at, updated_at
FROM shopee_products WHERE goods_id = ? AND deleted_at IS NULL`, goodsID).Scan(
&p.GoodsID, &title, &shopeeStatus, &mainSKUCode, &imageURL, &shopName, &imageSource, &imageObserved, &imageManual, &shopSource, &shopObserved, &shopManual, &source,
&pddGoodsURL, &pddGoodsID, &p.CreatedAt, &p.UpdatedAt)
&pddGoodsURL, &pddGoodsID, &deletedAt, &deletedBy, &p.CreatedAt, &p.UpdatedAt)
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
@@ -298,6 +301,8 @@ func GetShopeeProductByGoodsID(q Execer, goodsID string) (*model.ShopeeProduct,
p.Source = source.String
p.PddGoodsURL = pddGoodsURL.String
p.PddGoodsID = pddGoodsID.String
p.DeletedAt = deletedAt.String
p.DeletedByUserID = deletedBy.String
return &p, nil
}
@@ -314,7 +319,7 @@ func ListShopeePddLinksByGoodsIDs(q Execer, goodsIDs []string) (map[string]strin
args[i] = goodsID
}
rows, err := q.Query(`SELECT goods_id, COALESCE(pdd_goods_id, '')
FROM shopee_products WHERE goods_id IN (`+placeholders+`)`, args...)
FROM shopee_products WHERE deleted_at IS NULL AND goods_id IN (`+placeholders+`)`, args...)
if err != nil {
return nil, fmt.Errorf("批量查询蝦皮商品 PDD 关联失败: %w", err)
}
@@ -338,7 +343,7 @@ func UpdateShopeePddLink(q Execer, shopeeGoodsID, pddGoodsID, pddURL string) err
result, err := q.Exec(`
UPDATE shopee_products
SET pdd_goods_id = ?, pdd_goods_url = ?, updated_at = ?
WHERE goods_id = ?`,
WHERE goods_id = ? AND deleted_at IS NULL`,
pddGoodsID, pddURL, model.NowISO(), shopeeGoodsID)
if err != nil {
return fmt.Errorf("更新蝦皮商品 %s 的 PDD 关联失败: %w", shopeeGoodsID, err)
@@ -418,7 +423,8 @@ func GetShopeeSKUByID(q Execer, skuID string) (*model.ShopeeSKU, error) {
err := q.QueryRow(`
SELECT sku_id, COALESCE(shopee_sku_id,''), goods_id, spec_raw, color, size, advice, parse_ok, sku_code,
is_manual, created_at, updated_at
FROM shopee_skus WHERE sku_id = ?`, skuID).Scan(
FROM shopee_skus sk WHERE sku_id = ?
AND EXISTS (SELECT 1 FROM shopee_products sp WHERE sp.goods_id=sk.goods_id AND sp.deleted_at IS NULL)`, skuID).Scan(
&sk.RecordID, &sk.SKUID, &sk.GoodsID, &sk.SpecRaw, &color, &size, &advice,
&parseOK, &skuCode, &isManual, &sk.CreatedAt, &sk.UpdatedAt,
)
@@ -440,8 +446,66 @@ func GetShopeeSKUByID(q Execer, skuID string) (*model.ShopeeSKU, error) {
// CountShopeeProducts 统计蝦皮商品总数,供列表页判断"是否已导入过任何数据"。
func CountShopeeProducts(q Execer) (int, error) {
var n int
if err := q.QueryRow(`SELECT COUNT(*) FROM shopee_products`).Scan(&n); err != nil {
if err := q.QueryRow(`SELECT COUNT(*) FROM shopee_products WHERE deleted_at IS NULL`).Scan(&n); err != nil {
return 0, fmt.Errorf("统计蝦皮商品数量失败: %w", err)
}
return n, nil
}
// ShopeeProductsWithActiveTasks 返回仍有采集或采购任务执行中的商品编号。
func ShopeeProductsWithActiveTasks(q Execer, goodsIDs []string) ([]string, error) {
if len(goodsIDs) == 0 {
return nil, nil
}
placeholders := strings.TrimSuffix(strings.Repeat("?,", len(goodsIDs)), ",")
args := make([]any, len(goodsIDs))
for i, id := range goodsIDs {
args[i] = id
}
rows, err := q.Query(`SELECT DISTINCT sp.goods_id
FROM shopee_products sp JOIN tasks t
ON (t.task_type='purchase' AND t.goods_id=sp.goods_id)
OR (t.task_type='collect' AND t.goods_id=sp.pdd_goods_id)
WHERE sp.goods_id IN (`+placeholders+`)
AND t.status IN ('pending','assigned','claimed') ORDER BY sp.goods_id`, args...)
if err != nil {
return nil, fmt.Errorf("检查蝦皮商品进行中任务失败: %w", err)
}
defer rows.Close()
var ids []string
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, err
}
ids = append(ids, id)
}
return ids, rows.Err()
}
// SetShopeeProductsDeleted 批量设置或清除软删除标记。
func SetShopeeProductsDeleted(q Execer, goodsIDs []string, actorUserID, deletedAt string) (int64, error) {
if len(goodsIDs) == 0 {
return 0, nil
}
placeholders := strings.TrimSuffix(strings.Repeat("?,", len(goodsIDs)), ",")
args := make([]any, 0, len(goodsIDs)+3)
var sqlText string
if deletedAt != "" {
sqlText = `UPDATE shopee_products SET deleted_at=?,deleted_by_user_id=?,updated_at=?
WHERE deleted_at IS NULL AND goods_id IN (` + placeholders + `)`
args = append(args, deletedAt, actorUserID, deletedAt)
} else {
sqlText = `UPDATE shopee_products SET deleted_at=NULL,deleted_by_user_id=NULL,updated_at=?
WHERE deleted_at IS NOT NULL AND goods_id IN (` + placeholders + `)`
args = append(args, model.NowISO())
}
for _, id := range goodsIDs {
args = append(args, id)
}
result, err := q.Exec(sqlText, args...)
if err != nil {
return 0, fmt.Errorf("更新蝦皮商品删除状态失败: %w", err)
}
return result.RowsAffected()
}
+2 -2
View File
@@ -30,7 +30,7 @@ func ListShopSpecSources(q Execer, shopName string) ([]ShopSpecSource, error) {
pp.goods_id, pp.deleted_at, pp.skus_json
FROM shopee_products sp
LEFT JOIN pdd_products pp ON pp.goods_id = sp.pdd_goods_id
WHERE sp.shopee_shop_name = ?
WHERE sp.deleted_at IS NULL AND sp.shopee_shop_name = ?
ORDER BY sp.goods_id`, strings.TrimSpace(shopName))
if err != nil {
return nil, fmt.Errorf("读取店铺规格对比数据失败: %w", err)
@@ -65,7 +65,7 @@ func ListShopSpecSources(q Execer, shopName string) ([]ShopSpecSource, error) {
SELECT sk.goods_id, sk.color, sk.size, sk.parse_ok
FROM shopee_skus sk
JOIN shopee_products sp ON sp.goods_id = sk.goods_id
WHERE sp.shopee_shop_name = ?
WHERE sp.deleted_at IS NULL AND sp.shopee_shop_name = ?
ORDER BY sk.goods_id, sk.sku_id`, strings.TrimSpace(shopName))
if err != nil {
return nil, fmt.Errorf("读取店铺正式规格失败: %w", err)
+171 -14
View File
@@ -5,7 +5,9 @@
package repository
import (
"crypto/sha256"
"database/sql"
"encoding/hex"
"errors"
"fmt"
"strings"
@@ -351,19 +353,7 @@ func UpsertSybOrder(q Execer, o model.SybOrder) (created bool, err error) {
now := model.NowISO()
if strings.TrimSpace(o.ShopeeGoodsID) != "" {
if _, err := q.Exec(`
INSERT INTO shopee_products (goods_id, title, source, created_at, updated_at)
VALUES (?, ?, 'syb', ?, ?)
ON DUPLICATE KEY UPDATE
title = CASE
WHEN source = 'syb' AND TRIM(VALUES(title)) <> '' THEN VALUES(title)
ELSE title
END,
updated_at = CASE
WHEN source = 'syb' AND TRIM(VALUES(title)) <> '' THEN VALUES(updated_at)
ELSE updated_at
END`,
o.ShopeeGoodsID, o.Title, now, now); err != nil {
if err := upsertSybShopeeProduct(q, o.ShopeeGoodsID, o.Title, o.ShopName, o.ImageURL, now); err != nil {
return false, fmt.Errorf("为顺运宝明细 %s 补建蝦皮商品骨架失败: %w", o.SybID, err)
}
}
@@ -393,6 +383,173 @@ func UpsertSybOrder(q Execer, o model.SybOrder) (created bool, err error) {
return created, nil
}
// upsertSybShopeeProduct 用顺运宝观测补建蝦皮商品骨架。
//
// 店铺和图片是字段级低优先级数据:只能补空值,或更新原本同样来自 syb 的值;
// 人工字段和商品目录等权威来源永远不被顺运宝覆盖。空值也不能清除已有内容。
func upsertSybShopeeProduct(q Execer, goodsID, title, shopName, imageURL, observedAt string) error {
_, err := q.Exec(`
INSERT INTO shopee_products
(goods_id,title,image_url,shopee_shop_name,
image_source,image_observed_at,image_is_manual,
shop_name_source,shop_name_observed_at,shop_name_is_manual,
source,source_observed_at,created_at,updated_at)
VALUES (?, ?, NULLIF(TRIM(?),''), NULLIF(TRIM(?),''),
CASE WHEN TRIM(?)='' THEN NULL ELSE 'syb' END,
CASE WHEN TRIM(?)='' THEN NULL ELSE ? END, 0,
CASE WHEN TRIM(?)='' THEN NULL ELSE 'syb' END,
CASE WHEN TRIM(?)='' THEN NULL ELSE ? END, 0,
'syb', ?, ?, ?)
ON DUPLICATE KEY UPDATE
title = CASE
WHEN source='syb' AND TRIM(VALUES(title))<>'' THEN VALUES(title)
ELSE title
END,
source_observed_at = CASE
WHEN source='syb' AND TRIM(VALUES(title))<>'' THEN VALUES(source_observed_at)
ELSE source_observed_at
END,
image_url = CASE
WHEN image_is_manual=0 AND TRIM(VALUES(image_url))<>''
AND (image_url IS NULL OR TRIM(image_url)='' OR image_source='syb') THEN VALUES(image_url)
ELSE image_url
END,
image_observed_at = CASE
WHEN image_is_manual=0 AND TRIM(VALUES(image_url))<>''
AND (image_url IS NULL OR TRIM(image_url)='' OR image_source='syb') THEN VALUES(image_observed_at)
ELSE image_observed_at
END,
image_source = CASE
WHEN image_is_manual=0 AND TRIM(VALUES(image_url))<>''
AND (image_url IS NULL OR TRIM(image_url)='' OR image_source='syb') THEN 'syb'
ELSE image_source
END,
shopee_shop_name = CASE
WHEN shop_name_is_manual=0 AND TRIM(VALUES(shopee_shop_name))<>''
AND (shopee_shop_name IS NULL OR TRIM(shopee_shop_name)='' OR shop_name_source='syb') THEN VALUES(shopee_shop_name)
ELSE shopee_shop_name
END,
shop_name_observed_at = CASE
WHEN shop_name_is_manual=0 AND TRIM(VALUES(shopee_shop_name))<>''
AND (shopee_shop_name IS NULL OR TRIM(shopee_shop_name)='' OR shop_name_source='syb') THEN VALUES(shop_name_observed_at)
ELSE shop_name_observed_at
END,
shop_name_source = CASE
WHEN shop_name_is_manual=0 AND TRIM(VALUES(shopee_shop_name))<>''
AND (shopee_shop_name IS NULL OR TRIM(shopee_shop_name)='' OR shop_name_source='syb') THEN 'syb'
ELSE shop_name_source
END,
updated_at = CASE
WHEN source='syb'
OR (image_source='syb' AND TRIM(VALUES(image_url))<>'')
OR (shop_name_source='syb' AND TRIM(VALUES(shopee_shop_name))<>'') THEN VALUES(updated_at)
ELSE updated_at
END`,
goodsID, title, imageURL, shopName,
imageURL, imageURL, observedAt,
shopName, shopName, observedAt,
observedAt, observedAt, observedAt)
return err
}
// backfillSybProductMetadata 把历史货运单中每个商品最新的非空店铺、图片补入商品主表。
// 重放是安全的:实际覆盖规则仍由 upsertSybShopeeProduct 统一执行。
func backfillSybProductMetadata(db *sql.DB) error {
type metadata struct{ shopName, imageURL, observedAt string }
items := map[string]metadata{}
readLatest := func(column string, assign func(*metadata, string)) error {
query := fmt.Sprintf(`SELECT shopee_goods_id, value_text, updated_at FROM (
SELECT shopee_goods_id, %s AS value_text, updated_at,
ROW_NUMBER() OVER (PARTITION BY shopee_goods_id ORDER BY updated_at DESC, syb_id DESC) AS row_no
FROM syb_orders
WHERE TRIM(COALESCE(shopee_goods_id,''))<>'' AND TRIM(COALESCE(%s,''))<>''
) ranked WHERE row_no=1`, column, column)
rows, err := db.Query(query)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var goodsID, value, observedAt string
if err := rows.Scan(&goodsID, &value, &observedAt); err != nil {
return err
}
item := items[goodsID]
assign(&item, value)
if observedAt > item.observedAt {
item.observedAt = observedAt
}
items[goodsID] = item
}
return rows.Err()
}
if err := readLatest("shop_name", func(item *metadata, value string) { item.shopName = value }); err != nil {
return fmt.Errorf("读取历史顺运宝店铺失败: %w", err)
}
if err := readLatest("image_url", func(item *metadata, value string) { item.imageURL = value }); err != nil {
return fmt.Errorf("读取历史顺运宝图片失败: %w", err)
}
for goodsID, item := range items {
if err := upsertSybShopeeProduct(db, goodsID, "", item.shopName, item.imageURL, item.observedAt); err != nil {
return fmt.Errorf("回填蝦皮商品 %s 的顺运宝元数据失败: %w", goodsID, err)
}
}
return nil
}
// UpsertSybShopeeSKU 把格式明确的 SYB 规格作为低优先级蝦皮 SKU 观测写入。
// 返回 parsed=false 时不写库,避免把猜测结果污染商品主数据。
func UpsertSybShopeeSKU(q Execer, goodsID, specRaw, observedAt string) (parsed bool, err error) {
parsedSpec, ok := spec.ParseShopeeSpec(specRaw)
if !ok || strings.TrimSpace(goodsID) == "" {
return false, nil
}
specKey, err := spec.SpecKey(specRaw)
if err != nil {
return false, nil
}
sum := sha256.Sum256([]byte(goodsID + "\x00" + specKey))
recordID := "syb:" + hex.EncodeToString(sum[:])
now := model.NowISO()
_, err = UpsertCatalogShopeeSKU(q, CatalogShopeeSKUInput{
RecordID: recordID, GoodsID: goodsID, SpecRaw: specRaw, SpecKey: specKey,
Color: parsedSpec.Color, Size: parsedSpec.Size, Advice: parsedSpec.Advice,
ParseOK: true, Source: "syb", ObservedAt: observedAt, Now: now, UpdatePolicy: "fill_missing",
})
return true, err
}
func backfillSybShopeeSKUs(db *sql.DB) error {
rows, err := db.Query(`SELECT shopee_goods_id,product_spec,updated_at FROM (
SELECT shopee_goods_id,product_spec,updated_at,
ROW_NUMBER() OVER (PARTITION BY shopee_goods_id,product_spec ORDER BY updated_at DESC,syb_id DESC) row_no
FROM syb_orders
WHERE TRIM(COALESCE(shopee_goods_id,''))<>'' AND TRIM(COALESCE(product_spec,''))<>''
) ranked WHERE row_no=1`)
if err != nil {
return fmt.Errorf("读取历史顺运宝规格失败: %w", err)
}
defer rows.Close()
type observation struct{ goodsID, raw, observedAt string }
var observations []observation
for rows.Next() {
var item observation
if err := rows.Scan(&item.goodsID, &item.raw, &item.observedAt); err != nil {
return err
}
observations = append(observations, item)
}
if err := rows.Err(); err != nil {
return err
}
for _, item := range observations {
if _, err := UpsertSybShopeeSKU(db, item.goodsID, item.raw, item.observedAt); err != nil {
return fmt.Errorf("回填蝦皮商品 %s 的顺运宝规格失败: %w", item.goodsID, err)
}
}
return nil
}
// SybSpecObservation 是顺运宝对某个蝦皮商品规格的历史观测汇总。
// 它不是正式蝦皮 SKU,不得写回 shopee_skus。
type SybSpecObservation struct {
@@ -482,7 +639,7 @@ func sybOrderFilterClause(filter SybOrderFilter) (string, []any) {
const sybOrderContextFrom = `
FROM syb_orders so
LEFT JOIN shopee_products sp ON sp.goods_id = so.shopee_goods_id
LEFT JOIN shopee_products sp ON sp.goods_id = so.shopee_goods_id AND sp.deleted_at IS NULL
LEFT JOIN pdd_products pp ON pp.goods_id = sp.pdd_goods_id AND pp.deleted_at IS NULL
LEFT JOIN spec_mappings sm
ON sm.shopee_goods_id = so.shopee_goods_id
+62
View File
@@ -10,6 +10,7 @@ import (
"cmautobuy/admin/config"
"cmautobuy/admin/model"
"cmautobuy/admin/spec"
)
func newSybTestDB(t *testing.T) *sql.DB {
@@ -140,6 +141,67 @@ func TestUpsertSybOrder_只更新Syb骨架标题(t *testing.T) {
}
}
func TestUpsertSybOrder_店铺图片按字段来源保护(t *testing.T) {
db := newSybTestDB(t)
order := model.SybOrder{SybID: "META-1", OrderNo: "O-META", Title: "商品", ShopeeGoodsID: "SP-META", ProductSpec: "黑色,M", ShopName: "店铺一", ImageURL: "https://example.com/1.jpg", Quantity: 1, SybData: "{}"}
if _, err := UpsertSybOrder(db, order); err != nil {
t.Fatal(err)
}
var shop, image, shopSource, imageSource string
if err := db.QueryRow(`SELECT shopee_shop_name,image_url,shop_name_source,image_source FROM shopee_products WHERE goods_id='SP-META'`).Scan(&shop, &image, &shopSource, &imageSource); err != nil {
t.Fatal(err)
}
if shop != "店铺一" || image != "https://example.com/1.jpg" || shopSource != "syb" || imageSource != "syb" {
t.Fatalf("首次补全不正确:shop=%q image=%q sources=%q/%q", shop, image, shopSource, imageSource)
}
order.ShopName, order.ImageURL = "店铺二", "https://example.com/2.jpg"
if _, err := UpsertSybOrder(db, order); err != nil {
t.Fatal(err)
}
if _, err := db.Exec(`UPDATE shopee_products SET image_url='https://example.com/api.jpg',image_source='api',shop_name_is_manual=1 WHERE goods_id='SP-META'`); err != nil {
t.Fatal(err)
}
order.ShopName, order.ImageURL = "不应覆盖的店铺", "https://example.com/3.jpg"
if _, err := UpsertSybOrder(db, order); err != nil {
t.Fatal(err)
}
if err := db.QueryRow(`SELECT shopee_shop_name,image_url FROM shopee_products WHERE goods_id='SP-META'`).Scan(&shop, &image); err != nil {
t.Fatal(err)
}
if shop != "店铺二" || image != "https://example.com/api.jpg" {
t.Fatalf("字段保护失败:shop=%q image=%q", shop, image)
}
}
func TestUpsertSybShopeeSKU_确定性解析并允许权威来源升级(t *testing.T) {
db := newSybTestDB(t)
now := model.NowISO()
if _, err := db.Exec(`INSERT INTO shopee_products(goods_id,title,source,created_at,updated_at) VALUES('SP-SKU','商品','syb',?,?)`, now, now); err != nil {
t.Fatal(err)
}
parsed, err := UpsertSybShopeeSKU(db, "SP-SKU", "白色,L【建議50-60公斤】", now)
if err != nil || !parsed {
t.Fatalf("SYB 规格应解析成功:parsed=%v err=%v", parsed, err)
}
if parsed, err := UpsertSybShopeeSKU(db, "SP-SKU", "不明确规格", now); err != nil || parsed {
t.Fatalf("不明确规格不应写入:parsed=%v err=%v", parsed, err)
}
key, _ := spec.SpecKey("白色,L【建議50-60公斤】")
outcome, err := UpsertCatalogShopeeSKU(db, CatalogShopeeSKUInput{RecordID: "API-NEW", ShopeeSKUID: "REAL-1", GoodsID: "SP-SKU", SpecRaw: "白色,L【建議50-60公斤】", SpecKey: key, Color: "象牙白", Size: "L", Advice: "建议50-60公斤", ParseOK: true, Source: "partner", ObservedAt: now, Now: now, UpdatePolicy: "fill_missing"})
if err != nil || outcome != CatalogSKUFilled {
t.Fatalf("权威来源升级失败:outcome=%s err=%v", outcome, err)
}
var count int
var external, color, sourceJSON string
if err := db.QueryRow(`SELECT COUNT(*),MAX(COALESCE(shopee_sku_id,'')),MAX(COALESCE(color,'')),MAX(COALESCE(field_sources,'')) FROM shopee_skus WHERE goods_id='SP-SKU'`).Scan(&count, &external, &color, &sourceJSON); err != nil {
t.Fatal(err)
}
if count != 1 || external != "REAL-1" || color != "象牙白" || !strings.Contains(sourceJSON, `"color":"partner"`) {
t.Fatalf("升级后数据不正确:count=%d external=%q color=%q sources=%s", count, external, color, sourceJSON)
}
}
func TestUpsertSybOrder_ShopeeGoodsID会被同步更新(t *testing.T) {
db := newSybTestDB(t)