feat: 展示蝦皮主图和店铺 (#142)

This commit is contained in:
chengma
2026-08-11 11:18:51 +08:00
parent a441dfef87
commit 38d6a0a04b
22 changed files with 555 additions and 101 deletions
+1 -1
View File
@@ -57,7 +57,7 @@ func TestCatalogBatchAPI_按批次查询摘要(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
defer db.Close() defer db.Close()
_, err = db.Exec(`CREATE TABLE catalog_import_runs(source TEXT,batch_id TEXT,request_hash TEXT,status TEXT,request_count INTEGER,conflict_count INTEGER,observed_at TEXT,update_policy TEXT DEFAULT 'fill_missing',last_request_at TEXT,last_conflict_at TEXT,shopee_created INTEGER DEFAULT 0,shopee_updated INTEGER DEFAULT 0,sku_created INTEGER DEFAULT 0,sku_updated INTEGER DEFAULT 0,sku_filled INTEGER DEFAULT 0,sku_same_source_updated INTEGER DEFAULT 0,sku_skipped INTEGER DEFAULT 0,sku_manual_skipped INTEGER DEFAULT 0,sku_stale_skipped INTEGER DEFAULT 0,pdd_created INTEGER DEFAULT 0,pdd_updated INTEGER DEFAULT 0,association_created INTEGER DEFAULT 0,association_unchanged INTEGER DEFAULT 0,failure_count INTEGER DEFAULT 0,error_summary TEXT,response_body TEXT,created_at TEXT,finished_at TEXT,PRIMARY KEY(source,batch_id))`) _, err = db.Exec(`CREATE TABLE catalog_import_runs(source TEXT,batch_id TEXT,request_hash TEXT,status TEXT,request_count INTEGER,conflict_count INTEGER,observed_at TEXT,update_policy TEXT DEFAULT 'fill_missing',last_request_at TEXT,last_conflict_at TEXT,shopee_created INTEGER DEFAULT 0,shopee_updated INTEGER DEFAULT 0,shopee_fields_filled INTEGER DEFAULT 0,shopee_fields_same_source_updated INTEGER DEFAULT 0,shopee_fields_manual_skipped INTEGER DEFAULT 0,shopee_fields_stale_skipped INTEGER DEFAULT 0,sku_created INTEGER DEFAULT 0,sku_updated INTEGER DEFAULT 0,sku_filled INTEGER DEFAULT 0,sku_same_source_updated INTEGER DEFAULT 0,sku_skipped INTEGER DEFAULT 0,sku_manual_skipped INTEGER DEFAULT 0,sku_stale_skipped INTEGER DEFAULT 0,pdd_created INTEGER DEFAULT 0,pdd_updated INTEGER DEFAULT 0,association_created INTEGER DEFAULT 0,association_unchanged INTEGER DEFAULT 0,failure_count INTEGER DEFAULT 0,error_summary TEXT,response_body TEXT,created_at TEXT,finished_at TEXT,PRIMARY KEY(source,batch_id))`)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
+1 -1
View File
@@ -37,7 +37,7 @@ func (h *Handler) GetBatch(c *gin.Context) {
integrationError(c, 500, "INTERNAL_ERROR", "查询批次失败", true, nil) integrationError(c, 500, "INTERNAL_ERROR", "查询批次失败", true, nil)
return return
} }
c.JSON(200, gin.H{"source": run.Source, "batch_id": run.BatchID, "update_policy": run.UpdatePolicy, "status": run.Status, "request_count": run.RequestCount, "conflict_count": run.ConflictCount, "observed_at": run.ObservedAt, "created_at": run.CreatedAt, "finished_at": run.FinishedAt, "counts": gin.H{"shopee_created": run.ShopeeCreated, "shopee_updated": run.ShopeeUpdated, "sku_created": run.SKUCreated, "sku_updated": run.SKUUpdated, "sku_filled": run.SKUFilled, "sku_same_source_updated": run.SKUSameSourceUpdated, "sku_skipped": run.SKUSkipped, "sku_manual_skipped": run.SKUManualSkipped, "sku_stale_skipped": run.SKUStaleSkipped, "pdd_created": run.PddCreated, "pdd_updated": run.PddUpdated, "association_created": run.AssociationCreated, "association_unchanged": run.AssociationUnchanged}, "failure_count": run.FailureCount, "error_summary": run.ErrorSummary}) c.JSON(200, gin.H{"source": run.Source, "batch_id": run.BatchID, "update_policy": run.UpdatePolicy, "status": run.Status, "request_count": run.RequestCount, "conflict_count": run.ConflictCount, "observed_at": run.ObservedAt, "created_at": run.CreatedAt, "finished_at": run.FinishedAt, "counts": gin.H{"shopee_created": run.ShopeeCreated, "shopee_updated": run.ShopeeUpdated, "shopee_fields_filled": run.ShopeeFieldsFilled, "shopee_fields_same_source_updated": run.ShopeeFieldsSameSourceUpdated, "shopee_fields_manual_skipped": run.ShopeeFieldsManualSkipped, "shopee_fields_stale_skipped": run.ShopeeFieldsStaleSkipped, "sku_created": run.SKUCreated, "sku_updated": run.SKUUpdated, "sku_filled": run.SKUFilled, "sku_same_source_updated": run.SKUSameSourceUpdated, "sku_skipped": run.SKUSkipped, "sku_manual_skipped": run.SKUManualSkipped, "sku_stale_skipped": run.SKUStaleSkipped, "pdd_created": run.PddCreated, "pdd_updated": run.PddUpdated, "association_created": run.AssociationCreated, "association_unchanged": run.AssociationUnchanged}, "failure_count": run.FailureCount, "error_summary": run.ErrorSummary})
} }
func (h *Handler) CreateBatch(c *gin.Context) { func (h *Handler) CreateBatch(c *gin.Context) {
+25
View File
@@ -168,6 +168,31 @@ func TestCatalogHistory_展示更新策略和分级统计(t *testing.T) {
} }
} }
func TestShopeePage_展示主图店铺且保留标题宽度(t *testing.T) {
page, err := os.ReadFile("templates/shopee/list.html")
if err != nil {
t.Fatal(err)
}
detail, err := os.ReadFile("templates/shopee/detail_modal.html")
if err != nil {
t.Fatal(err)
}
css, err := os.ReadFile("static/css/app.css")
if err != nil {
t.Fatal(err)
}
js, err := os.ReadFile("static/js/app.js")
if err != nil {
t.Fatal(err)
}
content := string(page) + string(detail) + string(css) + string(js)
for _, want := range []string{"蝦皮店铺", "data-image-preview-url", "loading=\"lazy\"", "data-image-thumb-fallback", "ImageSourceText", "max-width: 18%", ".col-shop", "setupImageThumbnails"} {
if !strings.Contains(content, want) {
t.Errorf("蝦皮图片店铺界面缺少 %q", want)
}
}
}
func TestShopeeImportRoute_已移除(t *testing.T) { func TestShopeeImportRoute_已移除(t *testing.T) {
router, err := newRouter(nil) router, err := newRouter(nil)
if err != nil { if err != nil {
+32 -28
View File
@@ -11,32 +11,36 @@ const (
// CatalogImportRun 保存批次追踪信息,不保存 Token 或完整请求体。 // CatalogImportRun 保存批次追踪信息,不保存 Token 或完整请求体。
type CatalogImportRun struct { type CatalogImportRun struct {
Source string Source string
BatchID string BatchID string
RequestHash string RequestHash string
Status CatalogImportStatus Status CatalogImportStatus
RequestCount int RequestCount int
ConflictCount int ConflictCount int
ObservedAt string ObservedAt string
LastRequestAt string LastRequestAt string
LastConflictAt string LastConflictAt string
ShopeeCreated int ShopeeCreated int
ShopeeUpdated int ShopeeUpdated int
SKUCreated int ShopeeFieldsFilled int
SKUUpdated int ShopeeFieldsSameSourceUpdated int
UpdatePolicy string ShopeeFieldsManualSkipped int
SKUFilled int ShopeeFieldsStaleSkipped int
SKUSameSourceUpdated int SKUCreated int
SKUSkipped int SKUUpdated int
SKUManualSkipped int UpdatePolicy string
SKUStaleSkipped int SKUFilled int
PddCreated int SKUSameSourceUpdated int
PddUpdated int SKUSkipped int
AssociationCreated int SKUManualSkipped int
AssociationUnchanged int SKUStaleSkipped int
FailureCount int PddCreated int
ErrorSummary string PddUpdated int
ResponseBody string AssociationCreated int
CreatedAt string AssociationUnchanged int
FinishedAt string FailureCount int
ErrorSummary string
ResponseBody string
CreatedAt string
FinishedAt string
} }
+15 -9
View File
@@ -164,15 +164,21 @@ func (p PddProduct) IsDeleted() bool {
// //
// 采集结果和采集状态**不在这里**——它们属于 PDD 商品,见 PddProduct。 // 采集结果和采集状态**不在这里**——它们属于 PDD 商品,见 PddProduct。
type ShopeeProduct struct { type ShopeeProduct struct {
GoodsID string GoodsID string
Title string Title string
ShopeeStatus string ShopeeStatus string
MainSKUCode string MainSKUCode string
Source string // report=蝦皮报表完整行;syb=顺运宝同步补建的商品骨架 ImageURL string
PddGoodsURL string ShopeeShopName string
PddGoodsID string // 指向 PddProduct.GoodsID,为空表示还没填链接 ImageSource, ImageObservedAt string
CreatedAt string ImageIsManual bool
UpdatedAt string ShopNameSource, ShopNameObservedAt string
ShopNameIsManual bool
Source string // report=蝦皮报表完整行;syb=顺运宝同步补建的商品骨架
PddGoodsURL string
PddGoodsID string // 指向 PddProduct.GoodsID,为空表示还没填链接
CreatedAt string
UpdatedAt string
} }
// ShopeeSKU 是蝦皮的一个规格(SKU 级)。 // ShopeeSKU 是蝦皮的一个规格(SKU 级)。
+59 -18
View File
@@ -43,13 +43,13 @@ func GetCatalogImportRun(q Execer, source, batchID string) (model.CatalogImportR
var run model.CatalogImportRun var run model.CatalogImportRun
var observedAt, lastConflictAt, errorSummary, responseBody, finishedAt sql.NullString var observedAt, lastConflictAt, errorSummary, responseBody, finishedAt sql.NullString
err := q.QueryRow(`SELECT source,batch_id,request_hash,status,request_count,conflict_count, err := q.QueryRow(`SELECT source,batch_id,request_hash,status,request_count,conflict_count,
observed_at,update_policy,last_request_at,last_conflict_at,shopee_created,shopee_updated,sku_created, observed_at,update_policy,last_request_at,last_conflict_at,shopee_created,shopee_updated,shopee_fields_filled,shopee_fields_same_source_updated,shopee_fields_manual_skipped,shopee_fields_stale_skipped,sku_created,
sku_updated,sku_filled,sku_same_source_updated,sku_skipped,sku_manual_skipped,sku_stale_skipped, sku_updated,sku_filled,sku_same_source_updated,sku_skipped,sku_manual_skipped,sku_stale_skipped,
pdd_created,pdd_updated,association_created,association_unchanged,failure_count, pdd_created,pdd_updated,association_created,association_unchanged,failure_count,
error_summary,response_body,created_at,finished_at error_summary,response_body,created_at,finished_at
FROM catalog_import_runs WHERE source=? AND batch_id=?`, source, batchID).Scan( FROM catalog_import_runs WHERE source=? AND batch_id=?`, source, batchID).Scan(
&run.Source, &run.BatchID, &run.RequestHash, &run.Status, &run.RequestCount, &run.ConflictCount, &run.Source, &run.BatchID, &run.RequestHash, &run.Status, &run.RequestCount, &run.ConflictCount,
&observedAt, &run.UpdatePolicy, &run.LastRequestAt, &lastConflictAt, &run.ShopeeCreated, &run.ShopeeUpdated, &observedAt, &run.UpdatePolicy, &run.LastRequestAt, &lastConflictAt, &run.ShopeeCreated, &run.ShopeeUpdated, &run.ShopeeFieldsFilled, &run.ShopeeFieldsSameSourceUpdated, &run.ShopeeFieldsManualSkipped, &run.ShopeeFieldsStaleSkipped,
&run.SKUCreated, &run.SKUUpdated, &run.SKUFilled, &run.SKUSameSourceUpdated, &run.SKUSkipped, &run.SKUCreated, &run.SKUUpdated, &run.SKUFilled, &run.SKUSameSourceUpdated, &run.SKUSkipped,
&run.SKUManualSkipped, &run.SKUStaleSkipped, &run.PddCreated, &run.PddUpdated, &run.AssociationCreated, &run.SKUManualSkipped, &run.SKUStaleSkipped, &run.PddCreated, &run.PddUpdated, &run.AssociationCreated,
&run.AssociationUnchanged, &run.FailureCount, &errorSummary, &responseBody, &run.CreatedAt, &finishedAt) &run.AssociationUnchanged, &run.FailureCount, &errorSummary, &responseBody, &run.CreatedAt, &finishedAt)
@@ -149,37 +149,78 @@ func RecordCatalogImportConflict(q Execer, source, batchID, requestedAt string)
// CompleteCatalogImportRun 把处理结果摘要和安全的响应 JSON 固化,供幂等重放。 // CompleteCatalogImportRun 把处理结果摘要和安全的响应 JSON 固化,供幂等重放。
func CompleteCatalogImportRun(q Execer, run model.CatalogImportRun) error { func CompleteCatalogImportRun(q Execer, run model.CatalogImportRun) error {
result, err := q.Exec(`UPDATE catalog_import_runs SET status=?,shopee_created=?,shopee_updated=?, result, err := q.Exec(`UPDATE catalog_import_runs SET status=?,shopee_created=?,shopee_updated=?,shopee_fields_filled=?,shopee_fields_same_source_updated=?,shopee_fields_manual_skipped=?,shopee_fields_stale_skipped=?,
sku_created=?,sku_updated=?,sku_filled=?,sku_same_source_updated=?,sku_skipped=?, sku_created=?,sku_updated=?,sku_filled=?,sku_same_source_updated=?,sku_skipped=?,
sku_manual_skipped=?,sku_stale_skipped=?,pdd_created=?,pdd_updated=?,association_created=?, sku_manual_skipped=?,sku_stale_skipped=?,pdd_created=?,pdd_updated=?,association_created=?,
association_unchanged=?,failure_count=?,error_summary=NULLIF(?,''),response_body=NULLIF(?,''), association_unchanged=?,failure_count=?,error_summary=NULLIF(?,''),response_body=NULLIF(?,''),
finished_at=? WHERE source=? AND batch_id=?`, run.Status, run.ShopeeCreated, run.ShopeeUpdated, finished_at=? WHERE source=? AND batch_id=?`, run.Status, run.ShopeeCreated, run.ShopeeUpdated,
run.SKUCreated, run.SKUUpdated, run.SKUFilled, run.SKUSameSourceUpdated, run.SKUSkipped, run.ShopeeFieldsFilled, run.ShopeeFieldsSameSourceUpdated, run.ShopeeFieldsManualSkipped, run.ShopeeFieldsStaleSkipped, run.SKUCreated, run.SKUUpdated, run.SKUFilled, run.SKUSameSourceUpdated, run.SKUSkipped,
run.SKUManualSkipped, run.SKUStaleSkipped, run.PddCreated, run.PddUpdated, run.AssociationCreated, run.SKUManualSkipped, run.SKUStaleSkipped, run.PddCreated, run.PddUpdated, run.AssociationCreated,
run.AssociationUnchanged, run.FailureCount, run.ErrorSummary, run.ResponseBody, run.AssociationUnchanged, run.FailureCount, run.ErrorSummary, run.ResponseBody,
run.FinishedAt, run.Source, run.BatchID) run.FinishedAt, run.Source, run.BatchID)
return catalogRunUpdateResult(result, err, "完成商品目录批次") return catalogRunUpdateResult(result, err, "完成商品目录批次")
} }
// UpsertCatalogShopeeProduct 按上游观测时间更新来源字段,绝不覆盖人工 PDD 关联。 type CatalogShopeeProductInput struct{ GoodsID, Title, Status, MainSKU, ImageURL, ShopName, Source, ObservedAt, Now, UpdatePolicy string }
func UpsertCatalogShopeeProduct(q Execer, goodsID, title, status, mainSKU, observedAt, now string) (created, updated bool, err error) { type CatalogProductOutcome struct {
var oldObserved sql.NullString Created, Updated bool
err = q.QueryRow(`SELECT source_observed_at FROM shopee_products WHERE goods_id=?`, goodsID).Scan(&oldObserved) FieldsFilled, FieldsSameSourceUpdated, FieldsManualSkipped, FieldsStaleSkipped int
}
// UpsertCatalogShopeeProduct 独立保护主图和店铺字段,绝不覆盖人工 PDD 关联。
func UpsertCatalogShopeeProduct(q Execer, in CatalogShopeeProductInput) (out CatalogProductOutcome, err error) {
var oldObserved, imageURL, shopName, imageSource, imageObserved, shopSource, shopObserved sql.NullString
var imageManual, shopManual int
err = q.QueryRow(`SELECT source_observed_at,image_url,shopee_shop_name,image_source,image_observed_at,image_is_manual,shop_name_source,shop_name_observed_at,shop_name_is_manual FROM shopee_products WHERE goods_id=?`, in.GoodsID).Scan(&oldObserved, &imageURL, &shopName, &imageSource, &imageObserved, &imageManual, &shopSource, &shopObserved, &shopManual)
if errors.Is(err, sql.ErrNoRows) { if errors.Is(err, sql.ErrNoRows) {
_, err = q.Exec(`INSERT INTO shopee_products(goods_id,title,shopee_status,main_sku_code,source, _, err = q.Exec(`INSERT INTO shopee_products(goods_id,title,shopee_status,main_sku_code,image_url,shopee_shop_name,image_source,image_observed_at,image_is_manual,shop_name_source,shop_name_observed_at,shop_name_is_manual,source,source_observed_at,created_at,updated_at) VALUES(?,?,NULLIF(?,''),NULLIF(?,''),NULLIF(?,''),NULLIF(?,''),CASE WHEN TRIM(?)='' THEN NULL ELSE ? END,CASE WHEN TRIM(?)='' THEN NULL ELSE ? END,0,CASE WHEN TRIM(?)='' THEN NULL ELSE ? END,CASE WHEN TRIM(?)='' THEN NULL ELSE ? END,0,'api',?,?,?)`, in.GoodsID, in.Title, in.Status, in.MainSKU, in.ImageURL, in.ShopName, in.ImageURL, in.Source, in.ImageURL, in.ObservedAt, in.ShopName, in.Source, in.ShopName, in.ObservedAt, in.ObservedAt, in.Now, in.Now)
source_observed_at,created_at,updated_at) VALUES(?,?,NULLIF(?,''),NULLIF(?,''),'api',?,?,?)`, out.Created = err == nil
goodsID, title, status, mainSKU, observedAt, now, now) return out, err
return err == nil, false, err
} }
if err != nil { if err != nil {
return false, false, err return out, err
} }
if oldObserved.Valid && oldObserved.String > observedAt { baseUpdate := !oldObserved.Valid || oldObserved.String <= in.ObservedAt
return false, false, nil newImage, newShop := imageURL.String, shopName.String
imageChanged, shopChanged := false, false
apply := func(incoming string, current *string, source, observed sql.NullString, manual int) (changed bool) {
if strings.TrimSpace(incoming) == "" || in.UpdatePolicy == "insert_only" {
return false
}
if manual != 0 {
out.FieldsManualSkipped++
return false
}
if in.UpdatePolicy == "fill_missing" {
if *current == "" {
*current = incoming
out.FieldsFilled++
return true
}
return false
}
if !source.Valid || source.String != in.Source {
return false
}
if observed.Valid && observed.String > in.ObservedAt {
out.FieldsStaleSkipped++
return false
}
if *current != incoming {
*current = incoming
out.FieldsSameSourceUpdated++
return true
}
return false
} }
_, err = q.Exec(`UPDATE shopee_products SET title=?,shopee_status=NULLIF(?,''),main_sku_code=NULLIF(?,''), imageChanged = apply(in.ImageURL, &newImage, imageSource, imageObserved, imageManual)
source='api',source_observed_at=?,updated_at=? WHERE goods_id=?`, title, status, mainSKU, observedAt, now, goodsID) shopChanged = apply(in.ShopName, &newShop, shopSource, shopObserved, shopManual)
return false, err == nil, err if !baseUpdate && !imageChanged && !shopChanged {
return out, nil
}
_, err = q.Exec(`UPDATE shopee_products SET title=CASE WHEN ? THEN ? ELSE title END,shopee_status=CASE WHEN ? THEN NULLIF(?,'') ELSE shopee_status END,main_sku_code=CASE WHEN ? THEN NULLIF(?,'') ELSE main_sku_code END,source=CASE WHEN ? THEN 'api' ELSE source END,source_observed_at=CASE WHEN ? THEN ? ELSE source_observed_at END,image_url=NULLIF(?,''),shopee_shop_name=NULLIF(?,''),image_source=CASE WHEN ? THEN ? ELSE image_source END,image_observed_at=CASE WHEN ? THEN ? ELSE image_observed_at END,shop_name_source=CASE WHEN ? THEN ? ELSE shop_name_source END,shop_name_observed_at=CASE WHEN ? THEN ? ELSE shop_name_observed_at END,updated_at=? WHERE goods_id=?`, baseUpdate, in.Title, baseUpdate, in.Status, baseUpdate, in.MainSKU, baseUpdate, baseUpdate, in.ObservedAt, newImage, newShop, imageChanged, in.Source, imageChanged, in.ObservedAt, shopChanged, in.Source, shopChanged, in.ObservedAt, in.Now, in.GoodsID)
out.Updated = err == nil
return out, err
} }
type CatalogSKUOutcome string type CatalogSKUOutcome string
+1 -1
View File
@@ -20,7 +20,7 @@ func TestCatalogImportRun_登记查询重放和冲突(t *testing.T) {
if _, err := db.Exec(`CREATE TABLE catalog_import_runs ( if _, err := db.Exec(`CREATE TABLE catalog_import_runs (
source TEXT NOT NULL,batch_id TEXT NOT NULL,request_hash TEXT NOT NULL,status TEXT NOT NULL, source TEXT NOT NULL,batch_id TEXT NOT NULL,request_hash TEXT NOT NULL,status TEXT NOT NULL,
request_count INTEGER NOT NULL,conflict_count INTEGER NOT NULL,observed_at TEXT,update_policy TEXT NOT NULL DEFAULT 'fill_missing',last_request_at TEXT NOT NULL, request_count INTEGER NOT NULL,conflict_count INTEGER NOT NULL,observed_at TEXT,update_policy TEXT NOT NULL DEFAULT 'fill_missing',last_request_at TEXT NOT NULL,
last_conflict_at TEXT,shopee_created INTEGER NOT NULL DEFAULT 0,shopee_updated INTEGER NOT NULL DEFAULT 0, last_conflict_at TEXT,shopee_created INTEGER NOT NULL DEFAULT 0,shopee_updated INTEGER NOT NULL DEFAULT 0,shopee_fields_filled INTEGER NOT NULL DEFAULT 0,shopee_fields_same_source_updated INTEGER NOT NULL DEFAULT 0,shopee_fields_manual_skipped INTEGER NOT NULL DEFAULT 0,shopee_fields_stale_skipped INTEGER NOT NULL DEFAULT 0,
sku_created INTEGER NOT NULL DEFAULT 0,sku_updated INTEGER NOT NULL DEFAULT 0,sku_filled INTEGER NOT NULL DEFAULT 0,sku_same_source_updated INTEGER NOT NULL DEFAULT 0,sku_skipped INTEGER NOT NULL DEFAULT 0,sku_manual_skipped INTEGER NOT NULL DEFAULT 0,sku_stale_skipped INTEGER NOT NULL DEFAULT 0,pdd_created INTEGER NOT NULL DEFAULT 0, sku_created INTEGER NOT NULL DEFAULT 0,sku_updated INTEGER NOT NULL DEFAULT 0,sku_filled INTEGER NOT NULL DEFAULT 0,sku_same_source_updated INTEGER NOT NULL DEFAULT 0,sku_skipped INTEGER NOT NULL DEFAULT 0,sku_manual_skipped INTEGER NOT NULL DEFAULT 0,sku_stale_skipped INTEGER NOT NULL DEFAULT 0,pdd_created INTEGER NOT NULL DEFAULT 0,
pdd_updated INTEGER NOT NULL DEFAULT 0,association_created INTEGER NOT NULL DEFAULT 0, pdd_updated INTEGER NOT NULL DEFAULT 0,association_created INTEGER NOT NULL DEFAULT 0,
association_unchanged INTEGER NOT NULL DEFAULT 0,failure_count INTEGER NOT NULL DEFAULT 0, association_unchanged INTEGER NOT NULL DEFAULT 0,failure_count INTEGER NOT NULL DEFAULT 0,
+98 -2
View File
@@ -19,7 +19,7 @@ import (
"cmautobuy/admin/spec" "cmautobuy/admin/spec"
) )
const mysqlSchemaVersion = 8 const mysqlSchemaVersion = 9
// OpenMySQL 打开生产 MySQL 8 数据库。错误信息绝不包含完整 DSN 或密码。 // OpenMySQL 打开生产 MySQL 8 数据库。错误信息绝不包含完整 DSN 或密码。
func OpenMySQL(cfg config.DatabaseConfig) (*sql.DB, error) { func OpenMySQL(cfg config.DatabaseConfig) (*sql.DB, error) {
@@ -519,10 +519,65 @@ func MigrateMySQL(db *sql.DB) error {
if _, err := db.Exec(`INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)`, 8, time.Now().UTC().Format(time.RFC3339Nano)); err != nil { if _, err := db.Exec(`INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)`, 8, time.Now().UTC().Format(time.RFC3339Nano)); err != nil {
return fmt.Errorf("记录 MySQL schema v8 失败: %w", err) return fmt.Errorf("记录 MySQL schema v8 失败: %w", err)
} }
current = 8
}
if current < 9 {
if err := migrateMySQLV9(db); err != nil {
return fmt.Errorf("执行 MySQL schema v9 失败: %w", err)
}
if err := checkMySQLV9Shape(db); err != nil {
return fmt.Errorf("MySQL schema v9 自检失败,未记录版本: %w", err)
}
if _, err := db.Exec(`INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)`, 9, time.Now().UTC().Format(time.RFC3339Nano)); err != nil {
return fmt.Errorf("记录 MySQL schema v9 失败: %w", err)
}
} }
return CheckMySQLSchema(db) return CheckMySQLSchema(db)
} }
func migrateMySQLV9(db *sql.DB) error {
columns := []struct{ table, name, ddl string }{
{"shopee_products", "image_url", `ALTER TABLE shopee_products ADD COLUMN image_url VARCHAR(2048) NULL AFTER main_sku_code`},
{"shopee_products", "shopee_shop_name", `ALTER TABLE shopee_products ADD COLUMN shopee_shop_name VARCHAR(500) NULL AFTER image_url`},
{"shopee_products", "image_source", `ALTER TABLE shopee_products ADD COLUMN image_source VARCHAR(64) COLLATE utf8mb4_bin NULL AFTER shopee_shop_name`},
{"shopee_products", "image_observed_at", `ALTER TABLE shopee_products ADD COLUMN image_observed_at VARCHAR(35) NULL AFTER image_source`},
{"shopee_products", "image_is_manual", `ALTER TABLE shopee_products ADD COLUMN image_is_manual TINYINT NOT NULL DEFAULT 0 AFTER image_observed_at`},
{"shopee_products", "shop_name_source", `ALTER TABLE shopee_products ADD COLUMN shop_name_source VARCHAR(64) COLLATE utf8mb4_bin NULL AFTER image_is_manual`},
{"shopee_products", "shop_name_observed_at", `ALTER TABLE shopee_products ADD COLUMN shop_name_observed_at VARCHAR(35) NULL AFTER shop_name_source`},
{"shopee_products", "shop_name_is_manual", `ALTER TABLE shopee_products ADD COLUMN shop_name_is_manual TINYINT NOT NULL DEFAULT 0 AFTER shop_name_observed_at`},
{"catalog_import_runs", "shopee_fields_filled", `ALTER TABLE catalog_import_runs ADD COLUMN shopee_fields_filled INT UNSIGNED NOT NULL DEFAULT 0 AFTER shopee_updated`},
{"catalog_import_runs", "shopee_fields_same_source_updated", `ALTER TABLE catalog_import_runs ADD COLUMN shopee_fields_same_source_updated INT UNSIGNED NOT NULL DEFAULT 0 AFTER shopee_fields_filled`},
{"catalog_import_runs", "shopee_fields_manual_skipped", `ALTER TABLE catalog_import_runs ADD COLUMN shopee_fields_manual_skipped INT UNSIGNED NOT NULL DEFAULT 0 AFTER shopee_fields_same_source_updated`},
{"catalog_import_runs", "shopee_fields_stale_skipped", `ALTER TABLE catalog_import_runs ADD COLUMN shopee_fields_stale_skipped INT UNSIGNED NOT NULL DEFAULT 0 AFTER shopee_fields_manual_skipped`},
}
for _, column := range columns {
exists, err := mysqlColumnExists(db, column.table, column.name)
if err != nil {
return err
}
if !exists {
if _, err := db.Exec(column.ddl); err != nil {
return fmt.Errorf("增加 %s.%s 失败: %w", column.table, column.name, err)
}
}
}
for _, constraint := range []struct{ name, ddl string }{
{"chk_shopee_products_image_manual", `ALTER TABLE shopee_products ADD CONSTRAINT chk_shopee_products_image_manual CHECK (image_is_manual IN (0,1))`},
{"chk_shopee_products_shop_manual", `ALTER TABLE shopee_products ADD CONSTRAINT chk_shopee_products_shop_manual CHECK (shop_name_is_manual IN (0,1))`},
} {
exists, err := mysqlConstraintExists(db, "shopee_products", constraint.name)
if err != nil {
return err
}
if !exists {
if _, err := db.Exec(constraint.ddl); err != nil {
return fmt.Errorf("增加约束 %s 失败: %w", constraint.name, err)
}
}
}
return nil
}
// migrateMySQLV8 保留 sku_id 作为内部主键,并增加可空的真实蝦皮 SKU ID。 // migrateMySQLV8 保留 sku_id 作为内部主键,并增加可空的真实蝦皮 SKU ID。
// 每个 DDL 都先检查形状,支持 MySQL 隐式提交后的中断重放。 // 每个 DDL 都先检查形状,支持 MySQL 隐式提交后的中断重放。
func migrateMySQLV8(db *sql.DB) error { func migrateMySQLV8(db *sql.DB) error {
@@ -1056,7 +1111,48 @@ func CheckMySQLSchema(db *sql.DB) error {
if err := checkMySQLV7Shape(db); err != nil { if err := checkMySQLV7Shape(db); err != nil {
return err return err
} }
return checkMySQLV8Shape(db) if err := checkMySQLV8Shape(db); err != nil {
return err
}
return checkMySQLV9Shape(db)
}
func checkMySQLV9Shape(db *sql.DB) error {
for _, column := range []struct {
name string
length int64
nullable bool
collation, def string
}{
{"image_url", 2048, true, "utf8mb4_0900_ai_ci", ""}, {"shopee_shop_name", 500, true, "utf8mb4_0900_ai_ci", ""},
{"image_source", 64, true, "utf8mb4_bin", ""}, {"image_observed_at", 35, true, "utf8mb4_0900_ai_ci", ""},
{"shop_name_source", 64, true, "utf8mb4_bin", ""}, {"shop_name_observed_at", 35, true, "utf8mb4_0900_ai_ci", ""},
} {
if err := checkMySQLVarcharColumn(db, "shopee_products", column.name, column.length, column.nullable, column.collation, column.def); err != nil {
return err
}
}
for _, name := range []string{"image_is_manual", "shop_name_is_manual"} {
var dataType, isNullable string
var defaultValue sql.NullString
err := db.QueryRow(`SELECT data_type,is_nullable,column_default FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='shopee_products' AND column_name=?`, name).Scan(&dataType, &isNullable, &defaultValue)
if err != nil || dataType != "tinyint" || isNullable != "NO" || !defaultValue.Valid || defaultValue.String != "0" {
return fmt.Errorf("蝦皮商品字段 %s 形状不正确", name)
}
}
for _, name := range []string{"chk_shopee_products_image_manual", "chk_shopee_products_shop_manual"} {
exists, err := mysqlConstraintExists(db, "shopee_products", name)
if err != nil || !exists {
return fmt.Errorf("蝦皮商品约束 %s 缺失", name)
}
}
for _, name := range []string{"shopee_fields_filled", "shopee_fields_same_source_updated", "shopee_fields_manual_skipped", "shopee_fields_stale_skipped"} {
exists, err := mysqlColumnExists(db, "catalog_import_runs", name)
if err != nil || !exists {
return fmt.Errorf("目录记录字段 %s 缺失", name)
}
}
return nil
} }
func checkMySQLV8Shape(db *sql.DB) error { func checkMySQLV8Shape(db *sql.DB) error {
@@ -429,6 +429,45 @@ func TestMySQLMigrate_V8重复规格停止且不记版本(t *testing.T) {
} }
} }
func TestMySQLMigrate_V8升级V9且断点重放(t *testing.T) {
db := openMySQLMigrationTestDB(t)
defer db.Close()
cleanMySQLTestSchema(t, db)
defer cleanMySQLTestSchema(t, db)
prepareMySQLV7(t, db)
if err := migrateMySQLV8(db); err != nil {
t.Fatal(err)
}
mustExec(t, db, `INSERT INTO schema_migrations(version,applied_at) VALUES(8,'2026-08-11T00:00:00Z')`)
if err := migrateMySQLV9(db); err != nil {
t.Fatal(err)
}
if err := MigrateMySQL(db); err != nil {
t.Fatal(err)
}
if err := MigrateMySQL(db); err != nil {
t.Fatalf("v9 重放失败: %v", err)
}
mustExec(t, db, `INSERT INTO shopee_products(goods_id,title,source,image_url,shopee_shop_name,image_is_manual,shop_name_is_manual,created_at,updated_at) VALUES('S','商品','api','https://example.com/a.jpg','店铺',0,0,'2026-08-11T00:00:00Z','2026-08-11T00:00:00Z')`)
}
func TestMySQLMigrate_V9形状错误不记版本(t *testing.T) {
db := openMySQLMigrationTestDB(t)
defer db.Close()
cleanMySQLTestSchema(t, db)
defer cleanMySQLTestSchema(t, db)
prepareMySQLV8(t, db)
mustExec(t, db, `ALTER TABLE shopee_products ADD COLUMN image_url VARCHAR(10) NULL`)
if err := MigrateMySQL(db); err == nil {
t.Fatal("错误 image_url 形状必须阻止 v9")
}
var count int
db.QueryRow(`SELECT COUNT(*) FROM schema_migrations WHERE version=9`).Scan(&count)
if count != 0 {
t.Fatal("v9 自检失败不得记录版本")
}
}
func openMySQLMigrationTestDB(t *testing.T) *sql.DB { func openMySQLMigrationTestDB(t *testing.T) *sql.DB {
t.Helper() t.Helper()
if os.Getenv("CMAUTOBUY_MYSQL_TEST") != "1" { if os.Getenv("CMAUTOBUY_MYSQL_TEST") != "1" {
@@ -492,6 +531,15 @@ func prepareMySQLV7(t *testing.T, db *sql.DB) {
mustExec(t, db, `INSERT INTO schema_migrations(version,applied_at) VALUES (7,'2026-08-11T00:00:00Z')`) mustExec(t, db, `INSERT INTO schema_migrations(version,applied_at) VALUES (7,'2026-08-11T00:00:00Z')`)
} }
func prepareMySQLV8(t *testing.T, db *sql.DB) {
t.Helper()
prepareMySQLV7(t, db)
if err := migrateMySQLV8(db); err != nil {
t.Fatal(err)
}
mustExec(t, db, `INSERT INTO schema_migrations(version,applied_at) VALUES(8,'2026-08-11T00:00:00Z')`)
}
func mustExec(t *testing.T, db *sql.DB, query string, args ...any) { func mustExec(t *testing.T, db *sql.DB, query string, args ...any) {
t.Helper() t.Helper()
if _, err := db.Exec(query, args...); err != nil { if _, err := db.Exec(query, args...); err != nil {
+24 -6
View File
@@ -178,7 +178,7 @@ func shopeeFilterClause(filter ShopeeFilter) (string, []any) {
func ListShopeeProducts(q Execer, filter ShopeeFilter, limit, offset int) ([]ShopeeProductRow, error) { func ListShopeeProducts(q Execer, filter ShopeeFilter, limit, offset int) ([]ShopeeProductRow, error) {
where, args := shopeeFilterClause(filter) where, args := shopeeFilterClause(filter)
sqlText := ` sqlText := `
SELECT sp.goods_id, sp.title, sp.shopee_status, sp.main_sku_code, sp.source, SELECT sp.goods_id, sp.title, sp.shopee_status, sp.main_sku_code, sp.image_url,sp.shopee_shop_name,sp.image_source,sp.image_observed_at,sp.image_is_manual,sp.shop_name_source,sp.shop_name_observed_at,sp.shop_name_is_manual,sp.source,
sp.pdd_goods_url, sp.pdd_goods_id, sp.created_at, sp.updated_at, sp.pdd_goods_url, sp.pdd_goods_id, sp.created_at, sp.updated_at,
COUNT(DISTINCT CASE WHEN sk.parse_ok = 1 THEN sk.color END) AS color_count, COUNT(DISTINCT CASE WHEN sk.parse_ok = 1 THEN sk.color END) AS color_count,
COUNT(DISTINCT CASE WHEN sk.parse_ok = 1 THEN sk.size END) AS size_count, COUNT(DISTINCT CASE WHEN sk.parse_ok = 1 THEN sk.size END) AS size_count,
@@ -201,9 +201,10 @@ func ListShopeeProducts(q Execer, filter ShopeeFilter, limit, offset int) ([]Sho
var list []ShopeeProductRow var list []ShopeeProductRow
for rows.Next() { for rows.Next() {
var r ShopeeProductRow var r ShopeeProductRow
var title, shopeeStatus, mainSKUCode, source, pddGoodsURL, pddGoodsID sql.NullString var title, shopeeStatus, mainSKUCode, imageURL, shopName, imageSource, imageObserved, shopSource, shopObserved, source, pddGoodsURL, pddGoodsID sql.NullString
var imageManual, shopManual int
if err := rows.Scan( if err := rows.Scan(
&r.GoodsID, &title, &shopeeStatus, &mainSKUCode, &source, &r.GoodsID, &title, &shopeeStatus, &mainSKUCode, &imageURL, &shopName, &imageSource, &imageObserved, &imageManual, &shopSource, &shopObserved, &shopManual, &source,
&pddGoodsURL, &pddGoodsID, &r.CreatedAt, &r.UpdatedAt, &pddGoodsURL, &pddGoodsID, &r.CreatedAt, &r.UpdatedAt,
&r.ColorCount, &r.SizeCount, &r.SKUCount, &r.PendingCount, &r.ColorCount, &r.SizeCount, &r.SKUCount, &r.PendingCount,
&r.CollectStatus, &r.CollectMsg, &r.CollectStatus, &r.CollectMsg,
@@ -213,6 +214,14 @@ func ListShopeeProducts(q Execer, filter ShopeeFilter, limit, offset int) ([]Sho
r.Title = title.String r.Title = title.String
r.ShopeeStatus = shopeeStatus.String r.ShopeeStatus = shopeeStatus.String
r.MainSKUCode = mainSKUCode.String r.MainSKUCode = mainSKUCode.String
r.ImageURL = imageURL.String
r.ShopeeShopName = shopName.String
r.ImageSource = imageSource.String
r.ImageObservedAt = imageObserved.String
r.ImageIsManual = imageManual != 0
r.ShopNameSource = shopSource.String
r.ShopNameObservedAt = shopObserved.String
r.ShopNameIsManual = shopManual != 0
r.Source = source.String r.Source = source.String
r.PddGoodsURL = pddGoodsURL.String r.PddGoodsURL = pddGoodsURL.String
r.PddGoodsID = pddGoodsID.String r.PddGoodsID = pddGoodsID.String
@@ -240,12 +249,13 @@ func CountShopeeProductsFiltered(q Execer, filter ShopeeFilter) (int, error) {
// 弹窗组装商品信息时用。查不到返回 (nil, nil)。 // 弹窗组装商品信息时用。查不到返回 (nil, nil)。
func GetShopeeProductByGoodsID(q Execer, goodsID string) (*model.ShopeeProduct, error) { func GetShopeeProductByGoodsID(q Execer, goodsID string) (*model.ShopeeProduct, error) {
var p model.ShopeeProduct var p model.ShopeeProduct
var title, shopeeStatus, mainSKUCode, source, pddGoodsURL, pddGoodsID sql.NullString var title, shopeeStatus, mainSKUCode, imageURL, shopName, imageSource, imageObserved, shopSource, shopObserved, source, pddGoodsURL, pddGoodsID sql.NullString
var imageManual, shopManual int
err := q.QueryRow(` err := q.QueryRow(`
SELECT goods_id, title, shopee_status, main_sku_code, source, SELECT goods_id, title, shopee_status, main_sku_code,image_url,shopee_shop_name,image_source,image_observed_at,image_is_manual,shop_name_source,shop_name_observed_at,shop_name_is_manual, source,
pdd_goods_url, pdd_goods_id, created_at, updated_at pdd_goods_url, pdd_goods_id, created_at, updated_at
FROM shopee_products WHERE goods_id = ?`, goodsID).Scan( FROM shopee_products WHERE goods_id = ?`, goodsID).Scan(
&p.GoodsID, &title, &shopeeStatus, &mainSKUCode, &source, &p.GoodsID, &title, &shopeeStatus, &mainSKUCode, &imageURL, &shopName, &imageSource, &imageObserved, &imageManual, &shopSource, &shopObserved, &shopManual, &source,
&pddGoodsURL, &pddGoodsID, &p.CreatedAt, &p.UpdatedAt) &pddGoodsURL, &pddGoodsID, &p.CreatedAt, &p.UpdatedAt)
if errors.Is(err, sql.ErrNoRows) { if errors.Is(err, sql.ErrNoRows) {
return nil, nil return nil, nil
@@ -256,6 +266,14 @@ func GetShopeeProductByGoodsID(q Execer, goodsID string) (*model.ShopeeProduct,
p.Title = title.String p.Title = title.String
p.ShopeeStatus = shopeeStatus.String p.ShopeeStatus = shopeeStatus.String
p.MainSKUCode = mainSKUCode.String p.MainSKUCode = mainSKUCode.String
p.ImageURL = imageURL.String
p.ShopeeShopName = shopName.String
p.ImageSource = imageSource.String
p.ImageObservedAt = imageObserved.String
p.ImageIsManual = imageManual != 0
p.ShopNameSource = shopSource.String
p.ShopNameObservedAt = shopObserved.String
p.ShopNameIsManual = shopManual != 0
p.Source = source.String p.Source = source.String
p.PddGoodsURL = pddGoodsURL.String p.PddGoodsURL = pddGoodsURL.String
p.PddGoodsID = pddGoodsID.String p.PddGoodsID = pddGoodsID.String
+69 -16
View File
@@ -9,6 +9,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"net/http" "net/http"
"net/url"
"strings" "strings"
"time" "time"
"unicode/utf8" "unicode/utf8"
@@ -28,6 +29,8 @@ type CatalogShopeeProduct struct {
Title string `json:"title"` Title string `json:"title"`
Status string `json:"status"` Status string `json:"status"`
MainSKUCode string `json:"main_sku_code"` MainSKUCode string `json:"main_sku_code"`
ImageURL string `json:"image_url"`
ShopName string `json:"shop_name"`
} }
type CatalogShopeeSKU struct { type CatalogShopeeSKU struct {
SKUID string `json:"sku_id"` SKUID string `json:"sku_id"`
@@ -72,19 +75,23 @@ type CatalogBatchRequest struct {
Associations []CatalogAssociation `json:"associations"` Associations []CatalogAssociation `json:"associations"`
} }
type CatalogCounts struct { type CatalogCounts struct {
ShopeeCreated int `json:"shopee_created"` ShopeeCreated int `json:"shopee_created"`
ShopeeUpdated int `json:"shopee_updated"` ShopeeUpdated int `json:"shopee_updated"`
SKUCreated int `json:"sku_created"` ShopeeFieldsFilled int `json:"shopee_fields_filled"`
SKUUpdated int `json:"sku_updated"` ShopeeFieldsSameSourceUpdated int `json:"shopee_fields_same_source_updated"`
SKUFilled int `json:"sku_filled"` ShopeeFieldsManualSkipped int `json:"shopee_fields_manual_skipped"`
SKUSameSourceUpdated int `json:"sku_same_source_updated"` ShopeeFieldsStaleSkipped int `json:"shopee_fields_stale_skipped"`
SKUSkipped int `json:"sku_skipped"` SKUCreated int `json:"sku_created"`
SKUManualSkipped int `json:"sku_manual_skipped"` SKUUpdated int `json:"sku_updated"`
SKUStaleSkipped int `json:"sku_stale_skipped"` SKUFilled int `json:"sku_filled"`
PddCreated int `json:"pdd_created"` SKUSameSourceUpdated int `json:"sku_same_source_updated"`
PddUpdated int `json:"pdd_updated"` SKUSkipped int `json:"sku_skipped"`
AssociationCreated int `json:"association_created"` SKUManualSkipped int `json:"sku_manual_skipped"`
AssociationUnchanged int `json:"association_unchanged"` SKUStaleSkipped int `json:"sku_stale_skipped"`
PddCreated int `json:"pdd_created"`
PddUpdated int `json:"pdd_updated"`
AssociationCreated int `json:"association_created"`
AssociationUnchanged int `json:"association_unchanged"`
} }
type CatalogBatchResponse struct { type CatalogBatchResponse struct {
BatchID string `json:"batch_id"` BatchID string `json:"batch_id"`
@@ -132,6 +139,12 @@ func ValidateCatalogBatch(req CatalogBatchRequest) error {
return catalogInvalid("INVALID_SHOPEE_PRODUCT", "蝦皮商品 ID/标题不能为空且批内不能重复", map[string]any{"index": i, "goods_id": p.GoodsID}) return catalogInvalid("INVALID_SHOPEE_PRODUCT", "蝦皮商品 ID/标题不能为空且批内不能重复", map[string]any{"index": i, "goods_id": p.GoodsID})
} }
seenProducts[p.GoodsID] = true seenProducts[p.GoodsID] = true
if utf8.RuneCountInString(strings.TrimSpace(p.ShopName)) > 500 {
return catalogInvalid("INVALID_SHOPEE_SHOP_NAME", "蝦皮店铺名不能超过 500 个字符", map[string]any{"index": i})
}
if err := validateCatalogImageURL(p.ImageURL); err != nil {
return catalogInvalid("INVALID_SHOPEE_IMAGE_URL", err.Error(), map[string]any{"index": i})
}
} }
for i, s := range req.ShopeeSKUs { for i, s := range req.ShopeeSKUs {
key, keyErr := spec.SpecKey(s.SpecRaw) key, keyErr := spec.SpecKey(s.SpecRaw)
@@ -205,16 +218,20 @@ func ImportCatalogBatch(db *sql.DB, source string, req CatalogBatchRequest, raw
return CatalogBatchResponse{}, importErr return CatalogBatchResponse{}, importErr
} }
for _, p := range req.ShopeeProducts { for _, p := range req.ShopeeProducts {
c, u, e := repository.UpsertCatalogShopeeProduct(tx, p.GoodsID, p.Title, p.Status, p.MainSKUCode, req.ObservedAt, now) outcome, e := repository.UpsertCatalogShopeeProduct(tx, repository.CatalogShopeeProductInput{GoodsID: p.GoodsID, Title: p.Title, Status: p.Status, MainSKU: p.MainSKUCode, ImageURL: strings.TrimSpace(p.ImageURL), ShopName: strings.TrimSpace(p.ShopName), Source: source, ObservedAt: req.ObservedAt, Now: now, UpdatePolicy: policy})
if e != nil { if e != nil {
return fail(e) return fail(e)
} }
if c { if outcome.Created {
counts.ShopeeCreated++ counts.ShopeeCreated++
} }
if u { if outcome.Updated {
counts.ShopeeUpdated++ counts.ShopeeUpdated++
} }
counts.ShopeeFieldsFilled += outcome.FieldsFilled
counts.ShopeeFieldsSameSourceUpdated += outcome.FieldsSameSourceUpdated
counts.ShopeeFieldsManualSkipped += outcome.FieldsManualSkipped
counts.ShopeeFieldsStaleSkipped += outcome.FieldsStaleSkipped
} }
for _, p := range req.PddProducts { for _, p := range req.PddProducts {
var skus string var skus string
@@ -278,6 +295,10 @@ func ImportCatalogBatch(db *sql.DB, source string, req CatalogBatchRequest, raw
run.ResponseBody = string(body) run.ResponseBody = string(body)
run.ShopeeCreated = counts.ShopeeCreated run.ShopeeCreated = counts.ShopeeCreated
run.ShopeeUpdated = counts.ShopeeUpdated run.ShopeeUpdated = counts.ShopeeUpdated
run.ShopeeFieldsFilled = counts.ShopeeFieldsFilled
run.ShopeeFieldsSameSourceUpdated = counts.ShopeeFieldsSameSourceUpdated
run.ShopeeFieldsManualSkipped = counts.ShopeeFieldsManualSkipped
run.ShopeeFieldsStaleSkipped = counts.ShopeeFieldsStaleSkipped
run.SKUCreated = counts.SKUCreated run.SKUCreated = counts.SKUCreated
run.SKUUpdated = counts.SKUUpdated run.SKUUpdated = counts.SKUUpdated
run.SKUFilled = counts.SKUFilled run.SKUFilled = counts.SKUFilled
@@ -314,6 +335,38 @@ func catalogSKURecordID(goodsID, specKey string) string {
return "catalog:" + hex.EncodeToString(sum[:]) return "catalog:" + hex.EncodeToString(sum[:])
} }
func validateCatalogImageURL(raw string) error {
raw = strings.TrimSpace(raw)
if raw == "" {
return nil
}
if len(raw) > 2048 {
return fmt.Errorf("蝦皮主图片 URL 不能超过 2048 字节")
}
u, err := url.ParseRequestURI(raw)
if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
return fmt.Errorf("蝦皮主图片 URL 必须是完整的 HTTP/HTTPS 地址")
}
if u.User != nil {
return fmt.Errorf("蝦皮主图片 URL 不能包含用户名或密码")
}
parameters := u.Query()
if fragmentParameters, fragmentErr := url.ParseQuery(u.Fragment); fragmentErr == nil {
for key, values := range fragmentParameters {
parameters[key] = append(parameters[key], values...)
}
}
for key := range parameters {
lower := strings.ToLower(key)
for _, risky := range []string{"token", "cookie", "password", "passwd", "authorization", "credential", "secret", "session"} {
if strings.Contains(lower, risky) {
return fmt.Errorf("蝦皮主图片 URL 不能包含凭据参数")
}
}
}
return nil
}
func replayCatalogBatch(db *sql.DB, source, batchID, hash, now string) (CatalogBatchResponse, error) { func replayCatalogBatch(db *sql.DB, source, batchID, hash, now string) (CatalogBatchResponse, error) {
run, err := repository.GetCatalogImportRun(db, source, batchID) run, err := repository.GetCatalogImportRun(db, source, batchID)
if err != nil { if err != nil {
+96 -2
View File
@@ -19,10 +19,10 @@ func newCatalogTestDB(t *testing.T) *sql.DB {
} }
db.SetMaxOpenConns(1) db.SetMaxOpenConns(1)
statements := []string{ statements := []string{
`CREATE TABLE shopee_products(goods_id TEXT PRIMARY KEY,title TEXT NOT NULL,shopee_status TEXT,main_sku_code TEXT,source TEXT,source_observed_at TEXT,pdd_goods_url TEXT,pdd_goods_id TEXT,created_at TEXT,updated_at TEXT)`, `CREATE TABLE shopee_products(goods_id TEXT PRIMARY KEY,title TEXT NOT NULL,shopee_status TEXT,main_sku_code TEXT,image_url TEXT,shopee_shop_name TEXT,image_source TEXT,image_observed_at TEXT,image_is_manual INTEGER DEFAULT 0,shop_name_source TEXT,shop_name_observed_at TEXT,shop_name_is_manual INTEGER DEFAULT 0,source TEXT,source_observed_at TEXT,pdd_goods_url TEXT,pdd_goods_id TEXT,created_at TEXT,updated_at TEXT)`,
`CREATE TABLE shopee_skus(sku_id TEXT PRIMARY KEY,shopee_sku_id TEXT UNIQUE,goods_id TEXT NOT NULL,spec_raw TEXT,spec_key TEXT,color TEXT,size TEXT,advice TEXT,parse_ok INTEGER,sku_code TEXT,is_manual INTEGER,source TEXT,field_sources TEXT,field_observed_at TEXT,source_observed_at TEXT,created_at TEXT,updated_at TEXT,UNIQUE(goods_id,spec_key))`, `CREATE TABLE shopee_skus(sku_id TEXT PRIMARY KEY,shopee_sku_id TEXT UNIQUE,goods_id TEXT NOT NULL,spec_raw TEXT,spec_key TEXT,color TEXT,size TEXT,advice TEXT,parse_ok INTEGER,sku_code TEXT,is_manual INTEGER,source TEXT,field_sources TEXT,field_observed_at TEXT,source_observed_at TEXT,created_at TEXT,updated_at TEXT,UNIQUE(goods_id,spec_key))`,
`CREATE TABLE pdd_products(id INTEGER PRIMARY KEY AUTOINCREMENT,goods_id TEXT UNIQUE,url TEXT,title TEXT,shop_name TEXT,skus_json TEXT,collect_status TEXT,collect_msg TEXT,artifact_ref TEXT,collected_at TEXT,deleted_at TEXT,source TEXT,source_observed_at TEXT,created_at TEXT,updated_at TEXT)`, `CREATE TABLE pdd_products(id INTEGER PRIMARY KEY AUTOINCREMENT,goods_id TEXT UNIQUE,url TEXT,title TEXT,shop_name TEXT,skus_json TEXT,collect_status TEXT,collect_msg TEXT,artifact_ref TEXT,collected_at TEXT,deleted_at TEXT,source TEXT,source_observed_at TEXT,created_at TEXT,updated_at TEXT)`,
`CREATE TABLE catalog_import_runs(source TEXT,batch_id TEXT,request_hash TEXT,status TEXT,request_count INTEGER,conflict_count INTEGER,observed_at TEXT,update_policy TEXT DEFAULT 'fill_missing',last_request_at TEXT,last_conflict_at TEXT,shopee_created INTEGER DEFAULT 0,shopee_updated INTEGER DEFAULT 0,sku_created INTEGER DEFAULT 0,sku_updated INTEGER DEFAULT 0,sku_filled INTEGER DEFAULT 0,sku_same_source_updated INTEGER DEFAULT 0,sku_skipped INTEGER DEFAULT 0,sku_manual_skipped INTEGER DEFAULT 0,sku_stale_skipped INTEGER DEFAULT 0,pdd_created INTEGER DEFAULT 0,pdd_updated INTEGER DEFAULT 0,association_created INTEGER DEFAULT 0,association_unchanged INTEGER DEFAULT 0,failure_count INTEGER DEFAULT 0,error_summary TEXT,response_body TEXT,created_at TEXT,finished_at TEXT,PRIMARY KEY(source,batch_id))`, `CREATE TABLE catalog_import_runs(source TEXT,batch_id TEXT,request_hash TEXT,status TEXT,request_count INTEGER,conflict_count INTEGER,observed_at TEXT,update_policy TEXT DEFAULT 'fill_missing',last_request_at TEXT,last_conflict_at TEXT,shopee_created INTEGER DEFAULT 0,shopee_updated INTEGER DEFAULT 0,shopee_fields_filled INTEGER DEFAULT 0,shopee_fields_same_source_updated INTEGER DEFAULT 0,shopee_fields_manual_skipped INTEGER DEFAULT 0,shopee_fields_stale_skipped INTEGER DEFAULT 0,sku_created INTEGER DEFAULT 0,sku_updated INTEGER DEFAULT 0,sku_filled INTEGER DEFAULT 0,sku_same_source_updated INTEGER DEFAULT 0,sku_skipped INTEGER DEFAULT 0,sku_manual_skipped INTEGER DEFAULT 0,sku_stale_skipped INTEGER DEFAULT 0,pdd_created INTEGER DEFAULT 0,pdd_updated INTEGER DEFAULT 0,association_created INTEGER DEFAULT 0,association_unchanged INTEGER DEFAULT 0,failure_count INTEGER DEFAULT 0,error_summary TEXT,response_body TEXT,created_at TEXT,finished_at TEXT,PRIMARY KEY(source,batch_id))`,
} }
for _, statement := range statements { for _, statement := range statements {
if _, err := db.Exec(statement); err != nil { if _, err := db.Exec(statement); err != nil {
@@ -260,3 +260,97 @@ func TestImportCatalogBatch_同规格不同真实SKU冲突并回滚(t *testing.T
t.Fatalf("冲突批次未整体回滚:%q", title) t.Fatalf("冲突批次未整体回滚:%q", title)
} }
} }
func TestImportCatalogBatch_蝦皮图片店铺分级更新和人工保护(t *testing.T) {
db := newCatalogTestDB(t)
base := validCatalogBatch()
raw, _ := json.Marshal(base)
if _, err := ImportCatalogBatch(db, "script-a", base, raw); err != nil {
t.Fatal(err)
}
fill := validCatalogBatch()
fill.BatchID = "product-fill"
fill.ShopeeProducts[0].ImageURL = "https://img.example.com/a.jpg"
fill.ShopeeProducts[0].ShopName = "蝦皮店铺A"
raw, _ = json.Marshal(fill)
got, err := ImportCatalogBatch(db, "script-a", fill, raw)
if err != nil || got.Counts.ShopeeFieldsFilled != 2 {
t.Fatalf("补空失败:%+v %v", got, err)
}
keep := validCatalogBatch()
keep.BatchID = "product-keep"
keep.ShopeeProducts[0].ImageURL = "https://img.example.com/b.jpg"
keep.ShopeeProducts[0].ShopName = "店铺B"
raw, _ = json.Marshal(keep)
got, err = ImportCatalogBatch(db, "script-b", keep, raw)
if err != nil || got.Counts.ShopeeFieldsFilled != 0 {
t.Fatalf("默认策略不应覆盖:%+v %v", got, err)
}
cross := keep
cross.BatchID = "product-cross"
cross.UpdatePolicy = "overwrite_same_source"
cross.ObservedAt = "2026-08-12T07:00:00+08:00"
raw, _ = json.Marshal(cross)
got, err = ImportCatalogBatch(db, "script-b", cross, raw)
if err != nil || got.Counts.ShopeeFieldsSameSourceUpdated != 0 {
t.Fatalf("跨来源不应覆盖:%+v %v", got, err)
}
insertOnly := keep
insertOnly.BatchID = "product-insert-only"
insertOnly.UpdatePolicy = "insert_only"
raw, _ = json.Marshal(insertOnly)
got, err = ImportCatalogBatch(db, "script-a", insertOnly, raw)
if err != nil || got.Counts.ShopeeFieldsFilled != 0 || got.Counts.ShopeeFieldsSameSourceUpdated != 0 {
t.Fatalf("insert_only 不应修改已有商品:%+v %v", got, err)
}
overwrite := keep
overwrite.BatchID = "product-overwrite"
overwrite.UpdatePolicy = "overwrite_same_source"
overwrite.ObservedAt = "2026-08-12T08:00:00+08:00"
raw, _ = json.Marshal(overwrite)
got, err = ImportCatalogBatch(db, "script-a", overwrite, raw)
if err != nil || got.Counts.ShopeeFieldsSameSourceUpdated != 2 {
t.Fatalf("同来源更新失败:%+v %v", got, err)
}
var image, shop string
_ = db.QueryRow(`SELECT image_url,shopee_shop_name FROM shopee_products WHERE goods_id='S-1'`).Scan(&image, &shop)
if image != "https://img.example.com/b.jpg" || shop != "店铺B" {
t.Fatalf("字段未更新:%q %q", image, shop)
}
stale := overwrite
stale.BatchID = "product-stale"
stale.ObservedAt = "2026-08-11T09:00:00+08:00"
stale.ShopeeProducts[0].ImageURL = "https://img.example.com/old.jpg"
stale.ShopeeProducts[0].ShopName = "旧店铺"
raw, _ = json.Marshal(stale)
got, err = ImportCatalogBatch(db, "script-a", stale, raw)
if err != nil || got.Counts.ShopeeFieldsStaleSkipped != 2 {
t.Fatalf("旧观测保护失败:%+v %v", got, err)
}
_, _ = db.Exec(`UPDATE shopee_products SET image_is_manual=1,shop_name_is_manual=1 WHERE goods_id='S-1'`)
manual := overwrite
manual.BatchID = "product-manual"
manual.ObservedAt = "2026-08-13T08:00:00+08:00"
manual.ShopeeProducts[0].ImageURL = "https://img.example.com/c.jpg"
manual.ShopeeProducts[0].ShopName = "店铺C"
raw, _ = json.Marshal(manual)
got, err = ImportCatalogBatch(db, "script-a", manual, raw)
if err != nil || got.Counts.ShopeeFieldsManualSkipped != 2 {
t.Fatalf("人工保护失败:%+v %v", got, err)
}
}
func TestValidateCatalogBatch_拒绝危险图片URL(t *testing.T) {
for _, rawURL := range []string{"file:///tmp/a.jpg", "data:image/png;base64,AA", "https://user:pass@example.com/a.jpg", "https://example.com/a.jpg?access_token=secret", "javascript:alert(1)"} {
req := validCatalogBatch()
req.ShopeeProducts[0].ImageURL = rawURL
if err := ValidateCatalogBatch(req); err == nil {
t.Errorf("应拒绝 %q", rawURL)
}
}
req := validCatalogBatch()
req.ShopeeProducts[0].ImageURL = "https://img.example.com/a.jpg?v=1"
if err := ValidateCatalogBatch(req); err != nil {
t.Fatalf("合法图片被拒绝:%v", err)
}
}
+35 -11
View File
@@ -18,8 +18,10 @@ import (
// //
// 一行对应一个**商品**,不是一个 SKU——见工单 #41。 // 一行对应一个**商品**,不是一个 SKU——见工单 #41。
type ShopeeProductView struct { type ShopeeProductView struct {
GoodsID string GoodsID string
Title string Title string
ImageURL string
ShopName string
// ColorCount / SizeCount 只统计 parse_ok = 1 的 SKU(`[必须]`,见 #41)。 // ColorCount / SizeCount 只统计 parse_ok = 1 的 SKU(`[必须]`,见 #41)。
ColorCount int ColorCount int
@@ -107,6 +109,8 @@ func ListShopeeProducts(db *sql.DB, filter repository.ShopeeFilter, page int) (*
v := ShopeeProductView{ v := ShopeeProductView{
GoodsID: r.GoodsID, GoodsID: r.GoodsID,
Title: r.Title, Title: r.Title,
ImageURL: r.ImageURL,
ShopName: r.ShopeeShopName,
ColorCount: r.ColorCount, ColorCount: r.ColorCount,
SizeCount: r.SizeCount, SizeCount: r.SizeCount,
SKUCount: r.SKUCount, SKUCount: r.SKUCount,
@@ -204,10 +208,14 @@ type ShopeeSpecView struct {
// ShopeeProductDetail 是双击弹窗要显示的全部内容。 // ShopeeProductDetail 是双击弹窗要显示的全部内容。
type ShopeeProductDetail struct { type ShopeeProductDetail struct {
GoodsID string GoodsID string
Title string Title string
ShopeeStatus string ShopeeStatus string
MainSKUCode string MainSKUCode string
ImageURL string
ShopName string
ImageSourceText, ImageObservedAt string
ShopNameSourceText, ShopNameObservedAt string
PddURL string PddURL string
PddGoodsID string PddGoodsID string
@@ -238,11 +246,17 @@ func GetShopeeProductDetail(db *sql.DB, goodsID string) (*ShopeeProductDetail, e
} }
d := &ShopeeProductDetail{ d := &ShopeeProductDetail{
GoodsID: p.GoodsID, GoodsID: p.GoodsID,
Title: p.Title, Title: p.Title,
ShopeeStatus: p.ShopeeStatus, ShopeeStatus: p.ShopeeStatus,
MainSKUCode: p.MainSKUCode, MainSKUCode: p.MainSKUCode,
PddGoodsID: p.PddGoodsID, ImageURL: p.ImageURL,
ShopName: p.ShopeeShopName,
ImageSourceText: catalogFieldSourceText(p.ImageSource, p.ImageIsManual),
ImageObservedAt: formatLocalTime(p.ImageObservedAt),
ShopNameSourceText: catalogFieldSourceText(p.ShopNameSource, p.ShopNameIsManual),
ShopNameObservedAt: formatLocalTime(p.ShopNameObservedAt),
PddGoodsID: p.PddGoodsID,
} }
if d.Title == "" { if d.Title == "" {
d.Title = placeholder d.Title = placeholder
@@ -321,3 +335,13 @@ func GetShopeeProductDetail(db *sql.DB, goodsID string) (*ShopeeProductDetail, e
} }
return d, nil return d, nil
} }
func catalogFieldSourceText(source string, manual bool) string {
if manual {
return "人工维护"
}
if strings.TrimSpace(source) == "" {
return "—"
}
return source
}
+3
View File
@@ -249,6 +249,9 @@ tr.empty small { color: #aaa; }
.col-title { .col-title {
max-width: 18%; max-width: 18%;
} }
.col-shop { max-width: 150px; }
.thumb-placeholder, .thumb-fallback { color: #6b7280; font-size: 12px; white-space: nowrap; }
.detail-product-image { width: 120px; max-height: 120px; object-fit: contain; border: 1px solid #ccd1d6; border-radius: 4px; }
/* ── 弹窗 ─────────────────────────────── */ /* ── 弹窗 ─────────────────────────────── */
/* 用 hidden 属性控制显示隐藏,JS 只改这一个属性,不拼样式 */ /* 用 hidden 属性控制显示隐藏,JS 只改这一个属性,不拼样式 */
+14
View File
@@ -342,6 +342,7 @@
title.textContent = productTitle + " · 原图"; title.textContent = productTitle + " · 原图";
link.href = url; link.href = url;
image.alt = productTitle + " 原图"; image.alt = productTitle + " 原图";
image.referrerPolicy = "no-referrer";
image.hidden = true; image.hidden = true;
status.hidden = false; status.hidden = false;
status.classList.remove("missing"); status.classList.remove("missing");
@@ -363,6 +364,18 @@
}); });
} }
function setupImageThumbnails() {
document.querySelectorAll("[data-image-thumb]").forEach(function (image) {
function showFallback() {
image.hidden = true;
var fallback = image.parentElement && image.parentElement.querySelector("[data-image-thumb-fallback]");
if (fallback) fallback.hidden = false;
}
image.addEventListener("error", showFallback);
if (image.complete && image.naturalWidth === 0) showFallback();
});
}
/* ── 顺运宝登录弹窗:验证码"换一张" ───────────── /* ── 顺运宝登录弹窗:验证码"换一张" ─────────────
图片本身、以及旁边的"换一张"按钮,点了都重新请求验证码接口。 图片本身、以及旁边的"换一张"按钮,点了都重新请求验证码接口。
`[必须]` 每次请求带一个新的时间戳查询参数,绕开浏览器缓存—— `[必须]` 每次请求带一个新的时间戳查询参数,绕开浏览器缓存——
@@ -474,6 +487,7 @@
setupModals(); setupModals();
setupRowDetail(); setupRowDetail();
setupImagePreview(); setupImagePreview();
setupImageThumbnails();
setupCaptchaRefresh(); setupCaptchaRefresh();
setupSybSyncFeedback(); setupSybSyncFeedback();
setupSybHistoryRefresh(); setupSybHistoryRefresh();
+1 -1
View File
@@ -1 +1 @@
{{define "catalog/detail_modal"}}<div class="modal-body"><dl class="detail-grid"><dt>来源</dt><dd>{{.Run.Source}}</dd><dt>批次号</dt><dd>{{.Run.BatchID}}</dd><dt>更新策略</dt><dd>{{.Run.UpdatePolicy}}</dd><dt>状态</dt><dd>{{.Run.StatusText}}</dd><dt>上游观测时间</dt><dd>{{.Run.ObservedAt}}</dd><dt>SKU 新增/补空/同源更新</dt><dd>{{.Run.SKUCreated}} / {{.Run.SKUFilled}} / {{.Run.SKUSameSourceUpdated}}</dd><dt>SKU 跳过(人工/旧数据)</dt><dd>{{.Run.SKUSkipped}}({{.Run.SKUManualSkipped}} / {{.Run.SKUStaleSkipped}})</dd><dt>请求/冲突次数</dt><dd>{{.Run.RequestCount}} / {{.Run.ConflictCount}}</dd><dt>失败数</dt><dd>{{.Run.FailureCount}}</dd><dt>错误码</dt><dd>{{if .Run.ErrorCode}}{{.Run.ErrorCode}}{{else}}—{{end}}</dd><dt>可重试</dt><dd>{{if .Run.Retryable}}是{{else}}否{{end}}</dd><dt>错误摘要</dt><dd>{{if .Run.ErrorSummary}}{{.Run.ErrorSummary}}{{else}}—{{end}}</dd><dt>定位信息</dt><dd>{{if .Run.ErrorDetails}}{{.Run.ErrorDetails}}{{else}}—{{end}}</dd></dl><p class="hint">定位信息只包含数组位置和业务 ID;这里不保存完整请求 JSON 或任何凭据。</p></div>{{end}} {{define "catalog/detail_modal"}}<div class="modal-body"><dl class="detail-grid"><dt>来源</dt><dd>{{.Run.Source}}</dd><dt>批次号</dt><dd>{{.Run.BatchID}}</dd><dt>更新策略</dt><dd>{{.Run.UpdatePolicy}}</dd><dt>状态</dt><dd>{{.Run.StatusText}}</dd><dt>上游观测时间</dt><dd>{{.Run.ObservedAt}}</dd><dt>商品字段 补空/同源更新</dt><dd>{{.Run.ShopeeFieldsFilled}} / {{.Run.ShopeeFieldsSameSourceUpdated}}</dd><dt>商品字段 人工/旧数据跳过</dt><dd>{{.Run.ShopeeFieldsManualSkipped}} / {{.Run.ShopeeFieldsStaleSkipped}}</dd><dt>SKU 新增/补空/同源更新</dt><dd>{{.Run.SKUCreated}} / {{.Run.SKUFilled}} / {{.Run.SKUSameSourceUpdated}}</dd><dt>SKU 跳过(人工/旧数据)</dt><dd>{{.Run.SKUSkipped}}({{.Run.SKUManualSkipped}} / {{.Run.SKUStaleSkipped}})</dd><dt>请求/冲突次数</dt><dd>{{.Run.RequestCount}} / {{.Run.ConflictCount}}</dd><dt>失败数</dt><dd>{{.Run.FailureCount}}</dd><dt>错误码</dt><dd>{{if .Run.ErrorCode}}{{.Run.ErrorCode}}{{else}}—{{end}}</dd><dt>可重试</dt><dd>{{if .Run.Retryable}}是{{else}}否{{end}}</dd><dt>错误摘要</dt><dd>{{if .Run.ErrorSummary}}{{.Run.ErrorSummary}}{{else}}—{{end}}</dd><dt>定位信息</dt><dd>{{if .Run.ErrorDetails}}{{.Run.ErrorDetails}}{{else}}—{{end}}</dd></dl><p class="hint">定位信息只包含数组位置和业务 ID;这里不保存完整请求 JSON 或任何凭据。</p></div>{{end}}
+2 -2
View File
@@ -7,8 +7,8 @@
<button type="submit">筛选</button> <button type="submit">筛选</button>
</form> </form>
</div> </div>
<div class="table-wrap"><table><thead><tr><th>来源</th><th>批次号</th><th>策略</th><th>状态</th><th>请求</th><th>冲突</th><th>蝦皮 新/更</th><th>SKU 新/补/同源更/跳过</th><th>PDD 新/更</th><th>关联 新/同</th><th>首次接收</th><th>完成</th><th>耗时</th></tr></thead><tbody> <div class="table-wrap"><table><thead><tr><th>来源</th><th>批次号</th><th>策略</th><th>状态</th><th>请求</th><th>冲突</th><th>蝦皮 新/更/字段补/同源更</th><th>SKU 新/补/同源更/跳过</th><th>PDD 新/更</th><th>关联 新/同</th><th>首次接收</th><th>完成</th><th>耗时</th></tr></thead><tbody>
{{range .Rows}}<tr data-detail-id="{{.Source}}|{{.BatchID}}"><td>{{.Source}}</td><td>{{.BatchID}}</td><td>{{.UpdatePolicy}}</td><td>{{.StatusText}}</td><td>{{.RequestCount}}</td><td>{{.ConflictCount}}</td><td>{{.ShopeeCreated}} / {{.ShopeeUpdated}}</td><td>{{.SKUCreated}} / {{.SKUFilled}} / {{.SKUSameSourceUpdated}} / {{.SKUSkipped}}</td><td>{{.PddCreated}} / {{.PddUpdated}}</td><td>{{.AssociationCreated}} / {{.AssociationUnchanged}}</td><td>{{.CreatedText}}</td><td>{{.FinishedText}}</td><td>{{.DurationText}}</td></tr>{{else}}<tr class="empty"><td colspan="13">还没有符合条件的导入记录。</td></tr>{{end}}</tbody></table></div> {{range .Rows}}<tr data-detail-id="{{.Source}}|{{.BatchID}}"><td>{{.Source}}</td><td>{{.BatchID}}</td><td>{{.UpdatePolicy}}</td><td>{{.StatusText}}</td><td>{{.RequestCount}}</td><td>{{.ConflictCount}}</td><td>{{.ShopeeCreated}} / {{.ShopeeUpdated}} / {{.ShopeeFieldsFilled}} / {{.ShopeeFieldsSameSourceUpdated}}</td><td>{{.SKUCreated}} / {{.SKUFilled}} / {{.SKUSameSourceUpdated}} / {{.SKUSkipped}}</td><td>{{.PddCreated}} / {{.PddUpdated}}</td><td>{{.AssociationCreated}} / {{.AssociationUnchanged}}</td><td>{{.CreatedText}}</td><td>{{.FinishedText}}</td><td>{{.DurationText}}</td></tr>{{else}}<tr class="empty"><td colspan="13">还没有符合条件的导入记录。</td></tr>{{end}}</tbody></table></div>
<p class="hint">双击一行查看批次摘要;系统不会保存或展示 Token 和完整请求体。</p> <p class="hint">双击一行查看批次摘要;系统不会保存或展示 Token 和完整请求体。</p>
<div class="modal-backdrop" id="detail-modal" hidden><div class="modal" role="dialog" aria-modal="true"><div class="modal-head"><h2>导入批次详情</h2><button type="button" class="modal-x" data-modal-close>×</button></div><div id="detail-content"></div></div></div> <div class="modal-backdrop" id="detail-modal" hidden><div class="modal" role="dialog" aria-modal="true"><div class="modal-head"><h2>导入批次详情</h2><button type="button" class="modal-x" data-modal-close>×</button></div><div id="detail-content"></div></div></div>
<script>document.querySelectorAll('tr[data-detail-id]').forEach(function(row){row.addEventListener('dblclick',function(){var p=row.dataset.detailId.split('|');fetch('/integrations/catalog/detail?source='+encodeURIComponent(p[0])+'&batch_id='+encodeURIComponent(p[1])).then(function(r){return r.text()}).then(function(html){document.getElementById('detail-content').innerHTML=html;document.getElementById('detail-modal').hidden=false})})})</script> <script>document.querySelectorAll('tr[data-detail-id]').forEach(function(row){row.addEventListener('dblclick',function(){var p=row.dataset.detailId.split('|');fetch('/integrations/catalog/detail?source='+encodeURIComponent(p[0])+'&batch_id='+encodeURIComponent(p[1])).then(function(r){return r.text()}).then(function(html){document.getElementById('detail-content').innerHTML=html;document.getElementById('detail-modal').hidden=false})})})</script>
+4
View File
@@ -10,6 +10,10 @@
<dt>商品名称</dt><dd>{{.Title}}</dd> <dt>商品名称</dt><dd>{{.Title}}</dd>
<dt>蝦皮状态</dt><dd>{{.ShopeeStatus}}</dd> <dt>蝦皮状态</dt><dd>{{.ShopeeStatus}}</dd>
<dt>主商品货号</dt><dd>{{.MainSKUCode}}</dd> <dt>主商品货号</dt><dd>{{.MainSKUCode}}</dd>
<dt>蝦皮店铺</dt><dd>{{if .ShopName}}{{.ShopName}}{{else}}—{{end}}</dd>
<dt>店铺来源 / 观测时间</dt><dd>{{.ShopNameSourceText}} / {{if .ShopNameObservedAt}}{{.ShopNameObservedAt}}{{else}}—{{end}}</dd>
<dt>蝦皮主图</dt><dd>{{if .ImageURL}}<a href="{{.ImageURL}}" target="_blank" rel="noopener noreferrer"><img src="{{.ImageURL}}" alt="{{.Title}} 主图" class="detail-product-image" loading="lazy" referrerpolicy="no-referrer"></a>{{else}}无图{{end}}</dd>
<dt>图片来源 / 观测时间</dt><dd>{{.ImageSourceText}} / {{if .ImageObservedAt}}{{.ImageObservedAt}}{{else}}—{{end}}</dd>
<dt>PDD 商品 ID</dt><dd{{if .PddMissing}} class="missing"{{end}}>{{if .PddGoodsID}}{{.PddGoodsID}}{{else}}未关联{{end}}</dd> <dt>PDD 商品 ID</dt><dd{{if .PddMissing}} class="missing"{{end}}>{{if .PddGoodsID}}{{.PddGoodsID}}{{else}}未关联{{end}}</dd>
<dt>采集状态</dt><dd>{{.StatusText}}{{if .CollectMsg}}:{{.CollectMsg}}{{end}}</dd> <dt>采集状态</dt><dd>{{.StatusText}}{{if .CollectMsg}}:{{.CollectMsg}}{{end}}</dd>
</dl> </dl>
+7 -1
View File
@@ -35,7 +35,9 @@
<tr> <tr>
<th class="col-check"><input type="checkbox" data-check-all></th> <th class="col-check"><input type="checkbox" data-check-all></th>
<th>商品 ID</th> <th>商品 ID</th>
<th>图片</th>
<th>商品名称</th> <th>商品名称</th>
<th>蝦皮店铺</th>
<th>颜色</th> <th>颜色</th>
<th>尺码</th> <th>尺码</th>
{{/* SKU 数是这个商品报表里实际出现过的规格条数。 {{/* SKU 数是这个商品报表里实际出现过的规格条数。
@@ -57,10 +59,12 @@
<tr data-detail-id="{{.GoodsID}}"{{if gt .PendingCount 0}} class="row-warn"{{end}}> <tr data-detail-id="{{.GoodsID}}"{{if gt .PendingCount 0}} class="row-warn"{{end}}>
<td class="col-check"><input type="checkbox" name="ids" value="{{.GoodsID}}"></td> <td class="col-check"><input type="checkbox" name="ids" value="{{.GoodsID}}"></td>
<td>{{.GoodsID}}</td> <td>{{.GoodsID}}</td>
<td>{{if .ImageURL}}<button type="button" class="thumb-action" data-image-preview-url="{{.ImageURL}}" data-image-preview-title="{{.Title}}" title="查看蝦皮主图"><img src="{{.ImageURL}}" alt="{{.Title}} 缩略图" class="thumb" width="32" height="32" loading="lazy" referrerpolicy="no-referrer" data-image-thumb><span class="thumb-fallback" hidden data-image-thumb-fallback>加载失败</span></button>{{else}}<span class="thumb-placeholder">无图</span>{{end}}</td>
{{/* 商品名称列收窄,用独立的 .col-title 类设 max-width(具体数值见 app.css), {{/* 商品名称列收窄,用独立的 .col-title 类设 max-width(具体数值见 app.css),
不用 flex——表格单元格和工具条的 .search-narrow 那套不一样,见 #41。 不用 flex——表格单元格和工具条的 .search-narrow 那套不一样,见 #41。
title 属性让悬停能看完整标题,收窄之后必须留着。 */}} title 属性让悬停能看完整标题,收窄之后必须留着。 */}}
<td class="truncate col-title" title="{{.Title}}">{{.Title}}</td> <td class="truncate col-title" title="{{.Title}}">{{.Title}}</td>
<td class="truncate col-shop" title="{{.ShopName}}">{{if .ShopName}}{{.ShopName}}{{else}}—{{end}}</td>
<td>{{.ColorCount}}</td> <td>{{.ColorCount}}</td>
<td>{{.SizeCount}}</td> <td>{{.SizeCount}}</td>
<td>{{.SKUCount}}</td> <td>{{.SKUCount}}</td>
@@ -72,7 +76,7 @@
</tr> </tr>
{{else}} {{else}}
<tr class="empty"> <tr class="empty">
<td colspan="11"> <td colspan="13">
{{if .IsFiltered}} {{if .IsFiltered}}
没有匹配的数据,换个商品 ID 或商品名称试试。 没有匹配的数据,换个商品 ID 或商品名称试试。
{{else}} {{else}}
@@ -86,6 +90,8 @@
</table> </table>
</div> </div>
<div class="modal-backdrop" id="image-preview-modal" hidden><div class="modal image-preview-modal" role="dialog" aria-modal="true" aria-labelledby="image-preview-title"><div class="modal-head"><h2 id="image-preview-title" data-image-preview-title>蝦皮商品主图</h2><button type="button" class="modal-x" data-modal-close aria-label="关闭">×</button></div><div class="modal-body image-preview-body"><p class="hint image-preview-status" data-image-preview-status role="status">正在加载原图…</p><img data-image-preview-image alt="" hidden></div><div class="modal-foot"><a class="button-link" data-image-preview-link href="#" target="_blank" rel="noopener noreferrer">新窗口打开原图</a><button type="button" data-modal-close>关闭</button></div></div></div>
<p class="hint"> <p class="hint">
双击任意一行可以查看这个商品的完整规格表。 双击任意一行可以查看这个商品的完整规格表。
</p> </p>
+6
View File
@@ -438,6 +438,12 @@ MySQL schema v8 将 `shopee_skus.sku_id` 保留为系统内部记录主键(兼
来源表;`fill_missing` 补字段时分别登记,`overwrite_same_source` 只更新来源属于本次 来源表;`fill_missing` 补字段时分别登记,`overwrite_same_source` 只更新来源属于本次
调用方且观测时间不旧的字段,避免不同脚本各补一部分后互相覆盖。 调用方且观测时间不旧的字段,避免不同脚本各补一部分后互相覆盖。
MySQL schema v9 为 `shopee_products` 增加 `image_url` 和 `shopee_shop_name`。两者分别
使用 `image_source/image_observed_at/image_is_manual` 与
`shop_name_source/shop_name_observed_at/shop_name_is_manual` 记录来源边界。目录接口只
能补空或更新同来源的新观测,空值、跨来源、旧观测和人工字段均不覆盖。该图片是蝦皮
商品主图,与 `syb_orders.image_url` 的历史货运单观测图片是两个独立概念。
蝦皮详情把 `syb_orders` 按 `(shopee_goods_id, spec_key)` 聚合为“顺运宝观测规格”: 蝦皮详情把 `syb_orders` 按 `(shopee_goods_id, spec_key)` 聚合为“顺运宝观测规格”:
订单数按唯一 `syb_id` 行计数,累计数量求和,最近一行提供历史台币售价和图片。 订单数按唯一 `syb_id` 行计数,累计数量求和,最近一行提供历史台币售价和图片。
该读模型不写入 `shopee_skus`。只有同一商品下恰好一条正式 SKU 的 `spec_raw` 该读模型不写入 `shopee_skus`。只有同一商品下恰好一条正式 SKU 的 `spec_raw`
+6
View File
@@ -819,3 +819,9 @@ placeholder 写「任务编号 / 订单号 / 商品 ID」,**不要写全「PDD
商品详情在正式 SKU 表之后显示“顺运宝观测规格”。该区域必须明确标注它不是正式 商品详情在正式 SKU 表之后显示“顺运宝观测规格”。该区域必须明确标注它不是正式
蝦皮 SKU,价格是最近货运单台币售价、不是 PDD 人民币采购价;展示规格原文、订单 蝦皮 SKU,价格是最近货运单台币售价、不是 PDD 人民币采购价;展示规格原文、订单
数、累计数量、最近图片、最后出现时间及唯一正式 SKU 匹配结果,多条匹配交给人工。 数、累计数量、最近图片、最后出现时间及唯一正式 SKU 匹配结果,多条匹配交给人工。
蝦皮商品列表在商品 ID 后显示 32×32 懒加载主图,在标题后显示“蝦皮店铺”。主图可
打开通用原图弹窗和新窗口,加载失败显示文字占位;无图不打开空弹窗。店铺列最大约
150px并省略,完整值放在 `title` 和详情中。商品标题继续使用 `.col-title` 的 18%,
不得因新增列再次缩窄;1366×768 下通过表格横向滚动保留可读性。详情同时显示图片、
店铺、各自来源和观测时间,顺运宝观测图仍只出现在观测区域。
+8 -2
View File
@@ -21,7 +21,7 @@ Token 通过 `CMAUTOBUY_CATALOG_TOKEN` 或未提交的 `config.yaml` 配置。
"observed_at": "2026-08-11T10:00:00+08:00", "observed_at": "2026-08-11T10:00:00+08:00",
"update_policy": "fill_missing", "update_policy": "fill_missing",
"shopee_products": [ "shopee_products": [
{"goods_id": "S-1", "title": "蝦皮上衣", "status": "NORMAL", "main_sku_code": "A01"} {"goods_id": "S-1", "title": "蝦皮上衣", "status": "NORMAL", "main_sku_code": "A01", "image_url": "https://img.example.com/S-1.jpg", "shop_name": "蝦皮示例店铺"}
], ],
"shopee_skus": [ "shopee_skus": [
{"sku_id": null, "goods_id": "S-1", "spec_raw": "黑色,M", "color": "黑色", "size": "M", "parse_ok": true, "sku_code": "A01-B-M"} {"sku_id": null, "goods_id": "S-1", "spec_raw": "黑色,M", "color": "黑色", "size": "M", "parse_ok": true, "sku_code": "A01-B-M"}
@@ -47,6 +47,10 @@ Token 通过 `CMAUTOBUY_CATALOG_TOKEN` 或未提交的 `config.yaml` 配置。
不得自行生成。系统用内部主键和 `goods_id + spec_key` 保持记录身份,后续取得真实 ID 不得自行生成。系统用内部主键和 `goods_id + spec_key` 保持记录身份,后续取得真实 ID
会补到原记录,不会重复新增。 会补到原记录,不会重复新增。
`image_url` 和 `shop_name` 可选。图片只接受不含用户信息和明显凭据参数的完整
HTTP/HTTPS URL,最长 2048 字节;店铺名最长 500 个字符。Admin 只保存地址并由浏览器
懒加载,不会在服务端下载或代理图片。
## 3. 写入与幂等规则 ## 3. 写入与幂等规则
- 整个批次在一个事务中写入:实体、SKU、关联任一步失败就全部回滚。 - 整个批次在一个事务中写入:实体、SKU、关联任一步失败就全部回滚。
@@ -57,6 +61,8 @@ Token 通过 `CMAUTOBUY_CATALOG_TOKEN` 或未提交的 `config.yaml` 配置。
- `overwrite_same_source`:只有来源相同且 `observed_at` 更新时才覆盖来源字段。 - `overwrite_same_source`:只有来源相同且 `observed_at` 更新时才覆盖来源字段。
- 较旧的 `observed_at` 不覆盖较新的接口数据,任何策略都不能覆盖 `is_manual=1`。 - 较旧的 `observed_at` 不覆盖较新的接口数据,任何策略都不能覆盖 `is_manual=1`。
- `spec_raw` 和由它生成的规格身份不被更新;同商品同规格出现不同真实 SKU ID 返回 409。 - `spec_raw` 和由它生成的规格身份不被更新;同商品同规格出现不同真实 SKU ID 返回 409。
- 蝦皮主图和店铺名也遵循本批次策略,但各自独立记录来源、观测时间和人工保护;
空值永不清除现有值。顺运宝观测图片不参与蝦皮主图更新。
- 蝦皮 upsert 不覆盖采购员维护的 PDD 链接;SKU 的 `is_manual` 不被接口改写。 - 蝦皮 upsert 不覆盖采购员维护的 PDD 链接;SKU 的 `is_manual` 不被接口改写。
- 空关联可以建立、相同关联不重复写;已有不同 PDD 关联返回 409,绝不静默替换。 - 空关联可以建立、相同关联不重复写;已有不同 PDD 关联返回 409,绝不静默替换。
- 批次缺少某条记录不表示删除,接口没有“全量覆盖”语义。 - 批次缺少某条记录不表示删除,接口没有“全量覆盖”语义。
@@ -83,7 +89,7 @@ Token 通过 `CMAUTOBUY_CATALOG_TOKEN` 或未提交的 `config.yaml` 配置。
## 5. schema v8 发布与回退 ## 5. schema v8 发布与回退
发布前先备份生产库,并在 MySQL 8.4、库名以 `_test` 结尾的测试库演练 v7→v8。 发布前先备份生产库,并在 MySQL 8.4、库名以 `_test` 结尾的测试库演练 v7→v8→v9。
迁移只增加列和唯一索引,既有 `sku_id` 继续作为内部主键,因此历史引用不变;若检测 迁移只增加列和唯一索引,既有 `sku_id` 继续作为内部主键,因此历史引用不变;若检测
到同商品重复 `spec_key`,迁移会停止且不记录 v8,必须人工确认,不能自动合并。 到同商品重复 `spec_key`,迁移会停止且不记录 v8,必须人工确认,不能自动合并。