Files
cmautobuy/admin/service/catalog_import.go
T

455 lines
18 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package service
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"strings"
"time"
"unicode/utf8"
"cmautobuy/admin/model"
"cmautobuy/admin/repository"
"cmautobuy/admin/spec"
)
const (
CatalogMaxProducts = 500
CatalogMaxSKUs = 5000
)
type CatalogShopeeProduct struct {
GoodsID string `json:"goods_id"`
Title string `json:"title"`
Status string `json:"status"`
MainSKUCode string `json:"main_sku_code"`
ImageURL string `json:"image_url"`
ShopName string `json:"shop_name"`
}
type CatalogShopeeSKU struct {
SKUID string `json:"sku_id"`
GoodsID string `json:"goods_id"`
SpecRaw string `json:"spec_raw"`
Color string `json:"color"`
Size string `json:"size"`
Advice string `json:"advice"`
ParseOK bool `json:"parse_ok"`
SKUCode string `json:"sku_code"`
}
type CatalogPddDimension struct {
Key string `json:"key"`
Name string `json:"name"`
}
type CatalogPddSKU struct {
Options map[string]string `json:"options"`
PriceCent *int64 `json:"price_cent"`
ListPriceCent *int64 `json:"list_price_cent"`
Available bool `json:"available"`
}
type CatalogPddProduct struct {
GoodsID string `json:"goods_id"`
URL string `json:"url"`
Title string `json:"title"`
ShopName string `json:"shop_name"`
Dimensions []CatalogPddDimension `json:"dimensions"`
SKUs []CatalogPddSKU `json:"skus"`
}
type CatalogAssociation struct {
ShopeeGoodsID string `json:"shopee_goods_id"`
PddGoodsID string `json:"pdd_goods_id"`
}
type CatalogBatchRequest struct {
SchemaVersion int `json:"schema_version"`
BatchID string `json:"batch_id"`
ObservedAt string `json:"observed_at"`
UpdatePolicy string `json:"update_policy"`
ShopeeProducts []CatalogShopeeProduct `json:"shopee_products"`
ShopeeSKUs []CatalogShopeeSKU `json:"shopee_skus"`
PddProducts []CatalogPddProduct `json:"pdd_products"`
Associations []CatalogAssociation `json:"associations"`
// DryRun 只执行数据库预检;不得登记导入批次或写入任何业务数据。
DryRun bool `json:"dry_run"`
}
type CatalogCounts struct {
ShopeeCreated int `json:"shopee_created"`
ShopeeUpdated int `json:"shopee_updated"`
ShopeeFieldsFilled int `json:"shopee_fields_filled"`
ShopeeFieldsSameSourceUpdated int `json:"shopee_fields_same_source_updated"`
ShopeeFieldsManualSkipped int `json:"shopee_fields_manual_skipped"`
ShopeeFieldsStaleSkipped int `json:"shopee_fields_stale_skipped"`
SKUCreated int `json:"sku_created"`
SKUUpdated int `json:"sku_updated"`
SKUFilled int `json:"sku_filled"`
SKUSameSourceUpdated int `json:"sku_same_source_updated"`
SKUSkipped int `json:"sku_skipped"`
SKUManualSkipped int `json:"sku_manual_skipped"`
SKUStaleSkipped int `json:"sku_stale_skipped"`
PddCreated int `json:"pdd_created"`
PddUpdated int `json:"pdd_updated"`
AssociationCreated int `json:"association_created"`
AssociationUnchanged int `json:"association_unchanged"`
}
type CatalogBatchResponse struct {
BatchID string `json:"batch_id"`
Status string `json:"status"`
Replayed bool `json:"replayed"`
Counts CatalogCounts `json:"counts"`
FinishedAt string `json:"finished_at"`
Conflicts []CatalogAssociationConflict `json:"conflicts,omitempty"`
}
// CatalogAssociationConflict 是预检发现的既有人工关联冲突,不包含请求体或凭据。
type CatalogAssociationConflict struct {
ShopeeGoodsID string `json:"shopee_goods_id"`
ExistingPddID string `json:"existing_pdd_goods_id"`
IncomingPddID string `json:"incoming_pdd_goods_id"`
}
type CatalogError struct {
Status int
Code, Message string
Retryable bool
Details any
}
func (e *CatalogError) Error() string { return e.Message }
func catalogInvalid(code, message string, details any) error {
return &CatalogError{Status: http.StatusUnprocessableEntity, Code: code, Message: message, Details: details}
}
func ValidateCatalogBatch(req CatalogBatchRequest) error {
if req.SchemaVersion != 1 {
return catalogInvalid("UNSUPPORTED_SCHEMA", "schema_version 只支持 1", nil)
}
if len(req.BatchID) < 1 || len(req.BatchID) > 191 {
return catalogInvalid("INVALID_BATCH_ID", "batch_id 长度必须为 1-191", nil)
}
if _, err := time.Parse(time.RFC3339Nano, req.ObservedAt); err != nil {
return catalogInvalid("INVALID_OBSERVED_AT", "observed_at 必须是带时区 ISO 8601", nil)
}
if _, err := normalizeCatalogUpdatePolicy(req.UpdatePolicy); err != nil {
return err
}
if len(req.ShopeeProducts)+len(req.PddProducts) > CatalogMaxProducts {
return catalogInvalid("TOO_MANY_PRODUCTS", "每批商品总数不能超过 500", nil)
}
if len(req.ShopeeSKUs) > CatalogMaxSKUs {
return catalogInvalid("TOO_MANY_SKUS", "每批 SKU 不能超过 5000", nil)
}
seenProducts, seenSKUs, seenSpecs, seenPDD := map[string]bool{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
for i, p := range req.ShopeeProducts {
p.GoodsID = strings.TrimSpace(p.GoodsID)
if p.GoodsID == "" || strings.TrimSpace(p.Title) == "" || seenProducts[p.GoodsID] {
return catalogInvalid("INVALID_SHOPEE_PRODUCT", "蝦皮商品 ID/标题不能为空且批内不能重复", map[string]any{"index": i, "goods_id": p.GoodsID})
}
seenProducts[p.GoodsID] = true
if utf8.RuneCountInString(strings.TrimSpace(p.ShopName)) > 500 {
return catalogInvalid("INVALID_SHOPEE_SHOP_NAME", "蝦皮店铺名不能超过 500 个字符", map[string]any{"index": i})
}
if err := validateCatalogImageURL(p.ImageURL); err != nil {
return catalogInvalid("INVALID_SHOPEE_IMAGE_URL", err.Error(), map[string]any{"index": i})
}
}
for i, s := range req.ShopeeSKUs {
key, keyErr := spec.SpecKey(s.SpecRaw)
externalID := strings.TrimSpace(s.SKUID)
identity := strings.TrimSpace(s.GoodsID) + "\x00" + key
if strings.TrimSpace(s.GoodsID) == "" || keyErr != nil || utf8.RuneCountInString(key) > 191 || seenSpecs[identity] || (externalID != "" && seenSKUs[externalID]) {
return catalogInvalid("INVALID_SHOPEE_SKU", "商品 ID、有效 spec_raw 必填,批内规格身份和非空 SKU ID 不能重复", map[string]any{"index": i, "sku_id": externalID})
}
seenSpecs[identity] = true
if externalID != "" {
seenSKUs[externalID] = true
}
}
for i, p := range req.PddProducts {
if strings.TrimSpace(p.GoodsID) == "" || strings.TrimSpace(p.URL) == "" || seenPDD[p.GoodsID] {
return catalogInvalid("INVALID_PDD_PRODUCT", "PDD 商品 ID/URL 必填且批内不能重复", map[string]any{"index": i, "goods_id": p.GoodsID})
}
seenPDD[p.GoodsID] = true
for j, s := range p.SKUs {
if len(s.Options) == 0 || (s.PriceCent != nil && *s.PriceCent < 0) || (s.ListPriceCent != nil && *s.ListPriceCent < 0) {
return catalogInvalid("INVALID_PDD_SKU", "PDD 规格选项不能为空且金额不能为负", map[string]any{"product_index": i, "sku_index": j})
}
}
}
for i, a := range req.Associations {
if strings.TrimSpace(a.ShopeeGoodsID) == "" || strings.TrimSpace(a.PddGoodsID) == "" {
return catalogInvalid("INVALID_ASSOCIATION", "关联双方商品 ID 必填", map[string]any{"index": i})
}
}
return nil
}
// PreviewCatalogBatch 按当前数据库做只读预检。它不登记 catalog_import_runs,
// 也不写业务表;正式执行仍会重新检查,避免预检后人工修改造成覆盖。
func PreviewCatalogBatch(db *sql.DB, req CatalogBatchRequest) (CatalogBatchResponse, error) {
if err := ValidateCatalogBatch(req); err != nil {
return CatalogBatchResponse{}, err
}
tx, err := db.BeginTx(context.Background(), &sql.TxOptions{ReadOnly: true, Isolation: sql.LevelReadCommitted})
if err != nil {
return CatalogBatchResponse{}, err
}
defer tx.Rollback()
counts := CatalogCounts{}
for _, p := range req.ShopeeProducts {
exists, err := repository.CatalogEntityExists(tx, "shopee_products", p.GoodsID)
if err != nil { return CatalogBatchResponse{}, err }
if exists { counts.ShopeeUpdated++ } else { counts.ShopeeCreated++ }
}
for _, p := range req.PddProducts {
exists, err := repository.CatalogEntityExists(tx, "pdd_products", p.GoodsID)
if err != nil { return CatalogBatchResponse{}, err }
if exists { counts.PddUpdated++ } else { counts.PddCreated++ }
}
for _, s := range req.ShopeeSKUs {
key, _ := spec.SpecKey(s.SpecRaw)
exists, err := repository.CatalogSKUExists(tx, s.GoodsID, key)
if err != nil { return CatalogBatchResponse{}, err }
if exists { counts.SKUSkipped++ } else { counts.SKUCreated++ }
}
conflicts := make([]CatalogAssociationConflict, 0)
for _, a := range req.Associations {
current, exists, err := repository.CatalogAssociationCurrentPDD(tx, a.ShopeeGoodsID)
if err != nil { return CatalogBatchResponse{}, err }
if !exists || current == "" { counts.AssociationCreated++; continue }
if current == a.PddGoodsID { counts.AssociationUnchanged++; continue }
conflicts = append(conflicts, CatalogAssociationConflict{ShopeeGoodsID: a.ShopeeGoodsID, ExistingPddID: current, IncomingPddID: a.PddGoodsID})
}
return CatalogBatchResponse{BatchID: req.BatchID, Status: "previewed", Counts: counts, FinishedAt: model.NowISO(), Conflicts: conflicts}, nil
}
func ImportCatalogBatch(db *sql.DB, source string, req CatalogBatchRequest, raw []byte) (CatalogBatchResponse, error) {
if err := ValidateCatalogBatch(req); err != nil {
return CatalogBatchResponse{}, err
}
if req.DryRun {
return PreviewCatalogBatch(db, req)
}
observed, _ := time.Parse(time.RFC3339Nano, req.ObservedAt)
req.ObservedAt = observed.UTC().Format(model.TimeLayout)
policy, _ := normalizeCatalogUpdatePolicy(req.UpdatePolicy)
req.UpdatePolicy = policy
h := sha256.Sum256(raw)
hash := hex.EncodeToString(h[:])
now := model.NowISO()
tx, err := db.BeginTx(context.Background(), &sql.TxOptions{Isolation: sql.LevelReadCommitted})
if err != nil {
return CatalogBatchResponse{}, err
}
run := model.CatalogImportRun{Source: source, BatchID: req.BatchID, RequestHash: hash, Status: model.CatalogImportProcessing, UpdatePolicy: policy, ObservedAt: req.ObservedAt, LastRequestAt: now, CreatedAt: now}
if err = repository.InsertCatalogImportRun(tx, run); err != nil {
tx.Rollback()
if errors.Is(err, repository.ErrCatalogImportRunExists) {
return replayCatalogBatch(db, source, req.BatchID, hash, now)
}
return CatalogBatchResponse{}, err
}
counts := CatalogCounts{}
fail := func(importErr error) (CatalogBatchResponse, error) {
tx.Rollback()
run.Status = model.CatalogImportFailed
run.FailureCount = 1
run.ErrorSummary = truncateCatalogError(importErr.Error())
var catalogErr *CatalogError
if errors.As(importErr, &catalogErr) {
stored, _ := json.Marshal(catalogErr)
run.ResponseBody = string(stored)
}
run.FinishedAt = model.NowISO()
if insertErr := repository.InsertCatalogImportRun(db, run); insertErr == nil {
_ = repository.CompleteCatalogImportRun(db, run)
}
return CatalogBatchResponse{}, importErr
}
for _, p := range req.ShopeeProducts {
outcome, e := repository.UpsertCatalogShopeeProduct(tx, repository.CatalogShopeeProductInput{GoodsID: p.GoodsID, Title: p.Title, Status: p.Status, MainSKU: p.MainSKUCode, ImageURL: strings.TrimSpace(p.ImageURL), ShopName: strings.TrimSpace(p.ShopName), Source: source, ObservedAt: req.ObservedAt, Now: now, UpdatePolicy: policy})
if e != nil {
return fail(e)
}
if outcome.Created {
counts.ShopeeCreated++
}
if outcome.Updated {
counts.ShopeeUpdated++
}
counts.ShopeeFieldsFilled += outcome.FieldsFilled
counts.ShopeeFieldsSameSourceUpdated += outcome.FieldsSameSourceUpdated
counts.ShopeeFieldsManualSkipped += outcome.FieldsManualSkipped
counts.ShopeeFieldsStaleSkipped += outcome.FieldsStaleSkipped
}
for _, p := range req.PddProducts {
var skus string
if len(p.SKUs) > 0 {
b, _ := json.Marshal(map[string]any{"schema_version": 1, "goods_id": p.GoodsID, "title": p.Title, "shop_name": p.ShopName, "dimensions": p.Dimensions, "skus": p.SKUs})
skus = string(b)
}
c, u, e := repository.UpsertCatalogPddProduct(tx, p.GoodsID, p.URL, p.Title, p.ShopName, skus, req.ObservedAt, now, policy)
if e != nil {
return fail(e)
}
if c {
counts.PddCreated++
}
if u {
counts.PddUpdated++
}
}
for _, s := range req.ShopeeSKUs {
key, _ := spec.SpecKey(s.SpecRaw)
internalID := catalogSKURecordID(s.GoodsID, key)
outcome, e := repository.UpsertCatalogShopeeSKU(tx, repository.CatalogShopeeSKUInput{RecordID: internalID, ShopeeSKUID: strings.TrimSpace(s.SKUID), GoodsID: strings.TrimSpace(s.GoodsID), SpecRaw: s.SpecRaw, SpecKey: key, Color: s.Color, Size: s.Size, Advice: s.Advice, ParseOK: s.ParseOK, SKUCode: s.SKUCode, Source: source, ObservedAt: req.ObservedAt, Now: now, UpdatePolicy: policy})
if e != nil {
return fail(&CatalogError{Status: 409, Code: "SKU_OWNERSHIP_CONFLICT", Message: e.Error()})
}
switch outcome {
case repository.CatalogSKUCreated:
counts.SKUCreated++
case repository.CatalogSKUFilled:
counts.SKUFilled++
counts.SKUUpdated++
case repository.CatalogSKUSameSourceUpdated:
counts.SKUSameSourceUpdated++
counts.SKUUpdated++
case repository.CatalogSKUManualSkipped:
counts.SKUManualSkipped++
counts.SKUSkipped++
case repository.CatalogSKUStaleSkipped:
counts.SKUStaleSkipped++
counts.SKUSkipped++
default:
counts.SKUSkipped++
}
}
for _, a := range req.Associations {
c, u, e := repository.ApplyCatalogAssociation(tx, a.ShopeeGoodsID, a.PddGoodsID, now)
if e != nil {
return fail(&CatalogError{Status: 409, Code: "ASSOCIATION_CONFLICT", Message: e.Error()})
}
if c {
counts.AssociationCreated++
}
if u {
counts.AssociationUnchanged++
}
}
resp := CatalogBatchResponse{BatchID: req.BatchID, Status: "succeeded", Counts: counts, FinishedAt: model.NowISO()}
body, _ := json.Marshal(resp)
run.Status = model.CatalogImportSucceeded
run.FinishedAt = resp.FinishedAt
run.ResponseBody = string(body)
run.ShopeeCreated = counts.ShopeeCreated
run.ShopeeUpdated = counts.ShopeeUpdated
run.ShopeeFieldsFilled = counts.ShopeeFieldsFilled
run.ShopeeFieldsSameSourceUpdated = counts.ShopeeFieldsSameSourceUpdated
run.ShopeeFieldsManualSkipped = counts.ShopeeFieldsManualSkipped
run.ShopeeFieldsStaleSkipped = counts.ShopeeFieldsStaleSkipped
run.SKUCreated = counts.SKUCreated
run.SKUUpdated = counts.SKUUpdated
run.SKUFilled = counts.SKUFilled
run.SKUSameSourceUpdated = counts.SKUSameSourceUpdated
run.SKUSkipped = counts.SKUSkipped
run.SKUManualSkipped = counts.SKUManualSkipped
run.SKUStaleSkipped = counts.SKUStaleSkipped
run.PddCreated = counts.PddCreated
run.PddUpdated = counts.PddUpdated
run.AssociationCreated = counts.AssociationCreated
run.AssociationUnchanged = counts.AssociationUnchanged
if err = repository.CompleteCatalogImportRun(tx, run); err != nil {
return fail(err)
}
if err = tx.Commit(); err != nil {
return CatalogBatchResponse{}, err
}
return resp, nil
}
func normalizeCatalogUpdatePolicy(raw string) (string, error) {
switch strings.TrimSpace(raw) {
case "", "fill_missing":
return "fill_missing", nil
case "insert_only", "overwrite_same_source":
return strings.TrimSpace(raw), nil
default:
return "", catalogInvalid("INVALID_UPDATE_POLICY", "update_policy 只支持 insert_only、fill_missing、overwrite_same_source", nil)
}
}
func catalogSKURecordID(goodsID, specKey string) string {
sum := sha256.Sum256([]byte(goodsID + "\x00" + specKey))
return "catalog:" + hex.EncodeToString(sum[:])
}
func validateCatalogImageURL(raw string) error {
raw = strings.TrimSpace(raw)
if raw == "" {
return nil
}
if len(raw) > 2048 {
return fmt.Errorf("蝦皮主图片 URL 不能超过 2048 字节")
}
u, err := url.ParseRequestURI(raw)
if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
return fmt.Errorf("蝦皮主图片 URL 必须是完整的 HTTP/HTTPS 地址")
}
if u.User != nil {
return fmt.Errorf("蝦皮主图片 URL 不能包含用户名或密码")
}
parameters := u.Query()
if fragmentParameters, fragmentErr := url.ParseQuery(u.Fragment); fragmentErr == nil {
for key, values := range fragmentParameters {
parameters[key] = append(parameters[key], values...)
}
}
for key := range parameters {
lower := strings.ToLower(key)
for _, risky := range []string{"token", "cookie", "password", "passwd", "authorization", "credential", "secret", "session"} {
if strings.Contains(lower, risky) {
return fmt.Errorf("蝦皮主图片 URL 不能包含凭据参数")
}
}
}
return nil
}
func replayCatalogBatch(db *sql.DB, source, batchID, hash, now string) (CatalogBatchResponse, error) {
run, err := repository.GetCatalogImportRun(db, source, batchID)
if err != nil {
return CatalogBatchResponse{}, err
}
if run.RequestHash != hash {
_ = repository.RecordCatalogImportConflict(db, source, batchID, now)
return CatalogBatchResponse{}, &CatalogError{Status: 409, Code: "IDEMPOTENCY_CONFLICT", Message: "同一 batch_id 已提交不同内容"}
}
_ = repository.RecordCatalogImportReplay(db, source, batchID, now)
if run.Status == model.CatalogImportProcessing {
return CatalogBatchResponse{}, &CatalogError{Status: 409, Code: "BATCH_PROCESSING", Message: "批次仍在处理中", Retryable: true}
}
if run.Status == model.CatalogImportFailed {
var stored CatalogError
if json.Unmarshal([]byte(run.ResponseBody), &stored) == nil && stored.Code != "" {
return CatalogBatchResponse{}, &stored
}
return CatalogBatchResponse{}, &CatalogError{Status: 422, Code: "BATCH_FAILED", Message: run.ErrorSummary}
}
var resp CatalogBatchResponse
if err = json.Unmarshal([]byte(run.ResponseBody), &resp); err != nil {
return CatalogBatchResponse{}, fmt.Errorf("读取批次重放结果失败: %w", err)
}
resp.Replayed = true
return resp, nil
}
func truncateCatalogError(s string) string {
if len(s) > 1000 {
return s[:1000]
}
return s
}