fix: 增强顺运宝同步完整性保护 (#58)
This commit is contained in:
@@ -24,9 +24,9 @@ syb:
|
||||
# 列表接口每页条数。抓包里网页用的就是 20。
|
||||
page_size: 20
|
||||
|
||||
# 单次同步的条数上限,超过就报错而不是硬拉。
|
||||
# 防止日期范围手滑填成一整年,把整库拉下来。
|
||||
max_matches: 500
|
||||
# 整个日期范围的货运单数上限,默认 10000;超过就停止,不拉列表和明细。
|
||||
# 指定日期同步另有最多 31 天限制,自动增量不受天数限制。
|
||||
max_matches: 10000
|
||||
|
||||
# 首次同步的起始日期。之后按「上次同步时间」增量拉,不再用这个值。
|
||||
sync_from: "2026-07-01"
|
||||
|
||||
+148
-50
@@ -30,7 +30,11 @@ import (
|
||||
"cmautobuy/admin/syb"
|
||||
)
|
||||
|
||||
const dateLayout = "2006-01-02"
|
||||
const (
|
||||
dateLayout = "2006-01-02"
|
||||
maxSpecifiedSyncDays = 31
|
||||
defaultSybMaxMatches = 10000
|
||||
)
|
||||
|
||||
// SybSyncOptions 表示一次同步的日期选择。From/To 都为空是日常自动增量;
|
||||
// 两者都有值是操作员明确发起的指定日期补同步。
|
||||
@@ -72,9 +76,35 @@ func NewSybSyncOptions(fromRaw, toRaw string, now time.Time) (SybSyncOptions, er
|
||||
if to > dateOf(now) {
|
||||
return SybSyncOptions{}, fmt.Errorf("结束日期不能晚于顺运宝服务端今天(%s)", dateOf(now))
|
||||
}
|
||||
days := int(toDate.Sub(fromDate).Hours()/24) + 1
|
||||
if days > maxSpecifiedSyncDays {
|
||||
return SybSyncOptions{}, fmt.Errorf("指定日期同步最多选择 %d 天,当前范围为 %d 天,请分段同步",
|
||||
maxSpecifiedSyncDays, days)
|
||||
}
|
||||
return SybSyncOptions{From: from, To: to}, nil
|
||||
}
|
||||
|
||||
// splitDateRange 把闭区间拆成逐日查询段。顺运宝只支持日期粒度,按天拉取
|
||||
// 能把 offset 分页期间新增数据造成的位移限制在当天。
|
||||
func splitDateRange(from, to string) ([]string, error) {
|
||||
fromDate, err := time.Parse(dateLayout, from)
|
||||
if err != nil || fromDate.Format(dateLayout) != from {
|
||||
return nil, fmt.Errorf("开始日期格式不正确,请使用 YYYY-MM-DD")
|
||||
}
|
||||
toDate, err := time.Parse(dateLayout, to)
|
||||
if err != nil || toDate.Format(dateLayout) != to {
|
||||
return nil, fmt.Errorf("结束日期格式不正确,请使用 YYYY-MM-DD")
|
||||
}
|
||||
if fromDate.After(toDate) {
|
||||
return nil, fmt.Errorf("开始日期不能晚于结束日期")
|
||||
}
|
||||
dates := make([]string, 0, int(toDate.Sub(fromDate).Hours()/24)+1)
|
||||
for day := fromDate; !day.After(toDate); day = day.AddDate(0, 0, 1) {
|
||||
dates = append(dates, day.Format(dateLayout))
|
||||
}
|
||||
return dates, nil
|
||||
}
|
||||
|
||||
// sybLocation 是顺运宝服务端的时区。
|
||||
//
|
||||
// `[必须]` 日期范围筛的是服务端的 created,而它是 UTC+8——实测
|
||||
@@ -292,7 +322,7 @@ func RunSybSyncWithOptions(ctx context.Context, db *sql.DB, client *syb.Client,
|
||||
}
|
||||
maxMatches := cfg.MaxMatches
|
||||
if maxMatches <= 0 {
|
||||
maxMatches = 500
|
||||
maxMatches = defaultSybMaxMatches
|
||||
}
|
||||
|
||||
lastSyncedAt, _, err := repository.GetSybLastSyncedAt(db)
|
||||
@@ -327,20 +357,41 @@ func RunSybSyncWithOptions(ctx context.Context, db *sql.DB, client *syb.Client,
|
||||
}
|
||||
report.From, report.To = from, to
|
||||
|
||||
total, err := client.ListTotal(ctx, from, to, pageSize)
|
||||
dates, err := splitDateRange(from, to)
|
||||
if err != nil {
|
||||
report.Err = fmt.Errorf("查询货运单总数失败: %w", err)
|
||||
report.Err = err
|
||||
report.FinishedAt = time.Now().UTC()
|
||||
return report
|
||||
}
|
||||
if total > maxMatches {
|
||||
report.Err = fmt.Errorf(
|
||||
"日期范围 %s ~ %s 内有 %d 张货运单,超过单次同步上限 %d,"+
|
||||
"请缩小日期范围或联系维护者调大 max_matches", from, to, total, maxMatches)
|
||||
report.FinishedAt = time.Now().UTC()
|
||||
return report
|
||||
type dailyPlan struct {
|
||||
date string
|
||||
total int
|
||||
}
|
||||
if total == 0 {
|
||||
plans := make([]dailyPlan, 0, len(dates))
|
||||
allTotal := 0
|
||||
for _, date := range dates {
|
||||
total, totalErr := client.ListTotal(ctx, date, date, pageSize)
|
||||
if totalErr != nil {
|
||||
report.Err = fmt.Errorf("查询 %s 货运单总数失败: %w", date, totalErr)
|
||||
report.FinishedAt = time.Now().UTC()
|
||||
return report
|
||||
}
|
||||
if total < 0 {
|
||||
report.Err = fmt.Errorf("查询 %s 货运单总数返回负数 %d", date, total)
|
||||
report.FinishedAt = time.Now().UTC()
|
||||
return report
|
||||
}
|
||||
if total > maxMatches-allTotal {
|
||||
report.Err = fmt.Errorf(
|
||||
"日期范围 %s ~ %s 内货运单总数超过单次同步上限 %d,"+
|
||||
"请缩小日期范围或在确认机器和网络容量后调大 max_matches", from, to, maxMatches)
|
||||
report.FinishedAt = time.Now().UTC()
|
||||
return report
|
||||
}
|
||||
allTotal += total
|
||||
plans = append(plans, dailyPlan{date: date, total: total})
|
||||
}
|
||||
if allTotal == 0 {
|
||||
report.FinishedAt = time.Now().UTC()
|
||||
if advanceCursor {
|
||||
if err := repository.SetSybLastSyncedAt(db, model.NowISO()); err != nil {
|
||||
@@ -350,30 +401,7 @@ func RunSybSyncWithOptions(ctx context.Context, db *sql.DB, client *syb.Client,
|
||||
return report
|
||||
}
|
||||
|
||||
// ① 翻页拉全部货运单行。
|
||||
stockByID := map[int64]syb.StockRow{}
|
||||
var orderedIDs []int64
|
||||
for start := 0; start < total; start += pageSize {
|
||||
pageIndex := start/pageSize + 1
|
||||
rows, err := client.ListPage(ctx, from, to, start, pageIndex, pageSize)
|
||||
if err != nil {
|
||||
report.Err = fmt.Errorf("拉取货运单列表第 %d 页失败(已获取 %d/%d 张,本次同步整体作废,"+
|
||||
"下次会从同一个起始日期重新拉,靠 upsert 幂等不会重复计数): %w",
|
||||
pageIndex, len(orderedIDs), total, err)
|
||||
report.FinishedAt = time.Now().UTC()
|
||||
return report
|
||||
}
|
||||
for _, row := range rows {
|
||||
if _, dup := stockByID[row.ID]; dup {
|
||||
continue
|
||||
}
|
||||
stockByID[row.ID] = row
|
||||
orderedIDs = append(orderedIDs, row.ID)
|
||||
}
|
||||
}
|
||||
report.StockCount = len(orderedIDs)
|
||||
|
||||
// ② 按 100 个一批取明细,③ 逐张货运单写库。
|
||||
// ① 每天单独翻页,② 按 100 个一批取明细,③ 逐张货运单写库。
|
||||
//
|
||||
// `[必须]` 不放在一个大事务里——几千条明细的事务会长时间持锁;
|
||||
// 按货运单为单位提交,失败了已成功的部分保留(下次重拉会 upsert
|
||||
@@ -383,29 +411,71 @@ func RunSybSyncWithOptions(ctx context.Context, db *sql.DB, client *syb.Client,
|
||||
// 不更新 last_synced_at——已经成功写入的部分不回滚(它们本身是幂等
|
||||
// 的),但"这次同步整体算成功"这件事不能发生,否则漏掉的单永远补不回来。
|
||||
const detailBatch = 100
|
||||
for i := 0; i < len(orderedIDs); i += detailBatch {
|
||||
end := i + detailBatch
|
||||
if end > len(orderedIDs) {
|
||||
end = len(orderedIDs)
|
||||
for _, plan := range plans {
|
||||
if plan.total == 0 {
|
||||
continue
|
||||
}
|
||||
batch := orderedIDs[i:end]
|
||||
|
||||
details, err := client.DetailListByStock(ctx, batch)
|
||||
if err != nil {
|
||||
report.Err = fmt.Errorf("拉取货运单明细失败(本次同步整体作废,"+
|
||||
"已写入的数据保留,下次重拉会 upsert 覆盖): %w", err)
|
||||
stockByID := map[int64]syb.StockRow{}
|
||||
orderedIDs := make([]int64, 0, plan.total)
|
||||
for start := 0; start < plan.total; start += pageSize {
|
||||
pageIndex := start/pageSize + 1
|
||||
rows, responseTotal, listErr := client.ListPage(ctx, plan.date, plan.date, start, pageIndex, pageSize)
|
||||
if listErr != nil {
|
||||
report.Err = fmt.Errorf("拉取 %s 货运单列表第 %d 页失败(已获取 %d/%d 张,本次同步整体作废,"+
|
||||
"下次会从同一个起始日期重新拉,靠 upsert 幂等不会重复计数): %w",
|
||||
plan.date, pageIndex, len(orderedIDs), plan.total, listErr)
|
||||
report.FinishedAt = time.Now().UTC()
|
||||
return report
|
||||
}
|
||||
if responseTotal != plan.total {
|
||||
report.Err = fmt.Errorf("%s 货运单总数在分页期间从 %d 变为 %d,"+
|
||||
"为防止 offset 分页漏单,本次同步停止且不推进游标", plan.date, plan.total, responseTotal)
|
||||
report.FinishedAt = time.Now().UTC()
|
||||
return report
|
||||
}
|
||||
for _, row := range rows {
|
||||
if _, dup := stockByID[row.ID]; dup {
|
||||
continue
|
||||
}
|
||||
stockByID[row.ID] = row
|
||||
orderedIDs = append(orderedIDs, row.ID)
|
||||
}
|
||||
}
|
||||
if len(orderedIDs) != plan.total {
|
||||
report.Err = fmt.Errorf("%s 货运单列表不完整:预期 %d 张,分页后只有 %d 个唯一 ID,"+
|
||||
"本次同步停止且不推进游标", plan.date, plan.total, len(orderedIDs))
|
||||
report.FinishedAt = time.Now().UTC()
|
||||
return report
|
||||
}
|
||||
report.StockCount += len(orderedIDs)
|
||||
|
||||
for _, d := range details {
|
||||
stockRow := stockByID[d.ID]
|
||||
if err := writeStockDetail(db, cfg.BaseURL, stockRow, d, &report); err != nil {
|
||||
report.Err = fmt.Errorf("写入货运单 %s(id=%d)失败(本次同步整体作废,"+
|
||||
"已写入的数据保留): %w", d.Code, d.ID, err)
|
||||
for i := 0; i < len(orderedIDs); i += detailBatch {
|
||||
end := i + detailBatch
|
||||
if end > len(orderedIDs) {
|
||||
end = len(orderedIDs)
|
||||
}
|
||||
batch := orderedIDs[i:end]
|
||||
details, detailErr := client.DetailListByStock(ctx, batch)
|
||||
if detailErr != nil {
|
||||
report.Err = fmt.Errorf("拉取 %s 货运单明细失败(本次同步整体作废,"+
|
||||
"已写入的数据保留,下次重拉会 upsert 覆盖): %w", plan.date, detailErr)
|
||||
report.FinishedAt = time.Now().UTC()
|
||||
return report
|
||||
}
|
||||
if completeErr := validateDetailBatch(batch, details); completeErr != nil {
|
||||
report.Err = fmt.Errorf("%s 货运单明细不完整:%w;本次同步停止且不推进游标", plan.date, completeErr)
|
||||
report.FinishedAt = time.Now().UTC()
|
||||
return report
|
||||
}
|
||||
for _, d := range details {
|
||||
stockRow := stockByID[d.ID]
|
||||
if err := writeStockDetail(db, cfg.BaseURL, stockRow, d, &report); err != nil {
|
||||
report.Err = fmt.Errorf("写入货运单 %s(id=%d)失败(本次同步整体作废,"+
|
||||
"已写入的数据保留): %w", d.Code, d.ID, err)
|
||||
report.FinishedAt = time.Now().UTC()
|
||||
return report
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -420,6 +490,34 @@ func RunSybSyncWithOptions(ctx context.Context, db *sql.DB, client *syb.Client,
|
||||
return report
|
||||
}
|
||||
|
||||
// validateDetailBatch 确认批量明细响应与请求 ID 一一对应。任何缺失、重复、
|
||||
// 意外 ID 或空商品明细都会让同步失败,避免在数据不完整时推进游标。
|
||||
func validateDetailBatch(requested []int64, details []syb.StockDetail) error {
|
||||
wanted := make(map[int64]struct{}, len(requested))
|
||||
for _, id := range requested {
|
||||
wanted[id] = struct{}{}
|
||||
}
|
||||
seen := make(map[int64]struct{}, len(details))
|
||||
for _, detail := range details {
|
||||
if _, ok := wanted[detail.ID]; !ok {
|
||||
return fmt.Errorf("响应包含未请求的货运单 id=%d", detail.ID)
|
||||
}
|
||||
if _, duplicate := seen[detail.ID]; duplicate {
|
||||
return fmt.Errorf("响应重复返回货运单 id=%d", detail.ID)
|
||||
}
|
||||
if len(detail.Details) == 0 {
|
||||
return fmt.Errorf("货运单 id=%d 没有返回商品明细", detail.ID)
|
||||
}
|
||||
seen[detail.ID] = struct{}{}
|
||||
}
|
||||
for _, id := range requested {
|
||||
if _, ok := seen[id]; !ok {
|
||||
return fmt.Errorf("响应缺少货运单 id=%d", id)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeStockDetail 把一张货运单的全部商品明细写进 syb_orders,
|
||||
// 一张货运单一个事务(工单 #46「按货运单为单位提交」)。
|
||||
func writeStockDetail(db *sql.DB, baseURL string, stockRow syb.StockRow, detail syb.StockDetail, report *SyncReport) error {
|
||||
|
||||
+239
-9
@@ -92,6 +92,8 @@ func TestNewSybSyncOptions_指定日期校验(t *testing.T) {
|
||||
}{
|
||||
{name: "两端都空是自动增量"},
|
||||
{name: "合法闭区间", from: "2026-08-01", to: "2026-08-10"},
|
||||
{name: "三十一天合法", from: "2026-07-11", to: "2026-08-10"},
|
||||
{name: "超过三十一天", from: "2026-07-10", to: "2026-08-10", wantErr: "最多选择 31 天"},
|
||||
{name: "只填开始", from: "2026-08-01", wantErr: "同时填写"},
|
||||
{name: "开始格式错误", from: "2026/08/01", to: "2026-08-02", wantErr: "开始日期格式"},
|
||||
{name: "结束格式错误", from: "2026-08-01", to: "2026/08/02", wantErr: "结束日期格式"},
|
||||
@@ -157,6 +159,7 @@ type fakeStock struct {
|
||||
ID int64
|
||||
Code string
|
||||
ShopName string
|
||||
Created string
|
||||
Receiver string // 用来验证个人信息确实没有落库
|
||||
Details []fakeDetail
|
||||
}
|
||||
@@ -169,18 +172,30 @@ type fakeStock struct {
|
||||
func fakeSybServer(t *testing.T, stocks []fakeStock, failListPageIndex int) *httptest.Server {
|
||||
t.Helper()
|
||||
byID := map[int64]fakeStock{}
|
||||
for _, s := range stocks {
|
||||
for i := range stocks {
|
||||
if stocks[i].Created == "" {
|
||||
stocks[i].Created = "2026-07-28"
|
||||
}
|
||||
s := stocks[i]
|
||||
byID[s.ID] = s
|
||||
}
|
||||
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/am/stock/listTotal":
|
||||
writeEnvelope(t, w, true, "ok", len(stocks), nil)
|
||||
var body map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("解析 listTotal 请求失败: %v", err)
|
||||
}
|
||||
filtered := filterFakeStocksByRequest(t, stocks, body)
|
||||
writeEnvelope(t, w, true, "ok", len(filtered), nil)
|
||||
|
||||
case "/am/stock/list":
|
||||
var body map[string]any
|
||||
json.NewDecoder(r.Body).Decode(&body)
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("解析 list 请求失败: %v", err)
|
||||
}
|
||||
filtered := filterFakeStocksByRequest(t, stocks, body)
|
||||
pageIndex := int(body["pageIndex"].(float64))
|
||||
length := int(body["length"].(float64))
|
||||
start := int(body["start"].(float64))
|
||||
@@ -191,19 +206,19 @@ func fakeSybServer(t *testing.T, stocks []fakeStock, failListPageIndex int) *htt
|
||||
}
|
||||
|
||||
end := start + length
|
||||
if end > len(stocks) {
|
||||
end = len(stocks)
|
||||
if end > len(filtered) {
|
||||
end = len(filtered)
|
||||
}
|
||||
var list []map[string]any
|
||||
if start < len(stocks) {
|
||||
for _, s := range stocks[start:end] {
|
||||
if start < len(filtered) {
|
||||
for _, s := range filtered[start:end] {
|
||||
list = append(list, map[string]any{
|
||||
"id": s.ID, "code": s.Code, "shopName": s.ShopName,
|
||||
"receiver": s.Receiver, "orderStatus": "待出货",
|
||||
})
|
||||
}
|
||||
}
|
||||
writeEnvelope(t, w, true, "ok", map[string]any{"list": list}, nil)
|
||||
writeEnvelope(t, w, true, "ok", map[string]any{"list": list, "total": len(filtered)}, nil)
|
||||
|
||||
case "/am/stock/detail/listByStock":
|
||||
var body struct {
|
||||
@@ -238,6 +253,60 @@ func fakeSybServer(t *testing.T, stocks []fakeStock, failListPageIndex int) *htt
|
||||
}))
|
||||
}
|
||||
|
||||
func filterFakeStocksByRequest(t *testing.T, stocks []fakeStock, body map[string]any) []fakeStock {
|
||||
t.Helper()
|
||||
queries, ok := body["queries"].([]any)
|
||||
if !ok || len(queries) == 0 {
|
||||
t.Fatalf("同步请求缺少 queries: %#v", body["queries"])
|
||||
}
|
||||
query, ok := queries[0].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("同步请求 queries[0] 格式错误: %#v", queries[0])
|
||||
}
|
||||
rangeText, ok := query["dvalue"].(string)
|
||||
if !ok {
|
||||
t.Fatalf("同步请求缺少日期范围 dvalue: %#v", query["dvalue"])
|
||||
}
|
||||
parts := strings.SplitN(rangeText, ",", 2)
|
||||
if len(parts) != 2 {
|
||||
t.Fatalf("同步请求日期范围格式错误: %q", rangeText)
|
||||
}
|
||||
|
||||
filtered := make([]fakeStock, 0, len(stocks))
|
||||
for _, stock := range stocks {
|
||||
if stock.Created >= parts[0] && stock.Created <= parts[1] {
|
||||
filtered = append(filtered, stock)
|
||||
}
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
type integrityServerData struct {
|
||||
total int
|
||||
responseTotal int
|
||||
list []map[string]any
|
||||
details []map[string]any
|
||||
}
|
||||
|
||||
func fakeIntegritySybServer(t *testing.T, data integrityServerData) *httptest.Server {
|
||||
t.Helper()
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/am/stock/listTotal":
|
||||
writeEnvelope(t, w, true, "ok", data.total, nil)
|
||||
case "/am/stock/list":
|
||||
writeEnvelope(t, w, true, "ok", map[string]any{
|
||||
"list": data.list, "total": data.responseTotal,
|
||||
}, nil)
|
||||
case "/am/stock/detail/listByStock":
|
||||
writeEnvelope(t, w, true, "ok", map[string]any{"list": data.details}, nil)
|
||||
default:
|
||||
t.Errorf("测试假服务端没有实现这个路径: %s", r.URL.Path)
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
func writeEnvelope(t *testing.T, w http.ResponseWriter, status bool, msg string, data any, code any) {
|
||||
t.Helper()
|
||||
b, err := json.Marshal(map[string]any{"status": status, "msg": msg, "data": data, "code": code})
|
||||
@@ -297,7 +366,10 @@ func TestRunSybSync_已有的ShopeeSKUID同步后仍在(t *testing.T) {
|
||||
}
|
||||
|
||||
// 再同步一次(模拟顺运宝那边这张单信息有更新)。
|
||||
report2 := RunSybSync(context.Background(), db, client, cfg, now.Add(time.Hour))
|
||||
report2 := RunSybSyncWithOptions(context.Background(), db, client, cfg, now.Add(time.Hour), SybSyncOptions{
|
||||
From: "2026-07-28",
|
||||
To: "2026-07-28",
|
||||
})
|
||||
if report2.Err != nil {
|
||||
t.Fatalf("二次同步失败: %v", report2.Err)
|
||||
}
|
||||
@@ -521,6 +593,164 @@ func TestRunSybSync_超过最大条数报错不硬拉(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSybSync_全范围容量预检通过前不拉列表和明细(t *testing.T) {
|
||||
listCalls := 0
|
||||
detailCalls := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/am/stock/listTotal":
|
||||
writeEnvelope(t, w, true, "ok", 2, nil)
|
||||
case "/am/stock/list":
|
||||
listCalls++
|
||||
writeEnvelope(t, w, true, "ok", map[string]any{"list": []any{}, "total": 2}, nil)
|
||||
case "/am/stock/detail/listByStock":
|
||||
detailCalls++
|
||||
writeEnvelope(t, w, true, "ok", map[string]any{"list": []any{}}, nil)
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
db := newSyncTestDB(t)
|
||||
client, _ := syb.New(srv.URL)
|
||||
report := RunSybSyncWithOptions(context.Background(), db, client,
|
||||
config.SybConfig{BaseURL: srv.URL, PageSize: 20, MaxMatches: 3, SyncFrom: "2026-07-27"},
|
||||
time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC),
|
||||
SybSyncOptions{From: "2026-07-27", To: "2026-07-28"})
|
||||
if report.Err == nil || !strings.Contains(report.Err.Error(), "超过单次同步上限") {
|
||||
t.Fatalf("两天合计 4 张超过上限 3 应该失败,实际: %v", report.Err)
|
||||
}
|
||||
if listCalls != 0 || detailCalls != 0 {
|
||||
t.Fatalf("容量预检完成前不应拉列表或明细: list=%d detail=%d", listCalls, detailCalls)
|
||||
}
|
||||
if _, found, _ := repository.GetSybLastSyncedAt(db); found {
|
||||
t.Fatal("容量预检失败不得推进游标")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSybSync_默认上限允许同步超过五百张(t *testing.T) {
|
||||
const stockCount = 501
|
||||
stocks := make([]fakeStock, 0, stockCount)
|
||||
for i := 1; i <= stockCount; i++ {
|
||||
id := int64(i)
|
||||
stocks = append(stocks, fakeStock{
|
||||
ID: id, Code: "ORDER", Details: []fakeDetail{{
|
||||
ID: id, ProductID: 100000 + id, ProductQty: 1, ProductPrice: 1,
|
||||
}},
|
||||
})
|
||||
}
|
||||
srv := fakeSybServer(t, stocks, 0)
|
||||
defer srv.Close()
|
||||
|
||||
db := newSyncTestDB(t)
|
||||
client, _ := syb.New(srv.URL)
|
||||
report := RunSybSyncWithOptions(context.Background(), db, client,
|
||||
config.SybConfig{BaseURL: srv.URL, PageSize: 20, SyncFrom: "2026-07-28"},
|
||||
time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC),
|
||||
SybSyncOptions{From: "2026-07-28", To: "2026-07-28"})
|
||||
if report.Err != nil {
|
||||
t.Fatalf("默认上限应该允许 501 张货运单正常同步,实际失败: %v", report.Err)
|
||||
}
|
||||
if report.StockCount != stockCount || report.Created != stockCount {
|
||||
t.Fatalf("同步统计错误: StockCount=%d Created=%d", report.StockCount, report.Created)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSybSync_跨日范围按天同步并汇总(t *testing.T) {
|
||||
srv := fakeSybServer(t, []fakeStock{
|
||||
{ID: 1, Code: "DAY-1", Created: "2026-07-27", Details: []fakeDetail{{ID: 11, ProductID: 111, ProductQty: 1}}},
|
||||
{ID: 2, Code: "DAY-2", Created: "2026-07-28", Details: []fakeDetail{{ID: 22, ProductID: 222, ProductQty: 1}}},
|
||||
}, 0)
|
||||
defer srv.Close()
|
||||
|
||||
db := newSyncTestDB(t)
|
||||
client, _ := syb.New(srv.URL)
|
||||
report := RunSybSyncWithOptions(context.Background(), db, client,
|
||||
config.SybConfig{BaseURL: srv.URL, PageSize: 20, MaxMatches: 100, SyncFrom: "2026-07-27"},
|
||||
time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC),
|
||||
SybSyncOptions{From: "2026-07-27", To: "2026-07-28"})
|
||||
if report.Err != nil {
|
||||
t.Fatalf("跨日同步失败: %v", report.Err)
|
||||
}
|
||||
if report.StockCount != 2 || report.Created != 2 {
|
||||
t.Fatalf("跨日汇总错误: StockCount=%d Created=%d", report.StockCount, report.Created)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDetailBatch_必须与请求货运单一一对应(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
details []syb.StockDetail
|
||||
wantErr string
|
||||
}{
|
||||
{name: "完整", details: []syb.StockDetail{
|
||||
{ID: 1, Details: []syb.DetailItem{{ID: 11}}},
|
||||
{ID: 2, Details: []syb.DetailItem{{ID: 22}}},
|
||||
}},
|
||||
{name: "缺失", details: []syb.StockDetail{{ID: 1, Details: []syb.DetailItem{{ID: 11}}}}, wantErr: "缺少"},
|
||||
{name: "重复", details: []syb.StockDetail{
|
||||
{ID: 1, Details: []syb.DetailItem{{ID: 11}}},
|
||||
{ID: 1, Details: []syb.DetailItem{{ID: 12}}},
|
||||
}, wantErr: "重复"},
|
||||
{name: "意外ID", details: []syb.StockDetail{
|
||||
{ID: 1, Details: []syb.DetailItem{{ID: 11}}},
|
||||
{ID: 3, Details: []syb.DetailItem{{ID: 33}}},
|
||||
}, wantErr: "未请求"},
|
||||
{name: "空明细", details: []syb.StockDetail{
|
||||
{ID: 1, Details: []syb.DetailItem{{ID: 11}}}, {ID: 2},
|
||||
}, wantErr: "没有返回商品明细"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := validateDetailBatch([]int64{1, 2}, tc.details)
|
||||
if tc.wantErr == "" && err != nil {
|
||||
t.Fatalf("完整响应不应该报错: %v", err)
|
||||
}
|
||||
if tc.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tc.wantErr)) {
|
||||
t.Fatalf("错误应包含 %q,实际 %v", tc.wantErr, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSybSync_列表或明细不完整时不推进游标(t *testing.T) {
|
||||
completeList := []map[string]any{{"id": 1, "code": "A"}, {"id": 2, "code": "B"}}
|
||||
completeDetails := []map[string]any{
|
||||
{"id": 1, "code": "A", "details": []map[string]any{{"id": 11, "productId": 111, "productQty": 1}}},
|
||||
{"id": 2, "code": "B", "details": []map[string]any{{"id": 22, "productId": 222, "productQty": 1}}},
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
data integrityServerData
|
||||
wantErr string
|
||||
}{
|
||||
{name: "分页期间总数变化", data: integrityServerData{total: 2, responseTotal: 3, list: completeList}, wantErr: "分页期间"},
|
||||
{name: "列表存在重复ID", data: integrityServerData{total: 2, responseTotal: 2,
|
||||
list: []map[string]any{{"id": 1, "code": "A"}, {"id": 1, "code": "A"}}}, wantErr: "列表不完整"},
|
||||
{name: "明细缺少货运单", data: integrityServerData{total: 2, responseTotal: 2,
|
||||
list: completeList, details: completeDetails[:1]}, wantErr: "明细不完整"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
srv := fakeIntegritySybServer(t, tc.data)
|
||||
defer srv.Close()
|
||||
db := newSyncTestDB(t)
|
||||
client, _ := syb.New(srv.URL)
|
||||
report := RunSybSyncWithOptions(context.Background(), db, client,
|
||||
config.SybConfig{BaseURL: srv.URL, PageSize: 20, MaxMatches: 100, SyncFrom: "2026-07-28"},
|
||||
time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC),
|
||||
SybSyncOptions{From: "2026-07-28", To: "2026-07-28"})
|
||||
if report.Err == nil || !strings.Contains(report.Err.Error(), tc.wantErr) {
|
||||
t.Fatalf("错误应包含 %q,实际 %v", tc.wantErr, report.Err)
|
||||
}
|
||||
if _, found, err := repository.GetSybLastSyncedAt(db); err != nil || found {
|
||||
t.Fatalf("不完整响应不得推进游标: found=%v err=%v", found, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSybSync_一张货运单多个商品各落一行(t *testing.T) {
|
||||
srv := fakeSybServer(t, []fakeStock{
|
||||
{ID: 1, Code: "A", Details: []fakeDetail{
|
||||
|
||||
+11
-6
@@ -504,30 +504,35 @@ type StockRow struct {
|
||||
}
|
||||
|
||||
// ListPage 按日期范围翻一页货运单列表:POST /am/stock/list。
|
||||
func (c *Client) ListPage(ctx context.Context, dateFrom, dateTo string, start, pageIndex, pageSize int) ([]StockRow, error) {
|
||||
// 返回响应内的 total,供上层核对分页期间总数有没有漂移。
|
||||
func (c *Client) ListPage(ctx context.Context, dateFrom, dateTo string, start, pageIndex, pageSize int) ([]StockRow, int, error) {
|
||||
data, err := c.do(ctx, http.MethodPost, "/am/stock/list", nil,
|
||||
listPayload(dateFrom, dateTo, start, pageIndex, pageSize))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
var wrap struct {
|
||||
List []map[string]any `json:"list"`
|
||||
List []map[string]any `json:"list"`
|
||||
Total *int `json:"total"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &wrap); err != nil {
|
||||
return nil, fmt.Errorf("货运单列表响应格式错误: %w", err)
|
||||
return nil, 0, fmt.Errorf("货运单列表响应格式错误: %w", err)
|
||||
}
|
||||
if wrap.Total == nil || *wrap.Total < 0 {
|
||||
return nil, 0, fmt.Errorf("货运单列表响应缺少合法的 total")
|
||||
}
|
||||
|
||||
rows := make([]StockRow, 0, len(wrap.List))
|
||||
for _, raw := range wrap.List {
|
||||
id, ok := toInt64(raw["id"])
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("货运单列表里有一行缺少合法的 id: %v", raw)
|
||||
return nil, 0, fmt.Errorf("货运单列表里有一行缺少合法的 id: %v", raw)
|
||||
}
|
||||
code, _ := raw["code"].(string)
|
||||
rows = append(rows, StockRow{ID: id, Code: code, Raw: raw})
|
||||
}
|
||||
return rows, nil
|
||||
return rows, *wrap.Total, nil
|
||||
}
|
||||
|
||||
// ---------- 货运单明细 ----------
|
||||
|
||||
@@ -304,6 +304,7 @@ func TestClient_ListTotal和ListPage(t *testing.T) {
|
||||
w.Write(envelopeBody(t, true, "ok", 1, nil))
|
||||
case "/am/stock/list":
|
||||
w.Write(envelopeBody(t, true, "ok", map[string]any{
|
||||
"total": 1,
|
||||
"list": []map[string]any{
|
||||
{"id": 75104587, "code": "260728TB95MJTQ", "amtOrder": 61200},
|
||||
},
|
||||
@@ -321,15 +322,40 @@ func TestClient_ListTotal和ListPage(t *testing.T) {
|
||||
t.Fatalf("总数应该是 1,实际 %d", total)
|
||||
}
|
||||
|
||||
rows, err := c.ListPage(context.Background(), "2026-07-25", "2026-07-28", 0, 1, 20)
|
||||
rows, pageTotal, err := c.ListPage(context.Background(), "2026-07-25", "2026-07-28", 0, 1, 20)
|
||||
if err != nil {
|
||||
t.Fatalf("list 失败: %v", err)
|
||||
}
|
||||
if pageTotal != 1 {
|
||||
t.Fatalf("list 响应内 total 应该是 1,实际 %d", pageTotal)
|
||||
}
|
||||
if len(rows) != 1 || rows[0].ID != 75104587 || rows[0].Code != "260728TB95MJTQ" {
|
||||
t.Fatalf("列表结果不对: %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClient_ListPage要求合法Total(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
data map[string]any
|
||||
}{
|
||||
{"缺少total", map[string]any{"list": []any{}}},
|
||||
{"total类型错误", map[string]any{"list": []any{}, "total": "1"}},
|
||||
{"total为负数", map[string]any{"list": []any{}, "total": -1}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(envelopeBody(t, true, "ok", test.data, nil))
|
||||
}))
|
||||
defer srv.Close()
|
||||
client, _ := New(srv.URL)
|
||||
if _, _, err := client.ListPage(context.Background(), "2026-08-09", "2026-08-09", 0, 1, 20); err == nil {
|
||||
t.Fatal("非法 total 应返回错误")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClient_DetailListByStock_一单多商品(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Query().Get("hist") != "0" {
|
||||
|
||||
@@ -60,7 +60,7 @@
|
||||
</div>
|
||||
<p class="hint">
|
||||
按顺运宝货运单创建日期(UTC+8)同步,开始日和结束日都包含。
|
||||
日常同步请直接使用工具栏的“同步”。
|
||||
每次最多选择 31 天;日常同步请直接使用工具栏的“同步”。
|
||||
</p>
|
||||
</div>
|
||||
<div class="modal-foot">
|
||||
|
||||
@@ -445,12 +445,12 @@ Go 的 map 是无序的,不靠它定顺序的话,同一个商品每次刷新
|
||||
指定日期同步
|
||||
|
||||
开始日期 [2026-08-01] 结束日期 [2026-08-09]
|
||||
按货运单创建日期(UTC+8)同步,开始日和结束日都包含。
|
||||
按货运单创建日期(UTC+8)同步,开始日和结束日都包含。每次最多选择 31 天。
|
||||
|
||||
[取消] [开始同步]
|
||||
```
|
||||
|
||||
- 两端都必填,结束日期不得晚于 UTC+8 下的今天。
|
||||
- 两端都必填,结束日期不得晚于 UTC+8 下的今天,闭区间最多 31 天。
|
||||
- 指定历史范围用于首次分批拉取和补数据;日常操作仍点「同步」。
|
||||
- 指定范围经过自动 OCR 或手工验证码登录后必须原样保留。
|
||||
- 日期校验失败时重开弹窗、保留原输入,并在字段区域显示可读错误。
|
||||
|
||||
+17
-5
@@ -183,11 +183,18 @@ POST /am/stock/list → data.list 是数组
|
||||
|
||||
### 4.3 分页
|
||||
|
||||
`[必须]` 先调 `listTotal` 拿总数,再按 `length` 翻页调 `list`。
|
||||
`[必须]` 一个跨日范围要拆成逐日查询。先逐日调用 `listTotal` 做全范围容量
|
||||
预检,确认合计不超限后,再按天以 `length = 20` 翻页调用 `list`。明细仍按
|
||||
最多 100 个货运单 ID 一批读取。
|
||||
|
||||
`[必须]` **必须有单次同步的条数上限**,超了报错而不是硬拉。
|
||||
示例脚本用的是 `max_matches = 100`。日期范围拉全量时这个上限要调大,
|
||||
但不能没有——手滑填成一年的范围会把整库拉下来。
|
||||
Admin 默认 `max_matches = 10000`,可以在配置中调整;上限针对整个日期范围的
|
||||
逐日总数合计,不是每天各算一次。任何一天的列表或明细都不得在容量预检通过前
|
||||
开始拉取,避免超限后已经产生部分写入。
|
||||
|
||||
`[必须]` 每一页 `list` 响应里的 `total` 必须等于该日预检的 `listTotal`,
|
||||
全部页去重后的货运单 ID 数也必须完全相等。分页期间总数变化、短页造成缺失或
|
||||
重复 ID 都视为本次同步失败,不推进游标。
|
||||
|
||||
### 4.4 自动增量与指定日期补同步
|
||||
|
||||
@@ -196,7 +203,7 @@ POST /am/stock/list → data.list 是数组
|
||||
| 模式 | 日期范围 | `last_synced_at` |
|
||||
|---|---|---|
|
||||
| 自动增量「同步」 | 上次成功同步日期当天 ~ UTC+8 下的今天;首次从 `sync_from` 开始 | 全部成功后推进 |
|
||||
| 「指定日期同步」 | 操作员填写的开始日 ~ 结束日,两端都包含 | 见下方安全规则 |
|
||||
| 「指定日期同步」 | 操作员填写的开始日 ~ 结束日,两端都包含,最多 31 天 | 见下方安全规则 |
|
||||
|
||||
`[必须]` 指定范围只有在**完整覆盖自动增量本来应该同步的区间**时,全部成功后
|
||||
才可以推进 `last_synced_at`。局部历史补拉只 upsert 数据,不动游标。
|
||||
@@ -206,9 +213,14 @@ POST /am/stock/list → data.list 是数组
|
||||
日期可能永远不会再自动同步,而且不会报错。
|
||||
|
||||
`[必须]` 日期格式固定 `YYYY-MM-DD`,按 UTC+8 解释;两端必须同时填写,
|
||||
开始不得晚于结束,结束不得晚于 UTC+8 下的今天。中途失败或超过
|
||||
开始不得晚于结束,结束不得晚于 UTC+8 下的今天;闭区间最多 31 天。自动增量
|
||||
不受 31 天人工选择限制,但同样逐日查询。中途失败或超过
|
||||
`max_matches` 时,两种模式都不推进游标。
|
||||
|
||||
`[必须]` 每批明细响应必须与请求的货运单 ID 一一对应。缺失、重复、出现未请求
|
||||
ID,或某张货运单返回空商品明细,都视为不完整并停止同步;已经写入的幂等数据
|
||||
可以保留,但只有所有日期全部成功才推进游标。
|
||||
|
||||
`[必须]` 登录和验证码只是同步前置步骤。指定范围经过自动 OCR 降级、手工
|
||||
输入验证码和 303 跳转时必须原样保留,不能悄悄退化成自动增量。
|
||||
|
||||
|
||||
Reference in New Issue
Block a user