From 4b5cd13807b1cb8c841b3283c5683abd66cc9bdb Mon Sep 17 00:00:00 2001 From: chengma Date: Sun, 9 Aug 2026 17:17:42 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E5=A2=9E=E5=BC=BA=E9=A1=BA=E8=BF=90?= =?UTF-8?q?=E5=AE=9D=E5=90=8C=E6=AD=A5=E5=AE=8C=E6=95=B4=E6=80=A7=E4=BF=9D?= =?UTF-8?q?=E6=8A=A4=20(#58)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- admin/config.example.yaml | 6 +- admin/service/syb.go | 198 ++++++++++++++++++------ admin/service/syb_test.go | 248 ++++++++++++++++++++++++++++-- admin/syb/client.go | 17 +- admin/syb/client_test.go | 28 +++- admin/templates/syb/list.html | 2 +- docs/admin/05-ui-specification.md | 4 +- docs/admin/08-顺运宝接口.md | 22 ++- 8 files changed, 448 insertions(+), 77 deletions(-) diff --git a/admin/config.example.yaml b/admin/config.example.yaml index 37c0386..77b36ac 100644 --- a/admin/config.example.yaml +++ b/admin/config.example.yaml @@ -24,9 +24,9 @@ syb: # 列表接口每页条数。抓包里网页用的就是 20。 page_size: 20 - # 单次同步的条数上限，超过就报错而不是硬拉。 - # 防止日期范围手滑填成一整年，把整库拉下来。 - max_matches: 500 + # 整个日期范围的货运单数上限，默认 10000；超过就停止，不拉列表和明细。 + # 指定日期同步另有最多 31 天限制，自动增量不受天数限制。 + max_matches: 10000 # 首次同步的起始日期。之后按「上次同步时间」增量拉，不再用这个值。 sync_from: "2026-07-01" diff --git a/admin/service/syb.go b/admin/service/syb.go index 1e1e17a..f995d8a 100644 --- a/admin/service/syb.go +++ b/admin/service/syb.go @@ -30,7 +30,11 @@ import ( "cmautobuy/admin/syb" ) -const dateLayout = "2006-01-02" +const ( + dateLayout = "2006-01-02" + maxSpecifiedSyncDays = 31 + defaultSybMaxMatches = 10000 +) // SybSyncOptions 表示一次同步的日期选择。From/To 都为空是日常自动增量； // 两者都有值是操作员明确发起的指定日期补同步。 @@ -72,9 +76,35 @@ func NewSybSyncOptions(fromRaw, toRaw string, now time.Time) (SybSyncOptions, er if to > dateOf(now) { return SybSyncOptions{}, fmt.Errorf("结束日期不能晚于顺运宝服务端今天（%s）", dateOf(now)) } + days := int(toDate.Sub(fromDate).Hours()/24) + 1 + if days > maxSpecifiedSyncDays { + return SybSyncOptions{}, fmt.Errorf("指定日期同步最多选择 %d 天，当前范围为 %d 天，请分段同步", + maxSpecifiedSyncDays, days) + } return SybSyncOptions{From: from, To: to}, nil } +// splitDateRange 把闭区间拆成逐日查询段。顺运宝只支持日期粒度，按天拉取 +// 能把 offset 分页期间新增数据造成的位移限制在当天。 +func splitDateRange(from, to string) ([]string, error) { + fromDate, err := time.Parse(dateLayout, from) + if err != nil || fromDate.Format(dateLayout) != from { + return nil, fmt.Errorf("开始日期格式不正确，请使用 YYYY-MM-DD") + } + toDate, err := time.Parse(dateLayout, to) + if err != nil || toDate.Format(dateLayout) != to { + return nil, fmt.Errorf("结束日期格式不正确，请使用 YYYY-MM-DD") + } + if fromDate.After(toDate) { + return nil, fmt.Errorf("开始日期不能晚于结束日期") + } + dates := make([]string, 0, int(toDate.Sub(fromDate).Hours()/24)+1) + for day := fromDate; !day.After(toDate); day = day.AddDate(0, 0, 1) { + dates = append(dates, day.Format(dateLayout)) + } + return dates, nil +} + // sybLocation 是顺运宝服务端的时区。 // // `[必须]` 日期范围筛的是服务端的 created，而它是 UTC+8——实测 @@ -292,7 +322,7 @@ func RunSybSyncWithOptions(ctx context.Context, db *sql.DB, client *syb.Client, } maxMatches := cfg.MaxMatches if maxMatches <= 0 { - maxMatches = 500 + maxMatches = defaultSybMaxMatches } lastSyncedAt, _, err := repository.GetSybLastSyncedAt(db) @@ -327,20 +357,41 @@ func RunSybSyncWithOptions(ctx context.Context, db *sql.DB, client *syb.Client, } report.From, report.To = from, to - total, err := client.ListTotal(ctx, from, to, pageSize) + dates, err := splitDateRange(from, to) if err != nil { - report.Err = fmt.Errorf("查询货运单总数失败: %w", err) + report.Err = err report.FinishedAt = time.Now().UTC() return report } - if total > maxMatches { - report.Err = fmt.Errorf( - "日期范围 %s ~ %s 内有 %d 张货运单，超过单次同步上限 %d，"+ - "请缩小日期范围或联系维护者调大 max_matches", from, to, total, maxMatches) - report.FinishedAt = time.Now().UTC() - return report + type dailyPlan struct { + date string + total int } - if total == 0 { + plans := make([]dailyPlan, 0, len(dates)) + allTotal := 0 + for _, date := range dates { + total, totalErr := client.ListTotal(ctx, date, date, pageSize) + if totalErr != nil { + report.Err = fmt.Errorf("查询 %s 货运单总数失败: %w", date, totalErr) + report.FinishedAt = time.Now().UTC() + return report + } + if total < 0 { + report.Err = fmt.Errorf("查询 %s 货运单总数返回负数 %d", date, total) + report.FinishedAt = time.Now().UTC() + return report + } + if total > maxMatches-allTotal { + report.Err = fmt.Errorf( + "日期范围 %s ~ %s 内货运单总数超过单次同步上限 %d，"+ + "请缩小日期范围或在确认机器和网络容量后调大 max_matches", from, to, maxMatches) + report.FinishedAt = time.Now().UTC() + return report + } + allTotal += total + plans = append(plans, dailyPlan{date: date, total: total}) + } + if allTotal == 0 { report.FinishedAt = time.Now().UTC() if advanceCursor { if err := repository.SetSybLastSyncedAt(db, model.NowISO()); err != nil { @@ -350,30 +401,7 @@ func RunSybSyncWithOptions(ctx context.Context, db *sql.DB, client *syb.Client, return report } - // ① 翻页拉全部货运单行。 - stockByID := map[int64]syb.StockRow{} - var orderedIDs []int64 - for start := 0; start < total; start += pageSize { - pageIndex := start/pageSize + 1 - rows, err := client.ListPage(ctx, from, to, start, pageIndex, pageSize) - if err != nil { - report.Err = fmt.Errorf("拉取货运单列表第 %d 页失败（已获取 %d/%d 张，本次同步整体作废，"+ - "下次会从同一个起始日期重新拉，靠 upsert 幂等不会重复计数）: %w", - pageIndex, len(orderedIDs), total, err) - report.FinishedAt = time.Now().UTC() - return report - } - for _, row := range rows { - if _, dup := stockByID[row.ID]; dup { - continue - } - stockByID[row.ID] = row - orderedIDs = append(orderedIDs, row.ID) - } - } - report.StockCount = len(orderedIDs) - - // ② 按 100 个一批取明细，③ 逐张货运单写库。 + // ① 每天单独翻页，② 按 100 个一批取明细，③ 逐张货运单写库。 // // `[必须]` 不放在一个大事务里——几千条明细的事务会长时间持锁； // 按货运单为单位提交，失败了已成功的部分保留（下次重拉会 upsert @@ -383,29 +411,71 @@ func RunSybSyncWithOptions(ctx context.Context, db *sql.DB, client *syb.Client, // 不更新 last_synced_at——已经成功写入的部分不回滚（它们本身是幂等 // 的），但"这次同步整体算成功"这件事不能发生，否则漏掉的单永远补不回来。 const detailBatch = 100 - for i := 0; i < len(orderedIDs); i += detailBatch { - end := i + detailBatch - if end > len(orderedIDs) { - end = len(orderedIDs) + for _, plan := range plans { + if plan.total == 0 { + continue } - batch := orderedIDs[i:end] - - details, err := client.DetailListByStock(ctx, batch) - if err != nil { - report.Err = fmt.Errorf("拉取货运单明细失败（本次同步整体作废，"+ - "已写入的数据保留，下次重拉会 upsert 覆盖）: %w", err) + stockByID := map[int64]syb.StockRow{} + orderedIDs := make([]int64, 0, plan.total) + for start := 0; start < plan.total; start += pageSize { + pageIndex := start/pageSize + 1 + rows, responseTotal, listErr := client.ListPage(ctx, plan.date, plan.date, start, pageIndex, pageSize) + if listErr != nil { + report.Err = fmt.Errorf("拉取 %s 货运单列表第 %d 页失败（已获取 %d/%d 张，本次同步整体作废，"+ + "下次会从同一个起始日期重新拉，靠 upsert 幂等不会重复计数）: %w", + plan.date, pageIndex, len(orderedIDs), plan.total, listErr) + report.FinishedAt = time.Now().UTC() + return report + } + if responseTotal != plan.total { + report.Err = fmt.Errorf("%s 货运单总数在分页期间从 %d 变为 %d，"+ + "为防止 offset 分页漏单，本次同步停止且不推进游标", plan.date, plan.total, responseTotal) + report.FinishedAt = time.Now().UTC() + return report + } + for _, row := range rows { + if _, dup := stockByID[row.ID]; dup { + continue + } + stockByID[row.ID] = row + orderedIDs = append(orderedIDs, row.ID) + } + } + if len(orderedIDs) != plan.total { + report.Err = fmt.Errorf("%s 货运单列表不完整：预期 %d 张，分页后只有 %d 个唯一 ID，"+ + "本次同步停止且不推进游标", plan.date, plan.total, len(orderedIDs)) report.FinishedAt = time.Now().UTC() return report } + report.StockCount += len(orderedIDs) - for _, d := range details { - stockRow := stockByID[d.ID] - if err := writeStockDetail(db, cfg.BaseURL, stockRow, d, &report); err != nil { - report.Err = fmt.Errorf("写入货运单 %s（id=%d）失败（本次同步整体作废，"+ - "已写入的数据保留）: %w", d.Code, d.ID, err) + for i := 0; i < len(orderedIDs); i += detailBatch { + end := i + detailBatch + if end > len(orderedIDs) { + end = len(orderedIDs) + } + batch := orderedIDs[i:end] + details, detailErr := client.DetailListByStock(ctx, batch) + if detailErr != nil { + report.Err = fmt.Errorf("拉取 %s 货运单明细失败（本次同步整体作废，"+ + "已写入的数据保留，下次重拉会 upsert 覆盖）: %w", plan.date, detailErr) report.FinishedAt = time.Now().UTC() return report } + if completeErr := validateDetailBatch(batch, details); completeErr != nil { + report.Err = fmt.Errorf("%s 货运单明细不完整：%w；本次同步停止且不推进游标", plan.date, completeErr) + report.FinishedAt = time.Now().UTC() + return report + } + for _, d := range details { + stockRow := stockByID[d.ID] + if err := writeStockDetail(db, cfg.BaseURL, stockRow, d, &report); err != nil { + report.Err = fmt.Errorf("写入货运单 %s（id=%d）失败（本次同步整体作废，"+ + "已写入的数据保留）: %w", d.Code, d.ID, err) + report.FinishedAt = time.Now().UTC() + return report + } + } } } @@ -420,6 +490,34 @@ func RunSybSyncWithOptions(ctx context.Context, db *sql.DB, client *syb.Client, return report } +// validateDetailBatch 确认批量明细响应与请求 ID 一一对应。任何缺失、重复、 +// 意外 ID 或空商品明细都会让同步失败，避免在数据不完整时推进游标。 +func validateDetailBatch(requested []int64, details []syb.StockDetail) error { + wanted := make(map[int64]struct{}, len(requested)) + for _, id := range requested { + wanted[id] = struct{}{} + } + seen := make(map[int64]struct{}, len(details)) + for _, detail := range details { + if _, ok := wanted[detail.ID]; !ok { + return fmt.Errorf("响应包含未请求的货运单 id=%d", detail.ID) + } + if _, duplicate := seen[detail.ID]; duplicate { + return fmt.Errorf("响应重复返回货运单 id=%d", detail.ID) + } + if len(detail.Details) == 0 { + return fmt.Errorf("货运单 id=%d 没有返回商品明细", detail.ID) + } + seen[detail.ID] = struct{}{} + } + for _, id := range requested { + if _, ok := seen[id]; !ok { + return fmt.Errorf("响应缺少货运单 id=%d", id) + } + } + return nil +} + // writeStockDetail 把一张货运单的全部商品明细写进 syb_orders， // 一张货运单一个事务（工单 #46「按货运单为单位提交」）。 func writeStockDetail(db *sql.DB, baseURL string, stockRow syb.StockRow, detail syb.StockDetail, report *SyncReport) error { diff --git a/admin/service/syb_test.go b/admin/service/syb_test.go index e32eae0..100f478 100644 --- a/admin/service/syb_test.go +++ b/admin/service/syb_test.go @@ -92,6 +92,8 @@ func TestNewSybSyncOptions_指定日期校验(t *testing.T) { }{ {name: "两端都空是自动增量"}, {name: "合法闭区间", from: "2026-08-01", to: "2026-08-10"}, + {name: "三十一天合法", from: "2026-07-11", to: "2026-08-10"}, + {name: "超过三十一天", from: "2026-07-10", to: "2026-08-10", wantErr: "最多选择 31 天"}, {name: "只填开始", from: "2026-08-01", wantErr: "同时填写"}, {name: "开始格式错误", from: "2026/08/01", to: "2026-08-02", wantErr: "开始日期格式"}, {name: "结束格式错误", from: "2026-08-01", to: "2026/08/02", wantErr: "结束日期格式"}, @@ -157,6 +159,7 @@ type fakeStock struct { ID int64 Code string ShopName string + Created string Receiver string // 用来验证个人信息确实没有落库 Details []fakeDetail } @@ -169,18 +172,30 @@ type fakeStock struct { func fakeSybServer(t *testing.T, stocks []fakeStock, failListPageIndex int) *httptest.Server { t.Helper() byID := map[int64]fakeStock{} - for _, s := range stocks { + for i := range stocks { + if stocks[i].Created == "" { + stocks[i].Created = "2026-07-28" + } + s := stocks[i] byID[s.ID] = s } return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/am/stock/listTotal": - writeEnvelope(t, w, true, "ok", len(stocks), nil) + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatalf("解析 listTotal 请求失败: %v", err) + } + filtered := filterFakeStocksByRequest(t, stocks, body) + writeEnvelope(t, w, true, "ok", len(filtered), nil) case "/am/stock/list": var body map[string]any - json.NewDecoder(r.Body).Decode(&body) + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatalf("解析 list 请求失败: %v", err) + } + filtered := filterFakeStocksByRequest(t, stocks, body) pageIndex := int(body["pageIndex"].(float64)) length := int(body["length"].(float64)) start := int(body["start"].(float64)) @@ -191,19 +206,19 @@ func fakeSybServer(t *testing.T, stocks []fakeStock, failListPageIndex int) *htt } end := start + length - if end > len(stocks) { - end = len(stocks) + if end > len(filtered) { + end = len(filtered) } var list []map[string]any - if start < len(stocks) { - for _, s := range stocks[start:end] { + if start < len(filtered) { + for _, s := range filtered[start:end] { list = append(list, map[string]any{ "id": s.ID, "code": s.Code, "shopName": s.ShopName, "receiver": s.Receiver, "orderStatus": "待出货", }) } } - writeEnvelope(t, w, true, "ok", map[string]any{"list": list}, nil) + writeEnvelope(t, w, true, "ok", map[string]any{"list": list, "total": len(filtered)}, nil) case "/am/stock/detail/listByStock": var body struct { @@ -238,6 +253,60 @@ func fakeSybServer(t *testing.T, stocks []fakeStock, failListPageIndex int) *htt })) } +func filterFakeStocksByRequest(t *testing.T, stocks []fakeStock, body map[string]any) []fakeStock { + t.Helper() + queries, ok := body["queries"].([]any) + if !ok || len(queries) == 0 { + t.Fatalf("同步请求缺少 queries: %#v", body["queries"]) + } + query, ok := queries[0].(map[string]any) + if !ok { + t.Fatalf("同步请求 queries[0] 格式错误: %#v", queries[0]) + } + rangeText, ok := query["dvalue"].(string) + if !ok { + t.Fatalf("同步请求缺少日期范围 dvalue: %#v", query["dvalue"]) + } + parts := strings.SplitN(rangeText, ",", 2) + if len(parts) != 2 { + t.Fatalf("同步请求日期范围格式错误: %q", rangeText) + } + + filtered := make([]fakeStock, 0, len(stocks)) + for _, stock := range stocks { + if stock.Created >= parts[0] && stock.Created <= parts[1] { + filtered = append(filtered, stock) + } + } + return filtered +} + +type integrityServerData struct { + total int + responseTotal int + list []map[string]any + details []map[string]any +} + +func fakeIntegritySybServer(t *testing.T, data integrityServerData) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/am/stock/listTotal": + writeEnvelope(t, w, true, "ok", data.total, nil) + case "/am/stock/list": + writeEnvelope(t, w, true, "ok", map[string]any{ + "list": data.list, "total": data.responseTotal, + }, nil) + case "/am/stock/detail/listByStock": + writeEnvelope(t, w, true, "ok", map[string]any{"list": data.details}, nil) + default: + t.Errorf("测试假服务端没有实现这个路径: %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) +} + func writeEnvelope(t *testing.T, w http.ResponseWriter, status bool, msg string, data any, code any) { t.Helper() b, err := json.Marshal(map[string]any{"status": status, "msg": msg, "data": data, "code": code}) @@ -297,7 +366,10 @@ func TestRunSybSync_已有的ShopeeSKUID同步后仍在(t *testing.T) { } // 再同步一次（模拟顺运宝那边这张单信息有更新）。 - report2 := RunSybSync(context.Background(), db, client, cfg, now.Add(time.Hour)) + report2 := RunSybSyncWithOptions(context.Background(), db, client, cfg, now.Add(time.Hour), SybSyncOptions{ + From: "2026-07-28", + To: "2026-07-28", + }) if report2.Err != nil { t.Fatalf("二次同步失败: %v", report2.Err) } @@ -521,6 +593,164 @@ func TestRunSybSync_超过最大条数报错不硬拉(t *testing.T) { } } +func TestRunSybSync_全范围容量预检通过前不拉列表和明细(t *testing.T) { + listCalls := 0 + detailCalls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/am/stock/listTotal": + writeEnvelope(t, w, true, "ok", 2, nil) + case "/am/stock/list": + listCalls++ + writeEnvelope(t, w, true, "ok", map[string]any{"list": []any{}, "total": 2}, nil) + case "/am/stock/detail/listByStock": + detailCalls++ + writeEnvelope(t, w, true, "ok", map[string]any{"list": []any{}}, nil) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer srv.Close() + + db := newSyncTestDB(t) + client, _ := syb.New(srv.URL) + report := RunSybSyncWithOptions(context.Background(), db, client, + config.SybConfig{BaseURL: srv.URL, PageSize: 20, MaxMatches: 3, SyncFrom: "2026-07-27"}, + time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC), + SybSyncOptions{From: "2026-07-27", To: "2026-07-28"}) + if report.Err == nil || !strings.Contains(report.Err.Error(), "超过单次同步上限") { + t.Fatalf("两天合计 4 张超过上限 3 应该失败，实际: %v", report.Err) + } + if listCalls != 0 || detailCalls != 0 { + t.Fatalf("容量预检完成前不应拉列表或明细: list=%d detail=%d", listCalls, detailCalls) + } + if _, found, _ := repository.GetSybLastSyncedAt(db); found { + t.Fatal("容量预检失败不得推进游标") + } +} + +func TestRunSybSync_默认上限允许同步超过五百张(t *testing.T) { + const stockCount = 501 + stocks := make([]fakeStock, 0, stockCount) + for i := 1; i <= stockCount; i++ { + id := int64(i) + stocks = append(stocks, fakeStock{ + ID: id, Code: "ORDER", Details: []fakeDetail{{ + ID: id, ProductID: 100000 + id, ProductQty: 1, ProductPrice: 1, + }}, + }) + } + srv := fakeSybServer(t, stocks, 0) + defer srv.Close() + + db := newSyncTestDB(t) + client, _ := syb.New(srv.URL) + report := RunSybSyncWithOptions(context.Background(), db, client, + config.SybConfig{BaseURL: srv.URL, PageSize: 20, SyncFrom: "2026-07-28"}, + time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC), + SybSyncOptions{From: "2026-07-28", To: "2026-07-28"}) + if report.Err != nil { + t.Fatalf("默认上限应该允许 501 张货运单正常同步，实际失败: %v", report.Err) + } + if report.StockCount != stockCount || report.Created != stockCount { + t.Fatalf("同步统计错误: StockCount=%d Created=%d", report.StockCount, report.Created) + } +} + +func TestRunSybSync_跨日范围按天同步并汇总(t *testing.T) { + srv := fakeSybServer(t, []fakeStock{ + {ID: 1, Code: "DAY-1", Created: "2026-07-27", Details: []fakeDetail{{ID: 11, ProductID: 111, ProductQty: 1}}}, + {ID: 2, Code: "DAY-2", Created: "2026-07-28", Details: []fakeDetail{{ID: 22, ProductID: 222, ProductQty: 1}}}, + }, 0) + defer srv.Close() + + db := newSyncTestDB(t) + client, _ := syb.New(srv.URL) + report := RunSybSyncWithOptions(context.Background(), db, client, + config.SybConfig{BaseURL: srv.URL, PageSize: 20, MaxMatches: 100, SyncFrom: "2026-07-27"}, + time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC), + SybSyncOptions{From: "2026-07-27", To: "2026-07-28"}) + if report.Err != nil { + t.Fatalf("跨日同步失败: %v", report.Err) + } + if report.StockCount != 2 || report.Created != 2 { + t.Fatalf("跨日汇总错误: StockCount=%d Created=%d", report.StockCount, report.Created) + } +} + +func TestValidateDetailBatch_必须与请求货运单一一对应(t *testing.T) { + cases := []struct { + name string + details []syb.StockDetail + wantErr string + }{ + {name: "完整", details: []syb.StockDetail{ + {ID: 1, Details: []syb.DetailItem{{ID: 11}}}, + {ID: 2, Details: []syb.DetailItem{{ID: 22}}}, + }}, + {name: "缺失", details: []syb.StockDetail{{ID: 1, Details: []syb.DetailItem{{ID: 11}}}}, wantErr: "缺少"}, + {name: "重复", details: []syb.StockDetail{ + {ID: 1, Details: []syb.DetailItem{{ID: 11}}}, + {ID: 1, Details: []syb.DetailItem{{ID: 12}}}, + }, wantErr: "重复"}, + {name: "意外ID", details: []syb.StockDetail{ + {ID: 1, Details: []syb.DetailItem{{ID: 11}}}, + {ID: 3, Details: []syb.DetailItem{{ID: 33}}}, + }, wantErr: "未请求"}, + {name: "空明细", details: []syb.StockDetail{ + {ID: 1, Details: []syb.DetailItem{{ID: 11}}}, {ID: 2}, + }, wantErr: "没有返回商品明细"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := validateDetailBatch([]int64{1, 2}, tc.details) + if tc.wantErr == "" && err != nil { + t.Fatalf("完整响应不应该报错: %v", err) + } + if tc.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tc.wantErr)) { + t.Fatalf("错误应包含 %q，实际 %v", tc.wantErr, err) + } + }) + } +} + +func TestRunSybSync_列表或明细不完整时不推进游标(t *testing.T) { + completeList := []map[string]any{{"id": 1, "code": "A"}, {"id": 2, "code": "B"}} + completeDetails := []map[string]any{ + {"id": 1, "code": "A", "details": []map[string]any{{"id": 11, "productId": 111, "productQty": 1}}}, + {"id": 2, "code": "B", "details": []map[string]any{{"id": 22, "productId": 222, "productQty": 1}}}, + } + cases := []struct { + name string + data integrityServerData + wantErr string + }{ + {name: "分页期间总数变化", data: integrityServerData{total: 2, responseTotal: 3, list: completeList}, wantErr: "分页期间"}, + {name: "列表存在重复ID", data: integrityServerData{total: 2, responseTotal: 2, + list: []map[string]any{{"id": 1, "code": "A"}, {"id": 1, "code": "A"}}}, wantErr: "列表不完整"}, + {name: "明细缺少货运单", data: integrityServerData{total: 2, responseTotal: 2, + list: completeList, details: completeDetails[:1]}, wantErr: "明细不完整"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + srv := fakeIntegritySybServer(t, tc.data) + defer srv.Close() + db := newSyncTestDB(t) + client, _ := syb.New(srv.URL) + report := RunSybSyncWithOptions(context.Background(), db, client, + config.SybConfig{BaseURL: srv.URL, PageSize: 20, MaxMatches: 100, SyncFrom: "2026-07-28"}, + time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC), + SybSyncOptions{From: "2026-07-28", To: "2026-07-28"}) + if report.Err == nil || !strings.Contains(report.Err.Error(), tc.wantErr) { + t.Fatalf("错误应包含 %q，实际 %v", tc.wantErr, report.Err) + } + if _, found, err := repository.GetSybLastSyncedAt(db); err != nil || found { + t.Fatalf("不完整响应不得推进游标: found=%v err=%v", found, err) + } + }) + } +} + func TestRunSybSync_一张货运单多个商品各落一行(t *testing.T) { srv := fakeSybServer(t, []fakeStock{ {ID: 1, Code: "A", Details: []fakeDetail{ diff --git a/admin/syb/client.go b/admin/syb/client.go index e0bc884..052ecd6 100644 --- a/admin/syb/client.go +++ b/admin/syb/client.go @@ -504,30 +504,35 @@ type StockRow struct { } // ListPage 按日期范围翻一页货运单列表：POST /am/stock/list。 -func (c *Client) ListPage(ctx context.Context, dateFrom, dateTo string, start, pageIndex, pageSize int) ([]StockRow, error) { +// 返回响应内的 total，供上层核对分页期间总数有没有漂移。 +func (c *Client) ListPage(ctx context.Context, dateFrom, dateTo string, start, pageIndex, pageSize int) ([]StockRow, int, error) { data, err := c.do(ctx, http.MethodPost, "/am/stock/list", nil, listPayload(dateFrom, dateTo, start, pageIndex, pageSize)) if err != nil { - return nil, err + return nil, 0, err } var wrap struct { - List []map[string]any `json:"list"` + List []map[string]any `json:"list"` + Total *int `json:"total"` } if err := json.Unmarshal(data, &wrap); err != nil { - return nil, fmt.Errorf("货运单列表响应格式错误: %w", err) + return nil, 0, fmt.Errorf("货运单列表响应格式错误: %w", err) + } + if wrap.Total == nil || *wrap.Total < 0 { + return nil, 0, fmt.Errorf("货运单列表响应缺少合法的 total") } rows := make([]StockRow, 0, len(wrap.List)) for _, raw := range wrap.List { id, ok := toInt64(raw["id"]) if !ok { - return nil, fmt.Errorf("货运单列表里有一行缺少合法的 id: %v", raw) + return nil, 0, fmt.Errorf("货运单列表里有一行缺少合法的 id: %v", raw) } code, _ := raw["code"].(string) rows = append(rows, StockRow{ID: id, Code: code, Raw: raw}) } - return rows, nil + return rows, *wrap.Total, nil } // ---------- 货运单明细 ---------- diff --git a/admin/syb/client_test.go b/admin/syb/client_test.go index 2d9e66e..ec5dff1 100644 --- a/admin/syb/client_test.go +++ b/admin/syb/client_test.go @@ -304,6 +304,7 @@ func TestClient_ListTotal和ListPage(t *testing.T) { w.Write(envelopeBody(t, true, "ok", 1, nil)) case "/am/stock/list": w.Write(envelopeBody(t, true, "ok", map[string]any{ + "total": 1, "list": []map[string]any{ {"id": 75104587, "code": "260728TB95MJTQ", "amtOrder": 61200}, }, @@ -321,15 +322,40 @@ func TestClient_ListTotal和ListPage(t *testing.T) { t.Fatalf("总数应该是 1，实际 %d", total) } - rows, err := c.ListPage(context.Background(), "2026-07-25", "2026-07-28", 0, 1, 20) + rows, pageTotal, err := c.ListPage(context.Background(), "2026-07-25", "2026-07-28", 0, 1, 20) if err != nil { t.Fatalf("list 失败: %v", err) } + if pageTotal != 1 { + t.Fatalf("list 响应内 total 应该是 1，实际 %d", pageTotal) + } if len(rows) != 1 || rows[0].ID != 75104587 || rows[0].Code != "260728TB95MJTQ" { t.Fatalf("列表结果不对: %+v", rows) } } +func TestClient_ListPage要求合法Total(t *testing.T) { + for _, test := range []struct { + name string + data map[string]any + }{ + {"缺少total", map[string]any{"list": []any{}}}, + {"total类型错误", map[string]any{"list": []any{}, "total": "1"}}, + {"total为负数", map[string]any{"list": []any{}, "total": -1}}, + } { + t.Run(test.name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(envelopeBody(t, true, "ok", test.data, nil)) + })) + defer srv.Close() + client, _ := New(srv.URL) + if _, _, err := client.ListPage(context.Background(), "2026-08-09", "2026-08-09", 0, 1, 20); err == nil { + t.Fatal("非法 total 应返回错误") + } + }) + } +} + func TestClient_DetailListByStock_一单多商品(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Query().Get("hist") != "0" { diff --git a/admin/templates/syb/list.html b/admin/templates/syb/list.html index e921fec..af148e1 100644 --- a/admin/templates/syb/list.html +++ b/admin/templates/syb/list.html @@ -60,7 +60,7 @@

按顺运宝货运单创建日期（UTC+8）同步，开始日和结束日都包含。 - 日常同步请直接使用工具栏的“同步”。 + 每次最多选择 31 天；日常同步请直接使用工具栏的“同步”。

diff --git a/docs/admin/05-ui-specification.md b/docs/admin/05-ui-specification.md index 3d38d22..074500d 100644 --- a/docs/admin/05-ui-specification.md +++ b/docs/admin/05-ui-specification.md @@ -445,12 +445,12 @@ Go 的 map 是无序的，不靠它定顺序的话，同一个商品每次刷新 指定日期同步 开始日期 [2026-08-01] 结束日期 [2026-08-09] -按货运单创建日期（UTC+8）同步，开始日和结束日都包含。 +按货运单创建日期（UTC+8）同步，开始日和结束日都包含。每次最多选择 31 天。 [取消] [开始同步] ``` -- 两端都必填，结束日期不得晚于 UTC+8 下的今天。 +- 两端都必填，结束日期不得晚于 UTC+8 下的今天，闭区间最多 31 天。 - 指定历史范围用于首次分批拉取和补数据；日常操作仍点「同步」。 - 指定范围经过自动 OCR 或手工验证码登录后必须原样保留。 - 日期校验失败时重开弹窗、保留原输入，并在字段区域显示可读错误。 diff --git a/docs/admin/08-顺运宝接口.md b/docs/admin/08-顺运宝接口.md index 8883b65..8eaba7f 100644 --- a/docs/admin/08-顺运宝接口.md +++ b/docs/admin/08-顺运宝接口.md @@ -183,11 +183,18 @@ POST /am/stock/list → data.list 是数组 ### 4.3 分页 -`[必须]` 先调 `listTotal` 拿总数，再按 `length` 翻页调 `list`。 +`[必须]` 一个跨日范围要拆成逐日查询。先逐日调用 `listTotal` 做全范围容量 +预检，确认合计不超限后，再按天以 `length = 20` 翻页调用 `list`。明细仍按 +最多 100 个货运单 ID 一批读取。 `[必须]` **必须有单次同步的条数上限**，超了报错而不是硬拉。 -示例脚本用的是 `max_matches = 100`。日期范围拉全量时这个上限要调大， -但不能没有——手滑填成一年的范围会把整库拉下来。 +Admin 默认 `max_matches = 10000`，可以在配置中调整；上限针对整个日期范围的 +逐日总数合计，不是每天各算一次。任何一天的列表或明细都不得在容量预检通过前 +开始拉取，避免超限后已经产生部分写入。 + +`[必须]` 每一页 `list` 响应里的 `total` 必须等于该日预检的 `listTotal`， +全部页去重后的货运单 ID 数也必须完全相等。分页期间总数变化、短页造成缺失或 +重复 ID 都视为本次同步失败，不推进游标。 ### 4.4 自动增量与指定日期补同步 @@ -196,7 +203,7 @@ POST /am/stock/list → data.list 是数组 | 模式 | 日期范围 | `last_synced_at` | |---|---|---| | 自动增量「同步」 | 上次成功同步日期当天 ～ UTC+8 下的今天；首次从 `sync_from` 开始 | 全部成功后推进 | -| 「指定日期同步」 | 操作员填写的开始日 ～ 结束日，两端都包含 | 见下方安全规则 | +| 「指定日期同步」 | 操作员填写的开始日 ～ 结束日，两端都包含，最多 31 天 | 见下方安全规则 | `[必须]` 指定范围只有在**完整覆盖自动增量本来应该同步的区间**时，全部成功后 才可以推进 `last_synced_at`。局部历史补拉只 upsert 数据，不动游标。 @@ -206,9 +213,14 @@ POST /am/stock/list → data.list 是数组 日期可能永远不会再自动同步，而且不会报错。 `[必须]` 日期格式固定 `YYYY-MM-DD`，按 UTC+8 解释；两端必须同时填写， -开始不得晚于结束，结束不得晚于 UTC+8 下的今天。中途失败或超过 +开始不得晚于结束，结束不得晚于 UTC+8 下的今天；闭区间最多 31 天。自动增量 +不受 31 天人工选择限制，但同样逐日查询。中途失败或超过 `max_matches` 时，两种模式都不推进游标。 +`[必须]` 每批明细响应必须与请求的货运单 ID 一一对应。缺失、重复、出现未请求 +ID，或某张货运单返回空商品明细，都视为不完整并停止同步；已经写入的幂等数据 +可以保留，但只有所有日期全部成功才推进游标。 + `[必须]` 登录和验证码只是同步前置步骤。指定范围经过自动 OCR 降级、手工 输入验证码和 303 跳转时必须原样保留，不能悄悄退化成自动增量。