diff --git a/admin/service/syb.go b/admin/service/syb.go index d29c22d..0d8f3cf 100644 --- a/admin/service/syb.go +++ b/admin/service/syb.go @@ -621,7 +621,7 @@ func RunSybSyncWithOptions(ctx context.Context, db *sql.DB, client *syb.Client, orderedIDs := make([]int64, 0, plan.total) for start := 0; start < plan.total; start += pageSize { pageIndex := start/pageSize + 1 - rows, responseTotal, listErr := client.ListPage(ctx, plan.date, plan.date, start, pageIndex, pageSize) + rows, pageCount, listErr := client.ListPage(ctx, plan.date, plan.date, start, pageIndex, pageSize) if listErr != nil { report.Err = fmt.Errorf("拉取 %s 货运单列表第 %d 页失败(已获取 %d/%d 张,本次同步整体作废,"+ "下次会从同一个起始日期重新拉,靠 upsert 幂等不会重复计数): %w", @@ -629,9 +629,10 @@ func RunSybSyncWithOptions(ctx context.Context, db *sql.DB, client *syb.Client, report.FinishedAt = time.Now().UTC() return report } - if responseTotal != plan.total { - report.Err = fmt.Errorf("%s 货运单总数在分页期间从 %d 变为 %d,"+ - "为防止 offset 分页漏单,本次同步停止且不推进游标", plan.date, plan.total, responseTotal) + expectedPageCount := min(pageSize, plan.total-start) + if pageCount != expectedPageCount || len(rows) != expectedPageCount { + report.Err = fmt.Errorf("%s 货运单列表第 %d 页不完整:预期 %d 行,实际 %d 行,"+ + "本次同步停止且不推进游标", plan.date, pageIndex, expectedPageCount, len(rows)) report.FinishedAt = time.Now().UTC() return report } @@ -643,6 +644,18 @@ func RunSybSyncWithOptions(ctx context.Context, db *sql.DB, client *syb.Client, orderedIDs = append(orderedIDs, row.ID) } } + afterTotal, totalErr := client.ListTotal(ctx, plan.date, plan.date, pageSize) + if totalErr != nil { + report.Err = fmt.Errorf("分页后重新查询 %s 货运单总数失败: %w", plan.date, totalErr) + report.FinishedAt = time.Now().UTC() + return report + } + if afterTotal != plan.total { + report.Err = fmt.Errorf("%s 货运单总数在分页期间从 %d 变为 %d,"+ + "为防止 offset 分页漏单,本次同步停止且不推进游标", plan.date, plan.total, afterTotal) + report.FinishedAt = time.Now().UTC() + return report + } if len(orderedIDs) != plan.total { report.Err = fmt.Errorf("%s 货运单列表不完整:预期 %d 张,分页后只有 %d 个唯一 ID,"+ "本次同步停止且不推进游标", plan.date, plan.total, len(orderedIDs)) diff --git a/admin/service/syb_test.go b/admin/service/syb_test.go index 320b0e0..4938fc4 100644 --- a/admin/service/syb_test.go +++ b/admin/service/syb_test.go @@ -250,7 +250,8 @@ func fakeSybServer(t *testing.T, stocks []fakeStock, failListPageIndex int) *htt }) } } - writeEnvelope(t, w, true, "ok", map[string]any{"list": list, "total": len(filtered)}, nil) + // 真实 HAR 中 data.total 是当前页条数,不是筛选范围总数。 + writeEnvelope(t, w, true, "ok", map[string]any{"list": list, "total": len(list)}, nil) case "/am/stock/detail/listByStock": var body struct { @@ -314,21 +315,29 @@ func filterFakeStocksByRequest(t *testing.T, stocks []fakeStock, body map[string } type integrityServerData struct { - total int - responseTotal int - list []map[string]any - details []map[string]any + total int + afterTotal int + totalChanges bool + pageCount int + list []map[string]any + details []map[string]any } func fakeIntegritySybServer(t *testing.T, data integrityServerData) *httptest.Server { t.Helper() + totalCalls := 0 return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/am/stock/listTotal": - writeEnvelope(t, w, true, "ok", data.total, nil) + totalCalls++ + total := data.total + if data.totalChanges && totalCalls > 1 { + total = data.afterTotal + } + writeEnvelope(t, w, true, "ok", total, nil) case "/am/stock/list": writeEnvelope(t, w, true, "ok", map[string]any{ - "list": data.list, "total": data.responseTotal, + "list": data.list, "total": data.pageCount, }, nil) case "/am/stock/detail/listByStock": writeEnvelope(t, w, true, "ok", map[string]any{"list": data.details}, nil) @@ -813,10 +822,14 @@ func TestRunSybSync_列表或明细不完整时不推进游标(t *testing.T) { data integrityServerData wantErr string }{ - {name: "分页期间总数变化", data: integrityServerData{total: 2, responseTotal: 3, list: completeList}, wantErr: "分页期间"}, - {name: "列表存在重复ID", data: integrityServerData{total: 2, responseTotal: 2, + {name: "页内报告条数与实际列表不符", data: integrityServerData{total: 2, pageCount: 1, list: completeList}, wantErr: "当前页条数"}, + {name: "当前页短于预期", data: integrityServerData{total: 2, pageCount: 1, + list: []map[string]any{{"id": 1, "code": "A"}}}, wantErr: "第 1 页不完整"}, + {name: "分页前后总数变化", data: integrityServerData{total: 2, afterTotal: 3, totalChanges: true, + pageCount: 2, list: completeList}, wantErr: "分页期间"}, + {name: "列表存在重复ID", data: integrityServerData{total: 2, pageCount: 2, list: []map[string]any{{"id": 1, "code": "A"}, {"id": 1, "code": "A"}}}, wantErr: "列表不完整"}, - {name: "明细缺少货运单", data: integrityServerData{total: 2, responseTotal: 2, + {name: "明细缺少货运单", data: integrityServerData{total: 2, pageCount: 2, list: completeList, details: completeDetails[:1]}, wantErr: "明细不完整"}, } for _, tc := range cases { diff --git a/admin/syb/client.go b/admin/syb/client.go index adef764..286282e 100644 --- a/admin/syb/client.go +++ b/admin/syb/client.go @@ -504,7 +504,7 @@ type StockRow struct { } // ListPage 按日期范围翻一页货运单列表:POST /am/stock/list。 -// 返回响应内的 total,供上层核对分页期间总数有没有漂移。 +// 返回响应内的 total;HAR 证明它是当前页条数,不是筛选范围总数。 func (c *Client) ListPage(ctx context.Context, dateFrom, dateTo string, start, pageIndex, pageSize int) ([]StockRow, int, error) { data, err := c.do(ctx, http.MethodPost, "/am/stock/list", nil, listPayload(dateFrom, dateTo, start, pageIndex, pageSize)) @@ -522,6 +522,9 @@ func (c *Client) ListPage(ctx context.Context, dateFrom, dateTo string, start, p if wrap.Total == nil || *wrap.Total < 0 { return nil, 0, fmt.Errorf("货运单列表响应缺少合法的 total") } + if *wrap.Total != len(wrap.List) { + return nil, 0, fmt.Errorf("货运单列表响应的当前页条数为 %d,但实际返回 %d 行", *wrap.Total, len(wrap.List)) + } rows := make([]StockRow, 0, len(wrap.List)) for _, raw := range wrap.List { diff --git a/admin/syb/client_test.go b/admin/syb/client_test.go index ec5dff1..63cf554 100644 --- a/admin/syb/client_test.go +++ b/admin/syb/client_test.go @@ -327,7 +327,7 @@ func TestClient_ListTotal和ListPage(t *testing.T) { t.Fatalf("list 失败: %v", err) } if pageTotal != 1 { - t.Fatalf("list 响应内 total 应该是 1,实际 %d", pageTotal) + t.Fatalf("list 响应内当前页条数应该是 1,实际 %d", pageTotal) } if len(rows) != 1 || rows[0].ID != 75104587 || rows[0].Code != "260728TB95MJTQ" { t.Fatalf("列表结果不对: %+v", rows) @@ -342,6 +342,7 @@ func TestClient_ListPage要求合法Total(t *testing.T) { {"缺少total", map[string]any{"list": []any{}}}, {"total类型错误", map[string]any{"list": []any{}, "total": "1"}}, {"total为负数", map[string]any{"list": []any{}, "total": -1}}, + {"total与list长度不一致", map[string]any{"list": []any{map[string]any{"id": 1}}, "total": 0}}, } { t.Run(test.name, func(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/docs/admin/08-顺运宝接口.md b/docs/admin/08-顺运宝接口.md index bbaa12c..3837aec 100644 --- a/docs/admin/08-顺运宝接口.md +++ b/docs/admin/08-顺运宝接口.md @@ -127,9 +127,13 @@ GET /am/user/get?id=<登录响应里的 user.id> ```text POST /am/stock/listTotal → data 是裸整数,总条数 -POST /am/stock/list → data.list 是数组 +POST /am/stock/list → data.list 是数组,data.total 是当前页条数 ``` +`[必须]` 不要把 `list.data.total` 当成筛选范围总数。实测 HAR 中范围总数为 +3846 时,第一页返回 20 行且 `list.data.total = 20`;范围总数只能以 +`listTotal.data` 为准。 + ### 4.1 请求体 ```json @@ -192,9 +196,11 @@ Admin 默认 `max_matches = 10000`,可以在配置中调整;上限针对整 逐日总数合计,不是每天各算一次。任何一天的列表或明细都不得在容量预检通过前 开始拉取,避免超限后已经产生部分写入。 -`[必须]` 每一页 `list` 响应里的 `total` 必须等于该日预检的 `listTotal`, -全部页去重后的货运单 ID 数也必须完全相等。分页期间总数变化、短页造成缺失或 -重复 ID 都视为本次同步失败,不推进游标。 +`[必须]` 每一页 `list.data.total` 必须等于该页 `list` 数组长度。非最后一页 +必须返回 `length` 条,最后一页必须返回预检总数对应的剩余条数。每天翻页结束后 +再次调用 `listTotal`,前后总数必须一致;全部页去重后的货运单 ID 数还必须等于 +预检总数。前后总数变化、短页、重复 ID 或唯一 ID 不足都视为本次同步失败, +不推进游标。 ### 4.4 统一日期范围同步与覆盖游标