chengma/cmautobuy
2
0
Fork 0
Code Issues 89 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 统一顺运宝日期同步并记录历史 (#59)

Browse Source
This commit is contained in:
chengma
2026-08-09 18:22:58 +08:00
parent ea7932db63
commit 05a54b044d
18 changed files with 934 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
admin/service/syb.go
+232 -28
View File
@@ -36,8 +36,8 @@ const (
defaultSybMaxMatches = 10000
)
// SybSyncOptions 表示一次同步的日期选择。From/To 都为空是日常自动增量；
// 两者都有值是操作员明确发起的指定日期补同步。
// SybSyncOptions 表示一次同步的日期选择。页面必须明确传入两端日期；两端都空
// 仅保留给旧版内部调用兼容。
//
// `[必须]` 不允许只填一端。把“半个范围”悄悄退化成自动增量，会让操作员
// 以为补拉了指定日期，实际却跑了另一段数据。
@@ -46,7 +46,7 @@ type SybSyncOptions struct {
To string
}
// IsSpecified 判断这是不是指定日期同步。
// IsSpecified 判断是否明确给出了日期范围。
func (o SybSyncOptions) IsSpecified() bool {
return strings.TrimSpace(o.From) != "" || strings.TrimSpace(o.To) != ""
}
@@ -78,7 +78,7 @@ func NewSybSyncOptions(fromRaw, toRaw string, now time.Time) (SybSyncOptions, er
}
days := int(toDate.Sub(fromDate).Hours()/24) + 1
if days > maxSpecifiedSyncDays {
return SybSyncOptions{}, fmt.Errorf("指定日期同步最多选择 %d 天，当前范围为 %d 天，请分段同步",
return SybSyncOptions{}, fmt.Errorf("同步日期范围最多选择 %d 天，当前范围为 %d 天，请分段同步",
maxSpecifiedSyncDays, days)
}
return SybSyncOptions{From: from, To: to}, nil
@@ -136,6 +136,63 @@ func SybToday(now time.Time) string {
return dateOf(now)
}
// SybSyncDefaults 是页面首次打开时预填的同步范围和必要提示。
type SybSyncDefaults struct {
From string
To string
Warning string
}
// DefaultSybSyncRange 通常预填最近三天；如果覆盖游标落后，则优先从游标
// 当天连续补齐。缺口超过 31 天时只选最早一段，成功后下一次继续。
func DefaultSybSyncRange(db *sql.DB, now time.Time) (SybSyncDefaults, error) {
today, err := time.Parse(dateLayout, dateOf(now))
if err != nil {
return SybSyncDefaults{}, fmt.Errorf("计算顺运宝今天日期失败: %w", err)
}
recentStart := today.AddDate(0, 0, -2)
defaults := SybSyncDefaults{From: recentStart.Format(dateLayout), To: today.Format(dateLayout)}
lastSyncedAt, found, err := repository.GetSybLastSyncedAt(db)
if err != nil {
return SybSyncDefaults{}, err
}
if !found {
return defaults, nil
}
lastTime, ok := model.ParseISO(lastSyncedAt)
if !ok {
return SybSyncDefaults{}, fmt.Errorf("上次同步时间 %q 解析失败", lastSyncedAt)
}
coveredDate, err := time.Parse(dateLayout, dateOf(lastTime))
if err != nil || !coveredDate.Before(recentStart) {
return defaults, nil
}
segmentEnd := coveredDate.AddDate(0, 0, maxSpecifiedSyncDays-1)
defaults.From = coveredDate.Format(dateLayout)
if segmentEnd.Before(today) {
defaults.To = segmentEnd.Format(dateLayout)
defaults.Warning = fmt.Sprintf(
"存在较长的未同步区间，已先选择最早 %d 天（%s 至 %s）；本段成功后请继续同步下一段。",
maxSpecifiedSyncDays, defaults.From, defaults.To)
return defaults, nil
}
defaults.To = today.Format(dateLayout)
defaults.Warning = fmt.Sprintf("检测到上次同步停在 %s，已自动扩展开始日期以补齐缺口。", defaults.From)
return defaults, nil
}
// cursorISOForDate 把“已连续覆盖到哪一天”保存成 UTC ISO。下一次仍从该日
// 重拉，宁可重复当天，也不能漏掉该日稍晚创建的货运单。
func cursorISOForDate(date string) (string, error) {
t, err := time.ParseInLocation(dateLayout, date, sybLocation)
if err != nil || t.Format(dateLayout) != date {
return "", fmt.Errorf("同步游标日期格式不正确: %q", date)
}
return t.UTC().Format(model.TimeLayout), nil
}
// syncDateRange 算出这次同步该拉哪个日期范围。
//
// `[必须]` 增量必须从"上次同步日期当天"重新拉，不是第二天——created
@@ -175,17 +232,146 @@ type SkipNote struct {
// SyncReport 是一次同步的结果，供状态条显示。
type SyncReport struct {
From, To string
Specified bool // true 表示操作员发起的指定日期补同步
StockCount int // 拉到的货运单数
DetailCount int // 落库的商品明细行数（不含跳过的）
Created int
Updated int
SkippedZero int // quantity <= 0 被跳过的条数
Notes []SkipNote
Err error
StartedAt time.Time
FinishedAt time.Time
From, To string
Specified bool // true 表示操作员发起的指定日期补同步
StockCount int // 拉到的货运单数
DetailCount int // 落库的商品明细行数（不含跳过的）
Created int
Updated int
SkippedZero int // quantity <= 0 被跳过的条数
Notes []SkipNote
Err error
StartedAt time.Time
FinishedAt time.Time
CursorAdvanced bool
}
const sybSyncHistoryPageSize = 10
// SybSyncRunView 是同步记录弹窗的一行，时间和状态已经转成采购员可读文本。
type SybSyncRunView struct {
RunID string
Username string
DateRange string
StatusText string
StatusClass string
Summary string
ErrorMessage string
CursorText string
StartedAt string
FinishedAt string
}
// SybSyncHistoryResult 是同步记录弹窗的分页结果。
type SybSyncHistoryResult struct {
Rows []SybSyncRunView
Page int
TotalPages int
Total int
}
// CreateSybSyncRun 在后台任务启动前创建可审计记录。
func CreateSybSyncRun(db *sql.DB, actor *model.User, options SybSyncOptions, now time.Time) (string, error) {
if actor == nil || actor.UserID == "" {
return "", fmt.Errorf("无法确认当前操作账号，同步没有启动")
}
if !options.IsSpecified() {
return "", fmt.Errorf("同步必须明确填写开始日期和结束日期")
}
runID, err := randomID("SYB-", 16)
if err != nil {
return "", fmt.Errorf("生成同步记录编号失败: %w", err)
}
run := model.SybSyncRun{
RunID: runID, UserID: actor.UserID, DateFrom: options.From, DateTo: options.To,
Status: model.SybSyncRunning, StartedAt: now.UTC().Format(model.TimeLayout),
}
if err := repository.CreateSybSyncRun(db, run); err != nil {
return "", err
}
return runID, nil
}
// FinishSybSyncRun 把同步报告持久化到对应记录。
func FinishSybSyncRun(db *sql.DB, runID string, report SyncReport) error {
status := model.SybSyncSucceeded
errorMessage := ""
if report.Err != nil {
status = model.SybSyncFailed
errorMessage = report.Err.Error()
if len([]rune(errorMessage)) > 500 {
errorMessage = string([]rune(errorMessage)[:500]) + "…"
}
}
finishedAt := report.FinishedAt
if finishedAt.IsZero() {
finishedAt = time.Now().UTC()
}
return repository.FinishSybSyncRun(db, model.SybSyncRun{
RunID: runID, Status: status, StockCount: report.StockCount,
DetailCount: report.DetailCount, Created: report.Created, Updated: report.Updated,
Skipped: report.SkippedZero, ErrorMessage: errorMessage,
CursorAdvanced: report.CursorAdvanced, FinishedAt: finishedAt.UTC().Format(model.TimeLayout),
})
}
// InterruptRunningSybSyncRuns 收敛上次进程退出前未完成的同步记录。
func InterruptRunningSybSyncRuns(db *sql.DB, now time.Time) (int, error) {
return repository.InterruptRunningSybSyncRuns(db, now.UTC().Format(model.TimeLayout))
}
// ListSybSyncHistory 返回同步记录弹窗所需的分页视图。
func ListSybSyncHistory(db *sql.DB, page int) (*SybSyncHistoryResult, error) {
total, err := repository.CountSybSyncRuns(db)
if err != nil {
return nil, err
}
if page < 1 {
page = 1
}
totalPages := max(1, (total+sybSyncHistoryPageSize-1)/sybSyncHistoryPageSize)
if page > totalPages {
page = totalPages
}
runs, err := repository.ListSybSyncRuns(db, sybSyncHistoryPageSize, (page-1)*sybSyncHistoryPageSize)
if err != nil {
return nil, err
}
result := &SybSyncHistoryResult{Page: page, TotalPages: totalPages, Total: total}
for _, run := range runs {
statusText, statusClass := sybSyncRunStatusText(run.Status)
view := SybSyncRunView{
RunID: run.RunID, Username: run.Username,
DateRange: run.DateFrom + " ～ " + run.DateTo,
StatusText: statusText, StatusClass: statusClass,
Summary: fmt.Sprintf("货运单 %d，明细 %d（新增 %d，更新 %d，跳过 %d）",
run.StockCount, run.DetailCount, run.Created, run.Updated, run.Skipped),
ErrorMessage: run.ErrorMessage, StartedAt: formatLocalTime(run.StartedAt),
FinishedAt: formatLocalTime(run.FinishedAt),
}
if run.CursorAdvanced {
view.CursorText = "已推进"
} else {
view.CursorText = "未推进"
}
result.Rows = append(result.Rows, view)
}
return result, nil
}
func sybSyncRunStatusText(status model.SybSyncRunStatus) (string, string) {
switch status {
case model.SybSyncRunning:
return "同步中", "status-running"
case model.SybSyncSucceeded:
return "成功", "status-success"
case model.SybSyncFailed:
return "失败", "status-failed"
case model.SybSyncInterrupted:
return "已中断", "status-interrupted"
default:
return "未知", ""
}
}
// Summary 组装状态条文案，格式见工单 #46「报告要说清楚」：
@@ -196,11 +382,7 @@ func (r SyncReport) Summary() string {
if r.Err != nil {
return "同步失败：" + r.Err.Error()
}
prefix := "同步完成"
if r.Specified {
prefix = "指定日期同步完成"
}
msg := fmt.Sprintf("%s：日期范围 %s ~ %s，货运单 %d 张，商品明细 %d 条（新增 %d，更新 %d，跳过 %d）", prefix,
msg := fmt.Sprintf("同步完成：日期范围 %s ~ %s，货运单 %d 张，商品明细 %d 条（新增 %d，更新 %d，跳过 %d）",
r.From, r.To, r.StockCount, r.DetailCount, r.Created, r.Updated, r.SkippedZero)
if len(r.Notes) > 0 {
var reasons []string
@@ -309,7 +491,7 @@ func RunSybSync(ctx context.Context, db *sql.DB, client *syb.Client, cfg config.
return RunSybSyncWithOptions(ctx, db, client, cfg, now, SybSyncOptions{})
}
// RunSybSyncWithOptions 执行自动增量或指定日期同步。
// RunSybSyncWithOptions 执行日期范围同步；空范围仅兼容旧版内部自动增量调用。
//
// 指定日期只有完整覆盖“本来应该自动同步的范围”时才推进 last_synced_at。
// 局部历史补拉只 upsert 数据、不动游标，否则会让未覆盖的订单永久漏掉。
@@ -325,7 +507,7 @@ func RunSybSyncWithOptions(ctx context.Context, db *sql.DB, client *syb.Client,
maxMatches = defaultSybMaxMatches
}
lastSyncedAt, _, err := repository.GetSybLastSyncedAt(db)
lastSyncedAt, hasCursor, err := repository.GetSybLastSyncedAt(db)
if err != nil {
report.Err = fmt.Errorf("读取上次同步进度失败: %w", err)
report.FinishedAt = time.Now().UTC()
@@ -334,6 +516,7 @@ func RunSybSyncWithOptions(ctx context.Context, db *sql.DB, client *syb.Client,
from, to := options.From, options.To
advanceCursor := !options.IsSpecified()
cursorAt := model.NowISO()
if options.IsSpecified() {
validated, validateErr := NewSybSyncOptions(options.From, options.To, now)
if validateErr != nil {
@@ -343,10 +526,27 @@ func RunSybSyncWithOptions(ctx context.Context, db *sql.DB, client *syb.Client,
}
from, to = validated.From, validated.To
// 算得出自动范围时，只有完整覆盖它才允许推进游标。首次同步没有
// sync_from 时仍允许补拉，但不能凭局部范围建立一个不可靠的游标。
autoFrom, autoTo, rangeErr := syncDateRange(lastSyncedAt, cfg.SyncFrom, now)
advanceCursor = rangeErr == nil && from <= autoFrom && to >= autoTo
cursorAt, err = cursorISOForDate(to)
if err != nil {
report.Err = err
report.FinishedAt = time.Now().UTC()
return report
}
if !hasCursor {
// 统一日期入口第一次成功后，以操作员明确选择的结束日建立后续覆盖基线。
advanceCursor = true
} else {
lastTime, ok := model.ParseISO(lastSyncedAt)
if !ok {
report.Err = fmt.Errorf("上次同步时间 %q 解析失败", lastSyncedAt)
report.FinishedAt = time.Now().UTC()
return report
}
coveredDate := dateOf(lastTime)
// 只有范围衔接当前覆盖日期、且确实向后延伸时才推进。
// 历史补拉或跳过缺口的范围只 upsert 数据，不改变覆盖基线。
advanceCursor = from <= coveredDate && to > coveredDate
}
} else {
from, to, err = syncDateRange(lastSyncedAt, cfg.SyncFrom, now)
if err != nil {
@@ -394,8 +594,10 @@ func RunSybSyncWithOptions(ctx context.Context, db *sql.DB, client *syb.Client,
if allTotal == 0 {
report.FinishedAt = time.Now().UTC()
if advanceCursor {
if err := repository.SetSybLastSyncedAt(db, model.NowISO()); err != nil {
if err := repository.SetSybLastSyncedAt(db, cursorAt); err != nil {
report.Err = fmt.Errorf("更新同步进度失败: %w", err)
} else {
report.CursorAdvanced = true
}
}
return report
@@ -481,9 +683,11 @@ func RunSybSyncWithOptions(ctx context.Context, db *sql.DB, client *syb.Client,
// ④ 全部成功，才更新 last_synced_at。
if advanceCursor {
if err := repository.SetSybLastSyncedAt(db, model.NowISO()); err != nil {
if err := repository.SetSybLastSyncedAt(db, cursorAt); err != nil {
report.Err = fmt.Errorf("同步数据已全部写入，但更新同步进度失败，"+
"下次同步会重新拉这个日期范围（不会漏，但会重复拉一次）: %w", err)
} else {
report.CursorAdvanced = true
}
}
report.FinishedAt = time.Now().UTC()
admin/service/syb_test.go
+92 -4
View File
@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
@@ -119,6 +120,37 @@ func TestNewSybSyncOptions_指定日期校验(t *testing.T) {
}
}
func TestDefaultSybSyncRange_最近三天与缺口分段(t *testing.T) {
now := time.Date(2026, 8, 9, 12, 0, 0, 0, time.UTC)
t.Run("没有游标默认最近三天", func(t *testing.T) {
db := newSyncTestDB(t)
got, err := DefaultSybSyncRange(db, now)
if err != nil || got.From != "2026-08-07" || got.To != "2026-08-09" || got.Warning != "" {
t.Fatalf("默认范围错误: got=%+v err=%v", got, err)
}
})
t.Run("短缺口自动扩展到今天", func(t *testing.T) {
db := newSyncTestDB(t)
if err := repository.SetSybLastSyncedAt(db, "2026-08-05T00:00:00Z"); err != nil {
t.Fatal(err)
}
got, err := DefaultSybSyncRange(db, now)
if err != nil || got.From != "2026-08-05" || got.To != "2026-08-09" || got.Warning == "" {
t.Fatalf("短缺口范围错误: got=%+v err=%v", got, err)
}
})
t.Run("长缺口优先选择最早三十一天", func(t *testing.T) {
db := newSyncTestDB(t)
if err := repository.SetSybLastSyncedAt(db, "2026-07-01T00:00:00Z"); err != nil {
t.Fatal(err)
}
got, err := DefaultSybSyncRange(db, now)
if err != nil || got.From != "2026-07-01" || got.To != "2026-07-31" || !strings.Contains(got.Warning, "继续同步下一段") {
t.Fatalf("长缺口分段错误: got=%+v err=%v", got, err)
}
})
}
// ── 金额换算 ──────────────────────────────────────────
func TestRoundYuanToCent_先四舍五入再转整数(t *testing.T) {
@@ -489,8 +521,8 @@ func TestRunSybSyncWithOptions_历史补拉不推进游标(t *testing.T) {
if report.Err != nil {
t.Fatalf("历史补拉失败: %v", report.Err)
}
if !report.Specified || !strings.HasPrefix(report.Summary(), "指定日期同步完成") {
t.Fatalf("报告没有标明指定日期同步：%+v / %s", report, report.Summary())
if !report.Specified || !strings.HasPrefix(report.Summary(), "同步完成") {
t.Fatalf("明确日期范围的报告不正确：%+v / %s", report, report.Summary())
}
at, _, err := repository.GetSybLastSyncedAt(db)
if err != nil {
@@ -522,8 +554,64 @@ func TestRunSybSyncWithOptions_完整覆盖自动范围才推进游标(t *testin
if err != nil {
t.Fatalf("读取同步游标失败: %v", err)
}
if at == original || at == "" {
t.Fatalf("完整覆盖自动范围后应该推进游标，实际仍为 %q", at)
if at != "2026-07-27T16:00:00Z" || !report.CursorAdvanced {
t.Fatalf("连续覆盖后应该把游标推进到所选结束日，实际 %q，报告 %+v", at, report)
}
}
func TestRunSybSyncWithOptions_跳过缺口不推进游标(t *testing.T) {
srv := fakeSybServer(t, nil, 0)
defer srv.Close()
db := newSyncTestDB(t)
client, _ := syb.New(srv.URL)
const original = "2026-07-01T00:00:00Z"
if err := repository.SetSybLastSyncedAt(db, original); err != nil {
t.Fatal(err)
}
report := RunSybSyncWithOptions(context.Background(), db, client,
config.SybConfig{BaseURL: srv.URL, PageSize: 20, MaxMatches: 500},
time.Date(2026, 8, 9, 12, 0, 0, 0, time.UTC),
SybSyncOptions{From: "2026-07-05", To: "2026-07-10"})
if report.Err != nil {
t.Fatalf("同步失败: %v", report.Err)
}
at, _, err := repository.GetSybLastSyncedAt(db)
if err != nil || at != original || report.CursorAdvanced {
t.Fatalf("跳过缺口不应推进游标: at=%q report=%+v err=%v", at, report, err)
}
}
func TestSybSyncHistory_服务层创建失败记录并展示(t *testing.T) {
db := newSyncTestDB(t)
actor := model.User{
UserID: "USR-HISTORY", Username: "buyer", PasswordHash: "test-hash",
Role: model.RolePurchaser, Status: model.UserActive,
PasswordChangedAt: "2026-08-09T00:00:00Z", CreatedAt: "2026-08-09T00:00:00Z",
UpdatedAt: "2026-08-09T00:00:00Z",
}
if err := repository.CreateInitialAdmin(db, actor); err != nil {
t.Fatal(err)
}
runID, err := CreateSybSyncRun(db, &actor,
SybSyncOptions{From: "2026-08-07", To: "2026-08-09"},
time.Date(2026, 8, 9, 1, 0, 0, 0, time.UTC))
if err != nil {
t.Fatal(err)
}
if err := FinishSybSyncRun(db, runID, SyncReport{
From: "2026-08-07", To: "2026-08-09", Err: fmt.Errorf("测试失败"),
FinishedAt: time.Date(2026, 8, 9, 1, 1, 0, 0, time.UTC),
}); err != nil {
t.Fatal(err)
}
history, err := ListSybSyncHistory(db, 1)
if err != nil || history.Total != 1 || len(history.Rows) != 1 {
t.Fatalf("读取记录失败: history=%+v err=%v", history, err)
}
row := history.Rows[0]
if row.Username != "buyer" || row.StatusText != "失败" || row.ErrorMessage != "测试失败" || row.CursorText != "未推进" {
t.Fatalf("同步记录视图不正确: %+v", row)
}
}