chengma/cmroubao
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(t224): add incremental freight sync

Browse Source
This commit is contained in:
QiuSW
2026-07-29 00:26:05 +08:00
parent 433db45100
commit c69879650e
35 changed files with 2085 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-api/internal/repository/sqlite/auth_repository_test.go
+6 -3
View File
@@ -383,6 +383,9 @@ func TestAuthMigrationCanRollbackWithoutRebuildingPurchaseTasks(
if err != nil {
t.Fatalf("migration.New() error = %v", err)
}
if err := runner.Down(context.Background()); err != nil {
t.Fatalf("Down(v14) error = %v", err)
}
if err := runner.Down(context.Background()); err != nil {
t.Fatalf("Down(v13) error = %v", err)
}
@@ -429,9 +432,9 @@ func TestAuthMigrationCanRollbackWithoutRebuildingPurchaseTasks(
t.Fatal("purchase_tasks was lost during auth migration rollback")
}
if applied, err := runner.Up(context.Background()); err != nil {
t.Fatalf("Up(v3-v13) error = %v", err)
} else if applied != 11 {
t.Fatalf("Up(v3-v13) applied = %d, want 11", applied)
t.Fatalf("Up(v3-v14) error = %v", err)
} else if applied != 12 {
t.Fatalf("Up(v3-v14) applied = %d, want 12", applied)
}
}
backend-api/internal/repository/sqlite/freight_repository.go
+121 -7
View File
@@ -52,14 +52,18 @@ func (store *Store) CreateFreightSync(
ctx,
`INSERT INTO erp_sync_runs (
id, creator_subject, created_by_user_id, mode, order_number,
query_sha256, idempotency_key, request_sha256, status,
order_count, item_count, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, ?)`,
created_from, created_to, watermark_through, query_sha256,
idempotency_key, request_sha256, status, order_count, item_count,
created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, ?)`,
run.ID,
run.CreatorSubject,
run.CreatedByUserID,
run.Mode,
run.OrderNumber,
nullableFreightSyncValue(run.OrderNumber),
nullableFreightSyncValue(run.CreatedFrom),
nullableFreightSyncValue(run.CreatedTo),
nullableTimestamp(run.WatermarkThrough),
run.QuerySHA256,
idempotencyKey,
requestSHA256,
@@ -106,6 +110,32 @@ func (store *Store) CompleteFreightSync(
run domain.FreightSyncRun,
batch domain.FreightImportBatch,
finishedAt time.Time,
) error {
return store.completeFreightSync(ctx, run, batch, nil, finishedAt)
}
func (store *Store) CompleteFreightDateSync(
ctx context.Context,
run domain.FreightSyncRun,
batch domain.FreightImportBatch,
watermarkThrough time.Time,
finishedAt time.Time,
) error {
return store.completeFreightSync(
ctx,
run,
batch,
&watermarkThrough,
finishedAt,
)
}
func (store *Store) completeFreightSync(
ctx context.Context,
run domain.FreightSyncRun,
batch domain.FreightImportBatch,
watermarkThrough *time.Time,
finishedAt time.Time,
) error {
tx, err := store.db.BeginTx(ctx, nil)
if err != nil {
@@ -184,6 +214,28 @@ func (store *Store) CompleteFreightSync(
if changed != 1 {
return usecase.ErrTaskStateConflict
}
if watermarkThrough != nil {
if _, err := tx.ExecContext(
ctx,
`INSERT INTO erp_sync_watermarks (
creator_subject, source_system, last_successful_to,
last_successful_run_id, updated_at
) VALUES (?, 'SHUNYUNBAO', ?, ?, ?)
ON CONFLICT (creator_subject, source_system)
DO UPDATE SET
last_successful_to = excluded.last_successful_to,
last_successful_run_id = excluded.last_successful_run_id,
updated_at = excluded.updated_at
WHERE julianday(erp_sync_watermarks.last_successful_to)
< julianday(excluded.last_successful_to)`,
run.CreatorSubject,
formatTimestamp(*watermarkThrough),
run.ID,
formatTimestamp(finishedAt),
); err != nil {
return repositoryFailure(err)
}
}
if err := tx.Commit(); err != nil {
return repositoryFailure(err)
}
@@ -377,6 +429,43 @@ func (store *Store) GetFreightSync(
return run, nil
}
func (store *Store) GetFreightSyncWatermark(
ctx context.Context,
creatorSubject string,
) (*domain.FreightSyncWatermark, error) {
var watermark domain.FreightSyncWatermark
var lastSuccessfulTo, updatedAt string
err := store.db.QueryRowContext(
ctx,
`SELECT creator_subject, source_system, last_successful_to,
last_successful_run_id, updated_at
FROM erp_sync_watermarks
WHERE creator_subject = ? AND source_system = 'SHUNYUNBAO'`,
creatorSubject,
).Scan(
&watermark.CreatorSubject,
&watermark.SourceSystem,
&lastSuccessfulTo,
&watermark.LastSuccessfulRunID,
&updatedAt,
)
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
if err != nil {
return nil, repositoryFailure(err)
}
watermark.LastSuccessfulTo, err = parseTimestamp(lastSuccessfulTo)
if err != nil {
return nil, repositoryFailure(err)
}
watermark.UpdatedAt, err = parseTimestamp(updatedAt)
if err != nil {
return nil, repositoryFailure(err)
}
return &watermark, nil
}
func (store *Store) ListFreightOrders(
ctx context.Context,
creatorSubject string,
@@ -458,13 +547,15 @@ func (store *Store) GetFreightOrder(
const freightSyncSelect = `SELECT
id, creator_subject, created_by_user_id, mode, order_number,
query_sha256, status, error_code, order_count, item_count,
created_at, started_at, finished_at
created_from, created_to, watermark_through, query_sha256, status,
error_code, order_count, item_count, created_at, started_at, finished_at
FROM erp_sync_runs
`
func scanFreightSync(scanner rowScanner) (domain.FreightSyncRun, error) {
var run domain.FreightSyncRun
var orderNumber, createdFrom, createdTo sql.NullString
var watermarkThrough sql.NullString
var errorCode sql.NullString
var createdAt string
var startedAt sql.NullString
@@ -474,7 +565,10 @@ func scanFreightSync(scanner rowScanner) (domain.FreightSyncRun, error) {
&run.CreatorSubject,
&run.CreatedByUserID,
&run.Mode,
&run.OrderNumber,
&orderNumber,
&createdFrom,
&createdTo,
&watermarkThrough,
&run.QuerySHA256,
&run.Status,
&errorCode,
@@ -490,6 +584,19 @@ func scanFreightSync(scanner rowScanner) (domain.FreightSyncRun, error) {
if errorCode.Valid {
run.ErrorCode = &errorCode.String
}
if orderNumber.Valid {
run.OrderNumber = orderNumber.String
}
if createdFrom.Valid {
run.CreatedFrom = createdFrom.String
}
if createdTo.Valid {
run.CreatedTo = createdTo.String
}
run.WatermarkThrough, err = parseNullableTimestamp(watermarkThrough)
if err != nil {
return domain.FreightSyncRun{}, err
}
run.CreatedAt, err = parseTimestamp(createdAt)
if err != nil {
return domain.FreightSyncRun{}, err
@@ -622,6 +729,13 @@ func nullableFreightQuantity(value *int) any {
return *value
}
func nullableFreightSyncValue(value string) any {
if value == "" {
return nil
}
return value
}
func optionalString(value sql.NullString) *string {
if !value.Valid {
return nil
backend-api/internal/repository/sqlite/freight_repository_test.go
+124
View File
@@ -117,6 +117,9 @@ func TestFreightImportIsAtomicIdempotentAndRevisioned(t *testing.T) {
if err != nil {
t.Fatalf("migration.New() error = %v", err)
}
if err := runner.Down(ctx); err != nil {
t.Fatalf("date sync migration down: %v", err)
}
if err := runner.Down(ctx); err != nil {
t.Fatalf("procurement migration down: %v", err)
}
@@ -163,6 +166,106 @@ func TestFreightSyncRecoveryAndFailedBatchDoNotPersistOrders(t *testing.T) {
}
}
func TestFreightDateSyncAdvancesWatermarkOnlyOnWholeBatchSuccess(
t *testing.T,
) {
db := openDatabase(t)
store, _ := repository.New(db)
ctx := context.Background()
now := time.Date(2026, 7, 28, 8, 0, 0, 0, time.UTC)
userID := uuid(960)
seedFreightUser(t, db, userID, now)
firstThrough := now.Add(30 * time.Minute)
first := freightDateRun(
961,
userID,
now,
"2026-07-27",
"2026-07-28",
firstThrough,
)
createAndStartFreightRun(t, store, first, "date-success", "6")
if err := store.CompleteFreightDateSync(
ctx,
first,
domain.FreightImportBatch{},
firstThrough,
now.Add(time.Minute),
); err != nil {
t.Fatalf("CompleteFreightDateSync() error = %v", err)
}
watermark, err := store.GetFreightSyncWatermark(ctx, "local-admin")
if err != nil || watermark == nil ||
!watermark.LastSuccessfulTo.Equal(firstThrough) ||
watermark.LastSuccessfulRunID != first.ID {
t.Fatalf("first watermark = %+v, %v", watermark, err)
}
failed := freightDateRun(
962,
userID,
now.Add(time.Hour),
"2026-07-28",
"2026-07-28",
now.Add(2*time.Hour),
)
createAndStartFreightRun(t, store, failed, "date-failed", "7")
if err := store.FailFreightSync(
ctx,
failed.ID,
"ERP_CONNECTOR_UNAVAILABLE",
now.Add(time.Hour+time.Minute),
); err != nil {
t.Fatalf("FailFreightSync() error = %v", err)
}
watermark, _ = store.GetFreightSyncWatermark(ctx, "local-admin")
if !watermark.LastSuccessfulTo.Equal(firstThrough) ||
watermark.LastSuccessfulRunID != first.ID {
t.Fatalf("failed run advanced watermark = %+v", watermark)
}
olderThrough := now.Add(-time.Hour)
older := freightDateRun(
963,
userID,
now.Add(2*time.Hour),
"2026-07-26",
"2026-07-26",
olderThrough,
)
createAndStartFreightRun(t, store, older, "date-older", "8")
if err := store.CompleteFreightDateSync(
ctx,
older,
domain.FreightImportBatch{},
olderThrough,
now.Add(2*time.Hour+time.Minute),
); err != nil {
t.Fatalf("older CompleteFreightDateSync() error = %v", err)
}
watermark, _ = store.GetFreightSyncWatermark(ctx, "local-admin")
if !watermark.LastSuccessfulTo.Equal(firstThrough) ||
watermark.LastSuccessfulRunID != first.ID {
t.Fatalf("older run retreated watermark = %+v", watermark)
}
runner, _ := migration.New(db)
if err := runner.Down(ctx); err == nil {
t.Fatal("date sync migration down succeeded with retained watermark")
}
var foreignKeysEnabled int
if err := db.QueryRow(`PRAGMA foreign_keys`).Scan(
&foreignKeysEnabled,
); err != nil || foreignKeysEnabled != 1 {
t.Fatalf(
"failed down disabled foreign keys = %d, %v",
foreignKeysEnabled,
err,
)
}
}
func seedFreightUser(
t *testing.T,
db *sql.DB,
@@ -200,6 +303,27 @@ func freightRun(
}
}
func freightDateRun(
index int,
userID string,
now time.Time,
createdFrom, createdTo string,
watermarkThrough time.Time,
) domain.FreightSyncRun {
return domain.FreightSyncRun{
ID: uuid(index),
CreatorSubject: "local-admin",
CreatedByUserID: userID,
Mode: domain.FreightSyncCreatedRange,
CreatedFrom: createdFrom,
CreatedTo: createdTo,
WatermarkThrough: &watermarkThrough,
QuerySHA256: repeatHex("b"),
Status: domain.FreightSyncPending,
CreatedAt: now,
}
}
func freightBatch(index int, firstHash, secondHash string) domain.FreightImportBatch {
quantityOne := 1
quantityTwo := 2
backend-api/internal/repository/sqlite/procurement_repository_test.go
+3
View File
@@ -238,6 +238,9 @@ func TestProcurementRequestsArePerItemAndTaskSnapshotIsImmutable(
if err != nil {
t.Fatalf("migration.New() error = %v", err)
}
if err := runner.Down(ctx); err != nil {
t.Fatalf("date sync migration down: %v", err)
}
if err := runner.Down(ctx); err == nil {
t.Fatal("procurement migration down succeeded with retained requests")
}