feat(t224): add incremental freight sync
This commit is contained in:
QiuSW
@@ -0,0 +1,234 @@
|
||||
-- +goose NO TRANSACTION
|
||||
-- +goose Up
|
||||
PRAGMA foreign_keys = OFF;
|
||||
PRAGMA legacy_alter_table = ON;
|
||||
BEGIN IMMEDIATE;
|
||||
|
||||
ALTER TABLE erp_sync_runs RENAME TO erp_sync_runs_v13;
|
||||
|
||||
CREATE TABLE erp_sync_runs (
|
||||
id TEXT PRIMARY KEY NOT NULL CHECK (length(id) = 36),
|
||||
creator_subject TEXT NOT NULL,
|
||||
created_by_user_id TEXT NOT NULL
|
||||
REFERENCES users(id) ON UPDATE RESTRICT ON DELETE RESTRICT,
|
||||
mode TEXT NOT NULL CHECK (mode IN ('ORDER_NUMBER', 'CREATED_RANGE')),
|
||||
order_number TEXT
|
||||
CHECK (
|
||||
order_number IS NULL
|
||||
OR (
|
||||
length(trim(order_number)) > 0
|
||||
AND length(CAST(order_number AS BLOB)) <= 128
|
||||
)
|
||||
),
|
||||
created_from TEXT
|
||||
CHECK (
|
||||
created_from IS NULL
|
||||
OR (
|
||||
length(created_from) = 10
|
||||
AND date(created_from) = created_from
|
||||
)
|
||||
),
|
||||
created_to TEXT
|
||||
CHECK (
|
||||
created_to IS NULL
|
||||
OR (
|
||||
length(created_to) = 10
|
||||
AND date(created_to) = created_to
|
||||
)
|
||||
),
|
||||
watermark_through TEXT,
|
||||
query_sha256 TEXT NOT NULL
|
||||
CHECK (
|
||||
length(query_sha256) = 64
|
||||
AND query_sha256 NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
idempotency_key TEXT NOT NULL
|
||||
CHECK (
|
||||
length(trim(idempotency_key)) > 0
|
||||
AND length(CAST(idempotency_key AS BLOB)) <= 128
|
||||
),
|
||||
request_sha256 TEXT NOT NULL
|
||||
CHECK (
|
||||
length(request_sha256) = 64
|
||||
AND request_sha256 NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
status TEXT NOT NULL
|
||||
CHECK (status IN ('PENDING', 'RUNNING', 'SUCCEEDED', 'FAILED')),
|
||||
error_code TEXT
|
||||
CHECK (
|
||||
error_code IS NULL
|
||||
OR length(CAST(error_code AS BLOB)) <= 64
|
||||
),
|
||||
order_count INTEGER NOT NULL DEFAULT 0 CHECK (order_count >= 0),
|
||||
item_count INTEGER NOT NULL DEFAULT 0 CHECK (item_count >= 0),
|
||||
created_at TEXT NOT NULL,
|
||||
started_at TEXT,
|
||||
finished_at TEXT,
|
||||
CHECK (
|
||||
(
|
||||
mode = 'ORDER_NUMBER'
|
||||
AND order_number IS NOT NULL
|
||||
AND created_from IS NULL
|
||||
AND created_to IS NULL
|
||||
AND watermark_through IS NULL
|
||||
)
|
||||
OR (
|
||||
mode = 'CREATED_RANGE'
|
||||
AND order_number IS NULL
|
||||
AND created_from IS NOT NULL
|
||||
AND created_to IS NOT NULL
|
||||
AND created_from <= created_to
|
||||
AND watermark_through IS NOT NULL
|
||||
)
|
||||
),
|
||||
CHECK (
|
||||
(status = 'PENDING'
|
||||
AND started_at IS NULL
|
||||
AND finished_at IS NULL
|
||||
AND error_code IS NULL)
|
||||
OR (status = 'RUNNING'
|
||||
AND started_at IS NOT NULL
|
||||
AND finished_at IS NULL
|
||||
AND error_code IS NULL)
|
||||
OR (status = 'SUCCEEDED'
|
||||
AND started_at IS NOT NULL
|
||||
AND finished_at IS NOT NULL
|
||||
AND error_code IS NULL)
|
||||
OR (status = 'FAILED'
|
||||
AND finished_at IS NOT NULL
|
||||
AND error_code IS NOT NULL)
|
||||
),
|
||||
UNIQUE (creator_subject, idempotency_key)
|
||||
);
|
||||
|
||||
INSERT INTO erp_sync_runs (
|
||||
id, creator_subject, created_by_user_id, mode, order_number,
|
||||
created_from, created_to, watermark_through, query_sha256,
|
||||
idempotency_key, request_sha256, status, error_code, order_count,
|
||||
item_count, created_at, started_at, finished_at
|
||||
)
|
||||
SELECT
|
||||
id, creator_subject, created_by_user_id, mode, order_number,
|
||||
NULL, NULL, NULL, query_sha256, idempotency_key, request_sha256,
|
||||
status, error_code, order_count, item_count, created_at, started_at,
|
||||
finished_at
|
||||
FROM erp_sync_runs_v13;
|
||||
|
||||
DROP TABLE erp_sync_runs_v13;
|
||||
|
||||
CREATE INDEX erp_sync_runs_creator_created_idx
|
||||
ON erp_sync_runs (creator_subject, created_at DESC, id DESC);
|
||||
|
||||
CREATE TABLE erp_sync_watermarks (
|
||||
creator_subject TEXT NOT NULL,
|
||||
source_system TEXT NOT NULL CHECK (source_system = 'SHUNYUNBAO'),
|
||||
last_successful_to TEXT NOT NULL,
|
||||
last_successful_run_id TEXT NOT NULL
|
||||
REFERENCES erp_sync_runs(id) ON UPDATE RESTRICT ON DELETE RESTRICT,
|
||||
updated_at TEXT NOT NULL,
|
||||
PRIMARY KEY (creator_subject, source_system)
|
||||
);
|
||||
|
||||
COMMIT;
|
||||
PRAGMA legacy_alter_table = OFF;
|
||||
PRAGMA foreign_keys = ON;
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS erp_date_sync_v14_down_guard;
|
||||
CREATE TEMP TABLE erp_date_sync_v14_down_guard (
|
||||
allowed INTEGER NOT NULL CHECK (allowed = 1)
|
||||
);
|
||||
|
||||
INSERT INTO erp_date_sync_v14_down_guard (allowed)
|
||||
SELECT CASE
|
||||
WHEN EXISTS (
|
||||
SELECT 1 FROM erp_sync_runs WHERE mode = 'CREATED_RANGE'
|
||||
) OR EXISTS (SELECT 1 FROM erp_sync_watermarks)
|
||||
THEN 0
|
||||
ELSE 1
|
||||
END;
|
||||
|
||||
DROP TABLE erp_date_sync_v14_down_guard;
|
||||
PRAGMA foreign_keys = OFF;
|
||||
PRAGMA legacy_alter_table = ON;
|
||||
BEGIN IMMEDIATE;
|
||||
DROP TABLE erp_sync_watermarks;
|
||||
ALTER TABLE erp_sync_runs RENAME TO erp_sync_runs_v14;
|
||||
|
||||
CREATE TABLE erp_sync_runs (
|
||||
id TEXT PRIMARY KEY NOT NULL CHECK (length(id) = 36),
|
||||
creator_subject TEXT NOT NULL,
|
||||
created_by_user_id TEXT NOT NULL
|
||||
REFERENCES users(id) ON UPDATE RESTRICT ON DELETE RESTRICT,
|
||||
mode TEXT NOT NULL CHECK (mode = 'ORDER_NUMBER'),
|
||||
order_number TEXT NOT NULL
|
||||
CHECK (
|
||||
length(trim(order_number)) > 0
|
||||
AND length(CAST(order_number AS BLOB)) <= 128
|
||||
),
|
||||
query_sha256 TEXT NOT NULL
|
||||
CHECK (
|
||||
length(query_sha256) = 64
|
||||
AND query_sha256 NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
idempotency_key TEXT NOT NULL
|
||||
CHECK (
|
||||
length(trim(idempotency_key)) > 0
|
||||
AND length(CAST(idempotency_key AS BLOB)) <= 128
|
||||
),
|
||||
request_sha256 TEXT NOT NULL
|
||||
CHECK (
|
||||
length(request_sha256) = 64
|
||||
AND request_sha256 NOT GLOB '*[^0-9a-f]*'
|
||||
),
|
||||
status TEXT NOT NULL
|
||||
CHECK (status IN ('PENDING', 'RUNNING', 'SUCCEEDED', 'FAILED')),
|
||||
error_code TEXT
|
||||
CHECK (
|
||||
error_code IS NULL
|
||||
OR length(CAST(error_code AS BLOB)) <= 64
|
||||
),
|
||||
order_count INTEGER NOT NULL DEFAULT 0 CHECK (order_count >= 0),
|
||||
item_count INTEGER NOT NULL DEFAULT 0 CHECK (item_count >= 0),
|
||||
created_at TEXT NOT NULL,
|
||||
started_at TEXT,
|
||||
finished_at TEXT,
|
||||
CHECK (
|
||||
(status = 'PENDING'
|
||||
AND started_at IS NULL
|
||||
AND finished_at IS NULL
|
||||
AND error_code IS NULL)
|
||||
OR (status = 'RUNNING'
|
||||
AND started_at IS NOT NULL
|
||||
AND finished_at IS NULL
|
||||
AND error_code IS NULL)
|
||||
OR (status = 'SUCCEEDED'
|
||||
AND started_at IS NOT NULL
|
||||
AND finished_at IS NOT NULL
|
||||
AND error_code IS NULL)
|
||||
OR (status = 'FAILED'
|
||||
AND finished_at IS NOT NULL
|
||||
AND error_code IS NOT NULL)
|
||||
),
|
||||
UNIQUE (creator_subject, idempotency_key)
|
||||
);
|
||||
|
||||
INSERT INTO erp_sync_runs (
|
||||
id, creator_subject, created_by_user_id, mode, order_number,
|
||||
query_sha256, idempotency_key, request_sha256, status, error_code,
|
||||
order_count, item_count, created_at, started_at, finished_at
|
||||
)
|
||||
SELECT
|
||||
id, creator_subject, created_by_user_id, mode, order_number,
|
||||
query_sha256, idempotency_key, request_sha256, status, error_code,
|
||||
order_count, item_count, created_at, started_at, finished_at
|
||||
FROM erp_sync_runs_v14;
|
||||
|
||||
DROP TABLE erp_sync_runs_v14;
|
||||
|
||||
CREATE INDEX erp_sync_runs_creator_created_idx
|
||||
ON erp_sync_runs (creator_subject, created_at DESC, id DESC);
|
||||
|
||||
COMMIT;
|
||||
PRAGMA legacy_alter_table = OFF;
|
||||
PRAGMA foreign_keys = ON;
|
||||
Reference in New Issue
Block a user