feat: 任务改用采集采购独立序号 (#172)

This commit is contained in:
chengma
2026-08-12 09:21:21 +08:00
parent eca3a77084
commit 9ad067503e
15 changed files with 523 additions and 44 deletions
+235 -2
View File
@@ -5,6 +5,7 @@ import (
"crypto/tls"
"crypto/x509"
"database/sql"
"errors"
"fmt"
"log"
"net"
@@ -19,7 +20,7 @@ import (
"cmautobuy/admin/spec"
)
const mysqlSchemaVersion = 12
const mysqlSchemaVersion = 13
// OpenMySQL 打开生产 MySQL 8 数据库。错误信息绝不包含完整 DSN 或密码。
func OpenMySQL(cfg config.DatabaseConfig) (*sql.DB, error) {
@@ -567,10 +568,238 @@ func MigrateMySQL(db *sql.DB) error {
if _, err := db.Exec(`INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)`, 12, time.Now().UTC().Format(time.RFC3339Nano)); err != nil {
return fmt.Errorf("记录 MySQL schema v12 失败: %w", err)
}
current = 12
}
if current < 13 {
if err := migrateMySQLV13(db); err != nil {
return fmt.Errorf("执行 MySQL schema v13 失败: %w", err)
}
if err := checkMySQLV13Shape(db); err != nil {
return fmt.Errorf("MySQL schema v13 自检失败,未记录版本: %w", err)
}
if _, err := db.Exec(`INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)`, 13, time.Now().UTC().Format(time.RFC3339Nano)); err != nil {
return fmt.Errorf("记录 MySQL schema v13 失败: %w", err)
}
}
return CheckMySQLSchema(db)
}
// migrateMySQLV13 把任务真实主键迁为采集 cjN、采购 cgN 两套独立业务单号。
//
// 外键 DDL 可重放;主键和关联数据在单一事务中同时切换。若进程在记录版本前退出,
// 再次启动会识别已经合规的编号,只校准序列,不会二次重编号。
func migrateMySQLV13(db *sql.DB) error {
if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS task_sequences (
task_type VARCHAR(20) COLLATE utf8mb4_bin PRIMARY KEY,
current_value BIGINT NOT NULL DEFAULT 0,
CONSTRAINT chk_task_sequences_type CHECK (task_type IN ('collect','purchase')),
CONSTRAINT chk_task_sequences_value CHECK (current_value >= 0)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`); err != nil {
return fmt.Errorf("建立任务序列表失败: %w", err)
}
if _, err := db.Exec(`INSERT INTO task_sequences(task_type,current_value)
VALUES('collect',0),('purchase',0)
ON DUPLICATE KEY UPDATE task_type=VALUES(task_type)`); err != nil {
return fmt.Errorf("初始化任务序列失败: %w", err)
}
if err := ensureTaskForeignKeyCascade(db, "task_syb_sources", "fk_task_syb_sources_task"); err != nil {
return err
}
if err := migrateTaskIDsV13(db); err != nil {
return err
}
if err := ensureTaskForeignKeyCascade(db, "task_claims", "fk_task_claims_task"); err != nil {
return err
}
return nil
}
type taskIDMigrationV13 struct {
oldID, newID string
taskType model.TaskType
}
func migrateTaskIDsV13(db *sql.DB) error {
tx, err := db.Begin()
if err != nil {
return fmt.Errorf("开始任务编号迁移事务失败: %w", err)
}
defer tx.Rollback()
rows, err := tx.Query(`SELECT task_id,task_type FROM tasks ORDER BY task_type,created_at,task_id`)
if err != nil {
return fmt.Errorf("读取待迁移任务失败: %w", err)
}
var source []taskIDMigrationV13
var hasLegacy, hasBusinessID bool
lastValues := map[model.TaskType]int64{model.TaskCollect: 0, model.TaskPurchase: 0}
for rows.Next() {
var taskID string
var taskType model.TaskType
if err := rows.Scan(&taskID, &taskType); err != nil {
rows.Close()
return fmt.Errorf("读取待迁移任务编号失败: %w", err)
}
if number, ok := parseTaskIDNumber(taskType, taskID); ok {
hasBusinessID = true
if number > lastValues[taskType] {
lastValues[taskType] = number
}
} else {
hasLegacy = true
}
source = append(source, taskIDMigrationV13{oldID: taskID, taskType: taskType})
}
if err := rows.Close(); err != nil {
return fmt.Errorf("关闭待迁移任务结果失败: %w", err)
}
if hasLegacy && hasBusinessID {
return fmt.Errorf("任务编号同时包含旧格式和 cj/cg 格式,拒绝自动混合迁移")
}
if _, err := tx.Exec(`DELETE c FROM task_claims c LEFT JOIN tasks t ON t.task_id=c.task_id WHERE t.task_id IS NULL`); err != nil {
return fmt.Errorf("清理孤儿任务领取历史失败: %w", err)
}
if hasLegacy {
if _, err := tx.Exec(`CREATE TEMPORARY TABLE task_id_map_v13 (
old_task_id VARCHAR(191) COLLATE utf8mb4_bin PRIMARY KEY,
new_task_id VARCHAR(191) COLLATE utf8mb4_bin NOT NULL UNIQUE,
task_type VARCHAR(20) COLLATE utf8mb4_bin NOT NULL
) ENGINE=InnoDB`); err != nil {
return fmt.Errorf("建立任务编号临时映射失败: %w", err)
}
for i := range source {
lastValues[source[i].taskType]++
prefix, _ := taskIDPrefix(source[i].taskType)
source[i].newID = fmt.Sprintf("%s%d", prefix, lastValues[source[i].taskType])
if _, err := tx.Exec(`INSERT INTO task_id_map_v13(old_task_id,new_task_id,task_type) VALUES(?,?,?)`,
source[i].oldID, source[i].newID, source[i].taskType); err != nil {
return fmt.Errorf("保存任务编号映射 %s 失败: %w", source[i].oldID, err)
}
}
if _, err := tx.Exec(`UPDATE task_claims c JOIN task_id_map_v13 m ON m.old_task_id=c.task_id SET c.task_id=m.new_task_id`); err != nil {
return fmt.Errorf("迁移任务领取历史编号失败: %w", err)
}
if _, err := tx.Exec(`UPDATE tasks t JOIN task_id_map_v13 m ON m.old_task_id=t.task_id SET t.task_id=m.new_task_id`); err != nil {
return fmt.Errorf("迁移任务主键失败: %w", err)
}
if _, err := tx.Exec(`DROP TEMPORARY TABLE task_id_map_v13`); err != nil {
return fmt.Errorf("清理任务编号临时映射失败: %w", err)
}
}
for _, taskType := range []model.TaskType{model.TaskCollect, model.TaskPurchase} {
if _, err := tx.Exec(`UPDATE task_sequences SET current_value=? WHERE task_type=?`, lastValues[taskType], taskType); err != nil {
return fmt.Errorf("校准 %s 任务序列失败: %w", taskType, err)
}
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("提交任务编号迁移失败: %w", err)
}
return nil
}
func ensureTaskForeignKeyCascade(db *sql.DB, table, constraint string) error {
var deleteRule, updateRule string
err := db.QueryRow(`SELECT delete_rule,update_rule FROM information_schema.referential_constraints
WHERE constraint_schema=DATABASE() AND table_name=? AND constraint_name=?`, table, constraint).
Scan(&deleteRule, &updateRule)
if err == nil && deleteRule == "CASCADE" && updateRule == "CASCADE" {
return nil
}
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return fmt.Errorf("检查任务关联外键 %s 失败: %w", constraint, err)
}
if err == nil {
var dropStatement string
switch table {
case "task_syb_sources":
dropStatement = `ALTER TABLE task_syb_sources DROP FOREIGN KEY fk_task_syb_sources_task`
case "task_claims":
dropStatement = `ALTER TABLE task_claims DROP FOREIGN KEY fk_task_claims_task`
default:
return fmt.Errorf("不支持的任务关联表 %s", table)
}
if _, err := db.Exec(dropStatement); err != nil {
return fmt.Errorf("删除旧任务关联外键 %s 失败: %w", constraint, err)
}
}
var statement string
switch table {
case "task_syb_sources":
statement = `ALTER TABLE task_syb_sources ADD CONSTRAINT fk_task_syb_sources_task
FOREIGN KEY(task_id) REFERENCES tasks(task_id) ON DELETE CASCADE ON UPDATE CASCADE`
case "task_claims":
statement = `ALTER TABLE task_claims ADD CONSTRAINT fk_task_claims_task
FOREIGN KEY(task_id) REFERENCES tasks(task_id) ON DELETE CASCADE ON UPDATE CASCADE`
default:
return fmt.Errorf("不支持的任务关联表 %s", table)
}
if _, err := db.Exec(statement); err != nil {
return fmt.Errorf("建立任务关联外键 %s 失败: %w", constraint, err)
}
return nil
}
func checkMySQLV13Shape(db *sql.DB) error {
exists, err := mysqlTableExists(db, "task_sequences")
if err != nil || !exists {
return fmt.Errorf("任务序列表缺失")
}
if err := checkMySQLVarcharColumn(db, "task_sequences", "task_type", 20, false, "utf8mb4_bin", ""); err != nil {
return err
}
var valueType, valueNullable string
var valueDefault sql.NullString
if err := db.QueryRow(`SELECT data_type,is_nullable,column_default FROM information_schema.columns
WHERE table_schema=DATABASE() AND table_name='task_sequences' AND column_name='current_value'`).
Scan(&valueType, &valueNullable, &valueDefault); err != nil || valueType != "bigint" ||
valueNullable != "NO" || !valueDefault.Valid || valueDefault.String != "0" {
return fmt.Errorf("任务序列值列结构不正确")
}
for _, item := range []struct{ table, constraint string }{
{"task_claims", "fk_task_claims_task"},
{"task_syb_sources", "fk_task_syb_sources_task"},
} {
var deleteRule, updateRule string
if err := db.QueryRow(`SELECT delete_rule,update_rule FROM information_schema.referential_constraints
WHERE constraint_schema=DATABASE() AND table_name=? AND constraint_name=?`, item.table, item.constraint).
Scan(&deleteRule, &updateRule); err != nil || deleteRule != "CASCADE" || updateRule != "CASCADE" {
return fmt.Errorf("任务关联外键 %s 规则不正确", item.constraint)
}
}
var sequenceRows, invalidSequences int
if err := db.QueryRow(`SELECT COUNT(*),COALESCE(SUM(task_type NOT IN ('collect','purchase') OR current_value<0),0)
FROM task_sequences`).Scan(&sequenceRows, &invalidSequences); err != nil || sequenceRows != 2 || invalidSequences != 0 {
return fmt.Errorf("任务序列表必须且只能包含 collect/purchase 两条非负序列")
}
var invalidTasks, orphanClaims, orphanSources int
if err := db.QueryRow(`SELECT COUNT(*) FROM tasks WHERE
(task_type='collect' AND task_id NOT REGEXP '^cj[1-9][0-9]*$') OR
(task_type='purchase' AND task_id NOT REGEXP '^cg[1-9][0-9]*$')`).Scan(&invalidTasks); err != nil || invalidTasks != 0 {
return fmt.Errorf("存在不符合 cj/cg 规范的任务主键")
}
if err := db.QueryRow(`SELECT COUNT(*) FROM task_claims c LEFT JOIN tasks t ON t.task_id=c.task_id WHERE t.task_id IS NULL`).Scan(&orphanClaims); err != nil || orphanClaims != 0 {
return fmt.Errorf("存在孤儿任务领取历史")
}
if err := db.QueryRow(`SELECT COUNT(*) FROM task_syb_sources s LEFT JOIN tasks t ON t.task_id=s.task_id WHERE t.task_id IS NULL`).Scan(&orphanSources); err != nil || orphanSources != 0 {
return fmt.Errorf("存在孤儿顺运宝任务来源")
}
for _, taskType := range []model.TaskType{model.TaskCollect, model.TaskPurchase} {
prefix, _ := taskIDPrefix(taskType)
var lastValue, maxTaskNumber int64
if err := db.QueryRow(`SELECT current_value FROM task_sequences WHERE task_type=?`, taskType).Scan(&lastValue); err != nil {
return fmt.Errorf("读取 %s 任务序列失败: %w", taskType, err)
}
if err := db.QueryRow(`SELECT COALESCE(MAX(CAST(SUBSTRING(task_id,3) AS UNSIGNED)),0)
FROM tasks WHERE task_type=? AND task_id LIKE ?`, taskType, prefix+"%").Scan(&maxTaskNumber); err != nil {
return fmt.Errorf("读取 %s 最大任务编号失败: %w", taskType, err)
}
if lastValue < maxTaskNumber {
return fmt.Errorf("%s 任务序列 %d 小于现存最大编号 %d", taskType, lastValue, maxTaskNumber)
}
}
return nil
}
// migrateMySQLV12 保存采集任务的顺运宝来源,并回收没有有效任务的孤儿采集中状态。
// CREATE TABLE IF NOT EXISTS 和带条件的 UPDATE 都可重放,适合 MySQL DDL 隐式提交后的重启恢复。
func migrateMySQLV12(db *sql.DB) error {
@@ -1260,6 +1489,7 @@ func CheckMySQLSchema(db *sql.DB) error {
"spec_mapping_decisions",
"catalog_import_runs",
"task_syb_sources",
"task_sequences",
}
if err := checkMySQLSchema(db, mysqlRequiredTables); err != nil {
return err
@@ -1291,7 +1521,10 @@ func CheckMySQLSchema(db *sql.DB) error {
if err := checkMySQLV11Shape(db); err != nil {
return err
}
return checkMySQLV12Shape(db)
if err := checkMySQLV12Shape(db); err != nil {
return err
}
return checkMySQLV13Shape(db)
}
func checkMySQLV9Shape(db *sql.DB) error {
@@ -5,6 +5,7 @@ import (
"fmt"
"os"
"strings"
"sync"
"testing"
"cmautobuy/admin/config"
@@ -650,6 +651,154 @@ func TestMySQLMigrate_V12形状错误不记录版本(t *testing.T) {
}
}
func TestMySQLMigrate_V12升级V13迁移任务主键和关联(t *testing.T) {
db := openMySQLMigrationTestDB(t)
defer db.Close()
cleanMySQLTestSchema(t, db)
defer cleanMySQLTestSchema(t, db)
if err := MigrateMySQL(db); err != nil {
t.Fatal(err)
}
// 模拟生产 v12:移除 v13 表、版本和新增的级联更新外键。
mustExec(t, db, `DELETE FROM schema_migrations WHERE version=13`)
mustExec(t, db, `DROP TABLE task_sequences`)
mustExec(t, db, `ALTER TABLE task_claims DROP FOREIGN KEY fk_task_claims_task`)
mustExec(t, db, `ALTER TABLE task_syb_sources DROP FOREIGN KEY fk_task_syb_sources_task`)
mustExec(t, db, `ALTER TABLE task_syb_sources ADD CONSTRAINT fk_task_syb_sources_task
FOREIGN KEY(task_id) REFERENCES tasks(task_id) ON DELETE CASCADE`)
mustExec(t, db, `INSERT INTO syb_orders(syb_id,order_no,title,product_spec,quantity,syb_data,created_at,updated_at)
VALUES('SYB-V13','ORDER-SOURCE','商品','黑色,M',1,'{}','2026-08-12T00:00:00Z','2026-08-12T00:00:00Z')`)
mustExec(t, db, `INSERT INTO tasks(task_id,task_type,status,pdd_goods_url,pdd_goods_id,order_no,result_data,created_at,updated_at) VALUES
('OLD-C-LATE','collect','succeeded','https://example.invalid/c2','C-LATE',NULL,NULL,'2026-08-12T02:00:00Z','2026-08-12T02:00:00Z'),
('OLD-C-FIRST','collect','assigned','https://example.invalid/c1','C-FIRST',NULL,NULL,'2026-08-12T01:00:00Z','2026-08-12T01:00:00Z'),
('OLD-P-LATE','purchase','succeeded','https://example.invalid/p2','P-LATE','ORDER-P-LATE','{\"purchase\":{\"order_no\":\"PDD-2\",\"ordered_at\":\"2026-08-12T02:00:00Z\"}}','2026-08-12T04:00:00Z','2026-08-12T04:00:00Z'),
('OLD-P-FIRST','purchase','succeeded','https://example.invalid/p1','P-FIRST','ORDER-P-FIRST','{\"purchase\":{\"order_no\":\"PDD-1\",\"ordered_at\":\"2026-08-12T01:00:00Z\"}}','2026-08-12T03:00:00Z','2026-08-12T03:00:00Z')`)
mustExec(t, db, `INSERT INTO task_claims(task_id,client_id,claimed_at) VALUES
('OLD-C-FIRST','CLIENT-1','2026-08-12T01:10:00Z'),
('OLD-P-FIRST','CLIENT-1','2026-08-12T03:10:00Z'),
('DELETED-TASK','CLIENT-OLD','2026-08-01T00:00:00Z')`)
mustExec(t, db, `INSERT INTO task_syb_sources(task_id,syb_id,created_at)
VALUES('OLD-C-FIRST','SYB-V13','2026-08-12T01:00:00Z')`)
if err := MigrateMySQL(db); err != nil {
t.Fatalf("v12 升级 v13 失败: %v", err)
}
wants := map[string]string{"C-FIRST": "cj1", "C-LATE": "cj2", "P-FIRST": "cg1", "P-LATE": "cg2"}
for goodsID, wantID := range wants {
var taskID string
if err := db.QueryRow(`SELECT task_id FROM tasks WHERE pdd_goods_id=?`, goodsID).Scan(&taskID); err != nil || taskID != wantID {
t.Fatalf("商品 %s 任务编号=%q err=%v,期望 %s", goodsID, taskID, err, wantID)
}
}
var claimCount, orphanCount, sourceCount int
if err := db.QueryRow(`SELECT COUNT(*) FROM task_claims WHERE task_id IN ('cj1','cg1')`).Scan(&claimCount); err != nil || claimCount != 2 {
t.Fatalf("有效领取历史未迁移: count=%d err=%v", claimCount, err)
}
if err := db.QueryRow(`SELECT COUNT(*) FROM task_claims c LEFT JOIN tasks t ON t.task_id=c.task_id WHERE t.task_id IS NULL`).Scan(&orphanCount); err != nil || orphanCount != 0 {
t.Fatalf("孤儿领取历史未清理: count=%d err=%v", orphanCount, err)
}
if err := db.QueryRow(`SELECT COUNT(*) FROM task_syb_sources WHERE task_id='cj1' AND syb_id='SYB-V13'`).Scan(&sourceCount); err != nil || sourceCount != 1 {
t.Fatalf("顺运宝来源未迁移: count=%d err=%v", sourceCount, err)
}
var resultData string
if err := db.QueryRow(`SELECT result_data FROM tasks WHERE task_id='cg1'`).Scan(&resultData); err != nil || !strings.Contains(resultData, `"order_no":"PDD-1"`) {
t.Fatalf("采购结果不得变化: result=%q err=%v", resultData, err)
}
for taskType, want := range map[model.TaskType]int64{model.TaskCollect: 2, model.TaskPurchase: 2} {
var last int64
if err := db.QueryRow(`SELECT current_value FROM task_sequences WHERE task_type=?`, taskType).Scan(&last); err != nil || last != want {
t.Fatalf("%s 序列=%d err=%v,期望 %d", taskType, last, err, want)
}
}
if err := MigrateMySQL(db); err != nil {
t.Fatalf("v13 重放失败: %v", err)
}
var stableID string
if err := db.QueryRow(`SELECT task_id FROM tasks WHERE pdd_goods_id='C-FIRST'`).Scan(&stableID); err != nil || stableID != "cj1" {
t.Fatalf("重放不得二次改号: id=%s err=%v", stableID, err)
}
}
func TestNextTaskID_MySQL并发独立递增且回滚不耗号(t *testing.T) {
db := openMySQLMigrationTestDB(t)
defer db.Close()
cleanMySQLTestSchema(t, db)
defer cleanMySQLTestSchema(t, db)
if err := MigrateMySQL(db); err != nil {
t.Fatal(err)
}
const workers = 16
ids := make(map[string]bool, workers)
var mu sync.Mutex
errCh := make(chan error, workers)
var wg sync.WaitGroup
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
tx, err := db.Begin()
if err != nil {
errCh <- err
return
}
id, err := NextTaskID(tx, model.TaskCollect)
if err == nil {
err = tx.Commit()
} else {
tx.Rollback()
}
if err != nil {
errCh <- err
return
}
mu.Lock()
ids[id] = true
mu.Unlock()
}()
}
wg.Wait()
close(errCh)
for err := range errCh {
t.Fatal(err)
}
if len(ids) != workers {
t.Fatalf("并发分配得到 %d 个唯一编号,期望 %d", len(ids), workers)
}
for i := 1; i <= workers; i++ {
if !ids[fmt.Sprintf("cj%d", i)] {
t.Fatalf("并发编号缺少 cj%d: %+v", i, ids)
}
}
rollbackTx, err := db.Begin()
if err != nil {
t.Fatal(err)
}
rolledBackID, err := NextTaskID(rollbackTx, model.TaskPurchase)
if err != nil {
t.Fatal(err)
}
if err := rollbackTx.Rollback(); err != nil {
t.Fatal(err)
}
commitTx, err := db.Begin()
if err != nil {
t.Fatal(err)
}
committedID, err := NextTaskID(commitTx, model.TaskPurchase)
if err != nil {
t.Fatal(err)
}
if err := commitTx.Commit(); err != nil {
t.Fatal(err)
}
if rolledBackID != "cg1" || committedID != "cg1" {
t.Fatalf("回滚不应消耗采购序号: rollback=%s commit=%s", rolledBackID, committedID)
}
}
func openMySQLMigrationTestDB(t *testing.T) *sql.DB {
t.Helper()
if os.Getenv("CMAUTOBUY_MYSQL_TEST") != "1" {
+60
View File
@@ -5,11 +5,71 @@ import (
"database/sql"
"errors"
"fmt"
"strconv"
"strings"
"cmautobuy/admin/model"
)
const (
collectTaskIDPrefix = "cj"
purchaseTaskIDPrefix = "cg"
)
// NextTaskID 在当前事务内分配下一条任务业务单号。
//
// 采集和采购分别递增;更新序列表后再读取同一行,依赖事务行锁保证并发不重复。
// 调用方必须把分配和任务 INSERT 放在同一个事务里,失败回滚时不会消耗序号。
func NextTaskID(q Execer, taskType model.TaskType) (string, error) {
prefix, ok := taskIDPrefix(taskType)
if !ok {
return "", fmt.Errorf("不支持为任务类型 %q 分配编号", taskType)
}
result, err := q.Exec(`UPDATE task_sequences SET current_value=current_value+1 WHERE task_type=?`, taskType)
if err != nil {
return "", fmt.Errorf("递增 %s 任务序号失败: %w", taskType, err)
}
affected, err := result.RowsAffected()
if err != nil {
return "", fmt.Errorf("读取 %s 任务序号更新结果失败: %w", taskType, err)
}
if affected != 1 {
return "", fmt.Errorf("%s 任务序列表缺失,请先完成数据库迁移", taskType)
}
var number int64
if err := q.QueryRow(`SELECT current_value FROM task_sequences WHERE task_type=?`, taskType).Scan(&number); err != nil {
return "", fmt.Errorf("读取 %s 任务序号失败: %w", taskType, err)
}
if number <= 0 {
return "", fmt.Errorf("%s 任务序号不正确: %d", taskType, number)
}
return prefix + strconv.FormatInt(number, 10), nil
}
func taskIDPrefix(taskType model.TaskType) (string, bool) {
switch taskType {
case model.TaskCollect:
return collectTaskIDPrefix, true
case model.TaskPurchase:
return purchaseTaskIDPrefix, true
default:
return "", false
}
}
func parseTaskIDNumber(taskType model.TaskType, taskID string) (int64, bool) {
prefix, ok := taskIDPrefix(taskType)
if !ok || !strings.HasPrefix(taskID, prefix) || len(taskID) == len(prefix) {
return 0, false
}
numberText := taskID[len(prefix):]
if numberText[0] == '0' {
return 0, false
}
number, err := strconv.ParseInt(numberText, 10, 64)
return number, err == nil && number > 0
}
// ClaimNextTask 为指定客户端领取一个任务。
//
// 没有可领的任务时返回 (nil, nil) —— 调用方据此返回 204。