diff --git a/admin/handler/integration/auth_test.go b/admin/handler/integration/auth_test.go index bc55ead..d54fb12 100644 --- a/admin/handler/integration/auth_test.go +++ b/admin/handler/integration/auth_test.go @@ -2,6 +2,7 @@ package integration import ( "bytes" + "database/sql" "net/http" "net/http/httptest" "strings" @@ -10,6 +11,9 @@ import ( "github.com/gin-gonic/gin" "cmautobuy/admin/config" + "cmautobuy/admin/model" + "cmautobuy/admin/repository" + _ "modernc.org/sqlite" ) func TestRequireCatalogToken(t *testing.T) { @@ -47,6 +51,35 @@ func TestRequireCatalogToken(t *testing.T) { } } +func TestCatalogBatchAPI_按批次查询摘要(t *testing.T) { + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + _, err = db.Exec(`CREATE TABLE catalog_import_runs(source TEXT,batch_id TEXT,request_hash TEXT,status TEXT,request_count INTEGER,conflict_count INTEGER,observed_at TEXT,last_request_at TEXT,last_conflict_at TEXT,shopee_created INTEGER DEFAULT 0,shopee_updated INTEGER DEFAULT 0,sku_created INTEGER DEFAULT 0,sku_updated INTEGER DEFAULT 0,pdd_created INTEGER DEFAULT 0,pdd_updated INTEGER DEFAULT 0,association_created INTEGER DEFAULT 0,association_unchanged INTEGER DEFAULT 0,failure_count INTEGER DEFAULT 0,error_summary TEXT,response_body TEXT,created_at TEXT,finished_at TEXT,PRIMARY KEY(source,batch_id))`) + if err != nil { + t.Fatal(err) + } + run := model.CatalogImportRun{Source: "script", BatchID: "B-1", RequestHash: "abc", Status: model.CatalogImportSucceeded, LastRequestAt: model.NowISO(), CreatedAt: model.NowISO()} + if err := repository.InsertCatalogImportRun(db, run); err != nil { + t.Fatal(err) + } + const token = "0123456789abcdef0123456789abcdef" + r := gin.New() + Register(r, db, config.CatalogIntegrationConfig{Source: "script", Token: token}) + req := httptest.NewRequest(http.MethodGet, "/api/v1/integrations/catalog/batches/B-1", nil) + req.Header.Set("Authorization", "Bearer "+token) + resp := httptest.NewRecorder() + r.ServeHTTP(resp, req) + if resp.Code != 200 || !strings.Contains(resp.Body.String(), `"batch_id":"B-1"`) { + t.Fatalf("status=%d body=%s", resp.Code, resp.Body.String()) + } + if strings.Contains(resp.Body.String(), "request_hash") { + t.Fatal("查询响应不应暴露请求哈希") + } +} + func TestCatalogBatchAPI_鉴权JSON与体积限制(t *testing.T) { gin.SetMode(gin.TestMode) const token = "0123456789abcdef0123456789abcdef" diff --git a/admin/handler/integration/catalog_api.go b/admin/handler/integration/catalog_api.go index da41510..92fa8df 100644 --- a/admin/handler/integration/catalog_api.go +++ b/admin/handler/integration/catalog_api.go @@ -11,6 +11,7 @@ import ( "github.com/gin-gonic/gin" "cmautobuy/admin/config" + "cmautobuy/admin/repository" "cmautobuy/admin/service" ) @@ -23,6 +24,20 @@ func Register(r *gin.Engine, db *sql.DB, cfg config.CatalogIntegrationConfig) { h := &Handler{db: db} g := r.Group("/api/v1/integrations/catalog", RequireCatalogToken(cfg)) g.POST("/batches", h.CreateBatch) + g.GET("/batches/:batch_id", h.GetBatch) +} + +func (h *Handler) GetBatch(c *gin.Context) { + run, err := repository.GetCatalogImportRun(h.db, integrationSource(c), c.Param("batch_id")) + if errors.Is(err, repository.ErrCatalogImportRunNotFound) { + integrationError(c, 404, "BATCH_NOT_FOUND", "批次不存在", false, nil) + return + } + if err != nil { + integrationError(c, 500, "INTERNAL_ERROR", "查询批次失败", true, nil) + return + } + c.JSON(200, gin.H{"source": run.Source, "batch_id": run.BatchID, "status": run.Status, "request_count": run.RequestCount, "conflict_count": run.ConflictCount, "observed_at": run.ObservedAt, "created_at": run.CreatedAt, "finished_at": run.FinishedAt, "counts": gin.H{"shopee_created": run.ShopeeCreated, "shopee_updated": run.ShopeeUpdated, "sku_created": run.SKUCreated, "sku_updated": run.SKUUpdated, "pdd_created": run.PddCreated, "pdd_updated": run.PddUpdated, "association_created": run.AssociationCreated, "association_unchanged": run.AssociationUnchanged}, "failure_count": run.FailureCount, "error_summary": run.ErrorSummary}) } func (h *Handler) CreateBatch(c *gin.Context) { diff --git a/admin/handler/web/catalog.go b/admin/handler/web/catalog.go new file mode 100644 index 0000000..7e5aec9 --- /dev/null +++ b/admin/handler/web/catalog.go @@ -0,0 +1,40 @@ +package web + +import ( + "fmt" + "net/http" + "net/url" + + "github.com/gin-gonic/gin" + + "cmautobuy/admin/service" +) + +func (h *Handler) CatalogHistory(c *gin.Context) { + result, err := service.ListCatalogHistory(h.db, c.Query("source"), c.Query("status"), service.ParsePage(c.Query("page"))) + if err != nil { + fail(c, 500, "读取商品目录导入记录失败，数据没有被改动。请刷新后重试。") + return + } + values := url.Values{} + if result.Source != "" { + values.Set("source", result.Source) + } + if result.Status != "" { + values.Set("status", result.Status) + } + c.HTML(http.StatusOK, "catalog/list", page(c, "catalog", "商品目录导入记录", gin.H{"Rows": result.Rows, "Sources": result.Sources, "Source": result.Source, "StatusFilter": result.Status, "StatusOptions": service.CatalogStatusOptions(), "Status": fmt.Sprintf("共 %d 条 · 第 %d/%d 页", result.Total, result.Page, result.TotalPages), "Pagination": service.NewPaginationView(result.Page, result.TotalPages, values.Encode())})) +} + +func (h *Handler) CatalogHistoryDetail(c *gin.Context) { + view, err := service.GetCatalogRunView(h.db, c.Query("source"), c.Query("batch_id")) + if err != nil { + fail(c, 500, "读取导入批次详情失败。") + return + } + if view == nil { + fail(c, 404, "导入批次不存在，请刷新页面。") + return + } + c.HTML(200, "catalog/detail_modal", gin.H{"Run": view}) +} diff --git a/admin/handler/web/web.go b/admin/handler/web/web.go index 3372516..1e8b161 100644 --- a/admin/handler/web/web.go +++ b/admin/handler/web/web.go @@ -102,6 +102,11 @@ func Register(r *gin.Engine, db *sql.DB, onlineThreshold time.Duration) { users.POST("/create", h.UserCreate) users.POST("/status", h.UserSetStatus) users.POST("/reset-password", h.UserResetPassword) + + // 7. 第三方商品目录导入记录：系统级审计仅管理员可见。 + integrations := pages.Group("/integrations", AdminRequired()) + integrations.GET("/catalog", h.CatalogHistory) + integrations.GET("/catalog/detail", h.CatalogHistoryDetail) } // page 组装每个页面都要的公共数据（导航高亮、标题、CSRF token）。 diff --git a/admin/main_test.go b/admin/main_test.go index c20e48a..07b3413 100644 --- a/admin/main_test.go +++ b/admin/main_test.go @@ -157,7 +157,7 @@ func TestMainPagesReturnOK(t *testing.T) { request.AddCookie(&http.Cookie{Name: "cmautobuy_session", Value: token}) } - for _, path := range []string{"/shopee", "/pdd", "/syb", "/tasks", "/clients", "/users"} { + for _, path := range []string{"/shopee", "/pdd", "/syb", "/tasks", "/clients", "/users", "/integrations/catalog"} { t.Run(path, func(t *testing.T) { request := httptest.NewRequest(http.MethodGet, path, nil) addAuth(request) diff --git a/admin/repository/catalog_import.go b/admin/repository/catalog_import.go index 6d348fa..3c6c83a 100644 --- a/admin/repository/catalog_import.go +++ b/admin/repository/catalog_import.go @@ -64,6 +64,71 @@ func GetCatalogImportRun(q Execer, source, batchID string) (model.CatalogImportR return run, nil } +// ListCatalogImportRuns 在数据库中按来源、状态筛选并倒序分页。 +func ListCatalogImportRuns(q Execer, source string, status model.CatalogImportStatus, limit, offset int) ([]model.CatalogImportRun, int, error) { + where := " WHERE 1=1" + args := make([]any, 0, 4) + if source != "" { + where += " AND source=?" + args = append(args, source) + } + if status != "" { + where += " AND status=?" + args = append(args, status) + } + var total int + if err := q.QueryRow(`SELECT COUNT(*) FROM catalog_import_runs`+where, args...).Scan(&total); err != nil { + return nil, 0, fmt.Errorf("统计商品目录批次失败: %w", err) + } + listArgs := append(append([]any{}, args...), limit, offset) + rows, err := q.Query(`SELECT source,batch_id FROM catalog_import_runs`+where+` ORDER BY created_at DESC,source,batch_id LIMIT ? OFFSET ?`, listArgs...) + if err != nil { + return nil, 0, fmt.Errorf("查询商品目录批次列表失败: %w", err) + } + defer rows.Close() + keys := make([][2]string, 0, limit) + for rows.Next() { + var key [2]string + if err := rows.Scan(&key[0], &key[1]); err != nil { + return nil, 0, err + } + keys = append(keys, key) + } + if err := rows.Err(); err != nil { + return nil, 0, err + } + if err := rows.Close(); err != nil { + return nil, 0, err + } + list := make([]model.CatalogImportRun, 0, len(keys)) + for _, key := range keys { + run, err := GetCatalogImportRun(q, key[0], key[1]) + if err != nil { + return nil, 0, err + } + list = append(list, run) + } + return list, total, nil +} + +// ListCatalogImportSources 返回筛选下拉框所需的稳定来源名。 +func ListCatalogImportSources(q Execer) ([]string, error) { + rows, err := q.Query(`SELECT DISTINCT source FROM catalog_import_runs ORDER BY source`) + if err != nil { + return nil, err + } + defer rows.Close() + var list []string + for rows.Next() { + var source string + if err := rows.Scan(&source); err != nil { + return nil, err + } + list = append(list, source) + } + return list, rows.Err() +} + // RecordCatalogImportReplay 记录相同请求的重复提交，业务数据不会再次写入。 func RecordCatalogImportReplay(q Execer, source, batchID, requestedAt string) error { result, err := q.Exec(`UPDATE catalog_import_runs SET request_count=request_count+1,last_request_at=? diff --git a/admin/repository/catalog_import_test.go b/admin/repository/catalog_import_test.go index f729ccf..d5b9355 100644 --- a/admin/repository/catalog_import_test.go +++ b/admin/repository/catalog_import_test.go @@ -3,6 +3,7 @@ package repository import ( "database/sql" "errors" + "fmt" "testing" _ "modernc.org/sqlite" @@ -49,4 +50,29 @@ func TestCatalogImportRun_登记查询重放和冲突(t *testing.T) { if got.RequestCount != 3 || got.ConflictCount != 1 || got.LastConflictAt == "" { t.Fatalf("批次计数不正确：%+v", got) } + other := run + other.Source, other.BatchID = "script-b", "batch-2" + if err := InsertCatalogImportRun(db, other); err != nil { + t.Fatal(err) + } + list, total, err := ListCatalogImportRuns(db, "script-a", model.CatalogImportProcessing, 20, 0) + if err != nil || total != 1 || len(list) != 1 || list[0].BatchID != "batch-1" { + t.Fatalf("分页筛选错误：total=%d list=%+v err=%v", total, list, err) + } + sources, err := ListCatalogImportSources(db) + if err != nil || len(sources) != 2 { + t.Fatalf("来源列表错误：%v %v", sources, err) + } + for i := 3; i <= 1001; i++ { + bulk := run + bulk.Source = "script-b" + bulk.BatchID = fmt.Sprintf("batch-%04d", i) + if err := InsertCatalogImportRun(db, bulk); err != nil { + t.Fatal(err) + } + } + page, total, err := ListCatalogImportRuns(db, "script-b", "", 20, 20) + if err != nil || total != 1000 || len(page) != 20 { + t.Fatalf("1000 条数据库分页错误：total=%d rows=%d err=%v", total, len(page), err) + } } diff --git a/admin/service/catalog_history.go b/admin/service/catalog_history.go new file mode 100644 index 0000000..19987a2 --- /dev/null +++ b/admin/service/catalog_history.go @@ -0,0 +1,95 @@ +package service + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "cmautobuy/admin/model" + "cmautobuy/admin/repository" +) + +type CatalogHistoryResult struct { + Rows []CatalogRunView + Sources []string + Source string + Status string + Page, Total, TotalPages int +} +type CatalogRunView struct { + model.CatalogImportRun + StatusText, CreatedText, FinishedText, DurationText string + ErrorCode, ErrorDetails string + Retryable bool +} + +func ParseCatalogStatus(raw string) model.CatalogImportStatus { + switch model.CatalogImportStatus(strings.TrimSpace(raw)) { + case model.CatalogImportProcessing, model.CatalogImportSucceeded, model.CatalogImportFailed: + return model.CatalogImportStatus(raw) + } + return "" +} + +func ListCatalogHistory(q repository.Execer, source, statusRaw string, page int) (CatalogHistoryResult, error) { + status := ParseCatalogStatus(statusRaw) + totalPages := 1 + _, total, err := repository.ListCatalogImportRuns(q, strings.TrimSpace(source), status, 1, 0) + if err != nil { + return CatalogHistoryResult{}, err + } + totalPages = TotalPages(total) + page = ClampPage(page, totalPages) + runs, _, err := repository.ListCatalogImportRuns(q, strings.TrimSpace(source), status, PageSize, (page-1)*PageSize) + if err != nil { + return CatalogHistoryResult{}, err + } + sources, err := repository.ListCatalogImportSources(q) + if err != nil { + return CatalogHistoryResult{}, err + } + result := CatalogHistoryResult{Sources: sources, Source: strings.TrimSpace(source), Status: string(status), Page: page, Total: total, TotalPages: totalPages} + for _, run := range runs { + result.Rows = append(result.Rows, newCatalogRunView(run)) + } + return result, nil +} + +func GetCatalogRunView(q repository.Execer, source, batchID string) (*CatalogRunView, error) { + run, err := repository.GetCatalogImportRun(q, strings.TrimSpace(source), strings.TrimSpace(batchID)) + if errors.Is(err, repository.ErrCatalogImportRunNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + view := newCatalogRunView(run) + return &view, nil +} + +func newCatalogRunView(run model.CatalogImportRun) CatalogRunView { + v := CatalogRunView{CatalogImportRun: run, StatusText: map[model.CatalogImportStatus]string{model.CatalogImportProcessing: "处理中", model.CatalogImportSucceeded: "成功", model.CatalogImportFailed: "失败"}[run.Status], CreatedText: formatLocalTime(run.CreatedAt), FinishedText: formatLocalTime(run.FinishedAt), DurationText: "—"} + start, e1 := time.Parse(model.TimeLayout, run.CreatedAt) + end, e2 := time.Parse(model.TimeLayout, run.FinishedAt) + if e1 == nil && e2 == nil { + v.DurationText = fmt.Sprintf("%.1f 秒", end.Sub(start).Seconds()) + } + if run.Status == model.CatalogImportFailed && run.ResponseBody != "" { + var stored CatalogError + if json.Unmarshal([]byte(run.ResponseBody), &stored) == nil { + v.ErrorCode, v.Retryable = stored.Code, stored.Retryable + if detail, err := json.Marshal(stored.Details); err == nil && string(detail) != "null" { + v.ErrorDetails = truncateCatalogError(string(detail)) + } + } + } + return v +} + +type CatalogStatusOption struct{ Value, Text string } + +func CatalogStatusOptions() []CatalogStatusOption { + return []CatalogStatusOption{{Value: "", Text: "全部状态"}, {Value: "processing", Text: "处理中"}, {Value: "succeeded", Text: "成功"}, {Value: "failed", Text: "失败"}} +} diff --git a/admin/templates/catalog/detail_modal.html b/admin/templates/catalog/detail_modal.html new file mode 100644 index 0000000..551f526 --- /dev/null +++ b/admin/templates/catalog/detail_modal.html @@ -0,0 +1 @@ +{{define "catalog/detail_modal"}}
来源
{{.Run.Source}}
批次号
{{.Run.BatchID}}
状态
{{.Run.StatusText}}
上游观测时间
{{.Run.ObservedAt}}
请求/冲突次数
{{.Run.RequestCount}} / {{.Run.ConflictCount}}
失败数
{{.Run.FailureCount}}
错误码
{{if .Run.ErrorCode}}{{.Run.ErrorCode}}{{else}}—{{end}}
可重试
{{if .Run.Retryable}}是{{else}}否{{end}}
错误摘要
{{if .Run.ErrorSummary}}{{.Run.ErrorSummary}}{{else}}—{{end}}
定位信息
{{if .Run.ErrorDetails}}{{.Run.ErrorDetails}}{{else}}—{{end}}

定位信息只包含数组位置和业务 ID；这里不保存完整请求 JSON 或任何凭据。

{{end}} diff --git a/admin/templates/catalog/list.html b/admin/templates/catalog/list.html new file mode 100644 index 0000000..fa80bff --- /dev/null +++ b/admin/templates/catalog/list.html @@ -0,0 +1,15 @@ +{{define "catalog/list"}}{{template "header" .}} +
+ 返回蝦皮数据 +
+ + + +
+
+
+{{range .Rows}}{{else}}{{end}}
来源批次号状态请求冲突蝦皮 新/更SKU 新/更PDD 新/更关联 新/同首次接收完成耗时
{{.Source}}{{.BatchID}}{{.StatusText}}{{.RequestCount}}{{.ConflictCount}}{{.ShopeeCreated}} / {{.ShopeeUpdated}}{{.SKUCreated}} / {{.SKUUpdated}}{{.PddCreated}} / {{.PddUpdated}}{{.AssociationCreated}} / {{.AssociationUnchanged}}{{.CreatedText}}{{.FinishedText}}{{.DurationText}}
还没有符合条件的导入记录。
+

双击一行查看批次摘要；系统不会保存或展示 Token 和完整请求体。

+ + +{{template "footer" .}}{{end}} diff --git a/admin/templates/pdd/list.html b/admin/templates/pdd/list.html index f5febcd..8bf0800 100644 --- a/admin/templates/pdd/list.html +++ b/admin/templates/pdd/list.html @@ -4,6 +4,7 @@ {{/* 三段式布局的第一段：顶部工具条。 和另外四个页面保持一致，见 docs/admin/05-ui-specification.md §2。 */}}
+ {{if .CurrentUser.IsAdmin}}导入记录{{end}} {{/* 文案写全「添加 PDD 商品」：这条工具条上还有「创建采集任务」， 两个都以「创建」开头的话，第一次用的人分不清哪个是加商品、哪个是发起采集。 */}} diff --git a/admin/templates/shopee/list.html b/admin/templates/shopee/list.html index 1d7bac8..92e1efe 100644 --- a/admin/templates/shopee/list.html +++ b/admin/templates/shopee/list.html @@ -3,6 +3,7 @@ {{/* ── 第一段：顶部工具条 ────────────────────────────── */}}
+ {{if .CurrentUser.IsAdmin}}导入记录{{end}}