feat: 增加商品目录导入记录 (#134)

admin/handler/integration/auth_test.go

@@ -2,6 +2,7 @@ package integration
 
 import (
 	"bytes"
+	"database/sql"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -10,6 +11,9 @@ import (
 	"github.com/gin-gonic/gin"
 
 	"cmautobuy/admin/config"
+	"cmautobuy/admin/model"
+	"cmautobuy/admin/repository"
+	_ "modernc.org/sqlite"
 )
 
 func TestRequireCatalogToken(t *testing.T) {
@@ -47,6 +51,35 @@ func TestRequireCatalogToken(t *testing.T) {
 	}
 }
 
+func TestCatalogBatchAPI_按批次查询摘要(t *testing.T) {
+	db, err := sql.Open("sqlite", ":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+	_, err = db.Exec(`CREATE TABLE catalog_import_runs(source TEXT,batch_id TEXT,request_hash TEXT,status TEXT,request_count INTEGER,conflict_count INTEGER,observed_at TEXT,last_request_at TEXT,last_conflict_at TEXT,shopee_created INTEGER DEFAULT 0,shopee_updated INTEGER DEFAULT 0,sku_created INTEGER DEFAULT 0,sku_updated INTEGER DEFAULT 0,pdd_created INTEGER DEFAULT 0,pdd_updated INTEGER DEFAULT 0,association_created INTEGER DEFAULT 0,association_unchanged INTEGER DEFAULT 0,failure_count INTEGER DEFAULT 0,error_summary TEXT,response_body TEXT,created_at TEXT,finished_at TEXT,PRIMARY KEY(source,batch_id))`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	run := model.CatalogImportRun{Source: "script", BatchID: "B-1", RequestHash: "abc", Status: model.CatalogImportSucceeded, LastRequestAt: model.NowISO(), CreatedAt: model.NowISO()}
+	if err := repository.InsertCatalogImportRun(db, run); err != nil {
+		t.Fatal(err)
+	}
+	const token = "0123456789abcdef0123456789abcdef"
+	r := gin.New()
+	Register(r, db, config.CatalogIntegrationConfig{Source: "script", Token: token})
+	req := httptest.NewRequest(http.MethodGet, "/api/v1/integrations/catalog/batches/B-1", nil)
+	req.Header.Set("Authorization", "Bearer "+token)
+	resp := httptest.NewRecorder()
+	r.ServeHTTP(resp, req)
+	if resp.Code != 200 || !strings.Contains(resp.Body.String(), `"batch_id":"B-1"`) {
+		t.Fatalf("status=%d body=%s", resp.Code, resp.Body.String())
+	}
+	if strings.Contains(resp.Body.String(), "request_hash") {
+		t.Fatal("查询响应不应暴露请求哈希")
+	}
+}
+
 func TestCatalogBatchAPI_鉴权JSON与体积限制(t *testing.T) {
 	gin.SetMode(gin.TestMode)
 	const token = "0123456789abcdef0123456789abcdef"

admin/handler/integration/catalog_api.go

@@ -11,6 +11,7 @@ import (
 	"github.com/gin-gonic/gin"
 
 	"cmautobuy/admin/config"
+	"cmautobuy/admin/repository"
 	"cmautobuy/admin/service"
 )
 
@@ -23,6 +24,20 @@ func Register(r *gin.Engine, db *sql.DB, cfg config.CatalogIntegrationConfig) {
 	h := &Handler{db: db}
 	g := r.Group("/api/v1/integrations/catalog", RequireCatalogToken(cfg))
 	g.POST("/batches", h.CreateBatch)
+	g.GET("/batches/:batch_id", h.GetBatch)
+}
+
+func (h *Handler) GetBatch(c *gin.Context) {
+	run, err := repository.GetCatalogImportRun(h.db, integrationSource(c), c.Param("batch_id"))
+	if errors.Is(err, repository.ErrCatalogImportRunNotFound) {
+		integrationError(c, 404, "BATCH_NOT_FOUND", "批次不存在", false, nil)
+		return
+	}
+	if err != nil {
+		integrationError(c, 500, "INTERNAL_ERROR", "查询批次失败", true, nil)
+		return
+	}
+	c.JSON(200, gin.H{"source": run.Source, "batch_id": run.BatchID, "status": run.Status, "request_count": run.RequestCount, "conflict_count": run.ConflictCount, "observed_at": run.ObservedAt, "created_at": run.CreatedAt, "finished_at": run.FinishedAt, "counts": gin.H{"shopee_created": run.ShopeeCreated, "shopee_updated": run.ShopeeUpdated, "sku_created": run.SKUCreated, "sku_updated": run.SKUUpdated, "pdd_created": run.PddCreated, "pdd_updated": run.PddUpdated, "association_created": run.AssociationCreated, "association_unchanged": run.AssociationUnchanged}, "failure_count": run.FailureCount, "error_summary": run.ErrorSummary})
 }
 
 func (h *Handler) CreateBatch(c *gin.Context) {

admin/handler/web/catalog.go

@@ -0,0 +1,40 @@
+package web
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+
+	"github.com/gin-gonic/gin"
+
+	"cmautobuy/admin/service"
+)
+
+func (h *Handler) CatalogHistory(c *gin.Context) {
+	result, err := service.ListCatalogHistory(h.db, c.Query("source"), c.Query("status"), service.ParsePage(c.Query("page")))
+	if err != nil {
+		fail(c, 500, "读取商品目录导入记录失败，数据没有被改动。请刷新后重试。")
+		return
+	}
+	values := url.Values{}
+	if result.Source != "" {
+		values.Set("source", result.Source)
+	}
+	if result.Status != "" {
+		values.Set("status", result.Status)
+	}
+	c.HTML(http.StatusOK, "catalog/list", page(c, "catalog", "商品目录导入记录", gin.H{"Rows": result.Rows, "Sources": result.Sources, "Source": result.Source, "StatusFilter": result.Status, "StatusOptions": service.CatalogStatusOptions(), "Status": fmt.Sprintf("共 %d 条 · 第 %d/%d 页", result.Total, result.Page, result.TotalPages), "Pagination": service.NewPaginationView(result.Page, result.TotalPages, values.Encode())}))
+}
+
+func (h *Handler) CatalogHistoryDetail(c *gin.Context) {
+	view, err := service.GetCatalogRunView(h.db, c.Query("source"), c.Query("batch_id"))
+	if err != nil {
+		fail(c, 500, "读取导入批次详情失败。")
+		return
+	}
+	if view == nil {
+		fail(c, 404, "导入批次不存在，请刷新页面。")
+		return
+	}
+	c.HTML(200, "catalog/detail_modal", gin.H{"Run": view})
+}

admin/handler/web/web.go

@@ -102,6 +102,11 @@ func Register(r *gin.Engine, db *sql.DB, onlineThreshold time.Duration) {
 	users.POST("/create", h.UserCreate)
 	users.POST("/status", h.UserSetStatus)
 	users.POST("/reset-password", h.UserResetPassword)
+
+	// 7. 第三方商品目录导入记录：系统级审计仅管理员可见。
+	integrations := pages.Group("/integrations", AdminRequired())
+	integrations.GET("/catalog", h.CatalogHistory)
+	integrations.GET("/catalog/detail", h.CatalogHistoryDetail)
 }
 
 // page 组装每个页面都要的公共数据（导航高亮、标题、CSRF token）。

admin/main_test.go

@@ -157,7 +157,7 @@ func TestMainPagesReturnOK(t *testing.T) {
 		request.AddCookie(&http.Cookie{Name: "cmautobuy_session", Value: token})
 	}
 
-	for _, path := range []string{"/shopee", "/pdd", "/syb", "/tasks", "/clients", "/users"} {
+	for _, path := range []string{"/shopee", "/pdd", "/syb", "/tasks", "/clients", "/users", "/integrations/catalog"} {
 		t.Run(path, func(t *testing.T) {
 			request := httptest.NewRequest(http.MethodGet, path, nil)
 			addAuth(request)

admin/repository/catalog_import.go

@@ -64,6 +64,71 @@ func GetCatalogImportRun(q Execer, source, batchID string) (model.CatalogImportR
 	return run, nil
 }
 
+// ListCatalogImportRuns 在数据库中按来源、状态筛选并倒序分页。
+func ListCatalogImportRuns(q Execer, source string, status model.CatalogImportStatus, limit, offset int) ([]model.CatalogImportRun, int, error) {
+	where := " WHERE 1=1"
+	args := make([]any, 0, 4)
+	if source != "" {
+		where += " AND source=?"
+		args = append(args, source)
+	}
+	if status != "" {
+		where += " AND status=?"
+		args = append(args, status)
+	}
+	var total int
+	if err := q.QueryRow(`SELECT COUNT(*) FROM catalog_import_runs`+where, args...).Scan(&total); err != nil {
+		return nil, 0, fmt.Errorf("统计商品目录批次失败: %w", err)
+	}
+	listArgs := append(append([]any{}, args...), limit, offset)
+	rows, err := q.Query(`SELECT source,batch_id FROM catalog_import_runs`+where+` ORDER BY created_at DESC,source,batch_id LIMIT ? OFFSET ?`, listArgs...)
+	if err != nil {
+		return nil, 0, fmt.Errorf("查询商品目录批次列表失败: %w", err)
+	}
+	defer rows.Close()
+	keys := make([][2]string, 0, limit)
+	for rows.Next() {
+		var key [2]string
+		if err := rows.Scan(&key[0], &key[1]); err != nil {
+			return nil, 0, err
+		}
+		keys = append(keys, key)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, 0, err
+	}
+	if err := rows.Close(); err != nil {
+		return nil, 0, err
+	}
+	list := make([]model.CatalogImportRun, 0, len(keys))
+	for _, key := range keys {
+		run, err := GetCatalogImportRun(q, key[0], key[1])
+		if err != nil {
+			return nil, 0, err
+		}
+		list = append(list, run)
+	}
+	return list, total, nil
+}
+
+// ListCatalogImportSources 返回筛选下拉框所需的稳定来源名。
+func ListCatalogImportSources(q Execer) ([]string, error) {
+	rows, err := q.Query(`SELECT DISTINCT source FROM catalog_import_runs ORDER BY source`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var list []string
+	for rows.Next() {
+		var source string
+		if err := rows.Scan(&source); err != nil {
+			return nil, err
+		}
+		list = append(list, source)
+	}
+	return list, rows.Err()
+}
+
 // RecordCatalogImportReplay 记录相同请求的重复提交，业务数据不会再次写入。
 func RecordCatalogImportReplay(q Execer, source, batchID, requestedAt string) error {
 	result, err := q.Exec(`UPDATE catalog_import_runs SET request_count=request_count+1,last_request_at=?

admin/repository/catalog_import_test.go

@@ -3,6 +3,7 @@ package repository
 import (
 	"database/sql"
 	"errors"
+	"fmt"
 	"testing"
 
 	_ "modernc.org/sqlite"
@@ -49,4 +50,29 @@ func TestCatalogImportRun_登记查询重放和冲突(t *testing.T) {
 	if got.RequestCount != 3 || got.ConflictCount != 1 || got.LastConflictAt == "" {
 		t.Fatalf("批次计数不正确：%+v", got)
 	}
+	other := run
+	other.Source, other.BatchID = "script-b", "batch-2"
+	if err := InsertCatalogImportRun(db, other); err != nil {
+		t.Fatal(err)
+	}
+	list, total, err := ListCatalogImportRuns(db, "script-a", model.CatalogImportProcessing, 20, 0)
+	if err != nil || total != 1 || len(list) != 1 || list[0].BatchID != "batch-1" {
+		t.Fatalf("分页筛选错误：total=%d list=%+v err=%v", total, list, err)
+	}
+	sources, err := ListCatalogImportSources(db)
+	if err != nil || len(sources) != 2 {
+		t.Fatalf("来源列表错误：%v %v", sources, err)
+	}
+	for i := 3; i <= 1001; i++ {
+		bulk := run
+		bulk.Source = "script-b"
+		bulk.BatchID = fmt.Sprintf("batch-%04d", i)
+		if err := InsertCatalogImportRun(db, bulk); err != nil {
+			t.Fatal(err)
+		}
+	}
+	page, total, err := ListCatalogImportRuns(db, "script-b", "", 20, 20)
+	if err != nil || total != 1000 || len(page) != 20 {
+		t.Fatalf("1000 条数据库分页错误：total=%d rows=%d err=%v", total, len(page), err)
+	}
 }

admin/service/catalog_history.go

@@ -0,0 +1,95 @@
+package service
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"cmautobuy/admin/model"
+	"cmautobuy/admin/repository"
+)
+
+type CatalogHistoryResult struct {
+	Rows                    []CatalogRunView
+	Sources                 []string
+	Source                  string
+	Status                  string
+	Page, Total, TotalPages int
+}
+type CatalogRunView struct {
+	model.CatalogImportRun
+	StatusText, CreatedText, FinishedText, DurationText string
+	ErrorCode, ErrorDetails                             string
+	Retryable                                           bool
+}
+
+func ParseCatalogStatus(raw string) model.CatalogImportStatus {
+	switch model.CatalogImportStatus(strings.TrimSpace(raw)) {
+	case model.CatalogImportProcessing, model.CatalogImportSucceeded, model.CatalogImportFailed:
+		return model.CatalogImportStatus(raw)
+	}
+	return ""
+}
+
+func ListCatalogHistory(q repository.Execer, source, statusRaw string, page int) (CatalogHistoryResult, error) {
+	status := ParseCatalogStatus(statusRaw)
+	totalPages := 1
+	_, total, err := repository.ListCatalogImportRuns(q, strings.TrimSpace(source), status, 1, 0)
+	if err != nil {
+		return CatalogHistoryResult{}, err
+	}
+	totalPages = TotalPages(total)
+	page = ClampPage(page, totalPages)
+	runs, _, err := repository.ListCatalogImportRuns(q, strings.TrimSpace(source), status, PageSize, (page-1)*PageSize)
+	if err != nil {
+		return CatalogHistoryResult{}, err
+	}
+	sources, err := repository.ListCatalogImportSources(q)
+	if err != nil {
+		return CatalogHistoryResult{}, err
+	}
+	result := CatalogHistoryResult{Sources: sources, Source: strings.TrimSpace(source), Status: string(status), Page: page, Total: total, TotalPages: totalPages}
+	for _, run := range runs {
+		result.Rows = append(result.Rows, newCatalogRunView(run))
+	}
+	return result, nil
+}
+
+func GetCatalogRunView(q repository.Execer, source, batchID string) (*CatalogRunView, error) {
+	run, err := repository.GetCatalogImportRun(q, strings.TrimSpace(source), strings.TrimSpace(batchID))
+	if errors.Is(err, repository.ErrCatalogImportRunNotFound) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	view := newCatalogRunView(run)
+	return &view, nil
+}
+
+func newCatalogRunView(run model.CatalogImportRun) CatalogRunView {
+	v := CatalogRunView{CatalogImportRun: run, StatusText: map[model.CatalogImportStatus]string{model.CatalogImportProcessing: "处理中", model.CatalogImportSucceeded: "成功", model.CatalogImportFailed: "失败"}[run.Status], CreatedText: formatLocalTime(run.CreatedAt), FinishedText: formatLocalTime(run.FinishedAt), DurationText: "—"}
+	start, e1 := time.Parse(model.TimeLayout, run.CreatedAt)
+	end, e2 := time.Parse(model.TimeLayout, run.FinishedAt)
+	if e1 == nil && e2 == nil {
+		v.DurationText = fmt.Sprintf("%.1f 秒", end.Sub(start).Seconds())
+	}
+	if run.Status == model.CatalogImportFailed && run.ResponseBody != "" {
+		var stored CatalogError
+		if json.Unmarshal([]byte(run.ResponseBody), &stored) == nil {
+			v.ErrorCode, v.Retryable = stored.Code, stored.Retryable
+			if detail, err := json.Marshal(stored.Details); err == nil && string(detail) != "null" {
+				v.ErrorDetails = truncateCatalogError(string(detail))
+			}
+		}
+	}
+	return v
+}
+
+type CatalogStatusOption struct{ Value, Text string }
+
+func CatalogStatusOptions() []CatalogStatusOption {
+	return []CatalogStatusOption{{Value: "", Text: "全部状态"}, {Value: "processing", Text: "处理中"}, {Value: "succeeded", Text: "成功"}, {Value: "failed", Text: "失败"}}
+}

admin/templates/catalog/detail_modal.html

@@ -0,0 +1 @@
+{{define "catalog/detail_modal"}}<div class="modal-body"><dl class="detail-grid"><dt>来源</dt><dd>{{.Run.Source}}</dd><dt>批次号</dt><dd>{{.Run.BatchID}}</dd><dt>状态</dt><dd>{{.Run.StatusText}}</dd><dt>上游观测时间</dt><dd>{{.Run.ObservedAt}}</dd><dt>请求/冲突次数</dt><dd>{{.Run.RequestCount}} / {{.Run.ConflictCount}}</dd><dt>失败数</dt><dd>{{.Run.FailureCount}}</dd><dt>错误码</dt><dd>{{if .Run.ErrorCode}}{{.Run.ErrorCode}}{{else}}—{{end}}</dd><dt>可重试</dt><dd>{{if .Run.Retryable}}是{{else}}否{{end}}</dd><dt>错误摘要</dt><dd>{{if .Run.ErrorSummary}}{{.Run.ErrorSummary}}{{else}}—{{end}}</dd><dt>定位信息</dt><dd>{{if .Run.ErrorDetails}}{{.Run.ErrorDetails}}{{else}}—{{end}}</dd></dl><p class="hint">定位信息只包含数组位置和业务 ID；这里不保存完整请求 JSON 或任何凭据。</p></div>{{end}}

admin/templates/catalog/list.html

@@ -0,0 +1,15 @@
+{{define "catalog/list"}}{{template "header" .}}
+<div class="toolbar">
+  <a class="button" href="/shopee">返回蝦皮数据</a>
+  <form class="inline grow" method="get" action="/integrations/catalog">
+    <label for="catalog-source">来源</label><select id="catalog-source" name="source"><option value="">全部来源</option>{{range .Sources}}<option value="{{.}}" {{if eq . $.Source}}selected{{end}}>{{.}}</option>{{end}}</select>
+    <label for="catalog-status">状态</label><select id="catalog-status" name="status">{{range .StatusOptions}}<option value="{{.Value}}" {{if eq .Value $.StatusFilter}}selected{{end}}>{{.Text}}</option>{{end}}</select>
+    <button type="submit">筛选</button>
+  </form>
+</div>
+<div class="table-wrap"><table><thead><tr><th>来源</th><th>批次号</th><th>状态</th><th>请求</th><th>冲突</th><th>蝦皮 新/更</th><th>SKU 新/更</th><th>PDD 新/更</th><th>关联 新/同</th><th>首次接收</th><th>完成</th><th>耗时</th></tr></thead><tbody>
+{{range .Rows}}<tr data-detail-id="{{.Source}}|{{.BatchID}}"><td>{{.Source}}</td><td>{{.BatchID}}</td><td>{{.StatusText}}</td><td>{{.RequestCount}}</td><td>{{.ConflictCount}}</td><td>{{.ShopeeCreated}} / {{.ShopeeUpdated}}</td><td>{{.SKUCreated}} / {{.SKUUpdated}}</td><td>{{.PddCreated}} / {{.PddUpdated}}</td><td>{{.AssociationCreated}} / {{.AssociationUnchanged}}</td><td>{{.CreatedText}}</td><td>{{.FinishedText}}</td><td>{{.DurationText}}</td></tr>{{else}}<tr class="empty"><td colspan="12">还没有符合条件的导入记录。</td></tr>{{end}}</tbody></table></div>
+<p class="hint">双击一行查看批次摘要；系统不会保存或展示 Token 和完整请求体。</p>
+<div class="modal-backdrop" id="detail-modal" hidden><div class="modal" role="dialog" aria-modal="true"><div class="modal-head"><h2>导入批次详情</h2><button type="button" class="modal-x" data-modal-close>×</button></div><div id="detail-content"></div></div></div>
+<script>document.querySelectorAll('tr[data-detail-id]').forEach(function(row){row.addEventListener('dblclick',function(){var p=row.dataset.detailId.split('|');fetch('/integrations/catalog/detail?source='+encodeURIComponent(p[0])+'&batch_id='+encodeURIComponent(p[1])).then(function(r){return r.text()}).then(function(html){document.getElementById('detail-content').innerHTML=html;document.getElementById('detail-modal').hidden=false})})})</script>
+{{template "footer" .}}{{end}}

admin/templates/pdd/list.html

@@ -4,6 +4,7 @@
 {{/* 三段式布局的第一段：顶部工具条。
      和另外四个页面保持一致，见 docs/admin/05-ui-specification.md §2。 */}}
 <div class="toolbar">
+  {{if .CurrentUser.IsAdmin}}<a class="button" href="/integrations/catalog">导入记录</a>{{end}}
   {{/* 文案写全「添加 PDD 商品」：这条工具条上还有「创建采集任务」，
        两个都以「创建」开头的话，第一次用的人分不清哪个是加商品、哪个是发起采集。 */}}
   <button type="button" data-modal-open="create-modal">添加 PDD 商品</button>

admin/templates/shopee/list.html

@@ -3,6 +3,7 @@
 
 {{/* ── 第一段：顶部工具条 ────────────────────────────── */}}
 <div class="toolbar">
+  {{if .CurrentUser.IsAdmin}}<a class="button" href="/integrations/catalog">导入记录</a>{{end}}
   <form class="inline" method="post" action="/shopee/import" enctype="multipart/form-data">
     <input type="hidden" name="csrf_token" value="{{.CSRFToken}}">
     <input type="file" name="file" accept=".xlsx" required>