chengma/cmroubao
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(t233): add redacted ERP diagnostics

Browse Source
This commit is contained in:
QiuSW
2026-07-29 11:24:04 +08:00
parent 817a968233
commit 1e703ef257
14 changed files with 412 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-api/internal/config/config.go
+26
View File
@@ -23,6 +23,7 @@ const (
ShunyunbaoUsernameEnvironment = "CMROUBAO_SHUNYUNBAO_USERNAME"
ShunyunbaoPasswordEnvironment = "CMROUBAO_SHUNYUNBAO_PASSWORD"
OCRAPIURLEnvironment = "CMROUBAO_OCR_API_URL"
ERPDebugLogEnvironment = "CMROUBAO_ERP_DEBUG_LOG"
defaultHTTPAddress = "127.0.0.1:8080"
defaultDatabasePath = "var/cmroubao.db"
@@ -55,6 +56,7 @@ type Config struct {
ShunyunbaoPassword string
OCRAPIURL string
ShunyunbaoTimeout time.Duration
ERPDebugLog bool
}
func Load(lookup LookupEnvironment) (Config, error) {
@@ -186,6 +188,10 @@ func Load(lookup LookupEnvironment) (Config, error) {
if ocrAPISet && !validOCRAPIURL(ocrAPIURL) {
return Config{}, errors.New(OCRAPIURLEnvironment + " must be an approved OCR endpoint")
}
erpDebugLog, err := booleanEnvironment(lookup, ERPDebugLogEnvironment, false)
if err != nil {
return Config{}, err
}
return Config{
HTTPAddress: httpAddress,
@@ -207,6 +213,7 @@ func Load(lookup LookupEnvironment) (Config, error) {
ShunyunbaoPassword: shunyunbaoPassword,
OCRAPIURL: ocrAPIURL,
ShunyunbaoTimeout: 30 * time.Second,
ERPDebugLog: erpDebugLog,
}, nil
}
@@ -267,6 +274,25 @@ func durationEnvironment(
return duration, nil
}
func booleanEnvironment(
lookup LookupEnvironment,
name string,
defaultValue bool,
) (bool, error) {
value, exists := lookup(name)
if !exists {
return defaultValue, nil
}
switch strings.ToLower(strings.TrimSpace(value)) {
case "true":
return true, nil
case "false":
return false, nil
default:
return false, errors.New(name + " must be true or false")
}
}
func cleanOptionalPath(value string) string {
if value == "" {
return ""
backend-api/internal/config/config_test.go
+10 -2
View File
@@ -44,7 +44,7 @@ func TestLoadUsesSafeDefaults(t *testing.T) {
}
if cfg.ShunyunbaoURL != "https://www.shunyunbaoerp.com" ||
cfg.ShunyunbaoUsername != "" || cfg.ShunyunbaoPassword != "" ||
cfg.ShunyunbaoTimeout != 30*time.Second {
cfg.ShunyunbaoTimeout != 30*time.Second || cfg.ERPDebugLog {
t.Fatalf(
"shunyunbao defaults = %q / %q / %q / %s",
cfg.ShunyunbaoURL,
@@ -68,6 +68,7 @@ func TestLoadAcceptsExplicitConfiguration(t *testing.T) {
ShunyunbaoURLEnvironment: "https://erp.example.test:8443",
ShunyunbaoUsernameEnvironment: "service-user",
ShunyunbaoPasswordEnvironment: " pass with spaces ",
ERPDebugLogEnvironment: "true",
}
cfg, err := Load(mapEnvironment(values))
@@ -104,7 +105,8 @@ func TestLoadAcceptsExplicitConfiguration(t *testing.T) {
}
if cfg.ShunyunbaoURL != values[ShunyunbaoURLEnvironment] ||
cfg.ShunyunbaoUsername != values[ShunyunbaoUsernameEnvironment] ||
cfg.ShunyunbaoPassword != values[ShunyunbaoPasswordEnvironment] {
cfg.ShunyunbaoPassword != values[ShunyunbaoPasswordEnvironment] ||
!cfg.ERPDebugLog {
t.Fatalf("shunyunbao config was not preserved")
}
}
@@ -138,6 +140,12 @@ func TestLoadRejectsUnsafeOrInvalidValues(t *testing.T) {
ShunyunbaoPasswordEnvironment: "password",
},
},
{
name: "invalid ERP debug log",
values: map[string]string{
ERPDebugLogEnvironment: "yes",
},
},
{
name: "blank explicit address",
values: map[string]string{
backend-api/internal/config/envfile.go
+2 -1
View File
@@ -123,7 +123,8 @@ func isERPEnvironmentName(name string) bool {
case ShunyunbaoURLEnvironment,
ShunyunbaoUsernameEnvironment,
ShunyunbaoPasswordEnvironment,
OCRAPIURLEnvironment:
OCRAPIURLEnvironment,
ERPDebugLogEnvironment:
return true
default:
return false
backend-api/internal/config/envfile_test.go
+2 -1
View File
@@ -14,6 +14,7 @@ func TestWithERPEnvironmentFileUsesApprovedFallbackValues(t *testing.T) {
"CMROUBAO_SHUNYUNBAO_USERNAME=dotenv-user",
"CMROUBAO_SHUNYUNBAO_PASSWORD='dotenv password #1'",
"CMROUBAO_OCR_API_URL=http://127.0.0.1:8000/ocr",
"CMROUBAO_ERP_DEBUG_LOG=true",
}, "\n"))
lookup, err := WithERPEnvironmentFile(path, func(string) (string, bool) {
@@ -29,7 +30,7 @@ func TestWithERPEnvironmentFileUsesApprovedFallbackValues(t *testing.T) {
if cfg.ShunyunbaoURL != "https://erp.example.test" ||
cfg.ShunyunbaoUsername != "dotenv-user" ||
cfg.ShunyunbaoPassword != "dotenv password #1" ||
cfg.OCRAPIURL != "http://127.0.0.1:8000/ocr" {
cfg.OCRAPIURL != "http://127.0.0.1:8000/ocr" || !cfg.ERPDebugLog {
t.Fatalf(
"ERP config = %#v",
struct {
backend-api/internal/platform/shunyunbao/session.go
+224 -1
View File
@@ -25,6 +25,7 @@ const (
defaultCaptchaTTL = 5 * time.Minute
maxCaptchaBytes = 2 << 20
maxERPResponseBytes = 4 << 20
maxDiagnosticBytes = 4 << 10
)
var (
@@ -40,12 +41,17 @@ type SessionConfig struct {
CaptchaTTL time.Duration
AllowInsecureHTTP bool // Used only by isolated httptest contracts.
CaptchaRecognizer CaptchaRecognizer
DiagnosticLogger DiagnosticLogger
}
type CaptchaRecognizer interface {
Recognize(context.Context, []byte, string) (string, error)
}
// DiagnosticLogger receives only redacted request/response summaries when
// explicitly enabled by the API composition root.
type DiagnosticLogger func(string)
type SessionStatus struct {
Configured bool
Authenticated bool
@@ -69,6 +75,8 @@ type SessionManager struct {
headers http.Header
http *http.Client
recognizer CaptchaRecognizer
diagnosticLog DiagnosticLogger
diagnosticsOn bool
authenticated bool
captchaTicket string
captchaContent []byte
@@ -118,7 +126,9 @@ func NewSessionManager(config SessionConfig) (*SessionManager, error) {
return http.ErrUseLastResponse
},
},
recognizer: config.CaptchaRecognizer,
recognizer: config.CaptchaRecognizer,
diagnosticLog: config.DiagnosticLogger,
diagnosticsOn: config.DiagnosticLogger != nil,
}, nil
}
@@ -179,12 +189,15 @@ func (manager *SessionManager) FetchCaptcha(
return manager.statusLocked(), domain.ErrFreightSourceUnavailable
}
manager.applyHeaders(request)
manager.logERPRequest(request)
response, err := manager.http.Do(request)
if err != nil {
manager.logERPTransportFailure(request)
return manager.statusLocked(), domain.ErrFreightSourceUnavailable
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
manager.logERPResponsePreview(request, response)
return manager.statusLocked(), manager.responseErrorLocked(response.StatusCode)
}
contentType := strings.TrimSpace(
@@ -195,8 +208,10 @@ func (manager *SessionManager) FetchCaptcha(
}
content, err := readBounded(response.Body, maxCaptchaBytes)
if err != nil || len(content) == 0 {
manager.logERPResponseReadFailure(request, response)
return manager.statusLocked(), domain.ErrFreightSourceUnavailable
}
manager.logERPResponse(request, response, content, false)
ticket, err := newCaptchaTicket()
if err != nil {
return manager.statusLocked(), domain.ErrFreightSourceUnavailable
@@ -327,12 +342,15 @@ func (manager *SessionManager) requestJSONLocked(
if body != nil {
request.Header.Set("Content-Type", "application/json")
}
manager.logERPRequest(request)
response, err := manager.http.Do(request)
if err != nil {
manager.logERPTransportFailure(request)
return nil, domain.ErrFreightSourceUnavailable
}
defer response.Body.Close()
if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices {
manager.logERPResponsePreview(request, response)
if loginRequest {
return nil, ErrLoginRejected
}
@@ -340,8 +358,10 @@ func (manager *SessionManager) requestJSONLocked(
}
contentBytes, err := readBounded(response.Body, maxERPResponseBytes)
if err != nil {
manager.logERPResponseReadFailure(request, response)
return nil, domain.ErrFreightSourceUnavailable
}
manager.logERPResponse(request, response, contentBytes, false)
var envelope struct {
Status *bool `json:"status"`
Code json.RawMessage `json:"code"`
@@ -439,6 +459,209 @@ func (manager *SessionManager) clearCaptchaLocked() {
manager.captchaExpires = time.Time{}
}
func (manager *SessionManager) logERPRequest(request *http.Request) {
if !manager.diagnosticsOn {
return
}
manager.diagnosticLog(
"erp_request method=" + request.Method +
" path=" + request.URL.EscapedPath(),
)
}
func (manager *SessionManager) logERPTransportFailure(request *http.Request) {
if !manager.diagnosticsOn {
return
}
manager.diagnosticLog(
"erp_transport_failed method=" + request.Method +
" path=" + request.URL.EscapedPath() + " class=transport",
)
}
func (manager *SessionManager) logERPResponsePreview(
request *http.Request,
response *http.Response,
) {
if !manager.diagnosticsOn {
return
}
content, truncated, readable := readDiagnosticPreview(response.Body)
if !readable {
manager.logERPResponseReadFailure(request, response)
return
}
manager.logERPResponse(request, response, content, truncated)
}
func (manager *SessionManager) logERPResponseReadFailure(
request *http.Request,
response *http.Response,
) {
if !manager.diagnosticsOn {
return
}
manager.diagnosticLog(
"erp_response method=" + request.Method +
" path=" + request.URL.EscapedPath() +
" status=" + strconv.Itoa(response.StatusCode) +
" content_type=" + diagnosticContentType(response.Header.Get("Content-Type")) +
" body=unavailable",
)
}
func (manager *SessionManager) logERPResponse(
request *http.Request,
response *http.Response,
content []byte,
truncated bool,
) {
if !manager.diagnosticsOn {
return
}
byteCount := "bytes=" + strconv.Itoa(len(content))
body := "body=empty"
if truncated {
byteCount = "bytes_at_least=" + strconv.Itoa(len(content))
body = "body=omitted_truncated"
} else if len(content) > 0 {
body = "body=omitted_non_json"
if summary, ok := redactedDiagnosticJSON(content); ok {
body = "json=" + summary
}
}
manager.diagnosticLog(
"erp_response method=" + request.Method +
" path=" + request.URL.EscapedPath() +
" status=" + strconv.Itoa(response.StatusCode) +
" content_type=" + diagnosticContentType(response.Header.Get("Content-Type")) +
" " + byteCount + " " + body,
)
}
func readDiagnosticPreview(reader io.Reader) ([]byte, bool, bool) {
content, err := io.ReadAll(io.LimitReader(reader, maxDiagnosticBytes+1))
if err != nil {
return nil, false, false
}
if len(content) > maxDiagnosticBytes {
return content[:maxDiagnosticBytes], true, true
}
return content, false, true
}
func diagnosticContentType(value string) string {
value = strings.TrimSpace(strings.Split(value, ";")[0])
if value == "" || len(value) > 128 || !utf8.ValidString(value) || hasControl(value) {
return "unknown"
}
return value
}
func redactedDiagnosticJSON(content []byte) (string, bool) {
decoder := json.NewDecoder(bytes.NewReader(content))
decoder.UseNumber()
var value any
if err := decoder.Decode(&value); err != nil {
return "", false
}
var extra any
if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) {
return "", false
}
encoded, err := json.Marshal(redactDiagnosticValue(value))
if err != nil || len(encoded) > maxDiagnosticBytes {
return `{"summary":"omitted_large_json"}`, true
}
return string(encoded), true
}
func redactDiagnosticValue(value any) any {
switch typed := value.(type) {
case map[string]any:
redacted := make(map[string]any, len(typed))
for key, item := range typed {
if sensitiveDiagnosticKey(key) {
redacted[key] = "[REDACTED]"
continue
}
redacted[key] = redactDiagnosticValue(item)
}
return redacted
case []any:
limit := len(typed)
if limit > 20 {
limit = 20
}
redacted := make([]any, 0, limit+1)
for _, item := range typed[:limit] {
redacted = append(redacted, redactDiagnosticValue(item))
}
if len(typed) > limit {
redacted = append(redacted, "[TRUNCATED]")
}
return redacted
case string:
return redactDiagnosticString(typed)
default:
return value
}
}
func sensitiveDiagnosticKey(value string) bool {
var normalized strings.Builder
for _, character := range strings.ToLower(value) {
if (character >= 'a' && character <= 'z') ||
(character >= '0' && character <= '9') {
normalized.WriteRune(character)
}
}
key := normalized.String()
for _, marker := range []string{
"password", "passwd", "pwd", "username", "user", "captcha", "token",
"cookie", "authorization", "auth", "receiver", "recipient", "phone",
"mobile", "tel", "address", "email", "order", "stock", "tracking",
"track", "express", "shipment", "shop", "name", "id", "code", "remark",
"note", "detail",
} {
if strings.Contains(key, marker) {
return true
}
}
return false
}
func redactDiagnosticString(value string) string {
value = strings.ToValidUTF8(strings.TrimSpace(value), "?")
value = strings.NewReplacer("\r", " ", "\n", " ", "\t", " ").Replace(value)
if strings.Contains(value, "@") {
return "[REDACTED]"
}
var result strings.Builder
for index := 0; index < len(value); {
if value[index] < '0' || value[index] > '9' {
result.WriteByte(value[index])
index++
continue
}
end := index
for end < len(value) && value[end] >= '0' && value[end] <= '9' {
end++
}
if end-index >= 6 {
result.WriteString("[REDACTED]")
} else {
result.WriteString(value[index:end])
}
index = end
}
characters := []rune(result.String())
if len(characters) > 512 {
return string(characters[:512]) + "[TRUNCATED]"
}
return string(characters)
}
func hasUser(value any) bool {
data, ok := value.(map[string]any)
if !ok {
backend-api/internal/platform/shunyunbao/session_test.go
+59
View File
@@ -211,6 +211,65 @@ func TestSessionManagerEnsureAuthenticatedRequiresRecognizer(t *testing.T) {
}
}
func TestSessionManagerDiagnosticLogsAreRedacted(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case CaptchaPath:
w.Header().Set("Content-Type", "image/png")
_, _ = w.Write([]byte("private-captcha-image"))
case LoginPath:
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"status":false,
"msg":"ERP says order 123456789 is blocked",
"data":{"username":"test-user","password":"test-password","token":"secret-token","orderNumber":"order-123456789"}
}`))
default:
w.WriteHeader(http.StatusNotFound)
}
}))
defer server.Close()
var events []string
manager, err := NewSessionManager(SessionConfig{
BaseURL: server.URL,
Username: "test-user",
Password: "test-password",
Timeout: time.Second,
AllowInsecureHTTP: true,
CaptchaRecognizer: &fixedRecognizer{code: "1234"},
DiagnosticLogger: func(event string) {
events = append(events, event)
},
})
if err != nil {
t.Fatalf("NewSessionManager() error = %v", err)
}
if err := manager.EnsureAuthenticated(context.Background()); !errors.Is(err, domain.ErrFreightSourceLoginRejected) {
t.Fatalf("EnsureAuthenticated() error = %v", err)
}
actual := strings.Join(events, "\n")
for _, expected := range []string{
"erp_request method=GET path=/api/p/code1",
"erp_response method=GET path=/api/p/code1 status=200",
"body=omitted_non_json",
"erp_request method=POST path=/am/auth/login",
"erp_response method=POST path=/am/auth/login status=200",
`"status":false`,
`"password":"[REDACTED]"`,
} {
if !strings.Contains(actual, expected) {
t.Fatalf("diagnostic log missing %q: %s", expected, actual)
}
}
for _, secret := range []string{
"test-user", "test-password", "secret-token", "123456789", "private-captcha-image",
} {
if strings.Contains(actual, secret) {
t.Fatalf("diagnostic log leaked %q: %s", secret, actual)
}
}
}
type fixedRecognizer struct {
code string
err error
backend-api/internal/transport/webui/preflight_error_test.go
+18
View File
@@ -25,3 +25,21 @@ func TestMapFreightPreflightErrorUsesStablePublicErrors(t *testing.T) {
}
}
}
func TestMapFreightCreateErrorPreservesPreflightErrors(t *testing.T) {
for _, testCase := range []struct {
err error
want error
}{
{domain.ErrFreightSourceOCRInvalid, ErrOCRServiceInvalid},
{domain.ErrFreightSourceNotConfigured, ErrERPNotConfigured},
{domain.ErrFreightSourceLoginRejected, ErrERPLoginRejected},
{domain.ErrFreightSourceProtocol, ErrERPProtocol},
{domain.ErrFreightSourceUnavailable, ErrERPUnavailable},
} {
actual := mapFreightCreateError(testCase.err)
if !errors.Is(actual, testCase.want) || !errors.Is(actual, testCase.err) {
t.Fatalf("mapFreightCreateError(%v) = %v", testCase.err, actual)
}
}
}
backend-api/internal/transport/webui/usecase_adapter.go
+8 -4
View File
@@ -379,14 +379,18 @@ func (adapter *UsecaseAdapter) CreateFreightSync(
)
}
if err != nil {
if errors.Is(err, domain.ErrFreightSourceOCRInvalid) {
return FreightSync{}, &adapterError{public: ErrOCRServiceInvalid, cause: err}
}
return FreightSync{}, mapUsecaseError(err)
return FreightSync{}, mapFreightCreateError(err)
}
return freightSyncFrom(result.Run), nil
}
func mapFreightCreateError(err error) error {
if mapped := mapFreightPreflightError(err); mapped != nil {
return mapped
}
return mapUsecaseError(err)
}
func (adapter *UsecaseAdapter) GetFreightWatermark(
ctx context.Context,
) (*FreightWatermark, error) {