feat(t211): return SKU-matched image search candidates

2026-07-27 18:35:59 +08:00
parent 667a64f6c2
commit edfee2c35e
26 changed files with 1838 additions and 153 deletions
@@ -0,0 +1,149 @@
+package com.roubao.autopilot.pinduoduo
+
+import com.roubao.autopilot.readiness.PINDUODUO_PACKAGE
+import com.roubao.autopilot.workflow.SafetyStopReason
+import com.roubao.autopilot.workflow.WorkflowRunner
+import com.roubao.autopilot.workflow.WorkflowState
+import kotlinx.coroutines.test.runTest
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertFalse
+import org.junit.Assert.assertTrue
+import org.junit.Test
+
+class PinduoduoImageSearchAutomationTest {
+    @Test
+    fun `selects prepared image and verifies image results`() = runTest {
+        val driver = FakeImageDriver(PinduoduoPage.SEARCH_RESULTS)
+        val runner = WorkflowRunner(
+            PinduoduoImageSearchAutomation(
+                driver = driver,
+                pollIntervalMillis = 1,
+                unknownPageLimit = 3
+            )
+        )
+
+        val report = runner.run(PinduoduoImageSearchWorkflow.steps())
+
+        assertEquals(WorkflowState.SUCCEEDED, report.state)
+        assertTrue(driver.imageSearchOpened)
+        assertTrue(driver.preparedImageSelected)
+        assertEquals(PinduoduoPage.IMAGE_SEARCH_RESULTS, driver.page)
+    }
+
+    @Test
+    fun `selection verification failure blocks instead of using another image`() =
+        runTest {
+            val driver = FakeImageDriver(
+                page = PinduoduoPage.SEARCH_RESULTS,
+                selectionAllowed = false
+            )
+            val runner = WorkflowRunner(
+                PinduoduoImageSearchAutomation(
+                    driver = driver,
+                    pollIntervalMillis = 1,
+                    unknownPageLimit = 3
+                )
+            )
+
+            val report = runner.run(PinduoduoImageSearchWorkflow.steps())
+
+            assertEquals(WorkflowState.BLOCKED, report.state)
+            assertEquals(SafetyStopReason.UNKNOWN_PAGE, report.safetyStopReason)
+            assertFalse(driver.preparedImageSelected)
+        }
+
+    @Test
+    fun `recovers when camera result opens before prepared image selection`() =
+        runTest {
+            val driver = FakeImageDriver(
+                page = PinduoduoPage.SEARCH_RESULTS,
+                cameraResultRaceOnce = true
+            )
+            val runner = WorkflowRunner(
+                PinduoduoImageSearchAutomation(
+                    driver = driver,
+                    pollIntervalMillis = 1,
+                    unknownPageLimit = 3
+                )
+            )
+
+            val report = runner.run(PinduoduoImageSearchWorkflow.steps())
+
+            assertEquals(WorkflowState.SUCCEEDED, report.state)
+            assertEquals(1, driver.returnFromImageResultsCalls)
+            assertTrue(driver.preparedImageSelected)
+            assertEquals(PinduoduoPage.IMAGE_SEARCH_RESULTS, driver.page)
+        }
+
+    @Test
+    fun `payment marker stops before image entry`() = runTest {
+        val driver = FakeImageDriver(
+            page = PinduoduoPage.UNKNOWN,
+            safetyStopReason = SafetyStopReason.PAYMENT_BOUNDARY
+        )
+        val runner = WorkflowRunner(
+            PinduoduoImageSearchAutomation(
+                driver = driver,
+                pollIntervalMillis = 1,
+                unknownPageLimit = 3
+            )
+        )
+
+        val report = runner.run(PinduoduoImageSearchWorkflow.steps())
+
+        assertEquals(WorkflowState.BLOCKED, report.state)
+        assertFalse(driver.imageSearchOpened)
+    }
+
+    private class FakeImageDriver(
+        var page: PinduoduoPage,
+        private val selectionAllowed: Boolean = true,
+        private val safetyStopReason: SafetyStopReason? = null,
+        private val cameraResultRaceOnce: Boolean = false
+    ) : PinduoduoImageSearchDriver {
+        var imageSearchOpened = false
+        var preparedImageSelected = false
+        var returnFromImageResultsCalls = 0
+        private var cameraResultRaceConsumed = false
+
+        override suspend fun openApp(): Boolean = true
+
+        override suspend fun snapshot(): PinduoduoUiSnapshot =
+            PinduoduoUiSnapshot(
+                foregroundPackage = PINDUODUO_PACKAGE,
+                page = page,
+                safetyStopReason = safetyStopReason
+            )
+
+        override suspend fun openImageSearch(): Boolean {
+            imageSearchOpened = true
+            page = if (cameraResultRaceOnce && !cameraResultRaceConsumed) {
+                cameraResultRaceConsumed = true
+                PinduoduoPage.IMAGE_SEARCH_RESULTS
+            } else {
+                PinduoduoPage.IMAGE_SEARCH
+            }
+            return true
+        }
+
+        override suspend fun selectPreparedImage(): Boolean {
+            if (!selectionAllowed) {
+                return false
+            }
+            preparedImageSelected = true
+            page = PinduoduoPage.IMAGE_SEARCH_RESULTS
+            return true
+        }
+
+        override suspend fun returnFromCandidate(): Boolean {
+            page = PinduoduoPage.IMAGE_SEARCH_RESULTS
+            return true
+        }
+
+        override suspend fun returnFromImageResults(): Boolean {
+            returnFromImageResultsCalls += 1
+            page = PinduoduoPage.IMAGE_SEARCH
+            return true
+        }
+    }
+}
@@ -4,6 +4,7 @@ import com.roubao.autopilot.readiness.PINDUODUO_PACKAGE
 import com.roubao.autopilot.workflow.SafetyStopReason
 import org.junit.Assert.assertEquals
 import org.junit.Assert.assertNull
+import org.junit.Assert.assertTrue
 import org.junit.Test

 class PinduoduoPageClassifierTest {
@@ -102,6 +103,41 @@ class PinduoduoPageClassifierTest {
        assertEquals(PinduoduoPage.SEARCH_RESULTS_OTHER_QUERY, snapshot.page)
    }

+    @Test
+    fun `image search camera page requires album and camera markers`() {
+        val snapshot = classify(
+            element(text = "我的相册"),
+            element(text = "最近搜索"),
+            element(text = "历史浏览"),
+            element(text = "点击拍照")
+        )
+
+        assertEquals(PinduoduoPage.IMAGE_SEARCH, snapshot.page)
+    }
+
+    @Test
+    fun `image search results require header and sort controls`() {
+        val snapshot = classify(
+            element(text = "搜图片同款"),
+            element(text = "综合"),
+            element(text = "销量"),
+            element(text = "价格")
+        )
+
+        assertEquals(PinduoduoPage.IMAGE_SEARCH_RESULTS, snapshot.page)
+        assertTrue(snapshot.page.isCandidateResultsPage())
+    }
+
+    @Test
+    fun `partial image page markers remain unknown`() {
+        val snapshot = classify(
+            element(text = "我的相册"),
+            element(text = "最近项目")
+        )
+
+        assertEquals(PinduoduoPage.UNKNOWN, snapshot.page)
+    }
+
    @Test
    fun `login verification and risk markers remain distinct`() {
        assertEquals(
@@ -0,0 +1,40 @@
+package com.roubao.autopilot.pinduoduo
+
+import com.roubao.task.ProbeReferenceImage
+import org.junit.Assert.assertFalse
+import org.junit.Assert.assertTrue
+import org.junit.Test
+
+class PinduoduoReferenceImagePolicyTest {
+    @Test
+    fun `accepts only the declared jpeg bytes and hash`() {
+        val bytes = byteArrayOf(
+            0xff.toByte(),
+            0xd8.toByte(),
+            0xff.toByte(),
+            0xd9.toByte()
+        )
+        val reference = reference(bytes)
+
+        assertTrue(PinduoduoReferenceImagePolicy.isValid(bytes, reference))
+        assertFalse(
+            PinduoduoReferenceImagePolicy.isValid(
+                bytes + 0,
+                reference
+            )
+        )
+        assertFalse(
+            PinduoduoReferenceImagePolicy.isValid(
+                bytes,
+                reference.copy(sha256 = "0".repeat(64))
+            )
+        )
+    }
+
+    private fun reference(bytes: ByteArray) = ProbeReferenceImage(
+        relativePath = "reference.jpg",
+        mediaType = "image/jpeg",
+        sizeBytes = bytes.size.toLong(),
+        sha256 = PinduoduoEvidenceHash.sha256(bytes)
+    )
+}
@@ -53,6 +53,9 @@ class CandidateEvaluatorTest {
            assertFalse(prompt.contains("\"quantity\""))
            assertFalse(prompt.contains("order_submitted"))
            assertFalse(prompt.contains("payment_authorization"))
+            assertTrue(prompt.contains("\"hard_constraints\""))
+            assertTrue(prompt.contains("\"expected\":\"BLACK\""))
+            assertTrue(prompt.contains("\"expected\":\"L\""))
        }

    @Test
@@ -270,6 +273,45 @@ class CandidateEvaluatorTest {
        )
    }

+    @Test
+    fun `unresolved sku color or size never calls provider`() = runTest {
+        val calls = AtomicInteger()
+        val result = evaluator {
+            calls.incrementAndGet()
+            Result.success(validResponse(1))
+        }.evaluate(
+            CandidateEvaluationInput(
+                requirement = requirement().copy(sku = "BLACK"),
+                candidates = listOf(candidate(1))
+            )
+        ) as CandidateEvaluationResult.Failed
+
+        assertEquals(0, calls.get())
+        assertEquals(
+            CandidateEvaluationFailureCode.SKU_CONSTRAINTS_UNRESOLVED,
+            result.code
+        )
+    }
+
+    @Test
+    fun `unknown hard constraint cannot enter top five`() = runTest {
+        val result = evaluator {
+            Result.success(
+                validResponse(
+                    ordinal = 1,
+                    decision = CandidateDecision.MANUAL_REQUIRED,
+                    score = 0.95,
+                    missing = listOf("截图未显示尺码"),
+                    hardStatus = HardConstraintMatchStatus.UNKNOWN
+                )
+            )
+        }.evaluate(input(candidateCount = 1))
+            as CandidateEvaluationResult.Completed
+
+        assertTrue(CandidateTopFivePolicy.select(result.batch).isEmpty())
+        assertNull(result.batch.recommendedCandidateOrdinal)
+    }
+
    @Test
    fun `encoded review batch fixes order submitted to false`() = runTest {
        val result = evaluator { Result.success(validResponse(1)) }
@@ -320,7 +362,7 @@ class CandidateEvaluatorTest {
                )
            ),
            maxBudget = null,
-            sku = "BLACK",
+            sku = "BLACK-L",
            quantity = 2,
            confidence = 0.9,
            warnings = emptyList(),
@@ -338,10 +380,15 @@ class CandidateEvaluatorTest {
        matched: List<String> = listOf("颜色一致"),
        missing: List<String> = emptyList(),
        rejectionReasons: List<String> = emptyList(),
-        confidence: Double = 0.9
+        confidence: Double = 0.9,
+        hardStatus: HardConstraintMatchStatus = when (decision) {
+            CandidateDecision.REVIEW -> HardConstraintMatchStatus.MATCH
+            CandidateDecision.REJECT -> HardConstraintMatchStatus.MISMATCH
+            CandidateDecision.MANUAL_REQUIRED -> HardConstraintMatchStatus.UNKNOWN
+        }
    ): String =
        JSONObject()
-            .put("schema_version", 1)
+            .put("schema_version", CANDIDATE_EVALUATION_SCHEMA_VERSION)
            .put("candidate_index", ordinal)
            .put("decision", decision.name)
            .put("score", score)
@@ -349,6 +396,24 @@ class CandidateEvaluatorTest {
            .put("missing_or_uncertain", missing.toJsonArray())
            .put("rejection_reasons", rejectionReasons.toJsonArray())
            .put("confidence", confidence)
+            .put(
+                "hard_constraint_results",
+                JSONArray()
+                    .put(
+                        JSONObject()
+                            .put("kind", "COLOR")
+                            .put("expected", "BLACK")
+                            .put("status", hardStatus.name)
+                            .put("evidence", "截图颜色证据")
+                    )
+                    .put(
+                        JSONObject()
+                            .put("kind", "SIZE")
+                            .put("expected", "L")
+                            .put("status", hardStatus.name)
+                            .put("evidence", "截图尺码证据")
+                    )
+            )
            .toString()

    private fun List<String>.toJsonArray(): JSONArray =
@@ -0,0 +1,89 @@
+package com.roubao.autopilot.vlm
+
+import org.junit.Assert.assertEquals
+import org.junit.Test
+
+class CandidateTopFivePolicyTest {
+    @Test
+    fun `sorts eligible matches and limits result to five`() {
+        val selected = CandidateTopFivePolicy.select(
+            batch(
+                assessment(1, score = 0.80, confidence = 0.90),
+                assessment(2, score = 0.90, confidence = 0.80),
+                assessment(3, score = 0.90, confidence = 0.95),
+                assessment(
+                    4,
+                    score = 0.99,
+                    confidence = 0.99,
+                    hardStatus = HardConstraintMatchStatus.UNKNOWN
+                ),
+                assessment(5, score = 0.88, confidence = 0.88),
+                assessment(6, score = 0.87, confidence = 0.87),
+                assessment(7, score = 0.86, confidence = 0.86),
+                assessment(8, score = 0.85, confidence = 0.85)
+            )
+        )
+
+        assertEquals(listOf(3, 2, 5, 6, 7), selected.map { it.ordinal })
+    }
+
+    @Test
+    fun `does not fill missing slots with weak or unknown candidates`() {
+        val selected = CandidateTopFivePolicy.select(
+            batch(
+                assessment(1, score = 0.90, confidence = 0.90),
+                assessment(2, score = 0.74, confidence = 0.99),
+                assessment(
+                    3,
+                    score = 0.99,
+                    confidence = 0.99,
+                    hardStatus = HardConstraintMatchStatus.UNKNOWN
+                )
+            )
+        )
+
+        assertEquals(listOf(1), selected.map { it.ordinal })
+    }
+
+    private fun batch(vararg assessments: CandidateAssessment): CandidateReviewBatch =
+        CandidateReviewBatch(
+            assessments = assessments.toList(),
+            recommendedCandidateOrdinal = null,
+            conclusion = CandidateBatchConclusion.MANUAL_REQUIRED,
+            warnings = emptyList(),
+            providerId = "test-provider",
+            model = "test-model",
+            requirementReferenceImageSha256 = "f".repeat(64)
+        )
+
+    private fun assessment(
+        ordinal: Int,
+        score: Double,
+        confidence: Double,
+        hardStatus: HardConstraintMatchStatus = HardConstraintMatchStatus.MATCH
+    ): CandidateAssessment =
+        CandidateAssessment(
+            ordinal = ordinal,
+            decision = CandidateDecision.REVIEW,
+            score = score,
+            matched = listOf("参考图和规格匹配"),
+            missingOrUncertain = emptyList(),
+            rejectionReasons = emptyList(),
+            confidence = confidence,
+            evidenceSha256 = ordinal.toString(16).padStart(64, '0'),
+            hardConstraintResults = listOf(
+                CandidateHardConstraintResult(
+                    kind = SkuConstraintKind.COLOR,
+                    expected = "BLACK",
+                    status = hardStatus,
+                    evidence = "颜色证据"
+                ),
+                CandidateHardConstraintResult(
+                    kind = SkuConstraintKind.SIZE,
+                    expected = "L",
+                    status = hardStatus,
+                    evidence = "尺码证据"
+                )
+            )
+        )
+}
@@ -0,0 +1,59 @@
+package com.roubao.autopilot.vlm
+
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertFalse
+import org.junit.Assert.assertTrue
+import org.junit.Test
+
+class SkuHardConstraintExtractorTest {
+    @Test
+    fun `extracts normalized English color and size`() {
+        val result = SkuHardConstraintExtractor.extract("sku-black-xxl")
+
+        assertTrue(result.readyForAutomaticMatching)
+        assertEquals(
+            listOf(
+                SkuHardConstraint(SkuConstraintKind.COLOR, "BLACK"),
+                SkuHardConstraint(SkuConstraintKind.SIZE, "2XL")
+            ),
+            result.constraints
+        )
+    }
+
+    @Test
+    fun `extracts Chinese color and numeric size`() {
+        val result = SkuHardConstraintExtractor.extract("女裤-藏青-38码")
+
+        assertTrue(result.readyForAutomaticMatching)
+        assertEquals("BLUE", result.constraints[0].expected)
+        assertEquals("38码", result.constraints[1].expected)
+    }
+
+    @Test
+    fun `multiple colors are unresolved instead of guessed`() {
+        val result = SkuHardConstraintExtractor.extract("BLACK-WHITE-L")
+
+        assertFalse(result.readyForAutomaticMatching)
+        assertTrue(SkuConstraintKind.COLOR in result.unresolvedKinds)
+        assertEquals(
+            listOf(SkuHardConstraint(SkuConstraintKind.SIZE, "L")),
+            result.constraints
+        )
+    }
+
+    @Test
+    fun `missing size is unresolved`() {
+        val result = SkuHardConstraintExtractor.extract("黑色")
+
+        assertFalse(result.readyForAutomaticMatching)
+        assertTrue(SkuConstraintKind.SIZE in result.unresolvedKinds)
+    }
+
+    @Test
+    fun `English color fragment inside opaque code is not accepted`() {
+        val result = SkuHardConstraintExtractor.extract("PREDICT-L")
+
+        assertFalse(result.readyForAutomaticMatching)
+        assertTrue(SkuConstraintKind.COLOR in result.unresolvedKinds)
+    }
+}