feat(android): evaluate candidates before human confirmation

2026-07-25 21:54:52 +08:00
parent 32d5310000
commit 84f2da3cf7
25 changed files with 2127 additions and 43 deletions
@@ -0,0 +1,135 @@
+package com.roubao.autopilot.pinduoduo
+
+import java.io.File
+import java.nio.file.Files
+import kotlinx.coroutines.test.runTest
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertTrue
+import org.junit.Test
+
+class CandidateEvidenceSourceTest {
+    @Test
+    fun `loads only declared sequential evidence after metadata validation`() = runTest {
+        withEvidenceRoot { root ->
+            val first = writeCandidate(root, ordinal = 1, width = 1080, height = 2400)
+            val second = writeCandidate(root, ordinal = 2, width = 1080, height = 2400)
+
+            val result = CandidateEvidenceSource(root).load(listOf(first, second))
+
+            assertTrue(result.isSuccess)
+            assertEquals(listOf(1, 2), result.getOrThrow().map { it.ordinal })
+        }
+    }
+
+    @Test
+    fun `rejects non allowlisted filename before reading`() = runTest {
+        withEvidenceRoot { root ->
+            val evidence = writeCandidate(root, ordinal = 1, width = 10, height = 20)
+                .copy(screenshotFileName = "../candidate-01.png")
+
+            val result = CandidateEvidenceSource(root).load(listOf(evidence))
+
+            assertTrue(result.isFailure)
+        }
+    }
+
+    @Test
+    fun `rejects missing duplicate or non sequential ordinals`() = runTest {
+        withEvidenceRoot { root ->
+            val first = writeCandidate(root, ordinal = 1, width = 10, height = 20)
+            val second = writeCandidate(root, ordinal = 2, width = 10, height = 20)
+
+            assertTrue(
+                CandidateEvidenceSource(root).load(listOf(second)).isFailure
+            )
+            assertTrue(
+                CandidateEvidenceSource(root).load(listOf(first, first)).isFailure
+            )
+        }
+    }
+
+    @Test
+    fun `rejects byte hash and png dimension tampering`() = runTest {
+        withEvidenceRoot { root ->
+            val evidence = writeCandidate(root, ordinal = 1, width = 10, height = 20)
+            val source = CandidateEvidenceSource(root)
+
+            assertTrue(
+                source.load(
+                    listOf(evidence.copy(screenshotByteCount = evidence.screenshotByteCount + 1))
+                ).isFailure
+            )
+            assertTrue(
+                source.load(
+                    listOf(evidence.copy(screenshotSha256 = "0".repeat(64)))
+                ).isFailure
+            )
+            assertTrue(
+                source.load(
+                    listOf(evidence.copy(screenshotWidth = 11))
+                ).isFailure
+            )
+        }
+    }
+
+    @Test
+    fun `rejects invalid png header and declared size above limit`() = runTest {
+        withEvidenceRoot { root ->
+            val evidence = writeCandidate(root, ordinal = 1, width = 10, height = 20)
+            File(root, evidence.screenshotFileName).writeBytes(ByteArray(24))
+
+            assertTrue(CandidateEvidenceSource(root).load(listOf(evidence)).isFailure)
+            assertTrue(
+                CandidateEvidenceSource(root).load(
+                    listOf(evidence.copy(screenshotByteCount = 8 * 1024 * 1024 + 1))
+                ).isFailure
+            )
+        }
+    }
+
+    private suspend fun withEvidenceRoot(block: suspend (File) -> Unit) {
+        val root = Files.createTempDirectory("candidate-evidence-test").toFile()
+        try {
+            block(root)
+        } finally {
+            root.deleteRecursively()
+        }
+    }
+
+    private fun writeCandidate(
+        root: File,
+        ordinal: Int,
+        width: Int,
+        height: Int
+    ): PinduoduoCandidateEvidence {
+        val bytes = pngHeader(width, height)
+        val fileName = "candidate-%02d.png".format(ordinal)
+        File(root, fileName).writeBytes(bytes)
+        return PinduoduoCandidateEvidence(
+            ordinal = ordinal,
+            cardSignature = "card-$ordinal",
+            cardSemanticTextCount = 3,
+            detailSignature = "detail-$ordinal",
+            detailSemanticTextCount = 4,
+            screenshotFileName = fileName,
+            screenshotSha256 = PinduoduoEvidenceHash.sha256(bytes),
+            screenshotByteCount = bytes.size,
+            screenshotWidth = width,
+            screenshotHeight = height
+        )
+    }
+
+    private fun pngHeader(width: Int, height: Int): ByteArray =
+        byteArrayOf(
+            0x89.toByte(), 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
+            0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52,
+            (width ushr 24).toByte(),
+            (width ushr 16).toByte(),
+            (width ushr 8).toByte(),
+            width.toByte(),
+            (height ushr 24).toByte(),
+            (height ushr 16).toByte(),
+            (height ushr 8).toByte(),
+            height.toByte()
+        )
+}
@@ -85,7 +85,7 @@ class PinduoduoPageClassifierTest {
    }

    @Test
-    fun `results require the exact expected query`() {
+    fun `different result query is recognized but not accepted as current results`() {
        val snapshot = classify(
            element(
                contentDescription = "搜索",
@@ -97,7 +97,7 @@ class PinduoduoPageClassifierTest {
            element(text = "价格")
        )

-        assertEquals(PinduoduoPage.UNKNOWN, snapshot.page)
+        assertEquals(PinduoduoPage.SEARCH_RESULTS_OTHER_QUERY, snapshot.page)
    }

    @Test
@@ -107,6 +107,51 @@ class PinduoduoSearchAutomationTest {
        assertEquals(PinduoduoPage.SEARCH_RESULTS, driver.page)
    }

+    @Test
+    fun `dynamic requirement query replaces an existing results query`() = runTest {
+        val driver = FakeDriver(page = PinduoduoPage.SEARCH_RESULTS_OTHER_QUERY)
+        val dynamicQuery = "动态任务搜索词"
+        val runner = WorkflowRunner(
+            PinduoduoSearchAutomation(
+                driver = driver,
+                keyword = dynamicQuery,
+                forceKeywordEntry = true,
+                pollIntervalMillis = 1,
+                unknownPageLimit = 3
+            )
+        )
+
+        val report = runner.run(PinduoduoSearchWorkflow.steps())
+
+        assertEquals(WorkflowState.SUCCEEDED, report.state)
+        assertEquals(1, driver.openSearchCalls)
+        assertEquals(dynamicQuery, driver.enteredKeyword)
+        assertTrue(driver.searchSubmitted)
+    }
+
+    @Test
+    fun `dynamic requirement query returns from detail before replacing query`() =
+        runTest {
+            val driver = FakeDriver(page = PinduoduoPage.PRODUCT_DETAIL)
+            val dynamicQuery = "动态任务搜索词"
+            val runner = WorkflowRunner(
+                PinduoduoSearchAutomation(
+                    driver = driver,
+                    keyword = dynamicQuery,
+                    forceKeywordEntry = true,
+                    pollIntervalMillis = 1,
+                    unknownPageLimit = 3
+                )
+            )
+
+            val report = runner.run(PinduoduoSearchWorkflow.steps())
+
+            assertEquals(WorkflowState.SUCCEEDED, report.state)
+            assertEquals(1, driver.returnFromCandidateCalls)
+            assertEquals(1, driver.openSearchCalls)
+            assertEquals(dynamicQuery, driver.enteredKeyword)
+        }
+
    private class FakeDriver(
        var page: PinduoduoPage,
        private val safetyStopReason: SafetyStopReason? = null,
@@ -0,0 +1,358 @@
+package com.roubao.autopilot.vlm
+
+import java.util.concurrent.atomic.AtomicInteger
+import kotlinx.coroutines.CancellationException
+import kotlinx.coroutines.test.runTest
+import org.json.JSONArray
+import org.json.JSONObject
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertFalse
+import org.junit.Assert.assertNull
+import org.junit.Assert.assertTrue
+import org.junit.Test
+
+class CandidateEvaluatorTest {
+    @Test
+    fun `evaluates candidates once each and recommends deterministically`() = runTest {
+        val requests = mutableListOf<CandidateEvaluationVlmRequest>()
+        val responses = listOf(
+            validResponse(ordinal = 1, score = 0.9),
+            validResponse(ordinal = 2, score = 0.9)
+        )
+        val evaluator = evaluator { request ->
+            requests += request
+            Result.success(responses[requests.lastIndex])
+        }
+
+        val rawResult = evaluator.evaluate(input(candidateCount = 2))
+        assertTrue("result=$rawResult requests=${requests.size}", rawResult is CandidateEvaluationResult.Completed)
+        val result = rawResult as CandidateEvaluationResult.Completed
+
+        assertEquals(2, requests.size)
+        assertEquals(2, result.batch.assessments.size)
+        assertEquals(1, result.batch.recommendedCandidateOrdinal)
+        assertEquals(CandidateBatchConclusion.SUGGESTED, result.batch.conclusion)
+        assertTrue(result.batch.manualReviewRequired)
+        assertFalse(result.batch.orderSubmitted)
+        assertTrue(requests.all { request -> request.imageMediaType == "image/png" })
+    }
+
+    @Test
+    fun `prompt excludes local task identity quantity and payment authorization`() =
+        runTest {
+            lateinit var prompt: String
+            val evaluator = evaluator { request ->
+                prompt = request.prompt
+                Result.success(validResponse(ordinal = 1, score = 0.9))
+            }
+
+            evaluator.evaluate(input(candidateCount = 1))
+
+            assertFalse(prompt.contains("source_order_no"))
+            assertFalse(prompt.contains("source_store_name"))
+            assertFalse(prompt.contains("\"quantity\""))
+            assertFalse(prompt.contains("order_submitted"))
+            assertFalse(prompt.contains("payment_authorization"))
+        }
+
+    @Test
+    fun `invalid first output stops later paid calls and creates manual fallbacks`() =
+        runTest {
+            val calls = AtomicInteger()
+            val evaluator = evaluator {
+                calls.incrementAndGet()
+                Result.success("not-json")
+            }
+
+            val rawResult = evaluator.evaluate(input(candidateCount = 3))
+            assertTrue(
+                "result=$rawResult calls=${calls.get()}",
+                rawResult is CandidateEvaluationResult.Completed
+            )
+            val result = rawResult as CandidateEvaluationResult.Completed
+
+            assertEquals(1, calls.get())
+            assertEquals(3, result.batch.assessments.size)
+            assertTrue(
+                result.batch.assessments.all {
+                    it.decision == CandidateDecision.MANUAL_REQUIRED
+                }
+            )
+            assertEquals(CandidateBatchConclusion.MANUAL_REQUIRED, result.batch.conclusion)
+            assertNull(result.batch.recommendedCandidateOrdinal)
+        }
+
+    @Test
+    fun `invalid later output discards an earlier automatic recommendation`() = runTest {
+        val calls = AtomicInteger()
+        val evaluator = evaluator {
+            when (calls.incrementAndGet()) {
+                1 -> Result.success(validResponse(ordinal = 1, score = 0.95))
+                else -> Result.success("not-json")
+            }
+        }
+
+        val result = evaluator.evaluate(input(candidateCount = 3))
+            as CandidateEvaluationResult.Completed
+
+        assertEquals(2, calls.get())
+        assertEquals(CandidateBatchConclusion.MANUAL_REQUIRED, result.batch.conclusion)
+        assertNull(result.batch.recommendedCandidateOrdinal)
+        assertTrue(
+            result.batch.warnings.any {
+                it.code == CandidateEvaluationWarningCode.MODEL_OUTPUT_INVALID
+            }
+        )
+        assertEquals(
+            CandidateDecision.MANUAL_REQUIRED,
+            result.batch.assessments[2].decision
+        )
+    }
+
+    @Test
+    fun `provider failure stops the batch and preserves retryability`() = runTest {
+        val calls = AtomicInteger()
+        val evaluator = evaluator {
+            calls.incrementAndGet()
+            Result.failure(StructuredVlmException(retryable = false))
+        }
+
+        val result = evaluator.evaluate(input(candidateCount = 3))
+            as CandidateEvaluationResult.Failed
+
+        assertEquals("result=$result", 1, calls.get())
+        assertEquals(CandidateEvaluationFailureCode.PROVIDER_ERROR, result.code)
+        assertFalse(result.retryable)
+    }
+
+    @Test(expected = CancellationException::class)
+    fun `cancellation is propagated without starting another candidate`() = runTest {
+        val calls = AtomicInteger()
+        val evaluator = evaluator {
+            calls.incrementAndGet()
+            throw CancellationException("stop")
+        }
+
+        try {
+            evaluator.evaluate(input(candidateCount = 3))
+        } finally {
+            assertEquals(1, calls.get())
+        }
+    }
+
+    @Test
+    fun `all rejected candidates produce no match`() = runTest {
+        val calls = AtomicInteger()
+        val evaluator = evaluator {
+            val ordinal = calls.incrementAndGet()
+            Result.success(
+                validResponse(
+                    ordinal = ordinal,
+                    decision = CandidateDecision.REJECT,
+                    score = 0.1,
+                    matched = emptyList(),
+                    rejectionReasons = listOf("关键款式不符")
+                )
+            )
+        }
+
+        val result = evaluator.evaluate(input(candidateCount = 2))
+            as CandidateEvaluationResult.Completed
+
+        assertEquals(CandidateBatchConclusion.NO_MATCH, result.batch.conclusion)
+        assertNull(result.batch.recommendedCandidateOrdinal)
+    }
+
+    @Test
+    fun `low confidence review remains manual without recommendation`() = runTest {
+        val evaluator = evaluator {
+            Result.success(
+                validResponse(
+                    ordinal = 1,
+                    score = 0.9,
+                    confidence = 0.74
+                )
+            )
+        }
+
+        val result = evaluator.evaluate(input(candidateCount = 1))
+            as CandidateEvaluationResult.Completed
+
+        assertEquals(CandidateBatchConclusion.MANUAL_REQUIRED, result.batch.conclusion)
+        assertTrue(
+            result.batch.warnings.any {
+                it.code == CandidateEvaluationWarningCode.LOW_CONFIDENCE
+            }
+        )
+    }
+
+    @Test
+    fun `numeric strings extra fields and wrong ordinal are rejected`() = runTest {
+        val badResponses = listOf(
+            JSONObject(validResponse(1)).put("score", "0.9").toString(),
+            JSONObject(validResponse(1)).put("action", "review").toString(),
+            JSONObject(validResponse(2)).toString()
+        )
+
+        badResponses.forEach { response ->
+            val result = evaluator { Result.success(response) }
+                .evaluate(input(candidateCount = 1))
+                as CandidateEvaluationResult.Completed
+            assertEquals(
+                CandidateDecision.MANUAL_REQUIRED,
+                result.batch.assessments.single().decision
+            )
+        }
+    }
+
+    @Test
+    fun `action coordinate and payment strings are rejected in every reason list`() =
+        runTest {
+            val injected = listOf(
+                "click(20,30)",
+                "x=20",
+                "立即支付",
+                "提交订单"
+            )
+            injected.forEach { value ->
+                val response = JSONObject(validResponse(1))
+                    .put("matched", JSONArray().put(value))
+                    .toString()
+                val result = evaluator { Result.success(response) }
+                    .evaluate(input(candidateCount = 1))
+                    as CandidateEvaluationResult.Completed
+                assertEquals(
+                    CandidateDecision.MANUAL_REQUIRED,
+                    result.batch.assessments.single().decision
+                )
+            }
+        }
+
+    @Test
+    fun `model cannot claim price or budget match when task budget is absent`() =
+        runTest {
+            listOf("价格符合预算", "price within budget").forEach { value ->
+                val response = JSONObject(validResponse(1))
+                    .put("matched", JSONArray().put(value))
+                    .toString()
+                val result = evaluator { Result.success(response) }
+                    .evaluate(input(candidateCount = 1))
+                    as CandidateEvaluationResult.Completed
+
+                assertEquals(
+                    CandidateDecision.MANUAL_REQUIRED,
+                    result.batch.assessments.single().decision
+                )
+                assertNull(result.batch.recommendedCandidateOrdinal)
+            }
+        }
+
+    @Test
+    fun `invalid or manual requirement never calls provider`() = runTest {
+        val calls = AtomicInteger()
+        val evaluator = evaluator {
+            calls.incrementAndGet()
+            Result.success(validResponse(1))
+        }
+        val manualRequirement = requirement().copy(manualReviewRequired = true)
+
+        val result = evaluator.evaluate(
+            CandidateEvaluationInput(
+                requirement = manualRequirement,
+                candidates = listOf(candidate(1))
+            )
+        ) as CandidateEvaluationResult.Failed
+
+        assertEquals(0, calls.get())
+        assertEquals(
+            CandidateEvaluationFailureCode.REQUIREMENT_NOT_READY,
+            result.code
+        )
+    }
+
+    @Test
+    fun `encoded review batch fixes order submitted to false`() = runTest {
+        val result = evaluator { Result.success(validResponse(1)) }
+            .evaluate(input(candidateCount = 1))
+            as CandidateEvaluationResult.Completed
+
+        val json = JSONObject(CandidateReviewBatchJson.encode(result.batch))
+
+        assertFalse(json.getBoolean("order_submitted"))
+        assertTrue(json.getBoolean("manual_review_required"))
+        assertEquals(1, json.getJSONArray("candidates").length())
+    }
+
+    private fun evaluator(
+        complete: suspend (CandidateEvaluationVlmRequest) -> Result<String>
+    ): CandidateEvaluator =
+        CandidateEvaluator(
+            gateway = CandidateEvaluationVlmGateway { request ->
+                complete(request)
+            },
+            providerId = "fake",
+            model = "fake-model"
+        )
+
+    private fun input(candidateCount: Int): CandidateEvaluationInput =
+        CandidateEvaluationInput(
+            requirement = requirement(),
+            candidates = (1..candidateCount).map(::candidate)
+        )
+
+    private fun candidate(ordinal: Int): CandidateEvaluationImage =
+        CandidateEvaluationImage(
+            ordinal = ordinal,
+            mediaType = "image/png",
+            bytes = byteArrayOf(ordinal.toByte()),
+            sha256 = ordinal.toString(16).padStart(64, '0')
+        )
+
+    private fun requirement(): RequirementExtraction =
+        RequirementExtraction(
+            searchQuery = "黑色双肩包",
+            category = "双肩包",
+            attributes = listOf(
+                RequirementAttribute(
+                    name = "颜色",
+                    value = "黑色",
+                    source = RequirementAttributeSource.BOTH
+                )
+            ),
+            maxBudget = null,
+            sku = "BLACK",
+            quantity = 2,
+            confidence = 0.9,
+            warnings = emptyList(),
+            manualReviewRequired = false,
+            manualReviewReasons = emptyList(),
+            providerId = "fake",
+            model = "fake-model",
+            referenceImageSha256 = "f".repeat(64)
+        )
+
+    private fun validResponse(
+        ordinal: Int,
+        decision: CandidateDecision = CandidateDecision.REVIEW,
+        score: Double = 0.9,
+        matched: List<String> = listOf("颜色一致"),
+        missing: List<String> = emptyList(),
+        rejectionReasons: List<String> = emptyList(),
+        confidence: Double = 0.9
+    ): String =
+        JSONObject()
+            .put("schema_version", 1)
+            .put("candidate_index", ordinal)
+            .put("decision", decision.name)
+            .put("score", score)
+            .put("matched", matched.toJsonArray())
+            .put("missing_or_uncertain", missing.toJsonArray())
+            .put("rejection_reasons", rejectionReasons.toJsonArray())
+            .put("confidence", confidence)
+            .toString()
+
+    private fun List<String>.toJsonArray(): JSONArray =
+        JSONArray().apply {
+            this@toJsonArray.forEach { value -> put(value) }
+        }
+}
@@ -0,0 +1,100 @@
+package com.roubao.autopilot.vlm
+
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertFalse
+import org.junit.Test
+
+class CandidateHumanReviewPolicyTest {
+    @Test
+    fun `only a pending locally recommended candidate can be accepted`() {
+        val batch = batch(recommendedOrdinal = 1)
+
+        assertEquals(
+            CandidateEvaluationState.HUMAN_ACCEPTED,
+            CandidateHumanReviewPolicy.accept(
+                CandidateEvaluationState.AWAITING_CONFIRMATION,
+                batch
+            )
+        )
+        assertEquals(
+            CandidateEvaluationState.MANUAL_REVIEW,
+            CandidateHumanReviewPolicy.accept(
+                CandidateEvaluationState.MANUAL_REVIEW,
+                batch
+            )
+        )
+        assertEquals(
+            CandidateEvaluationState.AWAITING_CONFIRMATION,
+            CandidateHumanReviewPolicy.accept(
+                CandidateEvaluationState.AWAITING_CONFIRMATION,
+                batch(recommendedOrdinal = null)
+            )
+        )
+        assertEquals(
+            CandidateEvaluationState.AWAITING_CONFIRMATION,
+            CandidateHumanReviewPolicy.accept(
+                CandidateEvaluationState.AWAITING_CONFIRMATION,
+                batch(
+                    recommendedOrdinal = 1,
+                    recommendedDecision = CandidateDecision.REJECT
+                )
+            )
+        )
+        assertFalse(batch.orderSubmitted)
+    }
+
+    @Test
+    fun `reject is allowed only at human decision states`() {
+        listOf(
+            CandidateEvaluationState.AWAITING_CONFIRMATION,
+            CandidateEvaluationState.MANUAL_REVIEW,
+            CandidateEvaluationState.NO_MATCH
+        ).forEach { state ->
+            assertEquals(
+                CandidateEvaluationState.HUMAN_REJECTED,
+                CandidateHumanReviewPolicy.reject(state)
+            )
+        }
+        assertEquals(
+            CandidateEvaluationState.RUNNING,
+            CandidateHumanReviewPolicy.reject(CandidateEvaluationState.RUNNING)
+        )
+        assertEquals(
+            CandidateEvaluationState.HUMAN_ACCEPTED,
+            CandidateHumanReviewPolicy.reject(
+                CandidateEvaluationState.HUMAN_ACCEPTED
+            )
+        )
+    }
+
+    private fun batch(
+        recommendedOrdinal: Int?,
+        recommendedDecision: CandidateDecision = CandidateDecision.REVIEW
+    ): CandidateReviewBatch =
+        CandidateReviewBatch(
+            assessments = recommendedOrdinal?.let { ordinal ->
+                listOf(
+                    CandidateAssessment(
+                        ordinal = ordinal,
+                        decision = recommendedDecision,
+                        score = 0.9,
+                        matched = emptyList(),
+                        missingOrUncertain = emptyList(),
+                        rejectionReasons = emptyList(),
+                        confidence = 0.9,
+                        evidenceSha256 = "e".repeat(64)
+                    )
+                )
+            }.orEmpty(),
+            recommendedCandidateOrdinal = recommendedOrdinal,
+            conclusion = if (recommendedOrdinal == null) {
+                CandidateBatchConclusion.MANUAL_REQUIRED
+            } else {
+                CandidateBatchConclusion.SUGGESTED
+            },
+            warnings = emptyList(),
+            providerId = "fake",
+            model = "fake",
+            requirementReferenceImageSha256 = "f".repeat(64)
+        )
+}