feat(android): evaluate candidates before human confirmation

2026-07-25 21:54:52 +08:00
parent 32d5310000
commit 84f2da3cf7
25 changed files with 2127 additions and 43 deletions
@@ -50,6 +50,16 @@ import com.roubao.autopilot.vlm.RequirementExtractionResult
 import com.roubao.autopilot.vlm.RequirementExtractor
 import com.roubao.autopilot.vlm.RequirementProbeState
 import com.roubao.autopilot.vlm.RequirementProviderEndpointPolicy
+import com.roubao.autopilot.vlm.AndroidCandidateEvaluationVlmGateway
+import com.roubao.autopilot.vlm.CandidateBatchConclusion
+import com.roubao.autopilot.vlm.CandidateEvaluationFailureCode
+import com.roubao.autopilot.vlm.CandidateEvaluationImage
+import com.roubao.autopilot.vlm.CandidateEvaluationInput
+import com.roubao.autopilot.vlm.CandidateEvaluationResult
+import com.roubao.autopilot.vlm.CandidateEvaluationState
+import com.roubao.autopilot.vlm.CandidateEvaluator
+import com.roubao.autopilot.vlm.CandidateReviewBatch
+import com.roubao.autopilot.vlm.CandidateHumanReviewPolicy
 import kotlinx.coroutines.CancellationException
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.Job
@@ -65,6 +75,8 @@ import com.roubao.autopilot.pinduoduo.PinduoduoCandidateEvidence
 import com.roubao.autopilot.pinduoduo.PinduoduoCandidateWorkflow
 import com.roubao.autopilot.pinduoduo.PinduoduoProbeAutomation
 import com.roubao.autopilot.pinduoduo.PinduoduoSearchAutomation
+import com.roubao.autopilot.pinduoduo.CandidateEvidenceSource
+import com.roubao.autopilot.pinduoduo.SEARCH_PROBE_KEYWORD
 import com.roubao.autopilot.workflow.WorkflowReport
 import com.roubao.autopilot.workflow.WorkflowRunner
 import com.roubao.autopilot.workflow.WorkflowState
@@ -85,6 +97,7 @@ class MainActivity : ComponentActivity() {
    private lateinit var executionRepository: ExecutionRepository
    private lateinit var readinessChecker: DeviceReadinessChecker
    private lateinit var requirementProbeSource: RequirementProbeSource
+    private lateinit var candidateEvidenceSource: CandidateEvidenceSource

    private val mobileAgent = mutableStateOf<MobileAgent?>(null)
    private var shizukuAvailable = mutableStateOf(false)
@@ -94,13 +107,22 @@ class MainActivity : ComponentActivity() {
    private val searchProbeReport = mutableStateOf<WorkflowReport?>(null)
    private val candidateEvidence =
        mutableStateOf<List<PinduoduoCandidateEvidence>>(emptyList())
+    private val candidateSearchKeyword = mutableStateOf<String?>(null)
+    private val candidateRequirementSnapshot =
+        mutableStateOf<RequirementExtraction?>(null)
    private val requirementProbeState = mutableStateOf(RequirementProbeState.IDLE)
    private val requirementExtraction = mutableStateOf<RequirementExtraction?>(null)
    private val requirementFailureCode =
        mutableStateOf<RequirementExtractionFailureCode?>(null)
+    private val candidateEvaluationState =
+        mutableStateOf(CandidateEvaluationState.IDLE)
+    private val candidateReviewBatch = mutableStateOf<CandidateReviewBatch?>(null)
+    private val candidateEvaluationFailureCode =
+        mutableStateOf<CandidateEvaluationFailureCode?>(null)
    private var searchProbeRunner: WorkflowRunner? = null
    private var searchProbeJob: Job? = null
    private var requirementProbeJob: Job? = null
+    private var candidateEvaluationJob: Job? = null

    // 当前执行的协程 Job（用于停止任务）
    private var currentExecutionJob: kotlinx.coroutines.Job? = null
@@ -157,6 +179,7 @@ class MainActivity : ComponentActivity() {
        executionRepository = ExecutionRepository(this)
        readinessChecker = DeviceReadinessChecker(this)
        requirementProbeSource = RequirementProbeSource(this)
+        candidateEvidenceSource = CandidateEvidenceSource(this)
        refreshReadiness()

        // 加载执行记录
@@ -223,9 +246,14 @@ class MainActivity : ComponentActivity() {
        val probeStepId by remember { searchProbeStepId }
        val probeReport by remember { searchProbeReport }
        val evidence by remember { candidateEvidence }
+        val evidenceSearchKeyword by remember { candidateSearchKeyword }
+        val evidenceRequirement by remember { candidateRequirementSnapshot }
        val extractionState by remember { requirementProbeState }
        val extraction by remember { requirementExtraction }
        val extractionFailure by remember { requirementFailureCode }
+        val evaluationState by remember { candidateEvaluationState }
+        val reviewBatch by remember { candidateReviewBatch }
+        val evaluationFailure by remember { candidateEvaluationFailureCode }

        // 监听跳转事件
        LaunchedEffect(navigateToRecord, recordId) {
@@ -315,11 +343,35 @@ class MainActivity : ComponentActivity() {
                                currentStepId = probeStepId,
                                report = probeReport,
                                candidateEvidenceCount = evidence.size,
+                                searchKeyword = evidenceSearchKeyword
+                                    ?: extraction?.searchQuery
+                                    ?: SEARCH_PROBE_KEYWORD,
+                                usesRequirementSearch =
+                                    evidenceRequirement != null ||
+                                        extractionState == RequirementProbeState.READY,
                                requirementState = extractionState,
                                requirement = extraction,
                                requirementFailureCode = extractionFailure,
+                                candidateEvaluationState = evaluationState,
+                                candidateReviewBatch = reviewBatch,
+                                candidateEvaluationFailureCode = evaluationFailure,
+                                canStartCandidateEvaluation =
+                                    extractionState == RequirementProbeState.READY &&
+                                        extraction != null &&
+                                        evidenceRequirement == extraction &&
+                                        evidenceSearchKeyword == extraction?.searchQuery &&
+                                        probeReport?.state == WorkflowState.SUCCEEDED &&
+                                        evidence.isNotEmpty(),
                                onStartRequirement = { startRequirementProbe() },
                                onStopRequirement = { stopRequirementProbe() },
+                                onStartCandidateEvaluation = {
+                                    startCandidateEvaluation()
+                                },
+                                onStopCandidateEvaluation = {
+                                    stopCandidateEvaluation()
+                                },
+                                onAcceptCandidate = { acceptRecommendedCandidate() },
+                                onRejectCandidates = { rejectCandidateReview() },
                                onStart = { startSearchProbe() },
                                onStop = { stopSearchProbe() }
                            )
@@ -387,6 +439,7 @@ class MainActivity : ComponentActivity() {
    override fun onDestroy() {
        searchProbeRunner?.requestStop()
        requirementProbeJob?.cancel()
+        candidateEvaluationJob?.cancel()
        super.onDestroy()
        Shizuku.removeBinderReceivedListener(binderReceivedListener)
        Shizuku.removeBinderDeadListener(binderDeadListener)
@@ -418,7 +471,16 @@ class MainActivity : ComponentActivity() {
        if (searchProbeJob?.isActive == true) {
            return
        }
+        if (candidateEvaluationJob?.isActive == true) {
+            Toast.makeText(this, "请先停止候选评估", Toast.LENGTH_SHORT).show()
+            return
+        }

+        val boundRequirement = requirementExtraction.value?.takeIf {
+            requirementProbeState.value == RequirementProbeState.READY &&
+                !it.manualReviewRequired
+        }
+        val searchKeyword = boundRequirement?.searchQuery ?: SEARCH_PROBE_KEYWORD
        val candidateAutomation = PinduoduoCandidateAutomation(
            AndroidPinduoduoCandidateDriver(this)
        )
@@ -426,7 +488,9 @@ class MainActivity : ComponentActivity() {
        val runner = WorkflowRunner(
            PinduoduoProbeAutomation(
                searchAutomation = PinduoduoSearchAutomation(
-                    AndroidPinduoduoUiDriver(this)
+                    driver = AndroidPinduoduoUiDriver(this),
+                    keyword = searchKeyword,
+                    forceKeywordEntry = boundRequirement != null
                ),
                candidateAutomation = candidateAutomation
            )
@@ -436,6 +500,9 @@ class MainActivity : ComponentActivity() {
        searchProbeState.value = WorkflowState.IDLE
        searchProbeStepId.value = null
        candidateEvidence.value = emptyList()
+        candidateSearchKeyword.value = searchKeyword
+        candidateRequirementSnapshot.value = boundRequirement
+        clearCandidateEvaluation()
        searchProbeJob = lifecycleScope.launch {
            val stateCollector = launch {
                runner.state.collect { state -> searchProbeState.value = state }
@@ -482,6 +549,15 @@ class MainActivity : ComponentActivity() {
            Toast.makeText(this, "请先停止候选探针", Toast.LENGTH_SHORT).show()
            return
        }
+        if (candidateEvaluationJob?.isActive == true) {
+            Toast.makeText(this, "请先停止候选评估", Toast.LENGTH_SHORT).show()
+            return
+        }
+        candidateEvidence.value = emptyList()
+        candidateSearchKeyword.value = null
+        candidateRequirementSnapshot.value = null
+        searchProbeReport.value = null
+        clearCandidateEvaluation()
        requirementExtraction.value = null
        requirementFailureCode.value = null

@@ -578,6 +654,177 @@ class MainActivity : ComponentActivity() {
        requirementProbeState.value = RequirementProbeState.FAILED
    }

+    private fun startCandidateEvaluation() {
+        if (candidateEvaluationJob?.isActive == true) {
+            return
+        }
+        if (searchProbeJob?.isActive == true || requirementProbeJob?.isActive == true) {
+            Toast.makeText(this, "请先停止其他探针", Toast.LENGTH_SHORT).show()
+            return
+        }
+        val requirement = requirementExtraction.value
+        if (
+            requirement == null ||
+            requirementProbeState.value != RequirementProbeState.READY ||
+            requirement.manualReviewRequired
+        ) {
+            setCandidateEvaluationFailure(
+                CandidateEvaluationFailureCode.REQUIREMENT_NOT_READY
+            )
+            return
+        }
+        val evidence = candidateEvidence.value
+        if (
+            searchProbeReport.value?.state != WorkflowState.SUCCEEDED ||
+            evidence.isEmpty()
+        ) {
+            setCandidateEvaluationFailure(
+                CandidateEvaluationFailureCode.EVIDENCE_UNAVAILABLE
+            )
+            return
+        }
+        if (
+            candidateRequirementSnapshot.value != requirement ||
+            candidateSearchKeyword.value != requirement.searchQuery
+        ) {
+            setCandidateEvaluationFailure(
+                CandidateEvaluationFailureCode.EVIDENCE_REQUIREMENT_MISMATCH
+            )
+            return
+        }
+
+        val settings = settingsManager.settings.value
+        val provider = settings.currentProvider
+        when {
+            !provider.supportsRequirementExtraction -> {
+                setCandidateEvaluationFailure(
+                    CandidateEvaluationFailureCode.PROVIDER_UNSUPPORTED
+                )
+                return
+            }
+            settings.baseUrl.isBlank() || settings.model.isBlank() -> {
+                setCandidateEvaluationFailure(
+                    CandidateEvaluationFailureCode.PROVIDER_NOT_CONFIGURED
+                )
+                return
+            }
+            !RequirementProviderEndpointPolicy.isAllowed(
+                baseUrl = settings.baseUrl,
+                apiKey = settings.apiKey
+            ) -> {
+                setCandidateEvaluationFailure(
+                    CandidateEvaluationFailureCode.UNSAFE_PROVIDER_ENDPOINT
+                )
+                return
+            }
+            settings.apiKey.isNotBlank() &&
+                !settingsManager.isSecureCredentialStorageAvailable -> {
+                setCandidateEvaluationFailure(
+                    CandidateEvaluationFailureCode
+                        .SECURE_CREDENTIAL_STORAGE_UNAVAILABLE
+                )
+                return
+            }
+            provider.id != ApiProvider.CUSTOM.id && settings.apiKey.isBlank() -> {
+                setCandidateEvaluationFailure(
+                    CandidateEvaluationFailureCode.PROVIDER_NOT_CONFIGURED
+                )
+                return
+            }
+        }
+
+        candidateEvaluationState.value = CandidateEvaluationState.RUNNING
+        candidateReviewBatch.value = null
+        candidateEvaluationFailureCode.value = null
+        candidateEvaluationJob = lifecycleScope.launch {
+            try {
+                val validated = candidateEvidenceSource.load(evidence).getOrElse {
+                    setCandidateEvaluationFailure(
+                        CandidateEvaluationFailureCode.EVIDENCE_INVALID
+                    )
+                    return@launch
+                }
+                val client = VLMClient(
+                    apiKey = settings.apiKey,
+                    baseUrl = settings.baseUrl,
+                    model = settings.model
+                )
+                val result = CandidateEvaluator(
+                    gateway = AndroidCandidateEvaluationVlmGateway(client),
+                    providerId = provider.id,
+                    model = settings.model
+                ).evaluate(
+                    CandidateEvaluationInput(
+                        requirement = requirement,
+                        candidates = validated.map { candidate ->
+                            CandidateEvaluationImage(
+                                ordinal = candidate.ordinal,
+                                mediaType = "image/png",
+                                bytes = candidate.pngBytes,
+                                sha256 = candidate.sha256
+                            )
+                        }
+                    )
+                )
+                when (result) {
+                    is CandidateEvaluationResult.Completed -> {
+                        candidateReviewBatch.value = result.batch
+                        candidateEvaluationState.value = when (
+                            result.batch.conclusion
+                        ) {
+                            CandidateBatchConclusion.SUGGESTED ->
+                                CandidateEvaluationState.AWAITING_CONFIRMATION
+                            CandidateBatchConclusion.NO_MATCH ->
+                                CandidateEvaluationState.NO_MATCH
+                            CandidateBatchConclusion.MANUAL_REQUIRED ->
+                                CandidateEvaluationState.MANUAL_REVIEW
+                        }
+                    }
+                    is CandidateEvaluationResult.Failed -> {
+                        setCandidateEvaluationFailure(result.code)
+                    }
+                }
+            } catch (error: CancellationException) {
+                candidateEvaluationState.value = CandidateEvaluationState.STOPPED
+                throw error
+            } finally {
+                candidateEvaluationJob = null
+            }
+        }
+    }
+
+    private fun stopCandidateEvaluation() {
+        candidateEvaluationJob?.cancel()
+    }
+
+    private fun acceptRecommendedCandidate() {
+        candidateEvaluationState.value = CandidateHumanReviewPolicy.accept(
+            currentState = candidateEvaluationState.value,
+            batch = candidateReviewBatch.value
+        )
+    }
+
+    private fun rejectCandidateReview() {
+        candidateEvaluationState.value = CandidateHumanReviewPolicy.reject(
+            candidateEvaluationState.value
+        )
+    }
+
+    private fun setCandidateEvaluationFailure(
+        code: CandidateEvaluationFailureCode
+    ) {
+        candidateReviewBatch.value = null
+        candidateEvaluationFailureCode.value = code
+        candidateEvaluationState.value = CandidateEvaluationState.FAILED
+    }
+
+    private fun clearCandidateEvaluation() {
+        candidateEvaluationJob?.cancel()
+        candidateReviewBatch.value = null
+        candidateEvaluationFailureCode.value = null
+        candidateEvaluationState.value = CandidateEvaluationState.IDLE
+    }
+
    private fun checkShizukuPermission(): Boolean {
        return try {
            val granted = Shizuku.checkSelfPermission() == PackageManager.PERMISSION_GRANTED