From c25d63c730ede2307821ae8069ae879726d48e4a Mon Sep 17 00:00:00 2001 From: QiuSW <105186638@qq.com> Date: Sat, 25 Jul 2026 20:18:05 +0800 Subject: [PATCH] feat(android): collect bounded Pinduoduo candidates --- .../java/com/roubao/autopilot/MainActivity.kt | 39 +- .../accessibility/BuyerAccessibilityBridge.kt | 38 ++ .../BuyerAccessibilityService.kt | 399 ++++++++++++++++++ .../AndroidPinduoduoCandidateDriver.kt | 154 +++++++ .../pinduoduo/AndroidPinduoduoUiDriver.kt | 3 + .../pinduoduo/PinduoduoCandidateAutomation.kt | 205 +++++++++ .../pinduoduo/PinduoduoCandidateModels.kt | 59 +++ .../pinduoduo/PinduoduoPageClassifier.kt | 8 + .../pinduoduo/PinduoduoSearchAutomation.kt | 16 +- .../autopilot/ui/screens/SearchProbeScreen.kt | 56 ++- .../autopilot/workflow/WorkflowModels.kt | 3 +- .../res/xml/buyer_accessibility_service.xml | 1 + .../PinduoduoCandidateAutomationTest.kt | 260 ++++++++++++ .../pinduoduo/PinduoduoPageClassifierTest.kt | 13 + .../PinduoduoSearchAutomationTest.kt | 25 ++ docs/00-ai-start-here.md | 6 +- docs/02-requirements.md | 6 +- docs/03-tech-stack.md | 7 +- docs/04-architecture.md | 19 + docs/05-coding-rules.md | 6 + docs/current-state.md | 25 +- docs/tasks/T-102.md | 126 ++++++ progress.md | 8 + 23 files changed, 1449 insertions(+), 33 deletions(-) create mode 100644 android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/AndroidPinduoduoCandidateDriver.kt create mode 100644 android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/PinduoduoCandidateAutomation.kt create mode 100644 android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/PinduoduoCandidateModels.kt create mode 100644 android-buyer/app/src/test/java/com/roubao/autopilot/pinduoduo/PinduoduoCandidateAutomationTest.kt create mode 100644 docs/tasks/T-102.md diff --git a/android-buyer/app/src/main/java/com/roubao/autopilot/MainActivity.kt b/android-buyer/app/src/main/java/com/roubao/autopilot/MainActivity.kt index 74935a3..1f1c259 100644 --- a/android-buyer/app/src/main/java/com/roubao/autopilot/MainActivity.kt +++ b/android-buyer/app/src/main/java/com/roubao/autopilot/MainActivity.kt @@ -48,8 +48,13 @@ import kotlinx.coroutines.launch import rikka.shizuku.Shizuku import android.util.Log import com.roubao.autopilot.pinduoduo.AndroidPinduoduoUiDriver +import com.roubao.autopilot.pinduoduo.AndroidPinduoduoCandidateDriver +import com.roubao.autopilot.pinduoduo.PinduoduoCandidateAutomation +import com.roubao.autopilot.pinduoduo.CandidateBrowsePhase +import com.roubao.autopilot.pinduoduo.PinduoduoCandidateEvidence +import com.roubao.autopilot.pinduoduo.PinduoduoCandidateWorkflow +import com.roubao.autopilot.pinduoduo.PinduoduoProbeAutomation import com.roubao.autopilot.pinduoduo.PinduoduoSearchAutomation -import com.roubao.autopilot.pinduoduo.PinduoduoSearchWorkflow import com.roubao.autopilot.workflow.WorkflowReport import com.roubao.autopilot.workflow.WorkflowRunner import com.roubao.autopilot.workflow.WorkflowState @@ -76,6 +81,8 @@ class MainActivity : ComponentActivity() { private val searchProbeState = mutableStateOf(WorkflowState.IDLE) private val searchProbeStepId = mutableStateOf(null) private val searchProbeReport = mutableStateOf(null) + private val candidateEvidence = + mutableStateOf>(emptyList()) private var searchProbeRunner: WorkflowRunner? = null private var searchProbeJob: Job? = null @@ -198,6 +205,7 @@ class MainActivity : ComponentActivity() { val probeState by remember { searchProbeState } val probeStepId by remember { searchProbeStepId } val probeReport by remember { searchProbeReport } + val evidence by remember { candidateEvidence } // 监听跳转事件 LaunchedEffect(navigateToRecord, recordId) { @@ -286,6 +294,7 @@ class MainActivity : ComponentActivity() { state = probeState, currentStepId = probeStepId, report = probeReport, + candidateEvidenceCount = evidence.size, onStart = { startSearchProbe() }, onStop = { stopSearchProbe() } ) @@ -384,13 +393,23 @@ class MainActivity : ComponentActivity() { return } + val candidateAutomation = PinduoduoCandidateAutomation( + AndroidPinduoduoCandidateDriver(this) + ) + candidateAutomation.reset() val runner = WorkflowRunner( - PinduoduoSearchAutomation(AndroidPinduoduoUiDriver(this)) + PinduoduoProbeAutomation( + searchAutomation = PinduoduoSearchAutomation( + AndroidPinduoduoUiDriver(this) + ), + candidateAutomation = candidateAutomation + ) ) searchProbeRunner = runner searchProbeReport.value = null searchProbeState.value = WorkflowState.IDLE searchProbeStepId.value = null + candidateEvidence.value = emptyList() searchProbeJob = lifecycleScope.launch { val stateCollector = launch { runner.state.collect { state -> searchProbeState.value = state } @@ -398,13 +417,27 @@ class MainActivity : ComponentActivity() { val stepCollector = launch { runner.currentStepId.collect { stepId -> searchProbeStepId.value = stepId } } + val evidenceCollector = launch { + candidateAutomation.evidence.collect { evidence -> + candidateEvidence.value = evidence + } + } + val candidatePhaseCollector = launch { + candidateAutomation.phase.collect { phase -> + if (phase != CandidateBrowsePhase.IDLE) { + Log.d(TAG, "Candidate probe phase: ${phase.name}") + } + } + } try { - val report = runner.run(PinduoduoSearchWorkflow.steps()) + val report = runner.run(PinduoduoCandidateWorkflow.steps()) searchProbeReport.value = report searchProbeState.value = report.state } finally { stateCollector.cancel() stepCollector.cancel() + evidenceCollector.cancel() + candidatePhaseCollector.cancel() searchProbeRunner = null refreshReadiness() } diff --git a/android-buyer/app/src/main/java/com/roubao/autopilot/accessibility/BuyerAccessibilityBridge.kt b/android-buyer/app/src/main/java/com/roubao/autopilot/accessibility/BuyerAccessibilityBridge.kt index bd173a8..d83060b 100644 --- a/android-buyer/app/src/main/java/com/roubao/autopilot/accessibility/BuyerAccessibilityBridge.kt +++ b/android-buyer/app/src/main/java/com/roubao/autopilot/accessibility/BuyerAccessibilityBridge.kt @@ -1,9 +1,13 @@ package com.roubao.autopilot.accessibility import com.roubao.autopilot.pinduoduo.PinduoduoPage +import com.roubao.autopilot.pinduoduo.PinduoduoCandidateCard +import com.roubao.autopilot.pinduoduo.PinduoduoCandidateDetailEvidence +import com.roubao.autopilot.pinduoduo.PinduoduoScreenshotCapture import com.roubao.autopilot.pinduoduo.PinduoduoUiSnapshot import com.roubao.autopilot.readiness.DeviceObservationStore import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withTimeoutOrNull import kotlinx.coroutines.withContext object BuyerAccessibilityBridge { @@ -46,4 +50,38 @@ object BuyerAccessibilityBridge { suspend fun submitSearch(): Boolean = withContext(Dispatchers.Main.immediate) { service?.submitPinduoduoSearch() == true } + + suspend fun candidateCards(limit: Int): List = + withContext(Dispatchers.Main.immediate) { + service?.readPinduoduoCandidateCards(limit).orEmpty() + } + + suspend fun openCandidate(signature: String): Boolean = + withContext(Dispatchers.Main.immediate) { + service?.openPinduoduoCandidate(signature) == true + } + + suspend fun candidateDetailEvidence(): PinduoduoCandidateDetailEvidence? = + withContext(Dispatchers.Main.immediate) { + service?.readPinduoduoCandidateDetailEvidence() + } + + suspend fun captureScreenshot(): PinduoduoScreenshotCapture? = + withTimeoutOrNull(SCREENSHOT_TIMEOUT_MILLIS) { + withContext(Dispatchers.Main.immediate) { + service?.capturePinduoduoScreenshot() + } + } + + suspend fun returnToResults(): Boolean = + withContext(Dispatchers.Main.immediate) { + service?.returnFromPinduoduoCandidate() == true + } + + suspend fun scrollResults(): Boolean = + withContext(Dispatchers.Main.immediate) { + service?.scrollPinduoduoResults() == true + } + + private const val SCREENSHOT_TIMEOUT_MILLIS = 8_000L } diff --git a/android-buyer/app/src/main/java/com/roubao/autopilot/accessibility/BuyerAccessibilityService.kt b/android-buyer/app/src/main/java/com/roubao/autopilot/accessibility/BuyerAccessibilityService.kt index 67b8268..1fc1e93 100644 --- a/android-buyer/app/src/main/java/com/roubao/autopilot/accessibility/BuyerAccessibilityService.kt +++ b/android-buyer/app/src/main/java/com/roubao/autopilot/accessibility/BuyerAccessibilityService.kt @@ -1,11 +1,19 @@ package com.roubao.autopilot.accessibility import android.accessibilityservice.AccessibilityService +import android.graphics.Bitmap +import android.graphics.Rect +import android.os.Build import android.os.Bundle import android.os.SystemClock import android.util.Log +import android.view.Display import android.view.accessibility.AccessibilityEvent import android.view.accessibility.AccessibilityNodeInfo +import androidx.annotation.RequiresApi +import com.roubao.autopilot.pinduoduo.PinduoduoCandidateCard +import com.roubao.autopilot.pinduoduo.PinduoduoCandidateDetailEvidence +import com.roubao.autopilot.pinduoduo.PinduoduoEvidenceHash import com.roubao.autopilot.readiness.DeviceObservationStore import com.roubao.autopilot.readiness.LoginBlockerDetector import com.roubao.autopilot.readiness.PINDUODUO_PACKAGE @@ -14,11 +22,17 @@ import com.roubao.autopilot.pinduoduo.PinduoduoUiElement import com.roubao.autopilot.pinduoduo.PinduoduoUiSnapshot import com.roubao.autopilot.pinduoduo.PinduoduoPage import com.roubao.autopilot.pinduoduo.SEARCH_PROBE_KEYWORD +import com.roubao.autopilot.pinduoduo.PinduoduoScreenshotCapture +import java.io.ByteArrayOutputStream import java.util.ArrayDeque +import java.util.concurrent.Executors +import kotlin.coroutines.resume +import kotlinx.coroutines.suspendCancellableCoroutine class BuyerAccessibilityService : AccessibilityService() { private var lastPinduoduoScanAt = 0L private var expectedSearchQuery = SEARCH_PROBE_KEYWORD + private val screenshotExecutor = Executors.newSingleThreadExecutor() override fun onServiceConnected() { super.onServiceConnected() @@ -62,6 +76,7 @@ class BuyerAccessibilityService : AccessibilityService() { override fun onDestroy() { BuyerAccessibilityBridge.detach(this) DeviceObservationStore.setAccessibilityConnected(false) + screenshotExecutor.shutdownNow() super.onDestroy() } @@ -172,6 +187,165 @@ class BuyerAccessibilityService : AccessibilityService() { ) == true } ?: false + internal fun readPinduoduoCandidateCards( + limit: Int + ): List = + withPinduoduoRoot { root -> + val snapshot = classifyPinduoduoRoot(root) + if ( + snapshot.safetyStopReason != null || + snapshot.page != PinduoduoPage.SEARCH_RESULTS + ) { + return@withPinduoduoRoot emptyList() + } + candidateNodes(root, limit).map(CandidateNode::card) + } ?: emptyList() + + internal fun openPinduoduoCandidate(signature: String): Boolean = + withPinduoduoRoot { root -> + val snapshot = classifyPinduoduoRoot(root) + if ( + snapshot.safetyStopReason != null || + snapshot.page != PinduoduoPage.SEARCH_RESULTS + ) { + return@withPinduoduoRoot false + } + candidateNodes(root, MAX_CANDIDATE_NODE_SCAN) + .filter { candidate -> candidate.card.signature == signature } + .singleOrNull() + ?.node + ?.performAction(AccessibilityNodeInfo.ACTION_CLICK) == true + } ?: false + + internal fun readPinduoduoCandidateDetailEvidence(): + PinduoduoCandidateDetailEvidence? = + withPinduoduoRoot { root -> + if (!isVerifiedProductDetail(root)) { + return@withPinduoduoRoot null + } + val semanticTexts = collectSemanticTexts(root) + if (semanticTexts.isEmpty()) { + return@withPinduoduoRoot null + } + PinduoduoCandidateDetailEvidence( + signature = PinduoduoEvidenceHash.sha256( + semanticTexts.joinToString(TEXT_SIGNATURE_SEPARATOR) + ), + semanticTextCount = semanticTexts.size + ) + } + + internal fun returnFromPinduoduoCandidate(): Boolean = + withPinduoduoRoot { root -> + if (!isVerifiedProductDetail(root)) { + return@withPinduoduoRoot false + } + performGlobalAction(GLOBAL_ACTION_BACK) + } ?: false + + internal fun scrollPinduoduoResults(): Boolean = + withPinduoduoRoot { root -> + val snapshot = classifyPinduoduoRoot(root) + if ( + snapshot.safetyStopReason != null || + snapshot.page != PinduoduoPage.SEARCH_RESULTS + ) { + return@withPinduoduoRoot false + } + findMainResultsRecycler(root)?.performAction( + AccessibilityNodeInfo.ACTION_SCROLL_FORWARD + ) == true + } ?: false + + internal suspend fun capturePinduoduoScreenshot(): PinduoduoScreenshotCapture? { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) { + return null + } + return capturePinduoduoScreenshotApi30() + } + + @RequiresApi(Build.VERSION_CODES.R) + private suspend fun capturePinduoduoScreenshotApi30(): + PinduoduoScreenshotCapture? { + val root = rootInActiveWindow + if ( + root?.packageName?.toString() != PINDUODUO_PACKAGE || + !isVerifiedProductDetail(root) + ) { + return null + } + + return suspendCancellableCoroutine { continuation -> + val callback = object : TakeScreenshotCallback { + override fun onSuccess(screenshot: ScreenshotResult) { + val hardwareBuffer = screenshot.hardwareBuffer + var hardwareBitmap: Bitmap? = null + var softwareBitmap: Bitmap? = null + val capture = try { + hardwareBitmap = Bitmap.wrapHardwareBuffer( + hardwareBuffer, + screenshot.colorSpace + ) + softwareBitmap = hardwareBitmap?.copy( + Bitmap.Config.ARGB_8888, + false + ) + val bitmap = softwareBitmap + if (bitmap == null) { + null + } else { + val output = ByteArrayOutputStream() + if ( + !bitmap.compress( + Bitmap.CompressFormat.PNG, + 100, + output + ) + ) { + null + } else { + PinduoduoScreenshotCapture( + pngBytes = output.toByteArray(), + width = bitmap.width, + height = bitmap.height + ) + } + } + } catch (_: Exception) { + Log.w(TAG, "Pinduoduo screenshot conversion failed") + null + } finally { + softwareBitmap?.recycle() + hardwareBitmap?.recycle() + hardwareBuffer.close() + } + if (continuation.isActive) { + continuation.resume(capture) + } + } + + override fun onFailure(errorCode: Int) { + Log.w(TAG, "Pinduoduo screenshot failed with code $errorCode") + if (continuation.isActive) { + continuation.resume(null) + } + } + } + try { + takeScreenshot( + Display.DEFAULT_DISPLAY, + screenshotExecutor, + callback + ) + } catch (_: Exception) { + Log.w(TAG, "Pinduoduo screenshot request failed") + if (continuation.isActive) { + continuation.resume(null) + } + } + } + } + private fun classifyPinduoduoRoot( root: AccessibilityNodeInfo ): PinduoduoUiSnapshot { @@ -194,6 +368,205 @@ class BuyerAccessibilityService : AccessibilityService() { ) } + private fun candidateNodes( + root: AccessibilityNodeInfo, + limit: Int + ): List { + if (limit <= 0) { + return emptyList() + } + val recycler = findMainResultsRecycler(root) ?: return emptyList() + val rootBounds = Rect().also(root::getBoundsInScreen) + val recyclerBounds = Rect().also(recycler::getBoundsInScreen) + val minimumHeight = (rootBounds.height() / 8).coerceAtLeast(1) + val candidates = ArrayList() + + for (index in 0 until recycler.childCount) { + val node = recycler.getChild(index) ?: continue + if ( + !node.isVisibleToUser || + !node.isEnabled || + node.className?.toString()?.endsWith("FrameLayout") != true + ) { + continue + } + val bounds = Rect().also(node::getBoundsInScreen) + if ( + bounds.width() * 100 < recyclerBounds.width() * MIN_CARD_WIDTH_PERCENT || + bounds.width() * 100 > recyclerBounds.width() * MAX_CARD_WIDTH_PERCENT || + bounds.height() < minimumHeight + ) { + continue + } + val visibleBounds = Rect(bounds) + if ( + !visibleBounds.intersect(recyclerBounds) || + area(visibleBounds) * 100 < area(bounds) * MIN_VISIBLE_CARD_PERCENT + ) { + continue + } + + val descendants = collectNodes(node) + val semanticTexts = collectSemanticTexts(descendants) + val hasImage = descendants.any { descendant -> + descendant.isVisibleToUser && + descendant.isEnabled && + descendant.className?.toString()?.endsWith("ImageView") == true + } + val hasPrice = semanticTexts.any(::hasPriceSemantics) + val clickTarget = findSafeCandidateClickTarget(node, descendants) + if ( + semanticTexts.size < MIN_CANDIDATE_TEXTS || + !hasImage || + !hasPrice || + clickTarget == null + ) { + continue + } + val card = PinduoduoCandidateCard( + signature = PinduoduoEvidenceHash.sha256( + semanticTexts.joinToString(TEXT_SIGNATURE_SEPARATOR) + ), + semanticTextCount = semanticTexts.size, + hasImage = true + ) + candidates += CandidateNode(clickTarget, card) + if (candidates.size >= limit) { + break + } + } + return candidates + } + + private fun isVerifiedProductDetail(root: AccessibilityNodeInfo): Boolean { + val snapshot = classifyPinduoduoRoot(root) + if ( + snapshot.safetyStopReason != null || + snapshot.page != PinduoduoPage.PRODUCT_DETAIL + ) { + return false + } + val rootBounds = Rect().also(root::getBoundsInScreen) + val nodes = collectNodes(root) + val hasWideViewPager = nodes.any { node -> + if ( + !node.isVisibleToUser || + !node.isEnabled || + node.className?.toString()?.endsWith("ViewPager") != true + ) { + return@any false + } + val bounds = Rect().also(node::getBoundsInScreen) + bounds.width() * 100 >= + rootBounds.width() * MIN_DETAIL_CONTENT_WIDTH_PERCENT + } + val hasWideScrollableRecycler = nodes.any { node -> + if ( + !node.isVisibleToUser || + !node.isEnabled || + !node.isScrollable || + node.className?.toString()?.endsWith("RecyclerView") != true + ) { + return@any false + } + val bounds = Rect().also(node::getBoundsInScreen) + bounds.width() * 100 >= + rootBounds.width() * MIN_DETAIL_CONTENT_WIDTH_PERCENT && + collectNodes(node).size >= MIN_DETAIL_RECYCLER_NODES + } + return hasWideViewPager && hasWideScrollableRecycler + } + + private fun findMainResultsRecycler( + root: AccessibilityNodeInfo + ): AccessibilityNodeInfo? { + val rootBounds = Rect().also(root::getBoundsInScreen) + return collectNodes(root) + .asSequence() + .filter { node -> + node.isVisibleToUser && + node.isEnabled && + node.isScrollable && + node.className?.toString()?.endsWith("RecyclerView") == true + } + .map { node -> + val bounds = Rect().also(node::getBoundsInScreen) + RecyclerCandidate( + node = node, + bounds = bounds, + descendantCount = collectNodes(node).size + ) + } + .filter { candidate -> + candidate.bounds.width() * 100 >= + rootBounds.width() * MIN_MAIN_RECYCLER_WIDTH_PERCENT && + candidate.descendantCount >= MIN_MAIN_RECYCLER_NODES + } + .maxByOrNull { candidate -> area(candidate.bounds) } + ?.node + } + + private fun findSafeCandidateClickTarget( + cardRoot: AccessibilityNodeInfo, + descendants: Collection + ): AccessibilityNodeInfo? { + if (cardRoot.isClickable) { + return cardRoot + } + val cardBounds = Rect().also(cardRoot::getBoundsInScreen) + val coveringTargets = descendants.filter { node -> + if ( + node === cardRoot || + !node.isVisibleToUser || + !node.isEnabled || + !node.isClickable + ) { + return@filter false + } + val targetBounds = Rect().also(node::getBoundsInScreen) + val overlap = Rect(targetBounds) + overlap.intersect(cardBounds) && + area(overlap) * 100 >= + area(cardBounds) * MIN_CLICK_TARGET_COVER_PERCENT + } + return coveringTargets.singleOrNull() + } + + private fun hasPriceSemantics(value: String): Boolean = + value.contains('¥') || + value.contains('￥') || + DECIMAL_PRICE_PATTERN.matches(value) + + private fun area(bounds: Rect): Long = + bounds.width().toLong() * bounds.height().toLong() + + private fun collectSemanticTexts( + root: AccessibilityNodeInfo + ): List = collectSemanticTexts(collectNodes(root)) + + private fun collectSemanticTexts( + nodes: Collection + ): List = + nodes.asSequence() + .filter { node -> node.isVisibleToUser && node.isEnabled } + .flatMap { node -> + sequenceOf( + node.text?.toString(), + node.contentDescription?.toString() + ) + } + .filterNotNull() + .map(::normalizeEvidenceText) + .filter(String::isNotEmpty) + .distinct() + .take(MAX_EVIDENCE_TEXTS) + .toList() + + private fun normalizeEvidenceText(value: String): String = + value.trim() + .replace(Regex("\\s+"), " ") + .take(MAX_EVIDENCE_TEXT_LENGTH) + private fun shouldInspect(eventType: Int): Boolean = eventType == AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED || eventType == AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED || @@ -252,10 +625,36 @@ class BuyerAccessibilityService : AccessibilityService() { return false } + private data class CandidateNode( + val node: AccessibilityNodeInfo, + val card: PinduoduoCandidateCard + ) + + private data class RecyclerCandidate( + val node: AccessibilityNodeInfo, + val bounds: Rect, + val descendantCount: Int + ) + companion object { private const val TAG = "BuyerAccessibility" private const val MAX_NODES = 300 private const val MAX_CLICK_ANCESTORS = 4 + private const val MAX_CANDIDATE_NODE_SCAN = 20 + private const val MIN_CANDIDATE_TEXTS = 2 + private const val MIN_MAIN_RECYCLER_NODES = 20 + private const val MIN_MAIN_RECYCLER_WIDTH_PERCENT = 90 + private const val MIN_DETAIL_CONTENT_WIDTH_PERCENT = 90 + private const val MIN_DETAIL_RECYCLER_NODES = 40 + private const val MIN_CARD_WIDTH_PERCENT = 35 + private const val MAX_CARD_WIDTH_PERCENT = 60 + private const val MIN_VISIBLE_CARD_PERCENT = 80 + private const val MIN_CLICK_TARGET_COVER_PERCENT = 85 + private const val MAX_EVIDENCE_TEXTS = 100 + private const val MAX_EVIDENCE_TEXT_LENGTH = 500 + private const val TEXT_SIGNATURE_SEPARATOR = "\u001f" private const val MIN_SCAN_INTERVAL_MS = 300L + private val DECIMAL_PRICE_PATTERN = + Regex("^\\s*\\d{1,6}\\.\\d{1,2}\\s*$") } } diff --git a/android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/AndroidPinduoduoCandidateDriver.kt b/android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/AndroidPinduoduoCandidateDriver.kt new file mode 100644 index 0000000..759f31f --- /dev/null +++ b/android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/AndroidPinduoduoCandidateDriver.kt @@ -0,0 +1,154 @@ +package com.roubao.autopilot.pinduoduo + +import android.content.Context +import com.roubao.autopilot.accessibility.BuyerAccessibilityBridge +import java.io.File +import java.io.IOException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import org.json.JSONArray +import org.json.JSONObject + +class AndroidPinduoduoCandidateDriver( + context: Context +) : PinduoduoCandidateDriver { + private val evidenceStore = CandidateEvidenceStore(context.applicationContext) + + override suspend fun snapshot(): PinduoduoUiSnapshot = + BuyerAccessibilityBridge.snapshot() + + override suspend fun candidateCards( + limit: Int + ): List = + BuyerAccessibilityBridge.candidateCards(limit) + + override suspend fun openCandidate(signature: String): Boolean = + BuyerAccessibilityBridge.openCandidate(signature) + + override suspend fun captureCandidate( + ordinal: Int, + card: PinduoduoCandidateCard + ): PinduoduoCandidateEvidence? { + val detail = BuyerAccessibilityBridge.candidateDetailEvidence() + ?: return null + val screenshot = BuyerAccessibilityBridge.captureScreenshot() + ?: return null + return withContext(Dispatchers.IO) { + evidenceStore.save( + ordinal = ordinal, + card = card, + detail = detail, + screenshot = screenshot + ) + } + } + + override suspend fun returnToResults(): Boolean = + BuyerAccessibilityBridge.returnToResults() + + override suspend fun scrollResults(): Boolean = + BuyerAccessibilityBridge.scrollResults() + + override fun resetEvidence() { + evidenceStore.reset() + } +} + +private class CandidateEvidenceStore(context: Context) { + private val root = File(context.cacheDir, EVIDENCE_DIRECTORY) + private val evidenceByOrdinal = + linkedMapOf() + + fun reset() { + evidenceByOrdinal.clear() + if (root.exists()) { + root.deleteRecursively() + } + root.mkdirs() + } + + fun save( + ordinal: Int, + card: PinduoduoCandidateCard, + detail: PinduoduoCandidateDetailEvidence, + screenshot: PinduoduoScreenshotCapture + ): PinduoduoCandidateEvidence? { + if (ordinal !in 1..MAX_CANDIDATES_PER_PROBE) { + return null + } + return try { + if (!root.exists() && !root.mkdirs()) { + return null + } + val fileName = "candidate-%02d.png".format(ordinal) + val screenshotFile = File(root, fileName) + screenshotFile.writeBytes(screenshot.pngBytes) + val evidence = PinduoduoCandidateEvidence( + ordinal = ordinal, + cardSignature = card.signature, + cardSemanticTextCount = card.semanticTextCount, + detailSignature = detail.signature, + detailSemanticTextCount = detail.semanticTextCount, + screenshotFileName = fileName, + screenshotSha256 = PinduoduoEvidenceHash.sha256( + screenshot.pngBytes + ), + screenshotByteCount = screenshot.pngBytes.size, + screenshotWidth = screenshot.width, + screenshotHeight = screenshot.height + ) + evidenceByOrdinal[ordinal] = evidence + writeManifest() + evidence + } catch (_: IOException) { + null + } + } + + private fun writeManifest() { + val candidates = JSONArray() + evidenceByOrdinal.values.forEach { evidence -> + candidates.put( + JSONObject() + .put("ordinal", evidence.ordinal) + .put("card_signature", evidence.cardSignature) + .put( + "card_semantic_text_count", + evidence.cardSemanticTextCount + ) + .put("detail_signature", evidence.detailSignature) + .put( + "detail_semantic_text_count", + evidence.detailSemanticTextCount + ) + .put("screenshot_file", evidence.screenshotFileName) + .put("screenshot_sha256", evidence.screenshotSha256) + .put( + "screenshot_byte_count", + evidence.screenshotByteCount + ) + .put("screenshot_width", evidence.screenshotWidth) + .put("screenshot_height", evidence.screenshotHeight) + ) + } + val manifest = JSONObject() + .put("schema_version", 1) + .put("candidate_count", evidenceByOrdinal.size) + .put("candidates", candidates) + .toString(2) + val temporary = File(root, "$MANIFEST_FILE.tmp") + val destination = File(root, MANIFEST_FILE) + temporary.writeText(manifest, Charsets.UTF_8) + if (destination.exists() && !destination.delete()) { + throw IOException("Could not replace candidate evidence manifest") + } + if (!temporary.renameTo(destination)) { + throw IOException("Could not publish candidate evidence manifest") + } + } + + private companion object { + const val EVIDENCE_DIRECTORY = "pdd-candidate-probe" + const val MANIFEST_FILE = "manifest.json" + } +} diff --git a/android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/AndroidPinduoduoUiDriver.kt b/android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/AndroidPinduoduoUiDriver.kt index 30e6c92..ed49dc7 100644 --- a/android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/AndroidPinduoduoUiDriver.kt +++ b/android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/AndroidPinduoduoUiDriver.kt @@ -42,6 +42,9 @@ class AndroidPinduoduoUiDriver(context: Context) : PinduoduoUiDriver { override suspend fun submitSearch(): Boolean = BuyerAccessibilityBridge.submitSearch() + override suspend fun returnFromCandidate(): Boolean = + BuyerAccessibilityBridge.returnToResults() + private companion object { const val KEYWORD_VERIFY_ATTEMPTS = 5 const val KEYWORD_VERIFY_INTERVAL_MS = 100L diff --git a/android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/PinduoduoCandidateAutomation.kt b/android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/PinduoduoCandidateAutomation.kt new file mode 100644 index 0000000..22a68f2 --- /dev/null +++ b/android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/PinduoduoCandidateAutomation.kt @@ -0,0 +1,205 @@ +package com.roubao.autopilot.pinduoduo + +import com.roubao.autopilot.readiness.PINDUODUO_PACKAGE +import com.roubao.autopilot.workflow.AutomationGateway +import com.roubao.autopilot.workflow.AutomationResult +import com.roubao.autopilot.workflow.SafetyStopReason +import com.roubao.autopilot.workflow.WorkflowFailureCode +import com.roubao.autopilot.workflow.WorkflowStep +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow + +enum class CandidateBrowsePhase { + IDLE, + READING_RESULTS, + OPENING_CANDIDATE, + WAITING_DETAIL, + CAPTURING_EVIDENCE, + RETURNING_RESULTS, + SCROLLING_RESULTS, + COMPLETE +} + +class PinduoduoCandidateAutomation( + private val driver: PinduoduoCandidateDriver, + private val maxCandidates: Int = MAX_CANDIDATES_PER_PROBE, + private val maxResultScrolls: Int = MAX_RESULT_SCROLLS_PER_PROBE, + private val pagePollIntervalMillis: Long = 200, + private val unknownPageLimit: Int = 20 +) : AutomationGateway { + private val mutableEvidence = + MutableStateFlow>(emptyList()) + private val mutablePhase = MutableStateFlow(CandidateBrowsePhase.IDLE) + private val attemptedSignatures = linkedSetOf() + private var resultScrollCount = 0 + + val evidence: StateFlow> = + mutableEvidence.asStateFlow() + val phase: StateFlow = mutablePhase.asStateFlow() + + init { + require(maxCandidates in 1..MAX_CANDIDATES_PER_PROBE) + require(maxResultScrolls in 0..MAX_RESULT_SCROLLS_PER_PROBE) + require(pagePollIntervalMillis > 0) + require(unknownPageLimit > 0) + } + + fun reset() { + attemptedSignatures.clear() + resultScrollCount = 0 + mutableEvidence.value = emptyList() + mutablePhase.value = CandidateBrowsePhase.IDLE + driver.resetEvidence() + } + + override suspend fun execute(step: WorkflowStep): AutomationResult { + if (step.id != PinduoduoCandidateWorkflow.BROWSE_CANDIDATES) { + return AutomationResult.FatalFailure( + WorkflowFailureCode.AUTOMATION_EXCEPTION + ) + } + return browseCandidates() + } + + private suspend fun browseCandidates(): AutomationResult { + mutablePhase.value = CandidateBrowsePhase.READING_RESULTS + recoverDetailPageIfNeeded()?.let { return it } + + while ( + mutableEvidence.value.size < maxCandidates && + attemptedSignatures.size < maxCandidates + ) { + validateResultsPage()?.let { return it } + val nextCard = driver.candidateCards(maxCandidates * 2) + .firstOrNull { card -> card.signature !in attemptedSignatures } + + if (nextCard == null) { + if (resultScrollCount >= maxResultScrolls) { + return terminalCollectionResult() + } + resultScrollCount += 1 + mutablePhase.value = CandidateBrowsePhase.SCROLLING_RESULTS + if (!driver.scrollResults()) { + return terminalCollectionResult() + } + delay(pagePollIntervalMillis) + mutablePhase.value = CandidateBrowsePhase.READING_RESULTS + continue + } + + attemptedSignatures += nextCard.signature + mutablePhase.value = CandidateBrowsePhase.OPENING_CANDIDATE + if (!driver.openCandidate(nextCard.signature)) { + return AutomationResult.RetryableFailure( + WorkflowFailureCode.TRANSIENT_AUTOMATION + ) + } + + mutablePhase.value = CandidateBrowsePhase.WAITING_DETAIL + awaitPage(PinduoduoPage.PRODUCT_DETAIL)?.let { return it } + val ordinal = mutableEvidence.value.size + 1 + mutablePhase.value = CandidateBrowsePhase.CAPTURING_EVIDENCE + val captured = driver.captureCandidate(ordinal, nextCard) + ?: return AutomationResult.FatalFailure( + WorkflowFailureCode.EVIDENCE_CAPTURE_FAILED + ) + mutableEvidence.value = mutableEvidence.value + captured + + mutablePhase.value = CandidateBrowsePhase.RETURNING_RESULTS + if (!driver.returnToResults()) { + return AutomationResult.RetryableFailure( + WorkflowFailureCode.TRANSIENT_AUTOMATION + ) + } + awaitPage(PinduoduoPage.SEARCH_RESULTS)?.let { return it } + mutablePhase.value = CandidateBrowsePhase.READING_RESULTS + } + + return terminalCollectionResult() + } + + private suspend fun recoverDetailPageIfNeeded(): AutomationResult? { + val snapshot = driver.snapshot() + safetyResult(snapshot)?.let { return it } + if (snapshot.page != PinduoduoPage.PRODUCT_DETAIL) { + return null + } + if (!driver.returnToResults()) { + return AutomationResult.RetryableFailure( + WorkflowFailureCode.TRANSIENT_AUTOMATION + ) + } + return awaitPage(PinduoduoPage.SEARCH_RESULTS) + } + + private suspend fun validateResultsPage(): AutomationResult? { + val snapshot = driver.snapshot() + safetyResult(snapshot)?.let { return it } + return if (snapshot.page == PinduoduoPage.SEARCH_RESULTS) { + null + } else { + AutomationResult.Blocked(SafetyStopReason.UNKNOWN_PAGE) + } + } + + private suspend fun awaitPage(expected: PinduoduoPage): AutomationResult? { + var stableUnexpectedObservations = 0 + while (true) { + val snapshot = driver.snapshot() + safetyResult(snapshot)?.let { return it } + if (snapshot.page == expected) { + return null + } + + stableUnexpectedObservations = if ( + snapshot.foregroundPackage == PINDUODUO_PACKAGE + ) { + stableUnexpectedObservations + 1 + } else { + 0 + } + if (stableUnexpectedObservations >= unknownPageLimit) { + return AutomationResult.Blocked(SafetyStopReason.UNKNOWN_PAGE) + } + delay(pagePollIntervalMillis) + } + } + + private fun terminalCollectionResult(): AutomationResult { + mutablePhase.value = CandidateBrowsePhase.COMPLETE + return if (mutableEvidence.value.isNotEmpty()) { + AutomationResult.Success + } else { + AutomationResult.FatalFailure(WorkflowFailureCode.TARGET_NOT_READY) + } + } + + private fun safetyResult(snapshot: PinduoduoUiSnapshot): AutomationResult.Blocked? = + snapshot.safetyStopReason?.let(AutomationResult::Blocked) +} + +class PinduoduoProbeAutomation( + private val searchAutomation: PinduoduoSearchAutomation, + private val candidateAutomation: PinduoduoCandidateAutomation +) : AutomationGateway { + override suspend fun execute(step: WorkflowStep): AutomationResult = + if (step.id == PinduoduoCandidateWorkflow.BROWSE_CANDIDATES) { + candidateAutomation.execute(step) + } else { + searchAutomation.execute(step) + } +} + +object PinduoduoCandidateWorkflow { + const val BROWSE_CANDIDATES = "pdd_browse_candidates" + + fun steps(): List = + PinduoduoSearchWorkflow.steps() + + WorkflowStep( + id = BROWSE_CANDIDATES, + timeoutMillis = 120_000, + maxRetries = 1 + ) +} diff --git a/android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/PinduoduoCandidateModels.kt b/android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/PinduoduoCandidateModels.kt new file mode 100644 index 0000000..fdb5d1f --- /dev/null +++ b/android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/PinduoduoCandidateModels.kt @@ -0,0 +1,59 @@ +package com.roubao.autopilot.pinduoduo + +import java.security.MessageDigest + +const val MAX_CANDIDATES_PER_PROBE = 5 +const val MAX_RESULT_SCROLLS_PER_PROBE = 2 + +data class PinduoduoCandidateCard( + val signature: String, + val semanticTextCount: Int, + val hasImage: Boolean +) + +data class PinduoduoCandidateDetailEvidence( + val signature: String, + val semanticTextCount: Int +) + +data class PinduoduoScreenshotCapture( + val pngBytes: ByteArray, + val width: Int, + val height: Int +) + +data class PinduoduoCandidateEvidence( + val ordinal: Int, + val cardSignature: String, + val cardSemanticTextCount: Int, + val detailSignature: String, + val detailSemanticTextCount: Int, + val screenshotFileName: String, + val screenshotSha256: String, + val screenshotByteCount: Int, + val screenshotWidth: Int, + val screenshotHeight: Int +) + +object PinduoduoEvidenceHash { + fun sha256(value: String): String = + sha256(value.toByteArray(Charsets.UTF_8)) + + fun sha256(value: ByteArray): String = + MessageDigest.getInstance("SHA-256") + .digest(value) + .joinToString(separator = "") { byte -> "%02x".format(byte) } +} + +interface PinduoduoCandidateDriver { + suspend fun snapshot(): PinduoduoUiSnapshot + suspend fun candidateCards(limit: Int): List + suspend fun openCandidate(signature: String): Boolean + suspend fun captureCandidate( + ordinal: Int, + card: PinduoduoCandidateCard + ): PinduoduoCandidateEvidence? + suspend fun returnToResults(): Boolean + suspend fun scrollResults(): Boolean + fun resetEvidence() +} diff --git a/android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/PinduoduoPageClassifier.kt b/android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/PinduoduoPageClassifier.kt index 16710c8..ca9b712 100644 --- a/android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/PinduoduoPageClassifier.kt +++ b/android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/PinduoduoPageClassifier.kt @@ -20,6 +20,7 @@ enum class PinduoduoPage { HOME, SEARCH_INPUT, SEARCH_RESULTS, + PRODUCT_DETAIL, UNKNOWN } @@ -91,12 +92,19 @@ object PinduoduoPageClassifier { val sortControlCount = setOf("综合", "销量", "价格", "筛选") .count { marker -> normalized.any { it == marker } } val hasExactQuery = visibleTexts.any { it.trim() == expectedQuery } + val hasDetailBack = visibleElements.any { element -> + element.clickable && + normalize(element.contentDescription.orEmpty()) == "返回" + } + val detailMarkerCount = setOf("联系客服", "收藏", "店铺") + .count { marker -> normalized.any { it.contains(marker) } } val page = when { hasExactQuery && hasResultSearchHeader && sortControlCount >= 3 -> PinduoduoPage.SEARCH_RESULTS hasSearchInput && hasSubmitButton -> PinduoduoPage.SEARCH_INPUT hasHomeSearchEntry && hasHome -> PinduoduoPage.HOME + hasDetailBack && detailMarkerCount >= 2 -> PinduoduoPage.PRODUCT_DETAIL else -> PinduoduoPage.UNKNOWN } return PinduoduoUiSnapshot( diff --git a/android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/PinduoduoSearchAutomation.kt b/android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/PinduoduoSearchAutomation.kt index 433b00d..9a06f22 100644 --- a/android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/PinduoduoSearchAutomation.kt +++ b/android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/PinduoduoSearchAutomation.kt @@ -15,6 +15,7 @@ interface PinduoduoUiDriver { suspend fun openSearch(): Boolean suspend fun setSearchKeyword(keyword: String): Boolean suspend fun submitSearch(): Boolean + suspend fun returnFromCandidate(): Boolean } class PinduoduoSearchAutomation( @@ -45,7 +46,8 @@ class PinduoduoSearchAutomation( return awaitPage { it == PinduoduoPage.HOME || it == PinduoduoPage.SEARCH_INPUT || - it == PinduoduoPage.SEARCH_RESULTS + it == PinduoduoPage.SEARCH_RESULTS || + it == PinduoduoPage.PRODUCT_DETAIL } ?: AutomationResult.Success } @@ -53,7 +55,8 @@ class PinduoduoSearchAutomation( val ready = awaitPage { it == PinduoduoPage.HOME || it == PinduoduoPage.SEARCH_INPUT || - it == PinduoduoPage.SEARCH_RESULTS + it == PinduoduoPage.SEARCH_RESULTS || + it == PinduoduoPage.PRODUCT_DETAIL } if (ready != null) { return ready @@ -63,6 +66,15 @@ class PinduoduoSearchAutomation( if (currentPage == PinduoduoPage.SEARCH_RESULTS) { return AutomationResult.Success } + if (currentPage == PinduoduoPage.PRODUCT_DETAIL) { + if (!driver.returnFromCandidate()) { + return AutomationResult.RetryableFailure( + WorkflowFailureCode.TRANSIENT_AUTOMATION + ) + } + return awaitPage { it == PinduoduoPage.SEARCH_RESULTS } + ?: AutomationResult.Success + } if (currentPage != PinduoduoPage.SEARCH_INPUT) { if (!driver.openSearch()) { return AutomationResult.RetryableFailure( diff --git a/android-buyer/app/src/main/java/com/roubao/autopilot/ui/screens/SearchProbeScreen.kt b/android-buyer/app/src/main/java/com/roubao/autopilot/ui/screens/SearchProbeScreen.kt index 33335f8..eb4df0f 100644 --- a/android-buyer/app/src/main/java/com/roubao/autopilot/ui/screens/SearchProbeScreen.kt +++ b/android-buyer/app/src/main/java/com/roubao/autopilot/ui/screens/SearchProbeScreen.kt @@ -34,6 +34,8 @@ import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp import com.roubao.autopilot.pinduoduo.PinduoduoSearchWorkflow +import com.roubao.autopilot.pinduoduo.MAX_CANDIDATES_PER_PROBE +import com.roubao.autopilot.pinduoduo.PinduoduoCandidateWorkflow import com.roubao.autopilot.pinduoduo.SEARCH_PROBE_KEYWORD import com.roubao.autopilot.readiness.DeviceReadinessSnapshot import com.roubao.autopilot.ui.theme.BaoziTheme @@ -51,7 +53,8 @@ private val probeSteps = listOf( ProbeStepUi(PinduoduoSearchWorkflow.OPEN_APP, "打开拼多多"), ProbeStepUi(PinduoduoSearchWorkflow.ENTER_QUERY, "输入固定关键词"), ProbeStepUi(PinduoduoSearchWorkflow.SUBMIT_QUERY, "提交搜索"), - ProbeStepUi(PinduoduoSearchWorkflow.VERIFY_RESULTS, "确认结果页") + ProbeStepUi(PinduoduoSearchWorkflow.VERIFY_RESULTS, "确认结果页"), + ProbeStepUi(PinduoduoCandidateWorkflow.BROWSE_CANDIDATES, "采集候选证据") ) @Composable @@ -60,6 +63,7 @@ fun SearchProbeScreen( state: WorkflowState, currentStepId: String?, report: WorkflowReport?, + candidateEvidenceCount: Int, onStart: () -> Unit, onStop: () -> Unit ) { @@ -75,13 +79,13 @@ fun SearchProbeScreen( ) { item { Text( - text = "拼多多搜索探针", + text = "拼多多候选探针", fontSize = 28.sp, fontWeight = FontWeight.Bold, color = colors.textPrimary ) Text( - text = stateLabel(state, report), + text = stateLabel(state, report, candidateEvidenceCount), fontSize = 14.sp, color = stateColor(state) ) @@ -118,6 +122,41 @@ fun SearchProbeScreen( Divider(color = colors.surfaceVariant) } + item { + Row( + modifier = Modifier + .fillMaxWidth() + .height(58.dp), + verticalAlignment = Alignment.CenterVertically + ) { + Icon( + imageVector = Icons.Default.CheckCircle, + contentDescription = null, + tint = if (candidateEvidenceCount > 0) { + colors.success + } else { + colors.textHint + }, + modifier = Modifier.size(22.dp) + ) + Text( + text = "候选证据", + fontSize = 14.sp, + fontWeight = FontWeight.Medium, + color = colors.textPrimary, + modifier = Modifier + .weight(1f) + .padding(start = 12.dp) + ) + Text( + text = "$candidateEvidenceCount / $MAX_CANDIDATES_PER_PROBE", + fontSize = 13.sp, + color = colors.textSecondary + ) + } + Divider(color = colors.surfaceVariant) + } + items(probeSteps.size) { index -> val step = probeSteps[index] ProbeStepRow( @@ -151,7 +190,7 @@ fun SearchProbeScreen( ) { Icon(Icons.Default.PlayArrow, contentDescription = null) Spacer(modifier = Modifier.size(8.dp)) - Text("开始搜索探针") + Text("开始候选探针") } } if (!readiness.canStartProbe) { @@ -263,12 +302,16 @@ private fun stateColor(state: WorkflowState): Color { } } -private fun stateLabel(state: WorkflowState, report: WorkflowReport?): String = +private fun stateLabel( + state: WorkflowState, + report: WorkflowReport?, + candidateEvidenceCount: Int +): String = when (state) { WorkflowState.IDLE -> "等待开始" WorkflowState.RUNNING -> "正在执行" WorkflowState.RETRYING -> "正在重试" - WorkflowState.SUCCEEDED -> "搜索结果页已确认" + WorkflowState.SUCCEEDED -> "已采集 $candidateEvidenceCount 个候选" WorkflowState.STOPPED -> "已由用户停止" WorkflowState.FAILED -> failureLabel(report?.failureCode) WorkflowState.BLOCKED -> blockedLabel(report?.safetyStopReason) @@ -279,6 +322,7 @@ private fun failureLabel(code: WorkflowFailureCode?): String = when (code) { WorkflowFailureCode.TRANSIENT_AUTOMATION -> "页面操作失败" WorkflowFailureCode.AUTOMATION_EXCEPTION -> "自动化执行异常" WorkflowFailureCode.TARGET_NOT_READY -> "拼多多无法启动" + WorkflowFailureCode.EVIDENCE_CAPTURE_FAILED -> "候选证据保存失败" null -> "执行失败" } diff --git a/android-buyer/app/src/main/java/com/roubao/autopilot/workflow/WorkflowModels.kt b/android-buyer/app/src/main/java/com/roubao/autopilot/workflow/WorkflowModels.kt index 6a930c6..ec84a7e 100644 --- a/android-buyer/app/src/main/java/com/roubao/autopilot/workflow/WorkflowModels.kt +++ b/android-buyer/app/src/main/java/com/roubao/autopilot/workflow/WorkflowModels.kt @@ -30,7 +30,8 @@ enum class WorkflowFailureCode { TIMEOUT, TRANSIENT_AUTOMATION, AUTOMATION_EXCEPTION, - TARGET_NOT_READY + TARGET_NOT_READY, + EVIDENCE_CAPTURE_FAILED } enum class SafetyStopReason { diff --git a/android-buyer/app/src/main/res/xml/buyer_accessibility_service.xml b/android-buyer/app/src/main/res/xml/buyer_accessibility_service.xml index e6b9178..348df17 100644 --- a/android-buyer/app/src/main/res/xml/buyer_accessibility_service.xml +++ b/android-buyer/app/src/main/res/xml/buyer_accessibility_service.xml @@ -3,6 +3,7 @@ android:accessibilityEventTypes="typeWindowStateChanged|typeWindowContentChanged|typeViewScrolled" android:accessibilityFeedbackType="feedbackGeneric" android:accessibilityFlags="flagReportViewIds" + android:canTakeScreenshot="true" android:canRetrieveWindowContent="true" android:description="@string/buyer_accessibility_description" android:notificationTimeout="100" diff --git a/android-buyer/app/src/test/java/com/roubao/autopilot/pinduoduo/PinduoduoCandidateAutomationTest.kt b/android-buyer/app/src/test/java/com/roubao/autopilot/pinduoduo/PinduoduoCandidateAutomationTest.kt new file mode 100644 index 0000000..2fd07c7 --- /dev/null +++ b/android-buyer/app/src/test/java/com/roubao/autopilot/pinduoduo/PinduoduoCandidateAutomationTest.kt @@ -0,0 +1,260 @@ +package com.roubao.autopilot.pinduoduo + +import com.roubao.autopilot.readiness.PINDUODUO_PACKAGE +import com.roubao.autopilot.workflow.AutomationResult +import com.roubao.autopilot.workflow.SafetyStopReason +import com.roubao.autopilot.workflow.WorkflowFailureCode +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class PinduoduoCandidateAutomationTest { + @Test + fun `collects five distinct candidates with bounded scrolling`() = runTest { + val driver = FakeCandidateDriver( + cardPages = listOf( + listOf(card("a"), card("b"), card("c")), + listOf(card("c"), card("d"), card("e"), card("f")) + ) + ) + val automation = PinduoduoCandidateAutomation( + driver = driver, + pagePollIntervalMillis = 1 + ) + automation.reset() + + val result = automation.execute( + PinduoduoCandidateWorkflow.steps().last() + ) + + assertEquals(AutomationResult.Success, result) + assertEquals(listOf("a", "b", "c", "d", "e"), driver.openedSignatures) + assertEquals(5, automation.evidence.value.size) + assertEquals(1, driver.scrollCalls) + assertEquals(5, driver.savedEvidence.size) + } + + @Test + fun `configured candidate limit is a hard upper bound`() = runTest { + val driver = FakeCandidateDriver( + cardPages = listOf( + listOf(card("a"), card("b"), card("c"), card("d")) + ) + ) + val automation = PinduoduoCandidateAutomation( + driver = driver, + maxCandidates = 2, + pagePollIntervalMillis = 1 + ) + automation.reset() + + val result = automation.execute( + PinduoduoCandidateWorkflow.steps().last() + ) + + assertEquals(AutomationResult.Success, result) + assertEquals(2, driver.openedSignatures.size) + assertEquals(2, automation.evidence.value.size) + assertEquals(0, driver.scrollCalls) + } + + @Test + fun `bounded old-page observations are allowed during transitions`() = runTest { + val driver = FakeCandidateDriver( + cardPages = listOf(listOf(card("a"))), + transitionDelaySnapshots = 2 + ) + val automation = PinduoduoCandidateAutomation( + driver = driver, + maxCandidates = 1, + pagePollIntervalMillis = 1, + unknownPageLimit = 4 + ) + automation.reset() + + val result = automation.execute( + PinduoduoCandidateWorkflow.steps().last() + ) + + assertEquals(AutomationResult.Success, result) + assertEquals(1, automation.evidence.value.size) + } + + @Test + fun `empty result scanning never exceeds scroll budget`() = runTest { + val driver = FakeCandidateDriver( + cardPages = listOf(emptyList(), emptyList(), emptyList()) + ) + val automation = PinduoduoCandidateAutomation( + driver = driver, + maxResultScrolls = 2, + pagePollIntervalMillis = 1 + ) + automation.reset() + + val result = automation.execute( + PinduoduoCandidateWorkflow.steps().last() + ) + + assertEquals( + AutomationResult.FatalFailure(WorkflowFailureCode.TARGET_NOT_READY), + result + ) + assertEquals(2, driver.scrollCalls) + assertTrue(driver.openedSignatures.isEmpty()) + } + + @Test + fun `safety blocker prevents candidate actions`() = runTest { + val driver = FakeCandidateDriver( + cardPages = listOf(listOf(card("a"))), + safetyStopReason = SafetyStopReason.RISK_CONTROL + ) + val automation = PinduoduoCandidateAutomation( + driver = driver, + pagePollIntervalMillis = 1 + ) + automation.reset() + + val result = automation.execute( + PinduoduoCandidateWorkflow.steps().last() + ) + + assertEquals( + AutomationResult.Blocked(SafetyStopReason.RISK_CONTROL), + result + ) + assertTrue(driver.openedSignatures.isEmpty()) + assertTrue(driver.savedEvidence.isEmpty()) + } + + @Test + fun `missing screenshot evidence fails without opening another candidate`() = + runTest { + val driver = FakeCandidateDriver( + cardPages = listOf(listOf(card("a"), card("b"))), + failCapture = true + ) + val automation = PinduoduoCandidateAutomation( + driver = driver, + pagePollIntervalMillis = 1 + ) + automation.reset() + + val result = automation.execute( + PinduoduoCandidateWorkflow.steps().last() + ) + + assertEquals( + AutomationResult.FatalFailure( + WorkflowFailureCode.EVIDENCE_CAPTURE_FAILED + ), + result + ) + assertEquals(listOf("a"), driver.openedSignatures) + assertTrue(automation.evidence.value.isEmpty()) + } + + private fun card(signature: String) = PinduoduoCandidateCard( + signature = signature, + semanticTextCount = 3, + hasImage = true + ) + + private class FakeCandidateDriver( + private val cardPages: List>, + private val safetyStopReason: SafetyStopReason? = null, + private val failCapture: Boolean = false, + private val transitionDelaySnapshots: Int = 0 + ) : PinduoduoCandidateDriver { + private var cardPageIndex = 0 + private var page = PinduoduoPage.SEARCH_RESULTS + private var pendingPage: PinduoduoPage? = null + private var delayedSnapshotsRemaining = 0 + + val openedSignatures = mutableListOf() + val savedEvidence = mutableListOf() + var scrollCalls = 0 + + override suspend fun snapshot(): PinduoduoUiSnapshot { + if (pendingPage != null) { + if (delayedSnapshotsRemaining > 0) { + delayedSnapshotsRemaining -= 1 + } else { + page = pendingPage ?: page + pendingPage = null + } + } + return PinduoduoUiSnapshot( + foregroundPackage = PINDUODUO_PACKAGE, + page = page, + safetyStopReason = safetyStopReason + ) + } + + override suspend fun candidateCards( + limit: Int + ): List = + cardPages.getOrElse(cardPageIndex) { emptyList() }.take(limit) + + override suspend fun openCandidate(signature: String): Boolean { + openedSignatures += signature + transitionTo(PinduoduoPage.PRODUCT_DETAIL) + return true + } + + override suspend fun captureCandidate( + ordinal: Int, + card: PinduoduoCandidateCard + ): PinduoduoCandidateEvidence? { + if (failCapture) { + return null + } + return PinduoduoCandidateEvidence( + ordinal = ordinal, + cardSignature = card.signature, + cardSemanticTextCount = card.semanticTextCount, + detailSignature = "detail-${card.signature}", + detailSemanticTextCount = 10, + screenshotFileName = "candidate-%02d.png".format(ordinal), + screenshotSha256 = "hash-$ordinal", + screenshotByteCount = 100 + ordinal, + screenshotWidth = 1080, + screenshotHeight = 2400 + ).also(savedEvidence::add) + } + + override suspend fun returnToResults(): Boolean { + transitionTo(PinduoduoPage.SEARCH_RESULTS) + return true + } + + override suspend fun scrollResults(): Boolean { + scrollCalls += 1 + if (cardPageIndex + 1 < cardPages.size) { + cardPageIndex += 1 + } + return true + } + + override fun resetEvidence() { + openedSignatures.clear() + savedEvidence.clear() + scrollCalls = 0 + cardPageIndex = 0 + page = PinduoduoPage.SEARCH_RESULTS + pendingPage = null + delayedSnapshotsRemaining = 0 + } + + private fun transitionTo(nextPage: PinduoduoPage) { + if (transitionDelaySnapshots == 0) { + page = nextPage + } else { + pendingPage = nextPage + delayedSnapshotsRemaining = transitionDelaySnapshots + } + } + } +} diff --git a/android-buyer/app/src/test/java/com/roubao/autopilot/pinduoduo/PinduoduoPageClassifierTest.kt b/android-buyer/app/src/test/java/com/roubao/autopilot/pinduoduo/PinduoduoPageClassifierTest.kt index f38bf7b..57b9a03 100644 --- a/android-buyer/app/src/test/java/com/roubao/autopilot/pinduoduo/PinduoduoPageClassifierTest.kt +++ b/android-buyer/app/src/test/java/com/roubao/autopilot/pinduoduo/PinduoduoPageClassifierTest.kt @@ -100,6 +100,19 @@ class PinduoduoPageClassifierTest { assertEquals(PinduoduoPage.UNKNOWN, snapshot.page) } + @Test + fun `detail page requires clickable back and multiple stable markers`() { + val snapshot = classify( + element(contentDescription = "返回", clickable = true), + element(contentDescription = "联系客服"), + element(contentDescription = "收藏商品"), + element(text = "店铺") + ) + + assertEquals(PinduoduoPage.PRODUCT_DETAIL, snapshot.page) + assertNull(snapshot.safetyStopReason) + } + @Test fun `other foreground package is unknown and has no inferred blocker`() { val snapshot = PinduoduoPageClassifier.classify( diff --git a/android-buyer/app/src/test/java/com/roubao/autopilot/pinduoduo/PinduoduoSearchAutomationTest.kt b/android-buyer/app/src/test/java/com/roubao/autopilot/pinduoduo/PinduoduoSearchAutomationTest.kt index 7d240f0..6a843fb 100644 --- a/android-buyer/app/src/test/java/com/roubao/autopilot/pinduoduo/PinduoduoSearchAutomationTest.kt +++ b/android-buyer/app/src/test/java/com/roubao/autopilot/pinduoduo/PinduoduoSearchAutomationTest.kt @@ -89,6 +89,24 @@ class PinduoduoSearchAutomationTest { ) } + @Test + fun `workflow safely recovers from a verified candidate detail`() = runTest { + val driver = FakeDriver(page = PinduoduoPage.PRODUCT_DETAIL) + val runner = WorkflowRunner( + PinduoduoSearchAutomation( + driver = driver, + pollIntervalMillis = 1, + unknownPageLimit = 3 + ) + ) + + val report = runner.run(PinduoduoSearchWorkflow.steps()) + + assertEquals(WorkflowState.SUCCEEDED, report.state) + assertEquals(1, driver.returnFromCandidateCalls) + assertEquals(PinduoduoPage.SEARCH_RESULTS, driver.page) + } + private class FakeDriver( var page: PinduoduoPage, private val safetyStopReason: SafetyStopReason? = null, @@ -97,6 +115,7 @@ class PinduoduoSearchAutomationTest { var enteredKeyword: String? = null var searchSubmitted = false var openSearchCalls = 0 + var returnFromCandidateCalls = 0 override suspend fun openApp(): Boolean = true @@ -127,5 +146,11 @@ class PinduoduoSearchAutomationTest { page = PinduoduoPage.SEARCH_RESULTS return true } + + override suspend fun returnFromCandidate(): Boolean { + returnFromCandidateCalls += 1 + page = PinduoduoPage.SEARCH_RESULTS + return true + } } } diff --git a/docs/00-ai-start-here.md b/docs/00-ai-start-here.md index 31a96c0..21eac1a 100644 --- a/docs/00-ai-start-here.md +++ b/docs/00-ai-start-here.md @@ -51,9 +51,9 @@ ## 当前阶段与优先路径 -当前已完成 Phase 0 和 T-101：Android 可运行、设备就绪、workflow、私有样本导入 -以及固定脱敏词拼多多搜索均已在真机验证。下一步是 T-102，只浏览并采集最多 5 个 -候选，继续禁止订单提交和支付。 +当前已完成 Phase 0、T-101 和 T-102：Android 可运行、设备就绪、workflow、私有样本 +导入、固定词搜索以及最多 5 个候选截图采集均已在真机验证。下一步是 T-103，使用 +私有 ProbeTask 验证结构化需求提取，继续禁止订单提交和支付。 严格按以下顺序推进： diff --git a/docs/02-requirements.md b/docs/02-requirements.md index 134880b..909f720 100644 --- a/docs/02-requirements.md +++ b/docs/02-requirements.md @@ -8,7 +8,7 @@ | --- | --- | | 任务来源 | 其他管理后台或本项目管理 Web 采集商品标题、描述、图片、数量和预算。 | | 第一层样本来源 | 本机目录中的蝦皮订单文本和参考图；二者以蝦皮订单号作为同名文件名。 | -| 执行方式 | 采购人员使用 Android App 操作拼多多；固定词搜索探针已可运行，候选浏览和真实任务尚未实现。 | +| 执行方式 | 采购人员使用 Android App 操作拼多多；固定词搜索和最多 5 个候选证据采集已可运行，需求提取和匹配判断尚未实现。 | | 核心痛点 | 人工把图片和描述转成搜索词、逐条比较商品并记录结果，耗时且不一致。 | | 验证范围 | 一台设备、一个管理身份、一个采购执行人员、拼多多单平台。 | | 资金边界 | MVP 不提交订单、不支付，只验证到人工确认位置。 | @@ -140,8 +140,8 @@ T-004 已固定首版规则：推荐私有目录为被 Git 忽略的 `private-fi `T-001` 完成。 - 首份蝦皮样本已由 T-004 真实导入验证；当前只支持单 SKU 和 JPEG，扩展格式需新 样例和独立任务。 -- OnePlus PKG110、Android 16/API 36、拼多多 8.17.0 的首页、搜索输入和结果页已形成 - 可复现基线；候选列表、详情页及账号风控差异仍待 T-102 验证。 +- OnePlus PKG110、Android 16/API 36、拼多多 8.17.0 的首页、搜索输入、结果页、 + 候选卡和详情返回已形成可复现基线；不同账号、类目和页面实验的差异仍是风险。 - VLM 厂商、模型、成本上限、数据留存地区和图片隐私规则待确认。 - 拼多多平台条款、自动化允许范围和账号风控需要业务方确认；项目不实现绕过措施。 - 后续若允许提交订单，必须先明确 SKU、收货地址、运费、优惠、发票、金额审批、 diff --git a/docs/03-tech-stack.md b/docs/03-tech-stack.md index 3c84642..9e1c08e 100644 --- a/docs/03-tech-stack.md +++ b/docs/03-tech-stack.md @@ -11,7 +11,8 @@ | Android 构建链 | Gradle 8.2；AGP 8.2.0；Kotlin 1.9.20；JVM target 17 | 已验证 | 已补齐上游缺失的 `gradlew.bat`，不依赖全局 Gradle。 | | Android SDK | compileSdk/targetSdk 34；minSdk 26；SDK Build Tools 34.0.0 | 已验证 | 支持 Android 8.0+；本机使用 Command-line Tools 22.0。 | | Android UI | Jetpack Compose + Material 3；Compose Compiler 1.5.5 | 上游已核实 | Compose BOM 为 2023.10.01。 | -| Android 自动化 | `AccessibilityService` 语义节点动作；Shizuku 保留为上游兼容路径 | 固定词搜索已真机验证 | T-101 已用可见、启用、唯一节点完成打开搜索、`ACTION_SET_TEXT` 精确回读、提交和结果页确认；没有坐标或 shell 降级。上游 `main` 仍保留 Shizuku 13.1.5。 | +| Android 自动化 | `AccessibilityService` 语义节点动作；Shizuku 保留为上游兼容路径 | 搜索与 5 个候选已真机验证 | T-101/T-102 已完成精确输入、结果页确认、候选卡识别、详情截图和验证返回；没有坐标或 shell 降级。上游 `main` 仍保留 Shizuku 13.1.5。 | +| Android 候选证据 | API 30+ `AccessibilityService.takeScreenshot` + App cache JSON/PNG | 已真机验证 | 匿名 PNG 与只含 SHA-256、计数、尺寸的 manifest；转换/压缩使用独立 executor，文件 IO 使用 `Dispatchers.IO`。API 26-29 明确不支持该截图探针。 | | 第一层任务源 | UTF-8 无 BOM 四行蝦皮订单文本 + 同订单号 JPEG | 已实现 | `task-contract` 共享 `ProbeTask/TaskSource`；CLI 输出到 `.local/`，只有显式 Debug 属性才注入 APK，默认构建会清除私有资产。 | | Android 长任务 | 前台服务 + 持续通知 | 计划采用 | 降低执行中被系统挂起的风险，仍需处理进程死亡恢复。 | | 后端语言 | Go 1.23.0 | MVP 已定 | 与现有本机工具链一致；构建测试必须设置 `GOTOOLCHAIN=local` 防止静默升级。 | @@ -27,7 +28,7 @@ | VLM 接入 | 应用内统一适配器，优先兼容 OpenAI 风格多模态接口 | 接口已定，供应商待定 | 模型输出必须符合本项目 JSON Schema。 | | 通知 | MVP 不使用推送 | 已定 | 点击“获取任务”调用原子 claim API；V2 再评估厂商推送/WebSocket。 | | 后端测试 | 标准库 `testing` + `httptest` | MVP 已定 | 覆盖状态机、权限、幂等、SQLite 事务和输入校验。 | -| Android 测试 | Gradle `test` + `kotlinx-coroutines-test` 1.7.3 + 真实设备 smoke | 搜索探针已验证 | T-003 runner 与 T-101 页面分类/Fake driver 已覆盖；OnePlus PKG110 + 拼多多 8.17.0 固定词结果页 smoke 成功。 | +| Android 测试 | Gradle `test` + `kotlinx-coroutines-test` 1.7.3 + 真实设备 smoke | 候选探针已验证 | runner、页面分类、搜索和有界候选 Fake driver 已覆盖；OnePlus PKG110 + 拼多多 8.17.0 的 5 个截图及返回 smoke 成功。 | | 部署 | 单机局域网 Go 服务；容器化后置 | MVP 已定 | Android 测试机必须能通过 HTTPS 或受控测试网络访问。 | ## Roubao 上游版本基线 @@ -134,7 +135,7 @@ docs/ | --- | --- | --- | | Android 标准验证 | `.\init.ps1` | 已验证 | | Android 构建 | `android-buyer\gradlew.bat assembleDebug --no-daemon` | 已验证 | -| Android 测试任务 | `android-buyer\gradlew.bat test --no-daemon` | 已验证；当前全工程 62 次测试通过 | +| Android 测试任务 | `android-buyer\gradlew.bat test --no-daemon` | 已验证；当前全工程 78 次测试通过 | | Android 安装/启动 | `$env:RUN_START_COMMAND="1"; .\init.ps1` | 已在 Android 16 真机验证 | | 后端依赖 | `go mod download`（在 `backend-api/`） | 待 `T-201` | | 后端测试 | `go test ./...`（在 `backend-api/`） | 待 `T-201` | diff --git a/docs/04-architecture.md b/docs/04-architecture.md index 86a3e85..1f12ce1 100644 --- a/docs/04-architecture.md +++ b/docs/04-architecture.md @@ -169,6 +169,23 @@ SearchProbeScreen 登录、验证码、风控、订单或支付边界都终止 workflow。该路径不提供坐标、ADB、 Shizuku shell、OCR 或 VLM 动作降级。 +T-102 在搜索结果后追加一个有界候选步骤： + +```text +固定词结果页 + -> 最多 2 次滚动预算 + -> 最多 5 个去重商品卡 + -> 验证详情页 + -> 无障碍截图 + -> App cache/candidate-NN.png + manifest.json + -> 一次全局返回并复核固定词结果页 +``` + +候选卡只保留语义指纹和计数，详情截图保存在 App 内部 cache。manifest 记录匿名文件名、 +截图 SHA-256、字节数和尺寸，不保存商品标题或页面原文。T-104 使用这些证据时必须通过 +受控 evidence 边界读取，不能让 VLM adapter 自行遍历 cache。Android 10/API 29 及 +以下不能运行当前截图探针，应在预检时明确不支持，不使用媒体投影或 shell 绕过。 + ### 3.2 MVP 业务闭环 ```text @@ -327,6 +344,8 @@ IDLE 不能因资源 ID 重复而任取第一个节点。 - 已验证的首页、搜索输入和结果页路径禁止坐标降级；新页面必须先取得版本化真机 证据并在独立任务中定义识别条件。 +- 候选 session 的候选尝试与滚动预算在动作发出前扣减，retry 不得重置；详情动作 + 白名单只有截图和返回。 - 截图发送给模型前裁剪无关区域并按配置脱敏。 - 每个动作记录抽象步骤，不默认记录完整输入文本。 - 发现验证码、风险控制、支付、生物识别或系统权限页面立即停止。 diff --git a/docs/05-coding-rules.md b/docs/05-coding-rules.md index a3d9574..a3b7e3c 100644 --- a/docs/05-coding-rules.md +++ b/docs/05-coding-rules.md @@ -37,6 +37,12 @@ - 每个步骤定义 timeout、有限 retry、成功条件和停止条件。 - 禁止无界循环、无限滑动、无限候选遍历和随机点击。 - 默认最多检查 5 个候选；扩大范围必须先改需求。 +- 候选卡必须满足视口可见比例、单列尺寸、图片、价格和语义文本约束；根不可点击时 + 只允许唯一的大面积覆盖点击目标，禁止点击卡内小按钮。 +- 候选和滚动预算在发出动作前消耗，并跨 workflow retry 保持；返回 `false` 或取消 + 也不能恢复预算。 +- 详情截图只保存在 App 内部目录，文件名匿名；manifest 不保存标题/页面原文，并 + 校验文件 SHA-256、尺寸和字节数。 - 前台服务通知要展示任务编号和当前步骤，并提供安全停止入口。 - 验证码、风控、登录、支付、未知页面统一进入安全失败。 - MVP 不得实现最终提交订单或支付动作，包括“先写好但不调用”的隐藏代码。 diff --git a/docs/current-state.md b/docs/current-state.md index b86c3d3..6062f97 100644 --- a/docs/current-state.md +++ b/docs/current-state.md @@ -5,8 +5,8 @@ ## 当前快照 - 日期：2026-07-25 -- 阶段：T-101 已完成；准备 T-102 候选浏览和采集 -- Git：当前分支为 `main`；T-001 至 T-004 均已纳入 Git 历史 +- 阶段：T-102 已完成；准备 T-103 VLM 需求提取 +- Git：当前分支为 `main`；T-001 至 T-004、T-101 和 T-102 均已纳入 Git 历史 - 生产代码：`android-buyer/` 已接入 Roubao Android 源码 - Android：固定 `main@c8a6d7f03422eb01744b01f3ee77bf7757741f7e`；MIT 许可证已保留 - 后端：已决定使用 Go 1.23.0 + Gin 1.11.0；Go Blueprint v0.10.11 骨架尚未接入 @@ -14,21 +14,21 @@ Build Tools 34.0.0、Platform Tools/ADB 37.0.0；用户级 SDK 环境变量已设置 - Android Studio：未安装；`winget` 静默安装卡住后已终止，不阻塞命令行构建 - 测试：`lintDebug test assembleDebug` 成功；App 两个变体、task contract 和导入器 - 共执行 62 次测试，0 failure、0 error、0 skipped + 共执行 78 次测试，0 failure、0 error、0 skipped - Workflow：纯 Kotlin runner 已支持步骤 timeout、最多 3 次 retry、安全阻塞、 - 用户停止和单 runner 并发拒绝；T-101 已接入真实拼多多固定词搜索四步 + 用户停止和单 runner 并发拒绝；T-102 已接入搜索加有界候选采集五步 - TaskSource：严格 CLI 已生成并验证真实私有 ProbeTask；默认 APK 不含私有 fixture - 测试设备：OnePlus PKG110，Android 16/API 36；肉包 `1.4.2 (7)`；拼多多 `8.17.0 (81700)` -- 设备就绪：肉包采购无障碍已启用并连接；拼多多首页、搜索输入和固定词结果页已通过 - 8.17.0 真机验证，结果页精确词、搜索头和 4 个排序控件均确认 +- 设备就绪：肉包采购无障碍已启用并连接；拼多多首页、搜索输入、固定词结果页、 + 双列候选卡、详情截图和返回均已通过 8.17.0 真机验证 - 版本控制内测试数据：仅有脱敏、运行时生成的单元测试 fixture；没有真实订单内容 - 本地私有样本：仓库根目录存在一组未跟踪、已本地排除的同名蝦皮文本/JPEG； 已用 CLI 真实导入并逐字段/图片哈希验证，生成物位于被忽略的 `.local/` - 标准启动路径：`$env:RUN_START_COMMAND="1"; .\init.ps1` - 标准验证路径：`.\init.ps1` -- 当前 blocker：候选列表/详情页语义与有界返回路径尚待 T-102 验证；当前只支持单 - SKU/JPEG；VLM 供应商、模型和测试凭证未确认 +- 当前 blocker：VLM 供应商、模型、测试凭证、成本上限和数据留存尚未确认；当前只 + 支持单 SKU/JPEG；候选探针截图要求 Android 11/API 30+ ## 当前目录 @@ -41,6 +41,7 @@ | `docs/tasks/T-003.md` | DONE | 可注入 Fake automation 的受限 workflow runner | | `docs/tasks/T-004.md` | DONE | 私有蝦皮文本/JPEG 严格导入和 Debug TaskSource | | `docs/tasks/T-101.md` | DONE | 固定脱敏词拼多多搜索和结果页真机验证 | +| `docs/tasks/T-102.md` | DONE | 最多 5 个候选详情截图、证据 manifest 和结果页返回 | | `deepseek总结.txt` | 已有 | 历史讨论摘要，不是正式需求权威 | | `android-buyer/` | 已有 | Roubao `main` 固定 commit 的 Android 基线 | | `android-buyer/task-contract/` | 已有 | Android/CLI 共享 ProbeTask 与 TaskSource | @@ -50,9 +51,9 @@ ## 任务摘要 -- 已完成：T-001 至 T-004，以及 T-101 固定词搜索探针。 +- 已完成：T-001 至 T-004，以及 T-101、T-102。 - 正在进行：无。 -- 下一个可领取任务：T-102 浏览并采集最多 5 个候选。 +- 下一个可领取任务：T-103 接入 VLM 需求提取。 ## 当前可运行内容 @@ -64,8 +65,8 @@ $env:RUN_START_COMMAND = "1" ``` 2026-07-25 已在 PKG110、Android 16/API 36、拼多多 8.17.0 上完成 Debug APK -固定词搜索。默认首屏为搜索探针，用户点击后四步全部完成并确认结果页；设备页保留 -就绪检查入口，肉包采购无障碍服务已绑定，logcat 无崩溃或 ANR。 +固定词搜索和 5 个候选采集。默认首屏为候选探针，用户点击后五步全部完成，5 张 +匿名 PNG 与 manifest 哈希一致并返回结果页；logcat 无崩溃或 ANR。 ## 维护规则 diff --git a/docs/tasks/T-102.md b/docs/tasks/T-102.md new file mode 100644 index 0000000..0549a67 --- /dev/null +++ b/docs/tasks/T-102.md @@ -0,0 +1,126 @@ +--- +id: T-102 +title: 浏览并采集最多 5 个候选 +phase: 1 +deps: + - T-101 +status: DONE +created: 2026-07-25 +context_ref: f226d0f +work_branch: main +write_paths: + - android-buyer/app/src/main/java/com/roubao/autopilot/MainActivity.kt + - android-buyer/app/src/main/java/com/roubao/autopilot/accessibility/** + - android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/** + - android-buyer/app/src/main/java/com/roubao/autopilot/ui/screens/SearchProbeScreen.kt + - android-buyer/app/src/main/java/com/roubao/autopilot/workflow/WorkflowModels.kt + - android-buyer/app/src/main/res/xml/buyer_accessibility_service.xml + - android-buyer/app/src/test/java/com/roubao/autopilot/pinduoduo/** + - docs/00-ai-start-here.md + - docs/02-requirements.md + - docs/03-tech-stack.md + - docs/04-architecture.md + - docs/05-coding-rules.md + - docs/current-state.md + - docs/tasks/T-102.md + - progress.md +--- + +## 问题 / 背景 + +T-101 已能稳定进入固定词结果页，但尚未证明可以识别独立商品卡、进入详情、保存证据 +并返回同一结果页。T-102 只验证有界候选浏览，不进行匹配判断、下单或支付。 + +## 关联需求与交互 + +- 功能：F-005 的最多 5 个候选采集部分、F-007 安全失败。 +- 用户故事：US-004、US-006。 +- 交互：Android“探针”页展示候选计数和当前步骤。 +- 架构：`WorkflowRunner` + `PinduoduoCandidateAutomation` + 无障碍 driver + + App 内部缓存证据。 + +## 方案 + +1. 只从已确认的搜索结果页识别主 RecyclerView 的直属商品卡。 +2. 商品卡必须可见、启用、可点击，包含图片和有限文本；用文本 SHA-256 去重，不在 + 日志或 UI 展示标题。 +3. 每次只打开一个候选；详情页必须由稳定语义组合确认，保存全屏 PNG、文本指纹和 + 节点计数后立即返回结果页。 +4. 最多采集 5 个不同候选，最多滚动结果列表 2 次；没有无界循环或翻页。 +5. 登录、验证码、风控、未知页、订单和支付边界立即阻塞；绝不点击详情页购买控件。 + +## 验收要点 + +- [x] 从 T-101 结果页依次进入候选详情并返回。 +- [x] 采集数量始终不超过 5，结果滚动次数始终不超过 2。 +- [x] 每个候选有内部缓存 PNG、SHA-256、尺寸和语义节点证据。 +- [x] 证据文件名匿名，普通日志和 UI 不输出商品标题。 +- [x] 全流程不使用坐标，不点击购买、订单或支付控件。 +- [x] 页面未知或安全边界出现时产生可区分的 `BLOCKED`。 +- [x] Fake driver 自动化测试覆盖成功、去重、上限和安全停止。 +- [x] 拼多多 8.17.0 真机 smoke 采集 5 个候选并返回结果页。 +- [x] `lintDebug test assembleDebug` 通过，默认 APK 不含私有 fixture。 + +## 边界 + +- 不评估候选是否匹配；T-104 才接入候选评估。 +- 不把候选证据上传网络；T-202 以后再接后端证据接口。 +- 不点击购买、拼单、加入购物车、订单或支付相关节点。 +- 不保存真实蝦皮输入或把商品详情文本写入普通日志。 + +## 执行记录 + +### 2026-07-25：任务开始 + +- 基于 T-101 提交 `f226d0f` 开始。 +- 脱敏结构审计确认拼多多 8.17.0 结果页主 RecyclerView 和双列商品卡；详情页有 + 可点击返回语义以及联系客服、收藏、店铺语义，当前未出现订单或支付边界。 + +### 2026-07-25：实现 + +- 新增 `PinduoduoCandidateAutomation` 和复合探针 workflow。一次 session 最多尝试 + 5 个不同卡片、最多发出 2 次结果滚动；预算在动作前消耗，runner retry 不会重置。 +- 主结果 RecyclerView 必须可滚动、覆盖至少 90% 宽度且拥有足够节点。候选卡必须 + 至少 80% 可见、宽度在单列范围、含图片/价格/至少 2 个语义文本；根不可点击时 + 只接受唯一覆盖卡片至少 85% 面积的点击目标，小型嵌套按钮不会被选中。 +- 详情页同时要求可点击返回、联系客服/收藏/店铺至少两类语义、宽 ViewPager 和宽 + 可滚动主列表。详情页动作白名单只有截图和一次全局返回，返回后必须重新确认固定 + 查询结果页。 +- Android 11/API 30+ 使用声明了 `canTakeScreenshot` 的无障碍截图 API。Hardware + buffer 在独立执行器转换和压缩并始终关闭；8 秒超时后结构化失败，不绕过安全窗口。 +- 每个候选只保存匿名 PNG、卡片/详情语义 SHA-256、节点计数、文件 SHA-256、字节数 + 和尺寸到 App cache；manifest 不保存标题或页面原文，文件写入在 IO dispatcher。 +- 探针页增加候选 `0/5` 计数和第五步；仅记录 workflow/候选阶段枚举与截图错误码。 + +### 2026-07-25：自动化验证 + +- `gradlew.bat lintDebug test assembleDebug --no-daemon` 成功。 +- App Debug/Release、task contract 和导入器共 14 份报告、78 次测试，0 failure、 + 0 error、0 skipped。 +- Fake driver 覆盖 5 个去重候选、候选硬上限、旧页面有界过渡、两次滚动上限、 + 空列表、安全阻塞、截图失败以及从已验证详情页安全恢复。 +- 普通 Debug APK 中 `assets/probe-fixtures/` 条目数为 0。 + +### 2026-07-25：真机 smoke + +- OnePlus PKG110、Android 16/API 36、拼多多 8.17.0 上，从肉包按钮启动后 20 秒内 + 依次采集 5 个候选并返回固定词结果页；肉包显示 `5 / 5` 和五步完成。 +- cache manifest 为 5 条，匿名文件名合法；5 个 PNG 均为 `1080x2376`，字节数为 + 正数，文件 SHA-256 与 manifest 全部一致。 +- 最终结果页订单/支付边界计数为 0；logcat 没有肉包崩溃或 ANR，阶段日志不含商品 + 标题或页面原文。 +- 真机审计 XML/PNG 位于被忽略的 `.local/`，候选证据位于 App 内部 cache，均未纳入 + Git。 + +### 2026-07-25：修正记录 + +- 首次运行在候选点击后把短暂保留的旧结果页误判为未知。改为只在固定 20 次观测 + 预算内等待结果页和详情页过渡；任何安全阻塞仍立即停止。 +- 调试期间旧 Activity 造成终态读取混淆；最终 smoke 先清理肉包任务栈，再用 + `FLAG_ACTIVITY_REORDER_TO_FRONT` 返回原实例核验，未创建第二个 workflow。 + +## 后续 + +- T-103 使用 T-004 的私有 ProbeTask 接入结构化需求提取；不能把订单号或店铺名发给 + 模型。 +- T-104 才把候选截图交给评估适配器；T-102 不声明任何候选匹配结论。 diff --git a/progress.md b/progress.md index 4d9010e..bd28903 100644 --- a/progress.md +++ b/progress.md @@ -77,3 +77,11 @@ 和结果页确认，并在拼多多 8.17.0 真机验证四步 workflow。 - 影响：最高风险的基础搜索动作已证实可行；下一步 T-102 只扩展最多 5 个候选的 有界浏览和证据采集，继续禁止下单与支付。 + +## 2026-07-25 拼多多候选证据探针 + +- 类型：阶段完成 +- 内容：完成 T-102；用无障碍语义节点有界浏览 5 个候选，逐一验证详情、保存匿名 + PNG/SHA-256 证据并返回固定词结果页。 +- 影响：搜索到候选证据的 Android 风险闭环已证实可行；T-103 可开始验证私有任务的 + 结构化需求提取，T-104 再对候选证据做匹配判断。