feat(android): collect bounded Pinduoduo candidates

This commit is contained in:
QiuSW
2026-07-25 20:18:05 +08:00
parent f226d0f2ff
commit c25d63c730
23 changed files with 1449 additions and 33 deletions
@@ -48,8 +48,13 @@ import kotlinx.coroutines.launch
import rikka.shizuku.Shizuku import rikka.shizuku.Shizuku
import android.util.Log import android.util.Log
import com.roubao.autopilot.pinduoduo.AndroidPinduoduoUiDriver import com.roubao.autopilot.pinduoduo.AndroidPinduoduoUiDriver
import com.roubao.autopilot.pinduoduo.AndroidPinduoduoCandidateDriver
import com.roubao.autopilot.pinduoduo.PinduoduoCandidateAutomation
import com.roubao.autopilot.pinduoduo.CandidateBrowsePhase
import com.roubao.autopilot.pinduoduo.PinduoduoCandidateEvidence
import com.roubao.autopilot.pinduoduo.PinduoduoCandidateWorkflow
import com.roubao.autopilot.pinduoduo.PinduoduoProbeAutomation
import com.roubao.autopilot.pinduoduo.PinduoduoSearchAutomation import com.roubao.autopilot.pinduoduo.PinduoduoSearchAutomation
import com.roubao.autopilot.pinduoduo.PinduoduoSearchWorkflow
import com.roubao.autopilot.workflow.WorkflowReport import com.roubao.autopilot.workflow.WorkflowReport
import com.roubao.autopilot.workflow.WorkflowRunner import com.roubao.autopilot.workflow.WorkflowRunner
import com.roubao.autopilot.workflow.WorkflowState import com.roubao.autopilot.workflow.WorkflowState
@@ -76,6 +81,8 @@ class MainActivity : ComponentActivity() {
private val searchProbeState = mutableStateOf(WorkflowState.IDLE) private val searchProbeState = mutableStateOf(WorkflowState.IDLE)
private val searchProbeStepId = mutableStateOf<String?>(null) private val searchProbeStepId = mutableStateOf<String?>(null)
private val searchProbeReport = mutableStateOf<WorkflowReport?>(null) private val searchProbeReport = mutableStateOf<WorkflowReport?>(null)
private val candidateEvidence =
mutableStateOf<List<PinduoduoCandidateEvidence>>(emptyList())
private var searchProbeRunner: WorkflowRunner? = null private var searchProbeRunner: WorkflowRunner? = null
private var searchProbeJob: Job? = null private var searchProbeJob: Job? = null
@@ -198,6 +205,7 @@ class MainActivity : ComponentActivity() {
val probeState by remember { searchProbeState } val probeState by remember { searchProbeState }
val probeStepId by remember { searchProbeStepId } val probeStepId by remember { searchProbeStepId }
val probeReport by remember { searchProbeReport } val probeReport by remember { searchProbeReport }
val evidence by remember { candidateEvidence }
// 监听跳转事件 // 监听跳转事件
LaunchedEffect(navigateToRecord, recordId) { LaunchedEffect(navigateToRecord, recordId) {
@@ -286,6 +294,7 @@ class MainActivity : ComponentActivity() {
state = probeState, state = probeState,
currentStepId = probeStepId, currentStepId = probeStepId,
report = probeReport, report = probeReport,
candidateEvidenceCount = evidence.size,
onStart = { startSearchProbe() }, onStart = { startSearchProbe() },
onStop = { stopSearchProbe() } onStop = { stopSearchProbe() }
) )
@@ -384,13 +393,23 @@ class MainActivity : ComponentActivity() {
return return
} }
val candidateAutomation = PinduoduoCandidateAutomation(
AndroidPinduoduoCandidateDriver(this)
)
candidateAutomation.reset()
val runner = WorkflowRunner( val runner = WorkflowRunner(
PinduoduoSearchAutomation(AndroidPinduoduoUiDriver(this)) PinduoduoProbeAutomation(
searchAutomation = PinduoduoSearchAutomation(
AndroidPinduoduoUiDriver(this)
),
candidateAutomation = candidateAutomation
)
) )
searchProbeRunner = runner searchProbeRunner = runner
searchProbeReport.value = null searchProbeReport.value = null
searchProbeState.value = WorkflowState.IDLE searchProbeState.value = WorkflowState.IDLE
searchProbeStepId.value = null searchProbeStepId.value = null
candidateEvidence.value = emptyList()
searchProbeJob = lifecycleScope.launch { searchProbeJob = lifecycleScope.launch {
val stateCollector = launch { val stateCollector = launch {
runner.state.collect { state -> searchProbeState.value = state } runner.state.collect { state -> searchProbeState.value = state }
@@ -398,13 +417,27 @@ class MainActivity : ComponentActivity() {
val stepCollector = launch { val stepCollector = launch {
runner.currentStepId.collect { stepId -> searchProbeStepId.value = stepId } runner.currentStepId.collect { stepId -> searchProbeStepId.value = stepId }
} }
val evidenceCollector = launch {
candidateAutomation.evidence.collect { evidence ->
candidateEvidence.value = evidence
}
}
val candidatePhaseCollector = launch {
candidateAutomation.phase.collect { phase ->
if (phase != CandidateBrowsePhase.IDLE) {
Log.d(TAG, "Candidate probe phase: ${phase.name}")
}
}
}
try { try {
val report = runner.run(PinduoduoSearchWorkflow.steps()) val report = runner.run(PinduoduoCandidateWorkflow.steps())
searchProbeReport.value = report searchProbeReport.value = report
searchProbeState.value = report.state searchProbeState.value = report.state
} finally { } finally {
stateCollector.cancel() stateCollector.cancel()
stepCollector.cancel() stepCollector.cancel()
evidenceCollector.cancel()
candidatePhaseCollector.cancel()
searchProbeRunner = null searchProbeRunner = null
refreshReadiness() refreshReadiness()
} }
@@ -1,9 +1,13 @@
package com.roubao.autopilot.accessibility package com.roubao.autopilot.accessibility
import com.roubao.autopilot.pinduoduo.PinduoduoPage import com.roubao.autopilot.pinduoduo.PinduoduoPage
import com.roubao.autopilot.pinduoduo.PinduoduoCandidateCard
import com.roubao.autopilot.pinduoduo.PinduoduoCandidateDetailEvidence
import com.roubao.autopilot.pinduoduo.PinduoduoScreenshotCapture
import com.roubao.autopilot.pinduoduo.PinduoduoUiSnapshot import com.roubao.autopilot.pinduoduo.PinduoduoUiSnapshot
import com.roubao.autopilot.readiness.DeviceObservationStore import com.roubao.autopilot.readiness.DeviceObservationStore
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withTimeoutOrNull
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
object BuyerAccessibilityBridge { object BuyerAccessibilityBridge {
@@ -46,4 +50,38 @@ object BuyerAccessibilityBridge {
suspend fun submitSearch(): Boolean = withContext(Dispatchers.Main.immediate) { suspend fun submitSearch(): Boolean = withContext(Dispatchers.Main.immediate) {
service?.submitPinduoduoSearch() == true service?.submitPinduoduoSearch() == true
} }
suspend fun candidateCards(limit: Int): List<PinduoduoCandidateCard> =
withContext(Dispatchers.Main.immediate) {
service?.readPinduoduoCandidateCards(limit).orEmpty()
}
suspend fun openCandidate(signature: String): Boolean =
withContext(Dispatchers.Main.immediate) {
service?.openPinduoduoCandidate(signature) == true
}
suspend fun candidateDetailEvidence(): PinduoduoCandidateDetailEvidence? =
withContext(Dispatchers.Main.immediate) {
service?.readPinduoduoCandidateDetailEvidence()
}
suspend fun captureScreenshot(): PinduoduoScreenshotCapture? =
withTimeoutOrNull(SCREENSHOT_TIMEOUT_MILLIS) {
withContext(Dispatchers.Main.immediate) {
service?.capturePinduoduoScreenshot()
}
}
suspend fun returnToResults(): Boolean =
withContext(Dispatchers.Main.immediate) {
service?.returnFromPinduoduoCandidate() == true
}
suspend fun scrollResults(): Boolean =
withContext(Dispatchers.Main.immediate) {
service?.scrollPinduoduoResults() == true
}
private const val SCREENSHOT_TIMEOUT_MILLIS = 8_000L
} }
@@ -1,11 +1,19 @@
package com.roubao.autopilot.accessibility package com.roubao.autopilot.accessibility
import android.accessibilityservice.AccessibilityService import android.accessibilityservice.AccessibilityService
import android.graphics.Bitmap
import android.graphics.Rect
import android.os.Build
import android.os.Bundle import android.os.Bundle
import android.os.SystemClock import android.os.SystemClock
import android.util.Log import android.util.Log
import android.view.Display
import android.view.accessibility.AccessibilityEvent import android.view.accessibility.AccessibilityEvent
import android.view.accessibility.AccessibilityNodeInfo import android.view.accessibility.AccessibilityNodeInfo
import androidx.annotation.RequiresApi
import com.roubao.autopilot.pinduoduo.PinduoduoCandidateCard
import com.roubao.autopilot.pinduoduo.PinduoduoCandidateDetailEvidence
import com.roubao.autopilot.pinduoduo.PinduoduoEvidenceHash
import com.roubao.autopilot.readiness.DeviceObservationStore import com.roubao.autopilot.readiness.DeviceObservationStore
import com.roubao.autopilot.readiness.LoginBlockerDetector import com.roubao.autopilot.readiness.LoginBlockerDetector
import com.roubao.autopilot.readiness.PINDUODUO_PACKAGE import com.roubao.autopilot.readiness.PINDUODUO_PACKAGE
@@ -14,11 +22,17 @@ import com.roubao.autopilot.pinduoduo.PinduoduoUiElement
import com.roubao.autopilot.pinduoduo.PinduoduoUiSnapshot import com.roubao.autopilot.pinduoduo.PinduoduoUiSnapshot
import com.roubao.autopilot.pinduoduo.PinduoduoPage import com.roubao.autopilot.pinduoduo.PinduoduoPage
import com.roubao.autopilot.pinduoduo.SEARCH_PROBE_KEYWORD import com.roubao.autopilot.pinduoduo.SEARCH_PROBE_KEYWORD
import com.roubao.autopilot.pinduoduo.PinduoduoScreenshotCapture
import java.io.ByteArrayOutputStream
import java.util.ArrayDeque import java.util.ArrayDeque
import java.util.concurrent.Executors
import kotlin.coroutines.resume
import kotlinx.coroutines.suspendCancellableCoroutine
class BuyerAccessibilityService : AccessibilityService() { class BuyerAccessibilityService : AccessibilityService() {
private var lastPinduoduoScanAt = 0L private var lastPinduoduoScanAt = 0L
private var expectedSearchQuery = SEARCH_PROBE_KEYWORD private var expectedSearchQuery = SEARCH_PROBE_KEYWORD
private val screenshotExecutor = Executors.newSingleThreadExecutor()
override fun onServiceConnected() { override fun onServiceConnected() {
super.onServiceConnected() super.onServiceConnected()
@@ -62,6 +76,7 @@ class BuyerAccessibilityService : AccessibilityService() {
override fun onDestroy() { override fun onDestroy() {
BuyerAccessibilityBridge.detach(this) BuyerAccessibilityBridge.detach(this)
DeviceObservationStore.setAccessibilityConnected(false) DeviceObservationStore.setAccessibilityConnected(false)
screenshotExecutor.shutdownNow()
super.onDestroy() super.onDestroy()
} }
@@ -172,6 +187,165 @@ class BuyerAccessibilityService : AccessibilityService() {
) == true ) == true
} ?: false } ?: false
internal fun readPinduoduoCandidateCards(
limit: Int
): List<PinduoduoCandidateCard> =
withPinduoduoRoot { root ->
val snapshot = classifyPinduoduoRoot(root)
if (
snapshot.safetyStopReason != null ||
snapshot.page != PinduoduoPage.SEARCH_RESULTS
) {
return@withPinduoduoRoot emptyList()
}
candidateNodes(root, limit).map(CandidateNode::card)
} ?: emptyList()
internal fun openPinduoduoCandidate(signature: String): Boolean =
withPinduoduoRoot { root ->
val snapshot = classifyPinduoduoRoot(root)
if (
snapshot.safetyStopReason != null ||
snapshot.page != PinduoduoPage.SEARCH_RESULTS
) {
return@withPinduoduoRoot false
}
candidateNodes(root, MAX_CANDIDATE_NODE_SCAN)
.filter { candidate -> candidate.card.signature == signature }
.singleOrNull()
?.node
?.performAction(AccessibilityNodeInfo.ACTION_CLICK) == true
} ?: false
internal fun readPinduoduoCandidateDetailEvidence():
PinduoduoCandidateDetailEvidence? =
withPinduoduoRoot { root ->
if (!isVerifiedProductDetail(root)) {
return@withPinduoduoRoot null
}
val semanticTexts = collectSemanticTexts(root)
if (semanticTexts.isEmpty()) {
return@withPinduoduoRoot null
}
PinduoduoCandidateDetailEvidence(
signature = PinduoduoEvidenceHash.sha256(
semanticTexts.joinToString(TEXT_SIGNATURE_SEPARATOR)
),
semanticTextCount = semanticTexts.size
)
}
internal fun returnFromPinduoduoCandidate(): Boolean =
withPinduoduoRoot { root ->
if (!isVerifiedProductDetail(root)) {
return@withPinduoduoRoot false
}
performGlobalAction(GLOBAL_ACTION_BACK)
} ?: false
internal fun scrollPinduoduoResults(): Boolean =
withPinduoduoRoot { root ->
val snapshot = classifyPinduoduoRoot(root)
if (
snapshot.safetyStopReason != null ||
snapshot.page != PinduoduoPage.SEARCH_RESULTS
) {
return@withPinduoduoRoot false
}
findMainResultsRecycler(root)?.performAction(
AccessibilityNodeInfo.ACTION_SCROLL_FORWARD
) == true
} ?: false
internal suspend fun capturePinduoduoScreenshot(): PinduoduoScreenshotCapture? {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {
return null
}
return capturePinduoduoScreenshotApi30()
}
@RequiresApi(Build.VERSION_CODES.R)
private suspend fun capturePinduoduoScreenshotApi30():
PinduoduoScreenshotCapture? {
val root = rootInActiveWindow
if (
root?.packageName?.toString() != PINDUODUO_PACKAGE ||
!isVerifiedProductDetail(root)
) {
return null
}
return suspendCancellableCoroutine { continuation ->
val callback = object : TakeScreenshotCallback {
override fun onSuccess(screenshot: ScreenshotResult) {
val hardwareBuffer = screenshot.hardwareBuffer
var hardwareBitmap: Bitmap? = null
var softwareBitmap: Bitmap? = null
val capture = try {
hardwareBitmap = Bitmap.wrapHardwareBuffer(
hardwareBuffer,
screenshot.colorSpace
)
softwareBitmap = hardwareBitmap?.copy(
Bitmap.Config.ARGB_8888,
false
)
val bitmap = softwareBitmap
if (bitmap == null) {
null
} else {
val output = ByteArrayOutputStream()
if (
!bitmap.compress(
Bitmap.CompressFormat.PNG,
100,
output
)
) {
null
} else {
PinduoduoScreenshotCapture(
pngBytes = output.toByteArray(),
width = bitmap.width,
height = bitmap.height
)
}
}
} catch (_: Exception) {
Log.w(TAG, "Pinduoduo screenshot conversion failed")
null
} finally {
softwareBitmap?.recycle()
hardwareBitmap?.recycle()
hardwareBuffer.close()
}
if (continuation.isActive) {
continuation.resume(capture)
}
}
override fun onFailure(errorCode: Int) {
Log.w(TAG, "Pinduoduo screenshot failed with code $errorCode")
if (continuation.isActive) {
continuation.resume(null)
}
}
}
try {
takeScreenshot(
Display.DEFAULT_DISPLAY,
screenshotExecutor,
callback
)
} catch (_: Exception) {
Log.w(TAG, "Pinduoduo screenshot request failed")
if (continuation.isActive) {
continuation.resume(null)
}
}
}
}
private fun classifyPinduoduoRoot( private fun classifyPinduoduoRoot(
root: AccessibilityNodeInfo root: AccessibilityNodeInfo
): PinduoduoUiSnapshot { ): PinduoduoUiSnapshot {
@@ -194,6 +368,205 @@ class BuyerAccessibilityService : AccessibilityService() {
) )
} }
private fun candidateNodes(
root: AccessibilityNodeInfo,
limit: Int
): List<CandidateNode> {
if (limit <= 0) {
return emptyList()
}
val recycler = findMainResultsRecycler(root) ?: return emptyList()
val rootBounds = Rect().also(root::getBoundsInScreen)
val recyclerBounds = Rect().also(recycler::getBoundsInScreen)
val minimumHeight = (rootBounds.height() / 8).coerceAtLeast(1)
val candidates = ArrayList<CandidateNode>()
for (index in 0 until recycler.childCount) {
val node = recycler.getChild(index) ?: continue
if (
!node.isVisibleToUser ||
!node.isEnabled ||
node.className?.toString()?.endsWith("FrameLayout") != true
) {
continue
}
val bounds = Rect().also(node::getBoundsInScreen)
if (
bounds.width() * 100 < recyclerBounds.width() * MIN_CARD_WIDTH_PERCENT ||
bounds.width() * 100 > recyclerBounds.width() * MAX_CARD_WIDTH_PERCENT ||
bounds.height() < minimumHeight
) {
continue
}
val visibleBounds = Rect(bounds)
if (
!visibleBounds.intersect(recyclerBounds) ||
area(visibleBounds) * 100 < area(bounds) * MIN_VISIBLE_CARD_PERCENT
) {
continue
}
val descendants = collectNodes(node)
val semanticTexts = collectSemanticTexts(descendants)
val hasImage = descendants.any { descendant ->
descendant.isVisibleToUser &&
descendant.isEnabled &&
descendant.className?.toString()?.endsWith("ImageView") == true
}
val hasPrice = semanticTexts.any(::hasPriceSemantics)
val clickTarget = findSafeCandidateClickTarget(node, descendants)
if (
semanticTexts.size < MIN_CANDIDATE_TEXTS ||
!hasImage ||
!hasPrice ||
clickTarget == null
) {
continue
}
val card = PinduoduoCandidateCard(
signature = PinduoduoEvidenceHash.sha256(
semanticTexts.joinToString(TEXT_SIGNATURE_SEPARATOR)
),
semanticTextCount = semanticTexts.size,
hasImage = true
)
candidates += CandidateNode(clickTarget, card)
if (candidates.size >= limit) {
break
}
}
return candidates
}
private fun isVerifiedProductDetail(root: AccessibilityNodeInfo): Boolean {
val snapshot = classifyPinduoduoRoot(root)
if (
snapshot.safetyStopReason != null ||
snapshot.page != PinduoduoPage.PRODUCT_DETAIL
) {
return false
}
val rootBounds = Rect().also(root::getBoundsInScreen)
val nodes = collectNodes(root)
val hasWideViewPager = nodes.any { node ->
if (
!node.isVisibleToUser ||
!node.isEnabled ||
node.className?.toString()?.endsWith("ViewPager") != true
) {
return@any false
}
val bounds = Rect().also(node::getBoundsInScreen)
bounds.width() * 100 >=
rootBounds.width() * MIN_DETAIL_CONTENT_WIDTH_PERCENT
}
val hasWideScrollableRecycler = nodes.any { node ->
if (
!node.isVisibleToUser ||
!node.isEnabled ||
!node.isScrollable ||
node.className?.toString()?.endsWith("RecyclerView") != true
) {
return@any false
}
val bounds = Rect().also(node::getBoundsInScreen)
bounds.width() * 100 >=
rootBounds.width() * MIN_DETAIL_CONTENT_WIDTH_PERCENT &&
collectNodes(node).size >= MIN_DETAIL_RECYCLER_NODES
}
return hasWideViewPager && hasWideScrollableRecycler
}
private fun findMainResultsRecycler(
root: AccessibilityNodeInfo
): AccessibilityNodeInfo? {
val rootBounds = Rect().also(root::getBoundsInScreen)
return collectNodes(root)
.asSequence()
.filter { node ->
node.isVisibleToUser &&
node.isEnabled &&
node.isScrollable &&
node.className?.toString()?.endsWith("RecyclerView") == true
}
.map { node ->
val bounds = Rect().also(node::getBoundsInScreen)
RecyclerCandidate(
node = node,
bounds = bounds,
descendantCount = collectNodes(node).size
)
}
.filter { candidate ->
candidate.bounds.width() * 100 >=
rootBounds.width() * MIN_MAIN_RECYCLER_WIDTH_PERCENT &&
candidate.descendantCount >= MIN_MAIN_RECYCLER_NODES
}
.maxByOrNull { candidate -> area(candidate.bounds) }
?.node
}
private fun findSafeCandidateClickTarget(
cardRoot: AccessibilityNodeInfo,
descendants: Collection<AccessibilityNodeInfo>
): AccessibilityNodeInfo? {
if (cardRoot.isClickable) {
return cardRoot
}
val cardBounds = Rect().also(cardRoot::getBoundsInScreen)
val coveringTargets = descendants.filter { node ->
if (
node === cardRoot ||
!node.isVisibleToUser ||
!node.isEnabled ||
!node.isClickable
) {
return@filter false
}
val targetBounds = Rect().also(node::getBoundsInScreen)
val overlap = Rect(targetBounds)
overlap.intersect(cardBounds) &&
area(overlap) * 100 >=
area(cardBounds) * MIN_CLICK_TARGET_COVER_PERCENT
}
return coveringTargets.singleOrNull()
}
private fun hasPriceSemantics(value: String): Boolean =
value.contains('¥') ||
value.contains('¥') ||
DECIMAL_PRICE_PATTERN.matches(value)
private fun area(bounds: Rect): Long =
bounds.width().toLong() * bounds.height().toLong()
private fun collectSemanticTexts(
root: AccessibilityNodeInfo
): List<String> = collectSemanticTexts(collectNodes(root))
private fun collectSemanticTexts(
nodes: Collection<AccessibilityNodeInfo>
): List<String> =
nodes.asSequence()
.filter { node -> node.isVisibleToUser && node.isEnabled }
.flatMap { node ->
sequenceOf(
node.text?.toString(),
node.contentDescription?.toString()
)
}
.filterNotNull()
.map(::normalizeEvidenceText)
.filter(String::isNotEmpty)
.distinct()
.take(MAX_EVIDENCE_TEXTS)
.toList()
private fun normalizeEvidenceText(value: String): String =
value.trim()
.replace(Regex("\\s+"), " ")
.take(MAX_EVIDENCE_TEXT_LENGTH)
private fun shouldInspect(eventType: Int): Boolean = private fun shouldInspect(eventType: Int): Boolean =
eventType == AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED || eventType == AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED ||
eventType == AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED || eventType == AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED ||
@@ -252,10 +625,36 @@ class BuyerAccessibilityService : AccessibilityService() {
return false return false
} }
private data class CandidateNode(
val node: AccessibilityNodeInfo,
val card: PinduoduoCandidateCard
)
private data class RecyclerCandidate(
val node: AccessibilityNodeInfo,
val bounds: Rect,
val descendantCount: Int
)
companion object { companion object {
private const val TAG = "BuyerAccessibility" private const val TAG = "BuyerAccessibility"
private const val MAX_NODES = 300 private const val MAX_NODES = 300
private const val MAX_CLICK_ANCESTORS = 4 private const val MAX_CLICK_ANCESTORS = 4
private const val MAX_CANDIDATE_NODE_SCAN = 20
private const val MIN_CANDIDATE_TEXTS = 2
private const val MIN_MAIN_RECYCLER_NODES = 20
private const val MIN_MAIN_RECYCLER_WIDTH_PERCENT = 90
private const val MIN_DETAIL_CONTENT_WIDTH_PERCENT = 90
private const val MIN_DETAIL_RECYCLER_NODES = 40
private const val MIN_CARD_WIDTH_PERCENT = 35
private const val MAX_CARD_WIDTH_PERCENT = 60
private const val MIN_VISIBLE_CARD_PERCENT = 80
private const val MIN_CLICK_TARGET_COVER_PERCENT = 85
private const val MAX_EVIDENCE_TEXTS = 100
private const val MAX_EVIDENCE_TEXT_LENGTH = 500
private const val TEXT_SIGNATURE_SEPARATOR = "\u001f"
private const val MIN_SCAN_INTERVAL_MS = 300L private const val MIN_SCAN_INTERVAL_MS = 300L
private val DECIMAL_PRICE_PATTERN =
Regex("^\\s*\\d{1,6}\\.\\d{1,2}\\s*$")
} }
} }
@@ -0,0 +1,154 @@
package com.roubao.autopilot.pinduoduo
import android.content.Context
import com.roubao.autopilot.accessibility.BuyerAccessibilityBridge
import java.io.File
import java.io.IOException
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import org.json.JSONArray
import org.json.JSONObject
class AndroidPinduoduoCandidateDriver(
context: Context
) : PinduoduoCandidateDriver {
private val evidenceStore = CandidateEvidenceStore(context.applicationContext)
override suspend fun snapshot(): PinduoduoUiSnapshot =
BuyerAccessibilityBridge.snapshot()
override suspend fun candidateCards(
limit: Int
): List<PinduoduoCandidateCard> =
BuyerAccessibilityBridge.candidateCards(limit)
override suspend fun openCandidate(signature: String): Boolean =
BuyerAccessibilityBridge.openCandidate(signature)
override suspend fun captureCandidate(
ordinal: Int,
card: PinduoduoCandidateCard
): PinduoduoCandidateEvidence? {
val detail = BuyerAccessibilityBridge.candidateDetailEvidence()
?: return null
val screenshot = BuyerAccessibilityBridge.captureScreenshot()
?: return null
return withContext(Dispatchers.IO) {
evidenceStore.save(
ordinal = ordinal,
card = card,
detail = detail,
screenshot = screenshot
)
}
}
override suspend fun returnToResults(): Boolean =
BuyerAccessibilityBridge.returnToResults()
override suspend fun scrollResults(): Boolean =
BuyerAccessibilityBridge.scrollResults()
override fun resetEvidence() {
evidenceStore.reset()
}
}
private class CandidateEvidenceStore(context: Context) {
private val root = File(context.cacheDir, EVIDENCE_DIRECTORY)
private val evidenceByOrdinal =
linkedMapOf<Int, PinduoduoCandidateEvidence>()
fun reset() {
evidenceByOrdinal.clear()
if (root.exists()) {
root.deleteRecursively()
}
root.mkdirs()
}
fun save(
ordinal: Int,
card: PinduoduoCandidateCard,
detail: PinduoduoCandidateDetailEvidence,
screenshot: PinduoduoScreenshotCapture
): PinduoduoCandidateEvidence? {
if (ordinal !in 1..MAX_CANDIDATES_PER_PROBE) {
return null
}
return try {
if (!root.exists() && !root.mkdirs()) {
return null
}
val fileName = "candidate-%02d.png".format(ordinal)
val screenshotFile = File(root, fileName)
screenshotFile.writeBytes(screenshot.pngBytes)
val evidence = PinduoduoCandidateEvidence(
ordinal = ordinal,
cardSignature = card.signature,
cardSemanticTextCount = card.semanticTextCount,
detailSignature = detail.signature,
detailSemanticTextCount = detail.semanticTextCount,
screenshotFileName = fileName,
screenshotSha256 = PinduoduoEvidenceHash.sha256(
screenshot.pngBytes
),
screenshotByteCount = screenshot.pngBytes.size,
screenshotWidth = screenshot.width,
screenshotHeight = screenshot.height
)
evidenceByOrdinal[ordinal] = evidence
writeManifest()
evidence
} catch (_: IOException) {
null
}
}
private fun writeManifest() {
val candidates = JSONArray()
evidenceByOrdinal.values.forEach { evidence ->
candidates.put(
JSONObject()
.put("ordinal", evidence.ordinal)
.put("card_signature", evidence.cardSignature)
.put(
"card_semantic_text_count",
evidence.cardSemanticTextCount
)
.put("detail_signature", evidence.detailSignature)
.put(
"detail_semantic_text_count",
evidence.detailSemanticTextCount
)
.put("screenshot_file", evidence.screenshotFileName)
.put("screenshot_sha256", evidence.screenshotSha256)
.put(
"screenshot_byte_count",
evidence.screenshotByteCount
)
.put("screenshot_width", evidence.screenshotWidth)
.put("screenshot_height", evidence.screenshotHeight)
)
}
val manifest = JSONObject()
.put("schema_version", 1)
.put("candidate_count", evidenceByOrdinal.size)
.put("candidates", candidates)
.toString(2)
val temporary = File(root, "$MANIFEST_FILE.tmp")
val destination = File(root, MANIFEST_FILE)
temporary.writeText(manifest, Charsets.UTF_8)
if (destination.exists() && !destination.delete()) {
throw IOException("Could not replace candidate evidence manifest")
}
if (!temporary.renameTo(destination)) {
throw IOException("Could not publish candidate evidence manifest")
}
}
private companion object {
const val EVIDENCE_DIRECTORY = "pdd-candidate-probe"
const val MANIFEST_FILE = "manifest.json"
}
}
@@ -42,6 +42,9 @@ class AndroidPinduoduoUiDriver(context: Context) : PinduoduoUiDriver {
override suspend fun submitSearch(): Boolean = override suspend fun submitSearch(): Boolean =
BuyerAccessibilityBridge.submitSearch() BuyerAccessibilityBridge.submitSearch()
override suspend fun returnFromCandidate(): Boolean =
BuyerAccessibilityBridge.returnToResults()
private companion object { private companion object {
const val KEYWORD_VERIFY_ATTEMPTS = 5 const val KEYWORD_VERIFY_ATTEMPTS = 5
const val KEYWORD_VERIFY_INTERVAL_MS = 100L const val KEYWORD_VERIFY_INTERVAL_MS = 100L
@@ -0,0 +1,205 @@
package com.roubao.autopilot.pinduoduo
import com.roubao.autopilot.readiness.PINDUODUO_PACKAGE
import com.roubao.autopilot.workflow.AutomationGateway
import com.roubao.autopilot.workflow.AutomationResult
import com.roubao.autopilot.workflow.SafetyStopReason
import com.roubao.autopilot.workflow.WorkflowFailureCode
import com.roubao.autopilot.workflow.WorkflowStep
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
enum class CandidateBrowsePhase {
IDLE,
READING_RESULTS,
OPENING_CANDIDATE,
WAITING_DETAIL,
CAPTURING_EVIDENCE,
RETURNING_RESULTS,
SCROLLING_RESULTS,
COMPLETE
}
class PinduoduoCandidateAutomation(
private val driver: PinduoduoCandidateDriver,
private val maxCandidates: Int = MAX_CANDIDATES_PER_PROBE,
private val maxResultScrolls: Int = MAX_RESULT_SCROLLS_PER_PROBE,
private val pagePollIntervalMillis: Long = 200,
private val unknownPageLimit: Int = 20
) : AutomationGateway {
private val mutableEvidence =
MutableStateFlow<List<PinduoduoCandidateEvidence>>(emptyList())
private val mutablePhase = MutableStateFlow(CandidateBrowsePhase.IDLE)
private val attemptedSignatures = linkedSetOf<String>()
private var resultScrollCount = 0
val evidence: StateFlow<List<PinduoduoCandidateEvidence>> =
mutableEvidence.asStateFlow()
val phase: StateFlow<CandidateBrowsePhase> = mutablePhase.asStateFlow()
init {
require(maxCandidates in 1..MAX_CANDIDATES_PER_PROBE)
require(maxResultScrolls in 0..MAX_RESULT_SCROLLS_PER_PROBE)
require(pagePollIntervalMillis > 0)
require(unknownPageLimit > 0)
}
fun reset() {
attemptedSignatures.clear()
resultScrollCount = 0
mutableEvidence.value = emptyList()
mutablePhase.value = CandidateBrowsePhase.IDLE
driver.resetEvidence()
}
override suspend fun execute(step: WorkflowStep): AutomationResult {
if (step.id != PinduoduoCandidateWorkflow.BROWSE_CANDIDATES) {
return AutomationResult.FatalFailure(
WorkflowFailureCode.AUTOMATION_EXCEPTION
)
}
return browseCandidates()
}
private suspend fun browseCandidates(): AutomationResult {
mutablePhase.value = CandidateBrowsePhase.READING_RESULTS
recoverDetailPageIfNeeded()?.let { return it }
while (
mutableEvidence.value.size < maxCandidates &&
attemptedSignatures.size < maxCandidates
) {
validateResultsPage()?.let { return it }
val nextCard = driver.candidateCards(maxCandidates * 2)
.firstOrNull { card -> card.signature !in attemptedSignatures }
if (nextCard == null) {
if (resultScrollCount >= maxResultScrolls) {
return terminalCollectionResult()
}
resultScrollCount += 1
mutablePhase.value = CandidateBrowsePhase.SCROLLING_RESULTS
if (!driver.scrollResults()) {
return terminalCollectionResult()
}
delay(pagePollIntervalMillis)
mutablePhase.value = CandidateBrowsePhase.READING_RESULTS
continue
}
attemptedSignatures += nextCard.signature
mutablePhase.value = CandidateBrowsePhase.OPENING_CANDIDATE
if (!driver.openCandidate(nextCard.signature)) {
return AutomationResult.RetryableFailure(
WorkflowFailureCode.TRANSIENT_AUTOMATION
)
}
mutablePhase.value = CandidateBrowsePhase.WAITING_DETAIL
awaitPage(PinduoduoPage.PRODUCT_DETAIL)?.let { return it }
val ordinal = mutableEvidence.value.size + 1
mutablePhase.value = CandidateBrowsePhase.CAPTURING_EVIDENCE
val captured = driver.captureCandidate(ordinal, nextCard)
?: return AutomationResult.FatalFailure(
WorkflowFailureCode.EVIDENCE_CAPTURE_FAILED
)
mutableEvidence.value = mutableEvidence.value + captured
mutablePhase.value = CandidateBrowsePhase.RETURNING_RESULTS
if (!driver.returnToResults()) {
return AutomationResult.RetryableFailure(
WorkflowFailureCode.TRANSIENT_AUTOMATION
)
}
awaitPage(PinduoduoPage.SEARCH_RESULTS)?.let { return it }
mutablePhase.value = CandidateBrowsePhase.READING_RESULTS
}
return terminalCollectionResult()
}
private suspend fun recoverDetailPageIfNeeded(): AutomationResult? {
val snapshot = driver.snapshot()
safetyResult(snapshot)?.let { return it }
if (snapshot.page != PinduoduoPage.PRODUCT_DETAIL) {
return null
}
if (!driver.returnToResults()) {
return AutomationResult.RetryableFailure(
WorkflowFailureCode.TRANSIENT_AUTOMATION
)
}
return awaitPage(PinduoduoPage.SEARCH_RESULTS)
}
private suspend fun validateResultsPage(): AutomationResult? {
val snapshot = driver.snapshot()
safetyResult(snapshot)?.let { return it }
return if (snapshot.page == PinduoduoPage.SEARCH_RESULTS) {
null
} else {
AutomationResult.Blocked(SafetyStopReason.UNKNOWN_PAGE)
}
}
private suspend fun awaitPage(expected: PinduoduoPage): AutomationResult? {
var stableUnexpectedObservations = 0
while (true) {
val snapshot = driver.snapshot()
safetyResult(snapshot)?.let { return it }
if (snapshot.page == expected) {
return null
}
stableUnexpectedObservations = if (
snapshot.foregroundPackage == PINDUODUO_PACKAGE
) {
stableUnexpectedObservations + 1
} else {
0
}
if (stableUnexpectedObservations >= unknownPageLimit) {
return AutomationResult.Blocked(SafetyStopReason.UNKNOWN_PAGE)
}
delay(pagePollIntervalMillis)
}
}
private fun terminalCollectionResult(): AutomationResult {
mutablePhase.value = CandidateBrowsePhase.COMPLETE
return if (mutableEvidence.value.isNotEmpty()) {
AutomationResult.Success
} else {
AutomationResult.FatalFailure(WorkflowFailureCode.TARGET_NOT_READY)
}
}
private fun safetyResult(snapshot: PinduoduoUiSnapshot): AutomationResult.Blocked? =
snapshot.safetyStopReason?.let(AutomationResult::Blocked)
}
class PinduoduoProbeAutomation(
private val searchAutomation: PinduoduoSearchAutomation,
private val candidateAutomation: PinduoduoCandidateAutomation
) : AutomationGateway {
override suspend fun execute(step: WorkflowStep): AutomationResult =
if (step.id == PinduoduoCandidateWorkflow.BROWSE_CANDIDATES) {
candidateAutomation.execute(step)
} else {
searchAutomation.execute(step)
}
}
object PinduoduoCandidateWorkflow {
const val BROWSE_CANDIDATES = "pdd_browse_candidates"
fun steps(): List<WorkflowStep> =
PinduoduoSearchWorkflow.steps() +
WorkflowStep(
id = BROWSE_CANDIDATES,
timeoutMillis = 120_000,
maxRetries = 1
)
}
@@ -0,0 +1,59 @@
package com.roubao.autopilot.pinduoduo
import java.security.MessageDigest
const val MAX_CANDIDATES_PER_PROBE = 5
const val MAX_RESULT_SCROLLS_PER_PROBE = 2
data class PinduoduoCandidateCard(
val signature: String,
val semanticTextCount: Int,
val hasImage: Boolean
)
data class PinduoduoCandidateDetailEvidence(
val signature: String,
val semanticTextCount: Int
)
data class PinduoduoScreenshotCapture(
val pngBytes: ByteArray,
val width: Int,
val height: Int
)
data class PinduoduoCandidateEvidence(
val ordinal: Int,
val cardSignature: String,
val cardSemanticTextCount: Int,
val detailSignature: String,
val detailSemanticTextCount: Int,
val screenshotFileName: String,
val screenshotSha256: String,
val screenshotByteCount: Int,
val screenshotWidth: Int,
val screenshotHeight: Int
)
object PinduoduoEvidenceHash {
fun sha256(value: String): String =
sha256(value.toByteArray(Charsets.UTF_8))
fun sha256(value: ByteArray): String =
MessageDigest.getInstance("SHA-256")
.digest(value)
.joinToString(separator = "") { byte -> "%02x".format(byte) }
}
interface PinduoduoCandidateDriver {
suspend fun snapshot(): PinduoduoUiSnapshot
suspend fun candidateCards(limit: Int): List<PinduoduoCandidateCard>
suspend fun openCandidate(signature: String): Boolean
suspend fun captureCandidate(
ordinal: Int,
card: PinduoduoCandidateCard
): PinduoduoCandidateEvidence?
suspend fun returnToResults(): Boolean
suspend fun scrollResults(): Boolean
fun resetEvidence()
}
@@ -20,6 +20,7 @@ enum class PinduoduoPage {
HOME, HOME,
SEARCH_INPUT, SEARCH_INPUT,
SEARCH_RESULTS, SEARCH_RESULTS,
PRODUCT_DETAIL,
UNKNOWN UNKNOWN
} }
@@ -91,12 +92,19 @@ object PinduoduoPageClassifier {
val sortControlCount = setOf("综合", "销量", "价格", "筛选") val sortControlCount = setOf("综合", "销量", "价格", "筛选")
.count { marker -> normalized.any { it == marker } } .count { marker -> normalized.any { it == marker } }
val hasExactQuery = visibleTexts.any { it.trim() == expectedQuery } val hasExactQuery = visibleTexts.any { it.trim() == expectedQuery }
val hasDetailBack = visibleElements.any { element ->
element.clickable &&
normalize(element.contentDescription.orEmpty()) == "返回"
}
val detailMarkerCount = setOf("联系客服", "收藏", "店铺")
.count { marker -> normalized.any { it.contains(marker) } }
val page = when { val page = when {
hasExactQuery && hasResultSearchHeader && sortControlCount >= 3 -> hasExactQuery && hasResultSearchHeader && sortControlCount >= 3 ->
PinduoduoPage.SEARCH_RESULTS PinduoduoPage.SEARCH_RESULTS
hasSearchInput && hasSubmitButton -> PinduoduoPage.SEARCH_INPUT hasSearchInput && hasSubmitButton -> PinduoduoPage.SEARCH_INPUT
hasHomeSearchEntry && hasHome -> PinduoduoPage.HOME hasHomeSearchEntry && hasHome -> PinduoduoPage.HOME
hasDetailBack && detailMarkerCount >= 2 -> PinduoduoPage.PRODUCT_DETAIL
else -> PinduoduoPage.UNKNOWN else -> PinduoduoPage.UNKNOWN
} }
return PinduoduoUiSnapshot( return PinduoduoUiSnapshot(
@@ -15,6 +15,7 @@ interface PinduoduoUiDriver {
suspend fun openSearch(): Boolean suspend fun openSearch(): Boolean
suspend fun setSearchKeyword(keyword: String): Boolean suspend fun setSearchKeyword(keyword: String): Boolean
suspend fun submitSearch(): Boolean suspend fun submitSearch(): Boolean
suspend fun returnFromCandidate(): Boolean
} }
class PinduoduoSearchAutomation( class PinduoduoSearchAutomation(
@@ -45,7 +46,8 @@ class PinduoduoSearchAutomation(
return awaitPage { return awaitPage {
it == PinduoduoPage.HOME || it == PinduoduoPage.HOME ||
it == PinduoduoPage.SEARCH_INPUT || it == PinduoduoPage.SEARCH_INPUT ||
it == PinduoduoPage.SEARCH_RESULTS it == PinduoduoPage.SEARCH_RESULTS ||
it == PinduoduoPage.PRODUCT_DETAIL
} ?: AutomationResult.Success } ?: AutomationResult.Success
} }
@@ -53,7 +55,8 @@ class PinduoduoSearchAutomation(
val ready = awaitPage { val ready = awaitPage {
it == PinduoduoPage.HOME || it == PinduoduoPage.HOME ||
it == PinduoduoPage.SEARCH_INPUT || it == PinduoduoPage.SEARCH_INPUT ||
it == PinduoduoPage.SEARCH_RESULTS it == PinduoduoPage.SEARCH_RESULTS ||
it == PinduoduoPage.PRODUCT_DETAIL
} }
if (ready != null) { if (ready != null) {
return ready return ready
@@ -63,6 +66,15 @@ class PinduoduoSearchAutomation(
if (currentPage == PinduoduoPage.SEARCH_RESULTS) { if (currentPage == PinduoduoPage.SEARCH_RESULTS) {
return AutomationResult.Success return AutomationResult.Success
} }
if (currentPage == PinduoduoPage.PRODUCT_DETAIL) {
if (!driver.returnFromCandidate()) {
return AutomationResult.RetryableFailure(
WorkflowFailureCode.TRANSIENT_AUTOMATION
)
}
return awaitPage { it == PinduoduoPage.SEARCH_RESULTS }
?: AutomationResult.Success
}
if (currentPage != PinduoduoPage.SEARCH_INPUT) { if (currentPage != PinduoduoPage.SEARCH_INPUT) {
if (!driver.openSearch()) { if (!driver.openSearch()) {
return AutomationResult.RetryableFailure( return AutomationResult.RetryableFailure(
@@ -34,6 +34,8 @@ import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp import androidx.compose.ui.unit.sp
import com.roubao.autopilot.pinduoduo.PinduoduoSearchWorkflow import com.roubao.autopilot.pinduoduo.PinduoduoSearchWorkflow
import com.roubao.autopilot.pinduoduo.MAX_CANDIDATES_PER_PROBE
import com.roubao.autopilot.pinduoduo.PinduoduoCandidateWorkflow
import com.roubao.autopilot.pinduoduo.SEARCH_PROBE_KEYWORD import com.roubao.autopilot.pinduoduo.SEARCH_PROBE_KEYWORD
import com.roubao.autopilot.readiness.DeviceReadinessSnapshot import com.roubao.autopilot.readiness.DeviceReadinessSnapshot
import com.roubao.autopilot.ui.theme.BaoziTheme import com.roubao.autopilot.ui.theme.BaoziTheme
@@ -51,7 +53,8 @@ private val probeSteps = listOf(
ProbeStepUi(PinduoduoSearchWorkflow.OPEN_APP, "打开拼多多"), ProbeStepUi(PinduoduoSearchWorkflow.OPEN_APP, "打开拼多多"),
ProbeStepUi(PinduoduoSearchWorkflow.ENTER_QUERY, "输入固定关键词"), ProbeStepUi(PinduoduoSearchWorkflow.ENTER_QUERY, "输入固定关键词"),
ProbeStepUi(PinduoduoSearchWorkflow.SUBMIT_QUERY, "提交搜索"), ProbeStepUi(PinduoduoSearchWorkflow.SUBMIT_QUERY, "提交搜索"),
ProbeStepUi(PinduoduoSearchWorkflow.VERIFY_RESULTS, "确认结果页") ProbeStepUi(PinduoduoSearchWorkflow.VERIFY_RESULTS, "确认结果页"),
ProbeStepUi(PinduoduoCandidateWorkflow.BROWSE_CANDIDATES, "采集候选证据")
) )
@Composable @Composable
@@ -60,6 +63,7 @@ fun SearchProbeScreen(
state: WorkflowState, state: WorkflowState,
currentStepId: String?, currentStepId: String?,
report: WorkflowReport?, report: WorkflowReport?,
candidateEvidenceCount: Int,
onStart: () -> Unit, onStart: () -> Unit,
onStop: () -> Unit onStop: () -> Unit
) { ) {
@@ -75,13 +79,13 @@ fun SearchProbeScreen(
) { ) {
item { item {
Text( Text(
text = "拼多多搜索探针", text = "拼多多候选探针",
fontSize = 28.sp, fontSize = 28.sp,
fontWeight = FontWeight.Bold, fontWeight = FontWeight.Bold,
color = colors.textPrimary color = colors.textPrimary
) )
Text( Text(
text = stateLabel(state, report), text = stateLabel(state, report, candidateEvidenceCount),
fontSize = 14.sp, fontSize = 14.sp,
color = stateColor(state) color = stateColor(state)
) )
@@ -118,6 +122,41 @@ fun SearchProbeScreen(
Divider(color = colors.surfaceVariant) Divider(color = colors.surfaceVariant)
} }
item {
Row(
modifier = Modifier
.fillMaxWidth()
.height(58.dp),
verticalAlignment = Alignment.CenterVertically
) {
Icon(
imageVector = Icons.Default.CheckCircle,
contentDescription = null,
tint = if (candidateEvidenceCount > 0) {
colors.success
} else {
colors.textHint
},
modifier = Modifier.size(22.dp)
)
Text(
text = "候选证据",
fontSize = 14.sp,
fontWeight = FontWeight.Medium,
color = colors.textPrimary,
modifier = Modifier
.weight(1f)
.padding(start = 12.dp)
)
Text(
text = "$candidateEvidenceCount / $MAX_CANDIDATES_PER_PROBE",
fontSize = 13.sp,
color = colors.textSecondary
)
}
Divider(color = colors.surfaceVariant)
}
items(probeSteps.size) { index -> items(probeSteps.size) { index ->
val step = probeSteps[index] val step = probeSteps[index]
ProbeStepRow( ProbeStepRow(
@@ -151,7 +190,7 @@ fun SearchProbeScreen(
) { ) {
Icon(Icons.Default.PlayArrow, contentDescription = null) Icon(Icons.Default.PlayArrow, contentDescription = null)
Spacer(modifier = Modifier.size(8.dp)) Spacer(modifier = Modifier.size(8.dp))
Text("开始搜索探针") Text("开始候选探针")
} }
} }
if (!readiness.canStartProbe) { if (!readiness.canStartProbe) {
@@ -263,12 +302,16 @@ private fun stateColor(state: WorkflowState): Color {
} }
} }
private fun stateLabel(state: WorkflowState, report: WorkflowReport?): String = private fun stateLabel(
state: WorkflowState,
report: WorkflowReport?,
candidateEvidenceCount: Int
): String =
when (state) { when (state) {
WorkflowState.IDLE -> "等待开始" WorkflowState.IDLE -> "等待开始"
WorkflowState.RUNNING -> "正在执行" WorkflowState.RUNNING -> "正在执行"
WorkflowState.RETRYING -> "正在重试" WorkflowState.RETRYING -> "正在重试"
WorkflowState.SUCCEEDED -> "搜索结果页已确认" WorkflowState.SUCCEEDED -> "已采集 $candidateEvidenceCount 个候选"
WorkflowState.STOPPED -> "已由用户停止" WorkflowState.STOPPED -> "已由用户停止"
WorkflowState.FAILED -> failureLabel(report?.failureCode) WorkflowState.FAILED -> failureLabel(report?.failureCode)
WorkflowState.BLOCKED -> blockedLabel(report?.safetyStopReason) WorkflowState.BLOCKED -> blockedLabel(report?.safetyStopReason)
@@ -279,6 +322,7 @@ private fun failureLabel(code: WorkflowFailureCode?): String = when (code) {
WorkflowFailureCode.TRANSIENT_AUTOMATION -> "页面操作失败" WorkflowFailureCode.TRANSIENT_AUTOMATION -> "页面操作失败"
WorkflowFailureCode.AUTOMATION_EXCEPTION -> "自动化执行异常" WorkflowFailureCode.AUTOMATION_EXCEPTION -> "自动化执行异常"
WorkflowFailureCode.TARGET_NOT_READY -> "拼多多无法启动" WorkflowFailureCode.TARGET_NOT_READY -> "拼多多无法启动"
WorkflowFailureCode.EVIDENCE_CAPTURE_FAILED -> "候选证据保存失败"
null -> "执行失败" null -> "执行失败"
} }
@@ -30,7 +30,8 @@ enum class WorkflowFailureCode {
TIMEOUT, TIMEOUT,
TRANSIENT_AUTOMATION, TRANSIENT_AUTOMATION,
AUTOMATION_EXCEPTION, AUTOMATION_EXCEPTION,
TARGET_NOT_READY TARGET_NOT_READY,
EVIDENCE_CAPTURE_FAILED
} }
enum class SafetyStopReason { enum class SafetyStopReason {
@@ -3,6 +3,7 @@
android:accessibilityEventTypes="typeWindowStateChanged|typeWindowContentChanged|typeViewScrolled" android:accessibilityEventTypes="typeWindowStateChanged|typeWindowContentChanged|typeViewScrolled"
android:accessibilityFeedbackType="feedbackGeneric" android:accessibilityFeedbackType="feedbackGeneric"
android:accessibilityFlags="flagReportViewIds" android:accessibilityFlags="flagReportViewIds"
android:canTakeScreenshot="true"
android:canRetrieveWindowContent="true" android:canRetrieveWindowContent="true"
android:description="@string/buyer_accessibility_description" android:description="@string/buyer_accessibility_description"
android:notificationTimeout="100" android:notificationTimeout="100"
@@ -0,0 +1,260 @@
package com.roubao.autopilot.pinduoduo
import com.roubao.autopilot.readiness.PINDUODUO_PACKAGE
import com.roubao.autopilot.workflow.AutomationResult
import com.roubao.autopilot.workflow.SafetyStopReason
import com.roubao.autopilot.workflow.WorkflowFailureCode
import kotlinx.coroutines.test.runTest
import org.junit.Assert.assertEquals
import org.junit.Assert.assertTrue
import org.junit.Test
class PinduoduoCandidateAutomationTest {
@Test
fun `collects five distinct candidates with bounded scrolling`() = runTest {
val driver = FakeCandidateDriver(
cardPages = listOf(
listOf(card("a"), card("b"), card("c")),
listOf(card("c"), card("d"), card("e"), card("f"))
)
)
val automation = PinduoduoCandidateAutomation(
driver = driver,
pagePollIntervalMillis = 1
)
automation.reset()
val result = automation.execute(
PinduoduoCandidateWorkflow.steps().last()
)
assertEquals(AutomationResult.Success, result)
assertEquals(listOf("a", "b", "c", "d", "e"), driver.openedSignatures)
assertEquals(5, automation.evidence.value.size)
assertEquals(1, driver.scrollCalls)
assertEquals(5, driver.savedEvidence.size)
}
@Test
fun `configured candidate limit is a hard upper bound`() = runTest {
val driver = FakeCandidateDriver(
cardPages = listOf(
listOf(card("a"), card("b"), card("c"), card("d"))
)
)
val automation = PinduoduoCandidateAutomation(
driver = driver,
maxCandidates = 2,
pagePollIntervalMillis = 1
)
automation.reset()
val result = automation.execute(
PinduoduoCandidateWorkflow.steps().last()
)
assertEquals(AutomationResult.Success, result)
assertEquals(2, driver.openedSignatures.size)
assertEquals(2, automation.evidence.value.size)
assertEquals(0, driver.scrollCalls)
}
@Test
fun `bounded old-page observations are allowed during transitions`() = runTest {
val driver = FakeCandidateDriver(
cardPages = listOf(listOf(card("a"))),
transitionDelaySnapshots = 2
)
val automation = PinduoduoCandidateAutomation(
driver = driver,
maxCandidates = 1,
pagePollIntervalMillis = 1,
unknownPageLimit = 4
)
automation.reset()
val result = automation.execute(
PinduoduoCandidateWorkflow.steps().last()
)
assertEquals(AutomationResult.Success, result)
assertEquals(1, automation.evidence.value.size)
}
@Test
fun `empty result scanning never exceeds scroll budget`() = runTest {
val driver = FakeCandidateDriver(
cardPages = listOf(emptyList(), emptyList(), emptyList())
)
val automation = PinduoduoCandidateAutomation(
driver = driver,
maxResultScrolls = 2,
pagePollIntervalMillis = 1
)
automation.reset()
val result = automation.execute(
PinduoduoCandidateWorkflow.steps().last()
)
assertEquals(
AutomationResult.FatalFailure(WorkflowFailureCode.TARGET_NOT_READY),
result
)
assertEquals(2, driver.scrollCalls)
assertTrue(driver.openedSignatures.isEmpty())
}
@Test
fun `safety blocker prevents candidate actions`() = runTest {
val driver = FakeCandidateDriver(
cardPages = listOf(listOf(card("a"))),
safetyStopReason = SafetyStopReason.RISK_CONTROL
)
val automation = PinduoduoCandidateAutomation(
driver = driver,
pagePollIntervalMillis = 1
)
automation.reset()
val result = automation.execute(
PinduoduoCandidateWorkflow.steps().last()
)
assertEquals(
AutomationResult.Blocked(SafetyStopReason.RISK_CONTROL),
result
)
assertTrue(driver.openedSignatures.isEmpty())
assertTrue(driver.savedEvidence.isEmpty())
}
@Test
fun `missing screenshot evidence fails without opening another candidate`() =
runTest {
val driver = FakeCandidateDriver(
cardPages = listOf(listOf(card("a"), card("b"))),
failCapture = true
)
val automation = PinduoduoCandidateAutomation(
driver = driver,
pagePollIntervalMillis = 1
)
automation.reset()
val result = automation.execute(
PinduoduoCandidateWorkflow.steps().last()
)
assertEquals(
AutomationResult.FatalFailure(
WorkflowFailureCode.EVIDENCE_CAPTURE_FAILED
),
result
)
assertEquals(listOf("a"), driver.openedSignatures)
assertTrue(automation.evidence.value.isEmpty())
}
private fun card(signature: String) = PinduoduoCandidateCard(
signature = signature,
semanticTextCount = 3,
hasImage = true
)
private class FakeCandidateDriver(
private val cardPages: List<List<PinduoduoCandidateCard>>,
private val safetyStopReason: SafetyStopReason? = null,
private val failCapture: Boolean = false,
private val transitionDelaySnapshots: Int = 0
) : PinduoduoCandidateDriver {
private var cardPageIndex = 0
private var page = PinduoduoPage.SEARCH_RESULTS
private var pendingPage: PinduoduoPage? = null
private var delayedSnapshotsRemaining = 0
val openedSignatures = mutableListOf<String>()
val savedEvidence = mutableListOf<PinduoduoCandidateEvidence>()
var scrollCalls = 0
override suspend fun snapshot(): PinduoduoUiSnapshot {
if (pendingPage != null) {
if (delayedSnapshotsRemaining > 0) {
delayedSnapshotsRemaining -= 1
} else {
page = pendingPage ?: page
pendingPage = null
}
}
return PinduoduoUiSnapshot(
foregroundPackage = PINDUODUO_PACKAGE,
page = page,
safetyStopReason = safetyStopReason
)
}
override suspend fun candidateCards(
limit: Int
): List<PinduoduoCandidateCard> =
cardPages.getOrElse(cardPageIndex) { emptyList() }.take(limit)
override suspend fun openCandidate(signature: String): Boolean {
openedSignatures += signature
transitionTo(PinduoduoPage.PRODUCT_DETAIL)
return true
}
override suspend fun captureCandidate(
ordinal: Int,
card: PinduoduoCandidateCard
): PinduoduoCandidateEvidence? {
if (failCapture) {
return null
}
return PinduoduoCandidateEvidence(
ordinal = ordinal,
cardSignature = card.signature,
cardSemanticTextCount = card.semanticTextCount,
detailSignature = "detail-${card.signature}",
detailSemanticTextCount = 10,
screenshotFileName = "candidate-%02d.png".format(ordinal),
screenshotSha256 = "hash-$ordinal",
screenshotByteCount = 100 + ordinal,
screenshotWidth = 1080,
screenshotHeight = 2400
).also(savedEvidence::add)
}
override suspend fun returnToResults(): Boolean {
transitionTo(PinduoduoPage.SEARCH_RESULTS)
return true
}
override suspend fun scrollResults(): Boolean {
scrollCalls += 1
if (cardPageIndex + 1 < cardPages.size) {
cardPageIndex += 1
}
return true
}
override fun resetEvidence() {
openedSignatures.clear()
savedEvidence.clear()
scrollCalls = 0
cardPageIndex = 0
page = PinduoduoPage.SEARCH_RESULTS
pendingPage = null
delayedSnapshotsRemaining = 0
}
private fun transitionTo(nextPage: PinduoduoPage) {
if (transitionDelaySnapshots == 0) {
page = nextPage
} else {
pendingPage = nextPage
delayedSnapshotsRemaining = transitionDelaySnapshots
}
}
}
}
@@ -100,6 +100,19 @@ class PinduoduoPageClassifierTest {
assertEquals(PinduoduoPage.UNKNOWN, snapshot.page) assertEquals(PinduoduoPage.UNKNOWN, snapshot.page)
} }
@Test
fun `detail page requires clickable back and multiple stable markers`() {
val snapshot = classify(
element(contentDescription = "返回", clickable = true),
element(contentDescription = "联系客服"),
element(contentDescription = "收藏商品"),
element(text = "店铺")
)
assertEquals(PinduoduoPage.PRODUCT_DETAIL, snapshot.page)
assertNull(snapshot.safetyStopReason)
}
@Test @Test
fun `other foreground package is unknown and has no inferred blocker`() { fun `other foreground package is unknown and has no inferred blocker`() {
val snapshot = PinduoduoPageClassifier.classify( val snapshot = PinduoduoPageClassifier.classify(
@@ -89,6 +89,24 @@ class PinduoduoSearchAutomationTest {
) )
} }
@Test
fun `workflow safely recovers from a verified candidate detail`() = runTest {
val driver = FakeDriver(page = PinduoduoPage.PRODUCT_DETAIL)
val runner = WorkflowRunner(
PinduoduoSearchAutomation(
driver = driver,
pollIntervalMillis = 1,
unknownPageLimit = 3
)
)
val report = runner.run(PinduoduoSearchWorkflow.steps())
assertEquals(WorkflowState.SUCCEEDED, report.state)
assertEquals(1, driver.returnFromCandidateCalls)
assertEquals(PinduoduoPage.SEARCH_RESULTS, driver.page)
}
private class FakeDriver( private class FakeDriver(
var page: PinduoduoPage, var page: PinduoduoPage,
private val safetyStopReason: SafetyStopReason? = null, private val safetyStopReason: SafetyStopReason? = null,
@@ -97,6 +115,7 @@ class PinduoduoSearchAutomationTest {
var enteredKeyword: String? = null var enteredKeyword: String? = null
var searchSubmitted = false var searchSubmitted = false
var openSearchCalls = 0 var openSearchCalls = 0
var returnFromCandidateCalls = 0
override suspend fun openApp(): Boolean = true override suspend fun openApp(): Boolean = true
@@ -127,5 +146,11 @@ class PinduoduoSearchAutomationTest {
page = PinduoduoPage.SEARCH_RESULTS page = PinduoduoPage.SEARCH_RESULTS
return true return true
} }
override suspend fun returnFromCandidate(): Boolean {
returnFromCandidateCalls += 1
page = PinduoduoPage.SEARCH_RESULTS
return true
}
} }
} }
+3 -3
View File
@@ -51,9 +51,9 @@
## 当前阶段与优先路径 ## 当前阶段与优先路径
当前已完成 Phase 0 和 T-101:Android 可运行、设备就绪、workflow、私有样本导入 当前已完成 Phase 0、T-101 和 T-102:Android 可运行、设备就绪、workflow、私有样本
以及固定脱敏词拼多多搜索均已在真机验证。下一步是 T-102,只浏览并采集最多 5 个 导入、固定词搜索以及最多 5 个候选截图采集均已在真机验证。下一步是 T-103,使用
候选,继续禁止订单提交和支付。 私有 ProbeTask 验证结构化需求提取,继续禁止订单提交和支付。
严格按以下顺序推进: 严格按以下顺序推进:
+3 -3
View File
@@ -8,7 +8,7 @@
| --- | --- | | --- | --- |
| 任务来源 | 其他管理后台或本项目管理 Web 采集商品标题、描述、图片、数量和预算。 | | 任务来源 | 其他管理后台或本项目管理 Web 采集商品标题、描述、图片、数量和预算。 |
| 第一层样本来源 | 本机目录中的蝦皮订单文本和参考图;二者以蝦皮订单号作为同名文件名。 | | 第一层样本来源 | 本机目录中的蝦皮订单文本和参考图;二者以蝦皮订单号作为同名文件名。 |
| 执行方式 | 采购人员使用 Android App 操作拼多多;固定词搜索探针已可运行,候选浏览和真实任务尚未实现。 | | 执行方式 | 采购人员使用 Android App 操作拼多多;固定词搜索和最多 5 个候选证据采集已可运行,需求提取和匹配判断尚未实现。 |
| 核心痛点 | 人工把图片和描述转成搜索词、逐条比较商品并记录结果,耗时且不一致。 | | 核心痛点 | 人工把图片和描述转成搜索词、逐条比较商品并记录结果,耗时且不一致。 |
| 验证范围 | 一台设备、一个管理身份、一个采购执行人员、拼多多单平台。 | | 验证范围 | 一台设备、一个管理身份、一个采购执行人员、拼多多单平台。 |
| 资金边界 | MVP 不提交订单、不支付,只验证到人工确认位置。 | | 资金边界 | MVP 不提交订单、不支付,只验证到人工确认位置。 |
@@ -140,8 +140,8 @@ T-004 已固定首版规则:推荐私有目录为被 Git 忽略的 `private-fi
`T-001` 完成。 `T-001` 完成。
- 首份蝦皮样本已由 T-004 真实导入验证;当前只支持单 SKU 和 JPEG,扩展格式需新 - 首份蝦皮样本已由 T-004 真实导入验证;当前只支持单 SKU 和 JPEG,扩展格式需新
样例和独立任务。 样例和独立任务。
- OnePlus PKG110、Android 16/API 36、拼多多 8.17.0 的首页、搜索输入和结果页已形成 - OnePlus PKG110、Android 16/API 36、拼多多 8.17.0 的首页、搜索输入、结果页、
可复现基线;候选列表、详情页及账号风控差异仍待 T-102 验证。 候选卡和详情返回已形成可复现基线;不同账号、类目和页面实验的差异仍是风险。
- VLM 厂商、模型、成本上限、数据留存地区和图片隐私规则待确认。 - VLM 厂商、模型、成本上限、数据留存地区和图片隐私规则待确认。
- 拼多多平台条款、自动化允许范围和账号风控需要业务方确认;项目不实现绕过措施。 - 拼多多平台条款、自动化允许范围和账号风控需要业务方确认;项目不实现绕过措施。
- 后续若允许提交订单,必须先明确 SKU、收货地址、运费、优惠、发票、金额审批、 - 后续若允许提交订单,必须先明确 SKU、收货地址、运费、优惠、发票、金额审批、
+4 -3
View File
@@ -11,7 +11,8 @@
| Android 构建链 | Gradle 8.2;AGP 8.2.0;Kotlin 1.9.20;JVM target 17 | 已验证 | 已补齐上游缺失的 `gradlew.bat`,不依赖全局 Gradle。 | | Android 构建链 | Gradle 8.2;AGP 8.2.0;Kotlin 1.9.20;JVM target 17 | 已验证 | 已补齐上游缺失的 `gradlew.bat`,不依赖全局 Gradle。 |
| Android SDK | compileSdk/targetSdk 34;minSdk 26;SDK Build Tools 34.0.0 | 已验证 | 支持 Android 8.0+;本机使用 Command-line Tools 22.0。 | | Android SDK | compileSdk/targetSdk 34;minSdk 26;SDK Build Tools 34.0.0 | 已验证 | 支持 Android 8.0+;本机使用 Command-line Tools 22.0。 |
| Android UI | Jetpack Compose + Material 3;Compose Compiler 1.5.5 | 上游已核实 | Compose BOM 为 2023.10.01。 | | Android UI | Jetpack Compose + Material 3;Compose Compiler 1.5.5 | 上游已核实 | Compose BOM 为 2023.10.01。 |
| Android 自动化 | `AccessibilityService` 语义节点动作;Shizuku 保留为上游兼容路径 | 固定词搜索已真机验证 | T-101 已用可见、启用、唯一节点完成打开搜索、`ACTION_SET_TEXT` 精确回读、提交和结果页确认;没有坐标或 shell 降级。上游 `main` 仍保留 Shizuku 13.1.5。 | | Android 自动化 | `AccessibilityService` 语义节点动作;Shizuku 保留为上游兼容路径 | 搜索与 5 个候选已真机验证 | T-101/T-102 已完成精确输入、结果页确认、候选卡识别、详情截图和验证返回;没有坐标或 shell 降级。上游 `main` 仍保留 Shizuku 13.1.5。 |
| Android 候选证据 | API 30+ `AccessibilityService.takeScreenshot` + App cache JSON/PNG | 已真机验证 | 匿名 PNG 与只含 SHA-256、计数、尺寸的 manifest;转换/压缩使用独立 executor,文件 IO 使用 `Dispatchers.IO`。API 26-29 明确不支持该截图探针。 |
| 第一层任务源 | UTF-8 无 BOM 四行蝦皮订单文本 + 同订单号 JPEG | 已实现 | `task-contract` 共享 `ProbeTask/TaskSource`;CLI 输出到 `.local/`,只有显式 Debug 属性才注入 APK,默认构建会清除私有资产。 | | 第一层任务源 | UTF-8 无 BOM 四行蝦皮订单文本 + 同订单号 JPEG | 已实现 | `task-contract` 共享 `ProbeTask/TaskSource`;CLI 输出到 `.local/`,只有显式 Debug 属性才注入 APK,默认构建会清除私有资产。 |
| Android 长任务 | 前台服务 + 持续通知 | 计划采用 | 降低执行中被系统挂起的风险,仍需处理进程死亡恢复。 | | Android 长任务 | 前台服务 + 持续通知 | 计划采用 | 降低执行中被系统挂起的风险,仍需处理进程死亡恢复。 |
| 后端语言 | Go 1.23.0 | MVP 已定 | 与现有本机工具链一致;构建测试必须设置 `GOTOOLCHAIN=local` 防止静默升级。 | | 后端语言 | Go 1.23.0 | MVP 已定 | 与现有本机工具链一致;构建测试必须设置 `GOTOOLCHAIN=local` 防止静默升级。 |
@@ -27,7 +28,7 @@
| VLM 接入 | 应用内统一适配器,优先兼容 OpenAI 风格多模态接口 | 接口已定,供应商待定 | 模型输出必须符合本项目 JSON Schema。 | | VLM 接入 | 应用内统一适配器,优先兼容 OpenAI 风格多模态接口 | 接口已定,供应商待定 | 模型输出必须符合本项目 JSON Schema。 |
| 通知 | MVP 不使用推送 | 已定 | 点击“获取任务”调用原子 claim API;V2 再评估厂商推送/WebSocket。 | | 通知 | MVP 不使用推送 | 已定 | 点击“获取任务”调用原子 claim API;V2 再评估厂商推送/WebSocket。 |
| 后端测试 | 标准库 `testing` + `httptest` | MVP 已定 | 覆盖状态机、权限、幂等、SQLite 事务和输入校验。 | | 后端测试 | 标准库 `testing` + `httptest` | MVP 已定 | 覆盖状态机、权限、幂等、SQLite 事务和输入校验。 |
| Android 测试 | Gradle `test` + `kotlinx-coroutines-test` 1.7.3 + 真实设备 smoke | 搜索探针已验证 | T-003 runner 与 T-101 页面分类/Fake driver 已覆盖;OnePlus PKG110 + 拼多多 8.17.0 固定词结果页 smoke 成功。 | | Android 测试 | Gradle `test` + `kotlinx-coroutines-test` 1.7.3 + 真实设备 smoke | 候选探针已验证 | runner、页面分类、搜索和有界候选 Fake driver 已覆盖;OnePlus PKG110 + 拼多多 8.17.0 的 5 个截图及返回 smoke 成功。 |
| 部署 | 单机局域网 Go 服务;容器化后置 | MVP 已定 | Android 测试机必须能通过 HTTPS 或受控测试网络访问。 | | 部署 | 单机局域网 Go 服务;容器化后置 | MVP 已定 | Android 测试机必须能通过 HTTPS 或受控测试网络访问。 |
## Roubao 上游版本基线 ## Roubao 上游版本基线
@@ -134,7 +135,7 @@ docs/
| --- | --- | --- | | --- | --- | --- |
| Android 标准验证 | `.\init.ps1` | 已验证 | | Android 标准验证 | `.\init.ps1` | 已验证 |
| Android 构建 | `android-buyer\gradlew.bat assembleDebug --no-daemon` | 已验证 | | Android 构建 | `android-buyer\gradlew.bat assembleDebug --no-daemon` | 已验证 |
| Android 测试任务 | `android-buyer\gradlew.bat test --no-daemon` | 已验证;当前全工程 62 次测试通过 | | Android 测试任务 | `android-buyer\gradlew.bat test --no-daemon` | 已验证;当前全工程 78 次测试通过 |
| Android 安装/启动 | `$env:RUN_START_COMMAND="1"; .\init.ps1` | 已在 Android 16 真机验证 | | Android 安装/启动 | `$env:RUN_START_COMMAND="1"; .\init.ps1` | 已在 Android 16 真机验证 |
| 后端依赖 | `go mod download`(在 `backend-api/`) | 待 `T-201` | | 后端依赖 | `go mod download`(在 `backend-api/`) | 待 `T-201` |
| 后端测试 | `go test ./...`(在 `backend-api/`) | 待 `T-201` | | 后端测试 | `go test ./...`(在 `backend-api/`) | 待 `T-201` |
+19
View File
@@ -169,6 +169,23 @@ SearchProbeScreen
登录、验证码、风控、订单或支付边界都终止 workflow。该路径不提供坐标、ADB、 登录、验证码、风控、订单或支付边界都终止 workflow。该路径不提供坐标、ADB、
Shizuku shell、OCR 或 VLM 动作降级。 Shizuku shell、OCR 或 VLM 动作降级。
T-102 在搜索结果后追加一个有界候选步骤:
```text
固定词结果页
-> 最多 2 次滚动预算
-> 最多 5 个去重商品卡
-> 验证详情页
-> 无障碍截图
-> App cache/candidate-NN.png + manifest.json
-> 一次全局返回并复核固定词结果页
```
候选卡只保留语义指纹和计数,详情截图保存在 App 内部 cache。manifest 记录匿名文件名、
截图 SHA-256、字节数和尺寸,不保存商品标题或页面原文。T-104 使用这些证据时必须通过
受控 evidence 边界读取,不能让 VLM adapter 自行遍历 cache。Android 10/API 29 及
以下不能运行当前截图探针,应在预检时明确不支持,不使用媒体投影或 shell 绕过。
### 3.2 MVP 业务闭环 ### 3.2 MVP 业务闭环
```text ```text
@@ -327,6 +344,8 @@ IDLE
不能因资源 ID 重复而任取第一个节点。 不能因资源 ID 重复而任取第一个节点。
- 已验证的首页、搜索输入和结果页路径禁止坐标降级;新页面必须先取得版本化真机 - 已验证的首页、搜索输入和结果页路径禁止坐标降级;新页面必须先取得版本化真机
证据并在独立任务中定义识别条件。 证据并在独立任务中定义识别条件。
- 候选 session 的候选尝试与滚动预算在动作发出前扣减,retry 不得重置;详情动作
白名单只有截图和返回。
- 截图发送给模型前裁剪无关区域并按配置脱敏。 - 截图发送给模型前裁剪无关区域并按配置脱敏。
- 每个动作记录抽象步骤,不默认记录完整输入文本。 - 每个动作记录抽象步骤,不默认记录完整输入文本。
- 发现验证码、风险控制、支付、生物识别或系统权限页面立即停止。 - 发现验证码、风险控制、支付、生物识别或系统权限页面立即停止。
+6
View File
@@ -37,6 +37,12 @@
- 每个步骤定义 timeout、有限 retry、成功条件和停止条件。 - 每个步骤定义 timeout、有限 retry、成功条件和停止条件。
- 禁止无界循环、无限滑动、无限候选遍历和随机点击。 - 禁止无界循环、无限滑动、无限候选遍历和随机点击。
- 默认最多检查 5 个候选;扩大范围必须先改需求。 - 默认最多检查 5 个候选;扩大范围必须先改需求。
- 候选卡必须满足视口可见比例、单列尺寸、图片、价格和语义文本约束;根不可点击时
只允许唯一的大面积覆盖点击目标,禁止点击卡内小按钮。
- 候选和滚动预算在发出动作前消耗,并跨 workflow retry 保持;返回 `false` 或取消
也不能恢复预算。
- 详情截图只保存在 App 内部目录,文件名匿名;manifest 不保存标题/页面原文,并
校验文件 SHA-256、尺寸和字节数。
- 前台服务通知要展示任务编号和当前步骤,并提供安全停止入口。 - 前台服务通知要展示任务编号和当前步骤,并提供安全停止入口。
- 验证码、风控、登录、支付、未知页面统一进入安全失败。 - 验证码、风控、登录、支付、未知页面统一进入安全失败。
- MVP 不得实现最终提交订单或支付动作,包括“先写好但不调用”的隐藏代码。 - MVP 不得实现最终提交订单或支付动作,包括“先写好但不调用”的隐藏代码。
+13 -12
View File
@@ -5,8 +5,8 @@
## 当前快照 ## 当前快照
- 日期:2026-07-25 - 日期:2026-07-25
- 阶段:T-101 已完成;准备 T-102 候选浏览和采集 - 阶段:T-102 已完成;准备 T-103 VLM 需求提取
- Git:当前分支为 `main`;T-001 至 T-004 均已纳入 Git 历史 - Git:当前分支为 `main`;T-001 至 T-004、T-101 和 T-102 均已纳入 Git 历史
- 生产代码:`android-buyer/` 已接入 Roubao Android 源码 - 生产代码:`android-buyer/` 已接入 Roubao Android 源码
- Android:固定 `main@c8a6d7f03422eb01744b01f3ee77bf7757741f7e`;MIT 许可证已保留 - Android:固定 `main@c8a6d7f03422eb01744b01f3ee77bf7757741f7e`;MIT 许可证已保留
- 后端:已决定使用 Go 1.23.0 + Gin 1.11.0;Go Blueprint v0.10.11 骨架尚未接入 - 后端:已决定使用 Go 1.23.0 + Gin 1.11.0;Go Blueprint v0.10.11 骨架尚未接入
@@ -14,21 +14,21 @@
Build Tools 34.0.0、Platform Tools/ADB 37.0.0;用户级 SDK 环境变量已设置 Build Tools 34.0.0、Platform Tools/ADB 37.0.0;用户级 SDK 环境变量已设置
- Android Studio:未安装;`winget` 静默安装卡住后已终止,不阻塞命令行构建 - Android Studio:未安装;`winget` 静默安装卡住后已终止,不阻塞命令行构建
- 测试:`lintDebug test assembleDebug` 成功;App 两个变体、task contract 和导入器 - 测试:`lintDebug test assembleDebug` 成功;App 两个变体、task contract 和导入器
共执行 62 次测试,0 failure、0 error、0 skipped 共执行 78 次测试,0 failure、0 error、0 skipped
- Workflow:纯 Kotlin runner 已支持步骤 timeout、最多 3 次 retry、安全阻塞、 - Workflow:纯 Kotlin runner 已支持步骤 timeout、最多 3 次 retry、安全阻塞、
用户停止和单 runner 并发拒绝;T-101 已接入真实拼多多固定词搜索四步 用户停止和单 runner 并发拒绝;T-102 已接入搜索加有界候选采集五步
- TaskSource:严格 CLI 已生成并验证真实私有 ProbeTask;默认 APK 不含私有 fixture - TaskSource:严格 CLI 已生成并验证真实私有 ProbeTask;默认 APK 不含私有 fixture
- 测试设备:OnePlus PKG110,Android 16/API 36;肉包 `1.4.2 (7)`;拼多多 - 测试设备:OnePlus PKG110,Android 16/API 36;肉包 `1.4.2 (7)`;拼多多
`8.17.0 (81700)` `8.17.0 (81700)`
- 设备就绪:肉包采购无障碍已启用并连接;拼多多首页、搜索输入和固定词结果页已通过 - 设备就绪:肉包采购无障碍已启用并连接;拼多多首页、搜索输入、固定词结果页、
8.17.0 真机验证,结果页精确词、搜索头和 4 个排序控件均确认 双列候选卡、详情截图和返回均已通过 8.17.0 真机验证
- 版本控制内测试数据:仅有脱敏、运行时生成的单元测试 fixture;没有真实订单内容 - 版本控制内测试数据:仅有脱敏、运行时生成的单元测试 fixture;没有真实订单内容
- 本地私有样本:仓库根目录存在一组未跟踪、已本地排除的同名蝦皮文本/JPEG; - 本地私有样本:仓库根目录存在一组未跟踪、已本地排除的同名蝦皮文本/JPEG;
已用 CLI 真实导入并逐字段/图片哈希验证,生成物位于被忽略的 `.local/` 已用 CLI 真实导入并逐字段/图片哈希验证,生成物位于被忽略的 `.local/`
- 标准启动路径:`$env:RUN_START_COMMAND="1"; .\init.ps1` - 标准启动路径:`$env:RUN_START_COMMAND="1"; .\init.ps1`
- 标准验证路径:`.\init.ps1` - 标准验证路径:`.\init.ps1`
- 当前 blocker:候选列表/详情页语义与有界返回路径尚待 T-102 验证;当前只支持单 - 当前 blocker:VLM 供应商、模型、测试凭证、成本上限和数据留存尚未确认;当前只
SKU/JPEG;VLM 供应商、模型和测试凭证未确认 支持单 SKU/JPEG;候选探针截图要求 Android 11/API 30+
## 当前目录 ## 当前目录
@@ -41,6 +41,7 @@
| `docs/tasks/T-003.md` | DONE | 可注入 Fake automation 的受限 workflow runner | | `docs/tasks/T-003.md` | DONE | 可注入 Fake automation 的受限 workflow runner |
| `docs/tasks/T-004.md` | DONE | 私有蝦皮文本/JPEG 严格导入和 Debug TaskSource | | `docs/tasks/T-004.md` | DONE | 私有蝦皮文本/JPEG 严格导入和 Debug TaskSource |
| `docs/tasks/T-101.md` | DONE | 固定脱敏词拼多多搜索和结果页真机验证 | | `docs/tasks/T-101.md` | DONE | 固定脱敏词拼多多搜索和结果页真机验证 |
| `docs/tasks/T-102.md` | DONE | 最多 5 个候选详情截图、证据 manifest 和结果页返回 |
| `deepseek总结.txt` | 已有 | 历史讨论摘要,不是正式需求权威 | | `deepseek总结.txt` | 已有 | 历史讨论摘要,不是正式需求权威 |
| `android-buyer/` | 已有 | Roubao `main` 固定 commit 的 Android 基线 | | `android-buyer/` | 已有 | Roubao `main` 固定 commit 的 Android 基线 |
| `android-buyer/task-contract/` | 已有 | Android/CLI 共享 ProbeTask 与 TaskSource | | `android-buyer/task-contract/` | 已有 | Android/CLI 共享 ProbeTask 与 TaskSource |
@@ -50,9 +51,9 @@
## 任务摘要 ## 任务摘要
- 已完成:T-001 至 T-004,以及 T-101 固定词搜索探针。 - 已完成:T-001 至 T-004,以及 T-101、T-102。
- 正在进行:无。 - 正在进行:无。
- 下一个可领取任务:T-102 浏览并采集最多 5 个候选。 - 下一个可领取任务:T-103 接入 VLM 需求提取。
## 当前可运行内容 ## 当前可运行内容
@@ -64,8 +65,8 @@ $env:RUN_START_COMMAND = "1"
``` ```
2026-07-25 已在 PKG110、Android 16/API 36、拼多多 8.17.0 上完成 Debug APK 2026-07-25 已在 PKG110、Android 16/API 36、拼多多 8.17.0 上完成 Debug APK
固定词搜索。默认首屏为搜索探针,用户点击后四步全部完成并确认结果页;设备页保留 固定词搜索和 5 个候选采集。默认首屏为候选探针,用户点击后五步全部完成,5 张
就绪检查入口,肉包采购无障碍服务已绑定,logcat 无崩溃或 ANR。 匿名 PNG 与 manifest 哈希一致并返回结果页;logcat 无崩溃或 ANR。
## 维护规则 ## 维护规则
+126
View File
@@ -0,0 +1,126 @@
---
id: T-102
title: 浏览并采集最多 5 个候选
phase: 1
deps:
- T-101
status: DONE
created: 2026-07-25
context_ref: f226d0f
work_branch: main
write_paths:
- android-buyer/app/src/main/java/com/roubao/autopilot/MainActivity.kt
- android-buyer/app/src/main/java/com/roubao/autopilot/accessibility/**
- android-buyer/app/src/main/java/com/roubao/autopilot/pinduoduo/**
- android-buyer/app/src/main/java/com/roubao/autopilot/ui/screens/SearchProbeScreen.kt
- android-buyer/app/src/main/java/com/roubao/autopilot/workflow/WorkflowModels.kt
- android-buyer/app/src/main/res/xml/buyer_accessibility_service.xml
- android-buyer/app/src/test/java/com/roubao/autopilot/pinduoduo/**
- docs/00-ai-start-here.md
- docs/02-requirements.md
- docs/03-tech-stack.md
- docs/04-architecture.md
- docs/05-coding-rules.md
- docs/current-state.md
- docs/tasks/T-102.md
- progress.md
---
## 问题 / 背景
T-101 已能稳定进入固定词结果页,但尚未证明可以识别独立商品卡、进入详情、保存证据
并返回同一结果页。T-102 只验证有界候选浏览,不进行匹配判断、下单或支付。
## 关联需求与交互
- 功能:F-005 的最多 5 个候选采集部分、F-007 安全失败。
- 用户故事:US-004、US-006。
- 交互:Android“探针”页展示候选计数和当前步骤。
- 架构:`WorkflowRunner` + `PinduoduoCandidateAutomation` + 无障碍 driver +
App 内部缓存证据。
## 方案
1. 只从已确认的搜索结果页识别主 RecyclerView 的直属商品卡。
2. 商品卡必须可见、启用、可点击,包含图片和有限文本;用文本 SHA-256 去重,不在
日志或 UI 展示标题。
3. 每次只打开一个候选;详情页必须由稳定语义组合确认,保存全屏 PNG、文本指纹和
节点计数后立即返回结果页。
4. 最多采集 5 个不同候选,最多滚动结果列表 2 次;没有无界循环或翻页。
5. 登录、验证码、风控、未知页、订单和支付边界立即阻塞;绝不点击详情页购买控件。
## 验收要点
- [x] 从 T-101 结果页依次进入候选详情并返回。
- [x] 采集数量始终不超过 5,结果滚动次数始终不超过 2。
- [x] 每个候选有内部缓存 PNG、SHA-256、尺寸和语义节点证据。
- [x] 证据文件名匿名,普通日志和 UI 不输出商品标题。
- [x] 全流程不使用坐标,不点击购买、订单或支付控件。
- [x] 页面未知或安全边界出现时产生可区分的 `BLOCKED`。
- [x] Fake driver 自动化测试覆盖成功、去重、上限和安全停止。
- [x] 拼多多 8.17.0 真机 smoke 采集 5 个候选并返回结果页。
- [x] `lintDebug test assembleDebug` 通过,默认 APK 不含私有 fixture。
## 边界
- 不评估候选是否匹配;T-104 才接入候选评估。
- 不把候选证据上传网络;T-202 以后再接后端证据接口。
- 不点击购买、拼单、加入购物车、订单或支付相关节点。
- 不保存真实蝦皮输入或把商品详情文本写入普通日志。
## 执行记录
### 2026-07-25:任务开始
- 基于 T-101 提交 `f226d0f` 开始。
- 脱敏结构审计确认拼多多 8.17.0 结果页主 RecyclerView 和双列商品卡;详情页有
可点击返回语义以及联系客服、收藏、店铺语义,当前未出现订单或支付边界。
### 2026-07-25:实现
- 新增 `PinduoduoCandidateAutomation` 和复合探针 workflow。一次 session 最多尝试
5 个不同卡片、最多发出 2 次结果滚动;预算在动作前消耗,runner retry 不会重置。
- 主结果 RecyclerView 必须可滚动、覆盖至少 90% 宽度且拥有足够节点。候选卡必须
至少 80% 可见、宽度在单列范围、含图片/价格/至少 2 个语义文本;根不可点击时
只接受唯一覆盖卡片至少 85% 面积的点击目标,小型嵌套按钮不会被选中。
- 详情页同时要求可点击返回、联系客服/收藏/店铺至少两类语义、宽 ViewPager 和宽
可滚动主列表。详情页动作白名单只有截图和一次全局返回,返回后必须重新确认固定
查询结果页。
- Android 11/API 30+ 使用声明了 `canTakeScreenshot` 的无障碍截图 API。Hardware
buffer 在独立执行器转换和压缩并始终关闭;8 秒超时后结构化失败,不绕过安全窗口。
- 每个候选只保存匿名 PNG、卡片/详情语义 SHA-256、节点计数、文件 SHA-256、字节数
和尺寸到 App cache;manifest 不保存标题或页面原文,文件写入在 IO dispatcher。
- 探针页增加候选 `0/5` 计数和第五步;仅记录 workflow/候选阶段枚举与截图错误码。
### 2026-07-25:自动化验证
- `gradlew.bat lintDebug test assembleDebug --no-daemon` 成功。
- App Debug/Release、task contract 和导入器共 14 份报告、78 次测试,0 failure、
0 error、0 skipped。
- Fake driver 覆盖 5 个去重候选、候选硬上限、旧页面有界过渡、两次滚动上限、
空列表、安全阻塞、截图失败以及从已验证详情页安全恢复。
- 普通 Debug APK 中 `assets/probe-fixtures/` 条目数为 0。
### 2026-07-25:真机 smoke
- OnePlus PKG110、Android 16/API 36、拼多多 8.17.0 上,从肉包按钮启动后 20 秒内
依次采集 5 个候选并返回固定词结果页;肉包显示 `5 / 5` 和五步完成。
- cache manifest 为 5 条,匿名文件名合法;5 个 PNG 均为 `1080x2376`,字节数为
正数,文件 SHA-256 与 manifest 全部一致。
- 最终结果页订单/支付边界计数为 0;logcat 没有肉包崩溃或 ANR,阶段日志不含商品
标题或页面原文。
- 真机审计 XML/PNG 位于被忽略的 `.local/`,候选证据位于 App 内部 cache,均未纳入
Git。
### 2026-07-25:修正记录
- 首次运行在候选点击后把短暂保留的旧结果页误判为未知。改为只在固定 20 次观测
预算内等待结果页和详情页过渡;任何安全阻塞仍立即停止。
- 调试期间旧 Activity 造成终态读取混淆;最终 smoke 先清理肉包任务栈,再用
`FLAG_ACTIVITY_REORDER_TO_FRONT` 返回原实例核验,未创建第二个 workflow。
## 后续
- T-103 使用 T-004 的私有 ProbeTask 接入结构化需求提取;不能把订单号或店铺名发给
模型。
- T-104 才把候选截图交给评估适配器;T-102 不声明任何候选匹配结论。
+8
View File
@@ -77,3 +77,11 @@
和结果页确认,并在拼多多 8.17.0 真机验证四步 workflow。 和结果页确认,并在拼多多 8.17.0 真机验证四步 workflow。
- 影响:最高风险的基础搜索动作已证实可行;下一步 T-102 只扩展最多 5 个候选的 - 影响:最高风险的基础搜索动作已证实可行;下一步 T-102 只扩展最多 5 个候选的
有界浏览和证据采集,继续禁止下单与支付。 有界浏览和证据采集,继续禁止下单与支付。
## 2026-07-25 拼多多候选证据探针
- 类型:阶段完成
- 内容:完成 T-102;用无障碍语义节点有界浏览 5 个候选,逐一验证详情、保存匿名
PNG/SHA-256 证据并返回固定词结果页。
- 影响:搜索到候选证据的 Android 风险闭环已证实可行;T-103 可开始验证私有任务的
结构化需求提取,T-104 再对候选证据做匹配判断。