feat(t213): guard read-only specification verification

This commit is contained in:
QiuSW
2026-07-27 23:06:21 +08:00
parent 1316b9bb32
commit 9a8388fac6
25 changed files with 1540 additions and 207 deletions
+2 -2
View File
@@ -11,8 +11,8 @@ android {
applicationId = "com.roubao.autopilot" applicationId = "com.roubao.autopilot"
minSdk = 26 minSdk = 26
targetSdk = 34 targetSdk = 34
versionCode = 7 versionCode = 8
versionName = "1.4.2" versionName = "1.4.3"
vectorDrawables { vectorDrawables {
useSupportLibrary = true useSupportLibrary = true
@@ -65,6 +65,8 @@ import com.roubao.autopilot.vlm.CandidateEvaluator
import com.roubao.autopilot.vlm.CandidateReviewBatch import com.roubao.autopilot.vlm.CandidateReviewBatch
import com.roubao.autopilot.vlm.CandidateHumanReviewPolicy import com.roubao.autopilot.vlm.CandidateHumanReviewPolicy
import com.roubao.autopilot.vlm.CandidateTopFivePolicy import com.roubao.autopilot.vlm.CandidateTopFivePolicy
import com.roubao.autopilot.vlm.CandidateSpecificationHardConstraintPolicy
import com.roubao.autopilot.vlm.SkuHardConstraintExtractor
import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job import kotlinx.coroutines.Job
@@ -98,6 +100,7 @@ import com.roubao.autopilot.procurement.ExecutionCandidateSourceIdentity
import com.roubao.autopilot.procurement.ExecutionHardConstraintEvaluation import com.roubao.autopilot.procurement.ExecutionHardConstraintEvaluation
import com.roubao.autopilot.procurement.ExecutionRecommendation import com.roubao.autopilot.procurement.ExecutionRecommendation
import com.roubao.autopilot.procurement.ExecutionEvidenceDraft import com.roubao.autopilot.procurement.ExecutionEvidenceDraft
import com.roubao.autopilot.procurement.ExecutionEvidenceKind
import com.roubao.autopilot.procurement.ExecutionMode import com.roubao.autopilot.procurement.ExecutionMode
import com.roubao.autopilot.procurement.ExecutionProvenanceSnapshot import com.roubao.autopilot.procurement.ExecutionProvenanceSnapshot
import com.roubao.autopilot.procurement.ProcurementRepository import com.roubao.autopilot.procurement.ProcurementRepository
@@ -951,6 +954,8 @@ class MainActivity : ComponentActivity() {
baseUrl = settings.baseUrl, baseUrl = settings.baseUrl,
model = settings.model model = settings.model
) )
val hardConstraints =
SkuHardConstraintExtractor.extract(requirement.sku)
val result = CandidateEvaluator( val result = CandidateEvaluator(
gateway = AndroidCandidateEvaluationVlmGateway(client), gateway = AndroidCandidateEvaluationVlmGateway(client),
providerId = provider.id, providerId = provider.id,
@@ -962,8 +967,23 @@ class MainActivity : ComponentActivity() {
CandidateEvaluationImage( CandidateEvaluationImage(
ordinal = candidate.ordinal, ordinal = candidate.ordinal,
mediaType = "image/png", mediaType = "image/png",
bytes = candidate.pngBytes, bytes = candidate.detailPngBytes,
sha256 = candidate.sha256 sha256 = candidate.detailSha256,
specificationBytes =
candidate.specificationPngBytes,
specificationSha256 =
candidate.specificationSha256,
localHardConstraintResults =
CandidateSpecificationHardConstraintPolicy
.evaluate(
constraints =
hardConstraints.constraints,
evidence =
candidate.specification,
evidenceSha256 =
candidate
.specificationSha256
)
) )
} }
) )
@@ -1014,20 +1034,46 @@ class MainActivity : ComponentActivity() {
) )
) )
} }
val selectedEvidence = ranked.map { rankedCandidate -> val selectedEvidence = ranked.flatMap {
rankedCandidate ->
val candidate = requireNotNull( val candidate = requireNotNull(
evidenceByOrdinal[ evidenceByOrdinal[
rankedCandidate.sourceOrdinal rankedCandidate.sourceOrdinal
] ]
) )
require( require(
candidate.sha256 == candidate.detailSha256 ==
rankedCandidate.evidenceSha256 rankedCandidate.evidenceSha256
) )
require(
candidate.specificationSha256 ==
rankedCandidate
.specificationEvidenceSha256
)
listOf(
ExecutionEvidenceDraft( ExecutionEvidenceDraft(
ordinal = rankedCandidate.rankedOrdinal, ordinal =
pngBytes = candidate.pngBytes, rankedCandidate.rankedOrdinal,
sha256 = candidate.sha256 pngBytes =
candidate.detailPngBytes,
sha256 =
candidate.detailSha256,
kind =
ExecutionEvidenceKind.DETAIL
),
ExecutionEvidenceDraft(
ordinal =
rankedCandidate.rankedOrdinal,
pngBytes =
candidate
.specificationPngBytes,
sha256 =
candidate
.specificationSha256,
kind =
ExecutionEvidenceKind
.SPECIFICATION
)
) )
} }
val queued = procurementRepository.queueCandidateBatch( val queued = procurementRepository.queueCandidateBatch(
@@ -1078,7 +1124,10 @@ class MainActivity : ComponentActivity() {
rankedOrdinal = rankedOrdinal =
rankedCandidate.rankedOrdinal, rankedCandidate.rankedOrdinal,
evidenceSha256 = evidenceSha256 =
rankedCandidate.evidenceSha256 rankedCandidate.evidenceSha256,
specificationEvidenceSha256 =
rankedCandidate
.specificationEvidenceSha256
) )
}, },
candidates = queued candidates = queued
@@ -1139,11 +1188,20 @@ class MainActivity : ComponentActivity() {
) )
} }
), ),
validated.map { candidate -> validated.flatMap { candidate ->
listOf(
ExecutionEvidenceDraft( ExecutionEvidenceDraft(
ordinal = candidate.ordinal, ordinal = candidate.ordinal,
pngBytes = candidate.pngBytes, pngBytes = candidate.detailPngBytes,
sha256 = candidate.sha256 sha256 = candidate.detailSha256,
kind = ExecutionEvidenceKind.DETAIL
),
ExecutionEvidenceDraft(
ordinal = candidate.ordinal,
pngBytes = candidate.specificationPngBytes,
sha256 = candidate.specificationSha256,
kind = ExecutionEvidenceKind.SPECIFICATION
)
) )
} }
) ?: return ) ?: return
@@ -1153,7 +1211,9 @@ class MainActivity : ComponentActivity() {
ExecutionCandidateSourceIdentity( ExecutionCandidateSourceIdentity(
sourceOrdinal = candidate.ordinal, sourceOrdinal = candidate.ordinal,
rankedOrdinal = candidate.ordinal, rankedOrdinal = candidate.ordinal,
evidenceSha256 = candidate.sha256 evidenceSha256 = candidate.detailSha256,
specificationEvidenceSha256 =
candidate.specificationSha256
) )
}, },
candidates = queued candidates = queued
@@ -4,6 +4,7 @@ import com.roubao.autopilot.pinduoduo.PinduoduoPage
import com.roubao.autopilot.pinduoduo.PinduoduoCandidateCard import com.roubao.autopilot.pinduoduo.PinduoduoCandidateCard
import com.roubao.autopilot.pinduoduo.PinduoduoCandidateDetailEvidence import com.roubao.autopilot.pinduoduo.PinduoduoCandidateDetailEvidence
import com.roubao.autopilot.pinduoduo.PinduoduoScreenshotCapture import com.roubao.autopilot.pinduoduo.PinduoduoScreenshotCapture
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationEvidence
import com.roubao.autopilot.pinduoduo.PinduoduoUiSnapshot import com.roubao.autopilot.pinduoduo.PinduoduoUiSnapshot
import com.roubao.autopilot.readiness.DeviceObservationStore import com.roubao.autopilot.readiness.DeviceObservationStore
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
@@ -76,13 +77,37 @@ object BuyerAccessibilityBridge {
service?.readPinduoduoCandidateDetailEvidence() service?.readPinduoduoCandidateDetailEvidence()
} }
suspend fun captureScreenshot(): PinduoduoScreenshotCapture? = suspend fun captureDetailScreenshot(): PinduoduoScreenshotCapture? =
withTimeoutOrNull(SCREENSHOT_TIMEOUT_MILLIS) { withTimeoutOrNull(SCREENSHOT_TIMEOUT_MILLIS) {
withContext(Dispatchers.Main.immediate) { withContext(Dispatchers.Main.immediate) {
service?.capturePinduoduoScreenshot() service?.capturePinduoduoScreenshot(PinduoduoPage.PRODUCT_DETAIL)
} }
} }
suspend fun openSpecifications(): Boolean =
withContext(Dispatchers.Main.immediate) {
service?.openPinduoduoSpecifications() == true
}
suspend fun specificationEvidence(): PinduoduoSpecificationEvidence? =
withContext(Dispatchers.Main.immediate) {
service?.readPinduoduoSpecificationEvidence()
}
suspend fun captureSpecificationScreenshot(): PinduoduoScreenshotCapture? =
withTimeoutOrNull(SCREENSHOT_TIMEOUT_MILLIS) {
withContext(Dispatchers.Main.immediate) {
service?.capturePinduoduoScreenshot(
PinduoduoPage.SPECIFICATION_PANEL
)
}
}
suspend fun closeSpecifications(): Boolean =
withContext(Dispatchers.Main.immediate) {
service?.closePinduoduoSpecifications() == true
}
suspend fun returnToResults(): Boolean = suspend fun returnToResults(): Boolean =
withContext(Dispatchers.Main.immediate) { withContext(Dispatchers.Main.immediate) {
service?.returnFromPinduoduoCandidate() == true service?.returnFromPinduoduoCandidate() == true
@@ -24,6 +24,8 @@ import com.roubao.autopilot.pinduoduo.PinduoduoPage
import com.roubao.autopilot.pinduoduo.isCandidateResultsPage import com.roubao.autopilot.pinduoduo.isCandidateResultsPage
import com.roubao.autopilot.pinduoduo.SEARCH_PROBE_KEYWORD import com.roubao.autopilot.pinduoduo.SEARCH_PROBE_KEYWORD
import com.roubao.autopilot.pinduoduo.PinduoduoScreenshotCapture import com.roubao.autopilot.pinduoduo.PinduoduoScreenshotCapture
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationEvidence
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationParser
import java.io.ByteArrayOutputStream import java.io.ByteArrayOutputStream
import java.util.ArrayDeque import java.util.ArrayDeque
import java.util.concurrent.Executors import java.util.concurrent.Executors
@@ -321,6 +323,48 @@ class BuyerAccessibilityService : AccessibilityService() {
) )
} }
internal fun openPinduoduoSpecifications(): Boolean =
withPinduoduoRoot { root ->
if (!isVerifiedProductDetail(root)) {
return@withPinduoduoRoot false
}
val entries = collectNodes(root).filter { node ->
node.isVisibleToUser &&
node.isEnabled &&
PinduoduoSpecificationParser.isSafeEntryText(
semanticText(node)
)
}
entries.singleOrNull()?.let(::clickNodeOrAncestor) == true
} ?: false
internal fun readPinduoduoSpecificationEvidence():
PinduoduoSpecificationEvidence? =
withPinduoduoRoot { root ->
val snapshot = classifyPinduoduoRoot(root)
if (
snapshot.safetyStopReason != null ||
snapshot.page != PinduoduoPage.SPECIFICATION_PANEL
) {
return@withPinduoduoRoot null
}
PinduoduoSpecificationParser.parse(
collectNodes(root).map(::uiElement)
)
}
internal fun closePinduoduoSpecifications(): Boolean =
withPinduoduoRoot { root ->
val snapshot = classifyPinduoduoRoot(root)
if (
snapshot.safetyStopReason != null ||
snapshot.page != PinduoduoPage.SPECIFICATION_PANEL
) {
return@withPinduoduoRoot false
}
performGlobalAction(GLOBAL_ACTION_BACK)
} ?: false
internal fun returnFromPinduoduoCandidate(): Boolean = internal fun returnFromPinduoduoCandidate(): Boolean =
withPinduoduoRoot { root -> withPinduoduoRoot { root ->
if (!isVerifiedProductDetail(root)) { if (!isVerifiedProductDetail(root)) {
@@ -355,20 +399,38 @@ class BuyerAccessibilityService : AccessibilityService() {
) == true ) == true
} ?: false } ?: false
internal suspend fun capturePinduoduoScreenshot(): PinduoduoScreenshotCapture? { internal suspend fun capturePinduoduoScreenshot(
expectedPage: PinduoduoPage
): PinduoduoScreenshotCapture? {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {
return null return null
} }
return capturePinduoduoScreenshotApi30() if (
expectedPage !in setOf(
PinduoduoPage.PRODUCT_DETAIL,
PinduoduoPage.SPECIFICATION_PANEL
)
) {
return null
}
return capturePinduoduoScreenshotApi30(expectedPage)
} }
@RequiresApi(Build.VERSION_CODES.R) @RequiresApi(Build.VERSION_CODES.R)
private suspend fun capturePinduoduoScreenshotApi30(): private suspend fun capturePinduoduoScreenshotApi30(
expectedPage: PinduoduoPage
):
PinduoduoScreenshotCapture? { PinduoduoScreenshotCapture? {
val root = rootInActiveWindow val root = rootInActiveWindow
val snapshot = root?.let(::classifyPinduoduoRoot)
if ( if (
root?.packageName?.toString() != PINDUODUO_PACKAGE || root?.packageName?.toString() != PINDUODUO_PACKAGE ||
snapshot?.safetyStopReason != null ||
snapshot?.page != expectedPage ||
(
expectedPage == PinduoduoPage.PRODUCT_DETAIL &&
!isVerifiedProductDetail(root) !isVerifiedProductDetail(root)
)
) { ) {
return null return null
} }
@@ -447,8 +509,17 @@ class BuyerAccessibilityService : AccessibilityService() {
private fun classifyPinduoduoRoot( private fun classifyPinduoduoRoot(
root: AccessibilityNodeInfo root: AccessibilityNodeInfo
): PinduoduoUiSnapshot { ): PinduoduoUiSnapshot {
val elements = collectNodes(root).map { node -> val elements = collectNodes(root).map(::uiElement)
PinduoduoUiElement( return PinduoduoPageClassifier.classify(
foregroundPackage = root.packageName?.toString(),
elements = elements,
expectedQuery = expectedSearchQuery
)
}
private fun uiElement(node: AccessibilityNodeInfo): PinduoduoUiElement {
val bounds = Rect().also(node::getBoundsInScreen)
return PinduoduoUiElement(
text = node.text?.toString(), text = node.text?.toString(),
contentDescription = node.contentDescription?.toString(), contentDescription = node.contentDescription?.toString(),
className = node.className?.toString().orEmpty(), className = node.className?.toString().orEmpty(),
@@ -456,13 +527,11 @@ class BuyerAccessibilityService : AccessibilityService() {
clickable = node.isClickable, clickable = node.isClickable,
editable = node.isEditable, editable = node.isEditable,
enabled = node.isEnabled, enabled = node.isEnabled,
visibleToUser = node.isVisibleToUser visibleToUser = node.isVisibleToUser,
) selected = node.isSelected,
} scrollable = node.isScrollable,
return PinduoduoPageClassifier.classify( boundsLeft = bounds.left,
foregroundPackage = root.packageName?.toString(), boundsTop = bounds.top
elements = elements,
expectedQuery = expectedSearchQuery
) )
} }
@@ -668,6 +737,13 @@ class BuyerAccessibilityService : AccessibilityService() {
.replace(Regex("\\s+"), " ") .replace(Regex("\\s+"), " ")
.take(MAX_EVIDENCE_TEXT_LENGTH) .take(MAX_EVIDENCE_TEXT_LENGTH)
private fun semanticText(node: AccessibilityNodeInfo): String =
sequenceOf(node.text, node.contentDescription)
.filterNotNull()
.map { it.toString().trim() }
.firstOrNull(String::isNotEmpty)
.orEmpty()
private fun shouldInspect(eventType: Int): Boolean = private fun shouldInspect(eventType: Int): Boolean =
eventType == AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED || eventType == AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED ||
eventType == AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED || eventType == AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED ||
@@ -25,20 +25,41 @@ class AndroidPinduoduoCandidateDriver(
override suspend fun openCandidate(signature: String): Boolean = override suspend fun openCandidate(signature: String): Boolean =
BuyerAccessibilityBridge.openCandidate(signature) BuyerAccessibilityBridge.openCandidate(signature)
override suspend fun captureCandidate( override suspend fun captureDetail(): PinduoduoCandidateCapture? {
ordinal: Int,
card: PinduoduoCandidateCard
): PinduoduoCandidateEvidence? {
val detail = BuyerAccessibilityBridge.candidateDetailEvidence() val detail = BuyerAccessibilityBridge.candidateDetailEvidence()
?: return null ?: return null
val screenshot = BuyerAccessibilityBridge.captureScreenshot() val screenshot = BuyerAccessibilityBridge.captureDetailScreenshot()
?: return null ?: return null
return PinduoduoCandidateCapture(detail, screenshot)
}
override suspend fun openSpecifications(): Boolean =
BuyerAccessibilityBridge.openSpecifications()
override suspend fun captureSpecifications(): PinduoduoSpecificationCapture? {
val evidence = BuyerAccessibilityBridge.specificationEvidence()
?: return null
val screenshot =
BuyerAccessibilityBridge.captureSpecificationScreenshot()
?: return null
return PinduoduoSpecificationCapture(evidence, screenshot)
}
override suspend fun closeSpecifications(): Boolean =
BuyerAccessibilityBridge.closeSpecifications()
override suspend fun saveCandidate(
ordinal: Int,
card: PinduoduoCandidateCard,
detail: PinduoduoCandidateCapture,
specifications: PinduoduoSpecificationCapture
): PinduoduoCandidateEvidence? {
return withContext(Dispatchers.IO) { return withContext(Dispatchers.IO) {
evidenceStore.save( evidenceStore.save(
ordinal = ordinal, ordinal = ordinal,
card = card, card = card,
detail = detail, detail = detail,
screenshot = screenshot specifications = specifications
) )
} }
} }
@@ -70,8 +91,8 @@ private class CandidateEvidenceStore(context: Context) {
fun save( fun save(
ordinal: Int, ordinal: Int,
card: PinduoduoCandidateCard, card: PinduoduoCandidateCard,
detail: PinduoduoCandidateDetailEvidence, detail: PinduoduoCandidateCapture,
screenshot: PinduoduoScreenshotCapture specifications: PinduoduoSpecificationCapture
): PinduoduoCandidateEvidence? { ): PinduoduoCandidateEvidence? {
if (ordinal !in 1..MAX_CANDIDATES_PER_PROBE) { if (ordinal !in 1..MAX_CANDIDATES_PER_PROBE) {
return null return null
@@ -80,22 +101,25 @@ private class CandidateEvidenceStore(context: Context) {
if (!root.exists() && !root.mkdirs()) { if (!root.exists() && !root.mkdirs()) {
return null return null
} }
val fileName = "candidate-%02d.png".format(ordinal) val detailAsset = writeAsset(
val screenshotFile = File(root, fileName) ordinal = ordinal,
screenshotFile.writeBytes(screenshot.pngBytes) suffix = "detail",
screenshot = detail.screenshot
)
val specificationAsset = writeAsset(
ordinal = ordinal,
suffix = "specification",
screenshot = specifications.screenshot
)
val evidence = PinduoduoCandidateEvidence( val evidence = PinduoduoCandidateEvidence(
ordinal = ordinal, ordinal = ordinal,
cardSignature = card.signature, cardSignature = card.signature,
cardSemanticTextCount = card.semanticTextCount, cardSemanticTextCount = card.semanticTextCount,
detailSignature = detail.signature, detailSignature = detail.detail.signature,
detailSemanticTextCount = detail.semanticTextCount, detailSemanticTextCount = detail.detail.semanticTextCount,
screenshotFileName = fileName, detailAsset = detailAsset,
screenshotSha256 = PinduoduoEvidenceHash.sha256( specification = specifications.evidence,
screenshot.pngBytes specificationAsset = specificationAsset
),
screenshotByteCount = screenshot.pngBytes.size,
screenshotWidth = screenshot.width,
screenshotHeight = screenshot.height
) )
evidenceByOrdinal[ordinal] = evidence evidenceByOrdinal[ordinal] = evidence
writeManifest() writeManifest()
@@ -105,6 +129,22 @@ private class CandidateEvidenceStore(context: Context) {
} }
} }
private fun writeAsset(
ordinal: Int,
suffix: String,
screenshot: PinduoduoScreenshotCapture
): PinduoduoEvidenceAsset {
val fileName = "candidate-%02d-%s.png".format(ordinal, suffix)
File(root, fileName).writeBytes(screenshot.pngBytes)
return PinduoduoEvidenceAsset(
fileName = fileName,
sha256 = PinduoduoEvidenceHash.sha256(screenshot.pngBytes),
byteCount = screenshot.pngBytes.size,
width = screenshot.width,
height = screenshot.height
)
}
private fun writeManifest() { private fun writeManifest() {
val candidates = JSONArray() val candidates = JSONArray()
evidenceByOrdinal.values.forEach { evidence -> evidenceByOrdinal.values.forEach { evidence ->
@@ -121,18 +161,23 @@ private class CandidateEvidenceStore(context: Context) {
"detail_semantic_text_count", "detail_semantic_text_count",
evidence.detailSemanticTextCount evidence.detailSemanticTextCount
) )
.put("screenshot_file", evidence.screenshotFileName) .put("detail_asset", assetJson(evidence.detailAsset))
.put("screenshot_sha256", evidence.screenshotSha256)
.put( .put(
"screenshot_byte_count", "specification_signature",
evidence.screenshotByteCount evidence.specification.signature
)
.put(
"specification_semantic_text_count",
evidence.specification.semanticTextCount
)
.put(
"specification_asset",
assetJson(evidence.specificationAsset)
) )
.put("screenshot_width", evidence.screenshotWidth)
.put("screenshot_height", evidence.screenshotHeight)
) )
} }
val manifest = JSONObject() val manifest = JSONObject()
.put("schema_version", 1) .put("schema_version", 2)
.put("candidate_count", evidenceByOrdinal.size) .put("candidate_count", evidenceByOrdinal.size)
.put("candidates", candidates) .put("candidates", candidates)
.toString(2) .toString(2)
@@ -147,6 +192,14 @@ private class CandidateEvidenceStore(context: Context) {
} }
} }
private fun assetJson(asset: PinduoduoEvidenceAsset): JSONObject =
JSONObject()
.put("file", asset.fileName)
.put("sha256", asset.sha256)
.put("byte_count", asset.byteCount)
.put("width", asset.width)
.put("height", asset.height)
private companion object { private companion object {
const val MANIFEST_FILE = "manifest.json" const val MANIFEST_FILE = "manifest.json"
} }
@@ -9,8 +9,11 @@ import kotlinx.coroutines.withContext
data class ValidatedCandidateEvidence( data class ValidatedCandidateEvidence(
val ordinal: Int, val ordinal: Int,
val pngBytes: ByteArray, val detailPngBytes: ByteArray,
val sha256: String val detailSha256: String,
val specification: PinduoduoSpecificationEvidence,
val specificationPngBytes: ByteArray,
val specificationSha256: String
) )
class CandidateEvidenceSource( class CandidateEvidenceSource(
@@ -31,38 +34,63 @@ class CandidateEvidenceSource(
val canonicalRoot = root.canonicalFile val canonicalRoot = root.canonicalFile
var totalBytes = 0L var totalBytes = 0L
sorted.map { metadata -> sorted.map { metadata ->
val expectedFileName = "candidate-%02d.png".format(metadata.ordinal) require(
require(metadata.screenshotFileName == expectedFileName) metadata.specification.groups.map { it.kind }.toSet() ==
require(metadata.screenshotByteCount in 1..MAX_PNG_BYTES) PinduoduoSpecificationGroupKind.entries.toSet()
require(metadata.screenshotWidth in 1..MAX_SCREENSHOT_DIMENSION) )
require(metadata.screenshotHeight in 1..MAX_SCREENSHOT_DIMENSION) val detail = readAsset(
require(SHA256_PATTERN.matches(metadata.screenshotSha256)) canonicalRoot = canonicalRoot,
totalBytes += metadata.screenshotByteCount asset = metadata.detailAsset,
expectedFileName =
"candidate-%02d-detail.png".format(metadata.ordinal)
)
val specification = readAsset(
canonicalRoot = canonicalRoot,
asset = metadata.specificationAsset,
expectedFileName =
"candidate-%02d-specification.png".format(metadata.ordinal)
)
totalBytes += detail.size + specification.size
require(totalBytes <= MAX_TOTAL_PNG_BYTES) require(totalBytes <= MAX_TOTAL_PNG_BYTES)
val file = File(canonicalRoot, expectedFileName).canonicalFile
require(file.parentFile == canonicalRoot)
require(file.isFile && file.length() == metadata.screenshotByteCount.toLong())
val bytes = FileInputStream(file).use {
it.readBytesExact(metadata.screenshotByteCount)
}
require(bytes.hasPngHeader())
val dimensions = bytes.pngDimensions()
require(dimensions.first == metadata.screenshotWidth)
require(dimensions.second == metadata.screenshotHeight)
require(PinduoduoEvidenceHash.sha256(bytes) == metadata.screenshotSha256)
ValidatedCandidateEvidence( ValidatedCandidateEvidence(
ordinal = metadata.ordinal, ordinal = metadata.ordinal,
pngBytes = bytes, detailPngBytes = detail,
sha256 = metadata.screenshotSha256 detailSha256 = metadata.detailAsset.sha256,
specification = metadata.specification,
specificationPngBytes = specification,
specificationSha256 = metadata.specificationAsset.sha256
) )
} }
} }
} }
private fun readAsset(
canonicalRoot: File,
asset: PinduoduoEvidenceAsset,
expectedFileName: String
): ByteArray {
require(asset.fileName == expectedFileName)
require(asset.byteCount in 1..MAX_PNG_BYTES)
require(asset.width in 1..MAX_SCREENSHOT_DIMENSION)
require(asset.height in 1..MAX_SCREENSHOT_DIMENSION)
require(SHA256_PATTERN.matches(asset.sha256))
val file = File(canonicalRoot, expectedFileName).canonicalFile
require(file.parentFile == canonicalRoot)
require(file.isFile && file.length() == asset.byteCount.toLong())
val bytes = FileInputStream(file).use {
it.readBytesExact(asset.byteCount)
}
require(bytes.hasPngHeader())
val dimensions = bytes.pngDimensions()
require(dimensions.first == asset.width)
require(dimensions.second == asset.height)
require(PinduoduoEvidenceHash.sha256(bytes) == asset.sha256)
return bytes
}
private companion object { private companion object {
const val MAX_PNG_BYTES = 8 * 1024 * 1024 const val MAX_PNG_BYTES = 8 * 1024 * 1024
const val MAX_TOTAL_PNG_BYTES = 32L * 1024L * 1024L const val MAX_TOTAL_PNG_BYTES = 64L * 1024L * 1024L
const val MAX_SCREENSHOT_DIMENSION = 10_000 const val MAX_SCREENSHOT_DIMENSION = 10_000
val SHA256_PATTERN = Regex("^[0-9a-f]{64}$") val SHA256_PATTERN = Regex("^[0-9a-f]{64}$")
} }
@@ -16,7 +16,10 @@ enum class CandidateBrowsePhase {
READING_RESULTS, READING_RESULTS,
OPENING_CANDIDATE, OPENING_CANDIDATE,
WAITING_DETAIL, WAITING_DETAIL,
CAPTURING_EVIDENCE, CAPTURING_DETAIL,
OPENING_SPECIFICATIONS,
READING_SPECIFICATIONS,
CLOSING_SPECIFICATIONS,
RETURNING_RESULTS, RETURNING_RESULTS,
SCROLLING_RESULTS, SCROLLING_RESULTS,
COMPLETE COMPLETE
@@ -100,11 +103,34 @@ class PinduoduoCandidateAutomation(
mutablePhase.value = CandidateBrowsePhase.WAITING_DETAIL mutablePhase.value = CandidateBrowsePhase.WAITING_DETAIL
awaitPage(PinduoduoPage.PRODUCT_DETAIL)?.let { return it } awaitPage(PinduoduoPage.PRODUCT_DETAIL)?.let { return it }
val ordinal = mutableEvidence.value.size + 1 val ordinal = mutableEvidence.value.size + 1
mutablePhase.value = CandidateBrowsePhase.CAPTURING_EVIDENCE mutablePhase.value = CandidateBrowsePhase.CAPTURING_DETAIL
val captured = driver.captureCandidate(ordinal, nextCard) val detail = driver.captureDetail()
?: return AutomationResult.FatalFailure( ?: return AutomationResult.FatalFailure(
WorkflowFailureCode.EVIDENCE_CAPTURE_FAILED WorkflowFailureCode.EVIDENCE_CAPTURE_FAILED
) )
mutablePhase.value = CandidateBrowsePhase.OPENING_SPECIFICATIONS
if (!driver.openSpecifications()) {
return AutomationResult.Blocked(SafetyStopReason.UNKNOWN_PAGE)
}
awaitPage(PinduoduoPage.SPECIFICATION_PANEL)?.let { return it }
mutablePhase.value = CandidateBrowsePhase.READING_SPECIFICATIONS
val specifications = driver.captureSpecifications()
?: return AutomationResult.FatalFailure(
WorkflowFailureCode.EVIDENCE_CAPTURE_FAILED
)
mutablePhase.value = CandidateBrowsePhase.CLOSING_SPECIFICATIONS
if (!driver.closeSpecifications()) {
return AutomationResult.Blocked(SafetyStopReason.UNKNOWN_PAGE)
}
awaitPage(PinduoduoPage.PRODUCT_DETAIL)?.let { return it }
val captured = driver.saveCandidate(
ordinal = ordinal,
card = nextCard,
detail = detail,
specifications = specifications
) ?: return AutomationResult.FatalFailure(
WorkflowFailureCode.EVIDENCE_CAPTURE_FAILED
)
mutableEvidence.value = mutableEvidence.value + captured mutableEvidence.value = mutableEvidence.value + captured
mutablePhase.value = CandidateBrowsePhase.RETURNING_RESULTS mutablePhase.value = CandidateBrowsePhase.RETURNING_RESULTS
@@ -123,7 +149,17 @@ class PinduoduoCandidateAutomation(
private suspend fun recoverDetailPageIfNeeded(): AutomationResult? { private suspend fun recoverDetailPageIfNeeded(): AutomationResult? {
val snapshot = driver.snapshot() val snapshot = driver.snapshot()
safetyResult(snapshot)?.let { return it } safetyResult(snapshot)?.let { return it }
if (snapshot.page != PinduoduoPage.PRODUCT_DETAIL) { if (snapshot.page == PinduoduoPage.SPECIFICATION_PANEL) {
if (!driver.closeSpecifications()) {
return AutomationResult.RetryableFailure(
WorkflowFailureCode.TRANSIENT_AUTOMATION
)
}
awaitPage(PinduoduoPage.PRODUCT_DETAIL)?.let { return it }
}
val detailSnapshot = driver.snapshot()
safetyResult(detailSnapshot)?.let { return it }
if (detailSnapshot.page != PinduoduoPage.PRODUCT_DETAIL) {
return null return null
} }
if (!driver.returnToResults()) { if (!driver.returnToResults()) {
@@ -23,17 +23,23 @@ data class PinduoduoScreenshotCapture(
val height: Int val height: Int
) )
data class PinduoduoEvidenceAsset(
val fileName: String,
val sha256: String,
val byteCount: Int,
val width: Int,
val height: Int
)
data class PinduoduoCandidateEvidence( data class PinduoduoCandidateEvidence(
val ordinal: Int, val ordinal: Int,
val cardSignature: String, val cardSignature: String,
val cardSemanticTextCount: Int, val cardSemanticTextCount: Int,
val detailSignature: String, val detailSignature: String,
val detailSemanticTextCount: Int, val detailSemanticTextCount: Int,
val screenshotFileName: String, val detailAsset: PinduoduoEvidenceAsset,
val screenshotSha256: String, val specification: PinduoduoSpecificationEvidence,
val screenshotByteCount: Int, val specificationAsset: PinduoduoEvidenceAsset
val screenshotWidth: Int,
val screenshotHeight: Int
) )
object PinduoduoEvidenceHash { object PinduoduoEvidenceHash {
@@ -50,11 +56,27 @@ interface PinduoduoCandidateDriver {
suspend fun snapshot(): PinduoduoUiSnapshot suspend fun snapshot(): PinduoduoUiSnapshot
suspend fun candidateCards(limit: Int): List<PinduoduoCandidateCard> suspend fun candidateCards(limit: Int): List<PinduoduoCandidateCard>
suspend fun openCandidate(signature: String): Boolean suspend fun openCandidate(signature: String): Boolean
suspend fun captureCandidate( suspend fun captureDetail(): PinduoduoCandidateCapture?
suspend fun openSpecifications(): Boolean
suspend fun captureSpecifications(): PinduoduoSpecificationCapture?
suspend fun closeSpecifications(): Boolean
suspend fun saveCandidate(
ordinal: Int, ordinal: Int,
card: PinduoduoCandidateCard card: PinduoduoCandidateCard,
detail: PinduoduoCandidateCapture,
specifications: PinduoduoSpecificationCapture
): PinduoduoCandidateEvidence? ): PinduoduoCandidateEvidence?
suspend fun returnToResults(): Boolean suspend fun returnToResults(): Boolean
suspend fun scrollResults(): Boolean suspend fun scrollResults(): Boolean
fun resetEvidence() fun resetEvidence()
} }
data class PinduoduoCandidateCapture(
val detail: PinduoduoCandidateDetailEvidence,
val screenshot: PinduoduoScreenshotCapture
)
data class PinduoduoSpecificationCapture(
val evidence: PinduoduoSpecificationEvidence,
val screenshot: PinduoduoScreenshotCapture
)
@@ -13,7 +13,11 @@ data class PinduoduoUiElement(
val clickable: Boolean, val clickable: Boolean,
val editable: Boolean, val editable: Boolean,
val enabled: Boolean, val enabled: Boolean,
val visibleToUser: Boolean val visibleToUser: Boolean,
val selected: Boolean = false,
val scrollable: Boolean = false,
val boundsLeft: Int = 0,
val boundsTop: Int = 0
) )
enum class PinduoduoPage { enum class PinduoduoPage {
@@ -24,6 +28,7 @@ enum class PinduoduoPage {
IMAGE_SEARCH, IMAGE_SEARCH,
IMAGE_SEARCH_RESULTS, IMAGE_SEARCH_RESULTS,
PRODUCT_DETAIL, PRODUCT_DETAIL,
SPECIFICATION_PANEL,
UNKNOWN UNKNOWN
} }
@@ -142,8 +147,23 @@ object PinduoduoPageClassifier {
} }
val detailMarkerCount = setOf("联系客服", "收藏", "店铺") val detailMarkerCount = setOf("联系客服", "收藏", "店铺")
.count { marker -> normalized.any { it.contains(marker) } } .count { marker -> normalized.any { it.contains(marker) } }
val specificationGroupCount = setOf("颜色分类", "颜色", "尺码", "尺寸")
.count { marker -> normalized.any { it == marker } }
val specificationOptionCount = visibleElements.count { element ->
val semantic = element.text
?.takeIf(String::isNotBlank)
?: element.contentDescription.orEmpty()
element.clickable &&
semantic.isNotBlank() &&
normalize(semantic) !in specificationEntryMarkers &&
paymentMarkers.none(semantic::contains)
}
val hasSpecificationPanel =
specificationGroupCount >= 2 &&
specificationOptionCount >= 2
val page = when { val page = when {
hasSpecificationPanel -> PinduoduoPage.SPECIFICATION_PANEL
hasImageResultHeader && legacySortControlCount >= 3 -> hasImageResultHeader && legacySortControlCount >= 3 ->
PinduoduoPage.IMAGE_SEARCH_RESULTS PinduoduoPage.IMAGE_SEARCH_RESULTS
hasImageSearchPage -> PinduoduoPage.IMAGE_SEARCH hasImageSearchPage -> PinduoduoPage.IMAGE_SEARCH
@@ -200,6 +220,16 @@ object PinduoduoPageClassifier {
private fun Collection<String>.containsAny(markers: Collection<String>): Boolean = private fun Collection<String>.containsAny(markers: Collection<String>): Boolean =
any { text -> markers.any(text::contains) } any { text -> markers.any(text::contains) }
private val specificationEntryMarkers = setOf(
"选择规格",
"请选择规格",
"颜色分类",
"颜色款式",
"选择颜色",
"请选择颜色",
"选择尺码",
"请选择尺码"
)
private const val MINIMUM_QUERY_FRAGMENT_LENGTH = 12 private const val MINIMUM_QUERY_FRAGMENT_LENGTH = 12
private const val MINIMUM_ELIDED_FRAGMENT_LENGTH = 4 private const val MINIMUM_ELIDED_FRAGMENT_LENGTH = 4
} }
@@ -0,0 +1,178 @@
package com.roubao.autopilot.pinduoduo
enum class PinduoduoSpecificationGroupKind {
COLOR,
SIZE
}
data class PinduoduoSpecificationOption(
val text: String,
val selected: Boolean,
val enabled: Boolean
)
data class PinduoduoSpecificationGroup(
val kind: PinduoduoSpecificationGroupKind,
val title: String,
val options: List<PinduoduoSpecificationOption>,
val complete: Boolean
)
data class PinduoduoSpecificationEvidence(
val signature: String,
val semanticTextCount: Int,
val groups: List<PinduoduoSpecificationGroup>
)
object PinduoduoSpecificationParser {
fun parse(
elements: Collection<PinduoduoUiElement>
): PinduoduoSpecificationEvidence? {
val visible = elements
.asSequence()
.filter { it.visibleToUser }
.filter { semanticText(it).isNotEmpty() || it.scrollable }
.take(MAX_ELEMENTS + 1)
.toList()
if (visible.isEmpty() || visible.size > MAX_ELEMENTS) {
return null
}
val ordered = visible.sortedWith(
compareBy<PinduoduoUiElement> { it.boundsTop }
.thenBy { it.boundsLeft }
)
val headers = ordered.mapIndexedNotNull { index, element ->
groupKind(semanticText(element))?.let { kind ->
GroupHeader(index, kind, semanticText(element), element.boundsTop)
}
}
if (
headers.map { it.kind }.toSet() !=
PinduoduoSpecificationGroupKind.entries.toSet()
) {
return null
}
val groups = headers.mapIndexed { headerIndex, header ->
val nextTop = headers.getOrNull(headerIndex + 1)?.top ?: Int.MAX_VALUE
val options = ordered.asSequence()
.filter { element ->
element.boundsTop >= header.top &&
element.boundsTop < nextTop &&
groupKind(semanticText(element)) == null &&
isOption(element)
}
.map { element ->
PinduoduoSpecificationOption(
text = semanticText(element).take(MAX_OPTION_TEXT_LENGTH),
selected = element.selected,
enabled = element.enabled
)
}
.distinctBy { normalize(it.text) }
.take(MAX_OPTIONS_PER_GROUP + 1)
.toList()
val clipped =
options.size > MAX_OPTIONS_PER_GROUP ||
ordered.any { element ->
element.scrollable &&
element.boundsTop >= header.top &&
element.boundsTop < nextTop
}
PinduoduoSpecificationGroup(
kind = header.kind,
title = header.title.take(MAX_OPTION_TEXT_LENGTH),
options = options.take(MAX_OPTIONS_PER_GROUP),
complete = !clipped
)
}
if (groups.any { it.options.isEmpty() }) {
return null
}
val semantics = buildList {
groups.forEach { group ->
add("${group.kind.name}:${normalize(group.title)}:${group.complete}")
group.options.forEach { option ->
add(
"${normalize(option.text)}:" +
"${option.selected}:${option.enabled}"
)
}
}
}
return PinduoduoSpecificationEvidence(
signature = PinduoduoEvidenceHash.sha256(
semantics.joinToString("\u001f")
),
semanticTextCount = semantics.size,
groups = groups
)
}
fun isSafeEntryText(value: String): Boolean {
val normalized = normalize(value)
return normalized in SAFE_ENTRY_TEXTS ||
SAFE_ENTRY_PREFIXES.any(normalized::startsWith)
}
private fun isOption(element: PinduoduoUiElement): Boolean {
val text = semanticText(element)
val normalized = normalize(text)
return text.length <= MAX_OPTION_TEXT_LENGTH &&
normalized !in SAFE_ENTRY_TEXTS &&
SAFE_ENTRY_PREFIXES.none(normalized::startsWith) &&
TRANSACTION_MARKERS.none(normalized::contains) &&
(element.clickable || element.selected || !element.enabled)
}
private fun groupKind(value: String): PinduoduoSpecificationGroupKind? =
when (normalize(value).removeSuffix(":").removeSuffix(":")) {
"颜色", "颜色分类", "颜色选择" ->
PinduoduoSpecificationGroupKind.COLOR
"尺码", "尺寸", "大小", "尺码选择" ->
PinduoduoSpecificationGroupKind.SIZE
else -> null
}
private fun semanticText(element: PinduoduoUiElement): String =
sequenceOf(element.text, element.contentDescription)
.filterNotNull()
.map(String::trim)
.firstOrNull(String::isNotEmpty)
.orEmpty()
private fun normalize(value: String): String =
value.trim().lowercase().replace(Regex("\\s+"), "")
private data class GroupHeader(
val index: Int,
val kind: PinduoduoSpecificationGroupKind,
val title: String,
val top: Int
)
private val SAFE_ENTRY_TEXTS = setOf(
"选择规格",
"请选择规格",
"颜色分类",
"颜色款式",
"选择颜色",
"请选择颜色",
"选择尺码",
"请选择尺码"
)
private val SAFE_ENTRY_PREFIXES = setOf("请选择:", "请选择:", "已选:", "已选:")
private val TRANSACTION_MARKERS = setOf(
"购买",
"拼单",
"购物车",
"下单",
"支付",
"结算"
)
private const val MAX_ELEMENTS = 160
private const val MAX_OPTIONS_PER_GROUP = 30
private const val MAX_OPTION_TEXT_LENGTH = 80
}
@@ -5,9 +5,37 @@ import java.security.MessageDigest
data class ExecutionEvidenceDraft( data class ExecutionEvidenceDraft(
val ordinal: Int, val ordinal: Int,
val pngBytes: ByteArray, val pngBytes: ByteArray,
val sha256: String val sha256: String,
val kind: ExecutionEvidenceKind = ExecutionEvidenceKind.DETAIL
) )
enum class ExecutionEvidenceKind {
DETAIL,
SPECIFICATION
}
object ExecutionEvidenceAssociationPolicy {
fun group(
candidateOrdinals: List<Int>,
evidence: List<ExecutionEvidenceDraft>
): Map<Int, List<ExecutionEvidenceDraft>> {
require(candidateOrdinals == (1..candidateOrdinals.size).toList())
val grouped = evidence.groupBy { it.ordinal }
require(grouped.keys == candidateOrdinals.toSet())
require(
candidateOrdinals.all { ordinal ->
val assets = requireNotNull(grouped[ordinal])
assets.size == 2 &&
assets.map { it.kind }.toSet() ==
ExecutionEvidenceKind.entries.toSet()
}
)
return candidateOrdinals.associateWith { ordinal ->
requireNotNull(grouped[ordinal]).sortedBy { it.kind.ordinal }
}
}
}
data class ExecutionCandidateDraft( data class ExecutionCandidateDraft(
val ordinal: Int, val ordinal: Int,
val title: String, val title: String,
@@ -22,7 +50,8 @@ data class ExecutionCandidateDraft(
data class ExecutionCandidateSourceIdentity( data class ExecutionCandidateSourceIdentity(
val sourceOrdinal: Int, val sourceOrdinal: Int,
val rankedOrdinal: Int, val rankedOrdinal: Int,
val evidenceSha256: String val evidenceSha256: String,
val specificationEvidenceSha256: String = evidenceSha256
) )
data class RankedExecutionCandidateDraft( data class RankedExecutionCandidateDraft(
@@ -45,6 +74,11 @@ object ExecutionCandidateIdentityPolicy {
require(identities.all { it.sourceOrdinal > 0 }) require(identities.all { it.sourceOrdinal > 0 })
require(identities.map { it.sourceOrdinal }.distinct().size == identities.size) require(identities.map { it.sourceOrdinal }.distinct().size == identities.size)
require(identities.all { sha256Pattern.matches(it.evidenceSha256) }) require(identities.all { sha256Pattern.matches(it.evidenceSha256) })
require(
identities.all {
sha256Pattern.matches(it.specificationEvidenceSha256)
}
)
val candidatesByOrdinal = candidates.associateBy { it.ordinal } val candidatesByOrdinal = candidates.associateBy { it.ordinal }
require(candidatesByOrdinal.size == candidates.size) require(candidatesByOrdinal.size == candidates.size)
return identities.map { identity -> return identities.map { identity ->
@@ -343,11 +343,8 @@ class ProcurementRepository(
require(!execution.safetyStopped && !execution.isExpired()) { require(!execution.safetyStopped && !execution.isExpired()) {
"执行授权已到期,不能继续采集" "执行授权已到期,不能继续采集"
} }
require(evidence.size in 0..5) { "候选证据必须为 0 至 5 张" } require(evidence.size in 0..25) { "候选证据必须为 0 至 25 张" }
require(batch.candidates.size in 0..5) { "候选数量必须为 0 至 5 个" } require(batch.candidates.size in 0..5) { "候选数量必须为 0 至 5 个" }
require(evidence.size == batch.candidates.size) {
"候选和证据数量必须一致"
}
require(batch.candidates.map { it.ordinal } == (1..batch.candidates.size).toList()) { require(batch.candidates.map { it.ordinal } == (1..batch.candidates.size).toList()) {
"候选编号必须连续" "候选编号必须连续"
} }
@@ -359,9 +356,15 @@ class ProcurementRepository(
} else { } else {
require(batch.provenance == null) { "手工优先模式不能附带模型出处" } require(batch.provenance == null) { "手工优先模式不能附带模型出处" }
} }
val evidenceIDs = evidence.sortedBy { it.ordinal }.associate { draft -> val evidenceByOrdinal = ExecutionEvidenceAssociationPolicy.group(
require(draft.ordinal in 1..batch.candidates.size) candidateOrdinals = batch.candidates.map { it.ordinal },
evidence = evidence
)
evidence.forEach { draft ->
require(sha256(draft.pngBytes) == draft.sha256) require(sha256(draft.pngBytes) == draft.sha256)
}
val evidenceIDs = evidenceByOrdinal.mapValues { (_, drafts) ->
drafts.sortedBy { it.kind.ordinal }.map { draft ->
val localID = UUID.randomUUID().toString() val localID = UUID.randomUUID().toString()
persistEvidenceLocked(localID, draft.pngBytes) persistEvidenceLocked(localID, draft.pngBytes)
appendOutboxLocked( appendOutboxLocked(
@@ -370,17 +373,18 @@ class ProcurementRepository(
type = ExecutionOutboxType.EVIDENCE, type = ExecutionOutboxType.EVIDENCE,
idempotencyKey = newOpaqueSecret(), idempotencyKey = newOpaqueSecret(),
payload = "{}", payload = "{}",
evidenceRelativePath = "$OUTBOX_DIRECTORY/$localID.png" evidenceRelativePath =
"$OUTBOX_DIRECTORY/$localID.png"
) )
) )
draft.ordinal to localID localID
}
} }
val candidates = batch.candidates.map { candidate -> val candidates = batch.candidates.map { candidate ->
candidate.copy( candidate.copy(
evidenceLocalIDs = listOf( evidenceLocalIDs =
requireNotNull(evidenceIDs[candidate.ordinal]) requireNotNull(evidenceIDs[candidate.ordinal])
) )
)
} }
val payload = JSONObject() val payload = JSONObject()
.put("execution_id", execution.id) .put("execution_id", execution.id)
@@ -1,14 +1,18 @@
package com.roubao.autopilot.vlm package com.roubao.autopilot.vlm
const val CANDIDATE_EVALUATION_SCHEMA_VERSION = 2 const val CANDIDATE_EVALUATION_SCHEMA_VERSION = 3
const val CANDIDATE_EVALUATION_PROMPT_VERSION = "candidate-evaluation-v2" const val CANDIDATE_EVALUATION_PROMPT_VERSION = "candidate-evaluation-v3"
const val CANDIDATE_RECOMMENDATION_THRESHOLD = 0.75 const val CANDIDATE_RECOMMENDATION_THRESHOLD = 0.75
data class CandidateEvaluationImage( data class CandidateEvaluationImage(
val ordinal: Int, val ordinal: Int,
val mediaType: String, val mediaType: String,
val bytes: ByteArray, val bytes: ByteArray,
val sha256: String val sha256: String,
val specificationBytes: ByteArray = bytes,
val specificationSha256: String = sha256,
val localHardConstraintResults: List<CandidateHardConstraintResult> =
emptyList()
) )
data class CandidateEvaluationInput( data class CandidateEvaluationInput(
@@ -56,6 +60,7 @@ data class CandidateAssessment(
val rejectionReasons: List<String>, val rejectionReasons: List<String>,
val confidence: Double, val confidence: Double,
val evidenceSha256: String, val evidenceSha256: String,
val specificationEvidenceSha256: String = evidenceSha256,
val hardConstraintResults: List<CandidateHardConstraintResult> = emptyList() val hardConstraintResults: List<CandidateHardConstraintResult> = emptyList()
) )
@@ -63,12 +68,17 @@ data class RankedCandidateAssessment(
val sourceOrdinal: Int, val sourceOrdinal: Int,
val rankedOrdinal: Int, val rankedOrdinal: Int,
val evidenceSha256: String, val evidenceSha256: String,
val specificationEvidenceSha256: String,
val assessment: CandidateAssessment val assessment: CandidateAssessment
) { ) {
init { init {
require(sourceOrdinal == assessment.ordinal) require(sourceOrdinal == assessment.ordinal)
require(rankedOrdinal > 0) require(rankedOrdinal > 0)
require(evidenceSha256 == assessment.evidenceSha256) require(evidenceSha256 == assessment.evidenceSha256)
require(
specificationEvidenceSha256 ==
assessment.specificationEvidenceSha256
)
} }
} }
@@ -206,6 +216,8 @@ object CandidateTopFivePolicy {
sourceOrdinal = assessment.ordinal, sourceOrdinal = assessment.ordinal,
rankedOrdinal = index + 1, rankedOrdinal = index + 1,
evidenceSha256 = assessment.evidenceSha256, evidenceSha256 = assessment.evidenceSha256,
specificationEvidenceSha256 =
assessment.specificationEvidenceSha256,
assessment = assessment assessment = assessment
) )
} }
@@ -36,7 +36,7 @@ class CandidateEvaluator(
retryable = false retryable = false
) )
} }
if (!input.isValid()) { if (!input.isValid(hardConstraints.constraints)) {
return CandidateEvaluationResult.Failed( return CandidateEvaluationResult.Failed(
code = CandidateEvaluationFailureCode.EVIDENCE_INVALID, code = CandidateEvaluationFailureCode.EVIDENCE_INVALID,
retryable = false retryable = false
@@ -59,7 +59,8 @@ class CandidateEvaluator(
prompt = CandidateEvaluationPrompt.build( prompt = CandidateEvaluationPrompt.build(
requirement = input.requirement, requirement = input.requirement,
candidateOrdinal = candidate.ordinal, candidateOrdinal = candidate.ordinal,
hardConstraints = hardConstraints.constraints hardConstraintResults =
candidate.localHardConstraintResults
), ),
imageMediaType = candidate.mediaType, imageMediaType = candidate.mediaType,
imageBytes = candidate.bytes imageBytes = candidate.bytes
@@ -95,7 +96,10 @@ class CandidateEvaluator(
rawResponse = rawResponse, rawResponse = rawResponse,
expectedOrdinal = candidate.ordinal, expectedOrdinal = candidate.ordinal,
evidenceSha256 = candidate.sha256, evidenceSha256 = candidate.sha256,
expectedHardConstraints = hardConstraints.constraints specificationEvidenceSha256 =
candidate.specificationSha256,
expectedHardConstraintResults =
candidate.localHardConstraintResults
) )
} }
if (parsed == null) { if (parsed == null) {
@@ -165,7 +169,9 @@ class CandidateEvaluator(
) )
} }
private fun CandidateEvaluationInput.isValid(): Boolean { private fun CandidateEvaluationInput.isValid(
expectedConstraints: List<SkuHardConstraint>
): Boolean {
if (candidates.size !in 1..MAX_CANDIDATES) { if (candidates.size !in 1..MAX_CANDIDATES) {
return false return false
} }
@@ -175,7 +181,18 @@ class CandidateEvaluator(
candidates.all { candidate -> candidates.all { candidate ->
candidate.mediaType == SUPPORTED_MEDIA_TYPE && candidate.mediaType == SUPPORTED_MEDIA_TYPE &&
candidate.bytes.isNotEmpty() && candidate.bytes.isNotEmpty() &&
SHA256_PATTERN.matches(candidate.sha256) candidate.specificationBytes.isNotEmpty() &&
SHA256_PATTERN.matches(candidate.sha256) &&
SHA256_PATTERN.matches(candidate.specificationSha256) &&
candidate.localHardConstraintResults.size ==
expectedConstraints.size &&
candidate.localHardConstraintResults.zip(
expectedConstraints
).all { (result, expected) ->
result.kind == expected.kind &&
result.expected == expected.expected &&
result.evidence.isNotBlank()
}
} }
} }
@@ -191,7 +208,9 @@ class CandidateEvaluator(
rejectionReasons = emptyList(), rejectionReasons = emptyList(),
confidence = 0.0, confidence = 0.0,
evidenceSha256 = candidate.sha256, evidenceSha256 = candidate.sha256,
hardConstraintResults = emptyList() specificationEvidenceSha256 =
candidate.specificationSha256,
hardConstraintResults = candidate.localHardConstraintResults
) )
private companion object { private companion object {
@@ -205,7 +224,7 @@ object CandidateEvaluationPrompt {
fun build( fun build(
requirement: RequirementExtraction, requirement: RequirementExtraction,
candidateOrdinal: Int, candidateOrdinal: Int,
hardConstraints: List<SkuHardConstraint> hardConstraintResults: List<CandidateHardConstraintResult>
): String { ): String {
val requirementJson = JSONObject() val requirementJson = JSONObject()
.put("search_query", requirement.searchQuery) .put("search_query", requirement.searchQuery)
@@ -226,11 +245,13 @@ object CandidateEvaluationPrompt {
.put( .put(
"hard_constraints", "hard_constraints",
JSONArray().apply { JSONArray().apply {
hardConstraints.forEach { constraint -> hardConstraintResults.forEach { constraint ->
put( put(
JSONObject() JSONObject()
.put("kind", constraint.kind.name) .put("kind", constraint.kind.name)
.put("expected", constraint.expected) .put("expected", constraint.expected)
.put("status", constraint.status.name)
.put("evidence", constraint.evidence)
) )
} }
} }
@@ -244,13 +265,13 @@ object CandidateEvaluationPrompt {
Do not return coordinates, UI actions, state transitions, purchase decisions, order submission, or payment authorization. Do not return coordinates, UI actions, state transitions, purchase decisions, order submission, or payment authorization.
decision must be REVIEW, REJECT, or MANUAL_REQUIRED. decision must be REVIEW, REJECT, or MANUAL_REQUIRED.
REVIEW means potentially suitable for a person to inspect, never permission to buy. REVIEW means potentially suitable for a person to inspect, never permission to buy.
Evaluate every hard constraint using only visible screenshot evidence. Local read-only specification parsing already determined every hard constraint status.
hard constraint status must be MATCH, MISMATCH, or UNKNOWN. Copy each provided local status exactly; never override it from visual inference.
REVIEW requires every hard constraint to be MATCH. UNKNOWN requires MANUAL_REQUIRED. REVIEW requires every hard constraint to be MATCH. UNKNOWN requires MANUAL_REQUIRED.
A hard constraint MISMATCH requires REJECT. Never infer a hidden color or size. A hard constraint MISMATCH requires REJECT. Never infer a hidden color or size.
Required schema: Required schema:
{ {
"schema_version": 2, "schema_version": 3,
"candidate_index": $candidateOrdinal, "candidate_index": $candidateOrdinal,
"decision": "REVIEW", "decision": "REVIEW",
"score": 0.0, "score": 0.0,
@@ -295,7 +316,8 @@ private object CandidateAssessmentParser {
rawResponse: String, rawResponse: String,
expectedOrdinal: Int, expectedOrdinal: Int,
evidenceSha256: String, evidenceSha256: String,
expectedHardConstraints: List<SkuHardConstraint> specificationEvidenceSha256: String,
expectedHardConstraintResults: List<CandidateHardConstraintResult>
): CandidateAssessment? = ): CandidateAssessment? =
runCatching { runCatching {
val root = JSONObject(rawResponse.trim()) val root = JSONObject(rawResponse.trim())
@@ -313,7 +335,7 @@ private object CandidateAssessmentParser {
val confidence = root.strictUnitDouble("confidence") val confidence = root.strictUnitDouble("confidence")
val hardConstraintResults = root val hardConstraintResults = root
.getJSONArray("hard_constraint_results") .getJSONArray("hard_constraint_results")
.strictHardConstraintResults(expectedHardConstraints) .strictHardConstraintResults(expectedHardConstraintResults)
when (decision) { when (decision) {
CandidateDecision.REVIEW -> { CandidateDecision.REVIEW -> {
require(matched.isNotEmpty()) require(matched.isNotEmpty())
@@ -351,12 +373,14 @@ private object CandidateAssessmentParser {
rejectionReasons = rejectionReasons, rejectionReasons = rejectionReasons,
confidence = confidence, confidence = confidence,
evidenceSha256 = evidenceSha256, evidenceSha256 = evidenceSha256,
specificationEvidenceSha256 =
specificationEvidenceSha256,
hardConstraintResults = hardConstraintResults hardConstraintResults = hardConstraintResults
) )
}.getOrNull() }.getOrNull()
private fun JSONArray.strictHardConstraintResults( private fun JSONArray.strictHardConstraintResults(
expected: List<SkuHardConstraint> expected: List<CandidateHardConstraintResult>
): List<CandidateHardConstraintResult> { ): List<CandidateHardConstraintResult> {
require(length() == expected.size) require(length() == expected.size)
return buildList(length()) { return buildList(length()) {
@@ -375,16 +399,13 @@ private object CandidateAssessmentParser {
val expectedValue = item.getString("expected").trim() val expectedValue = item.getString("expected").trim()
require(kind == constraint.kind) require(kind == constraint.kind)
require(expectedValue == constraint.expected) require(expectedValue == constraint.expected)
add( val status = HardConstraintMatchStatus.valueOf(
CandidateHardConstraintResult(
kind = kind,
expected = expectedValue,
status = HardConstraintMatchStatus.valueOf(
item.getString("status") item.getString("status")
),
evidence = item.getString("evidence")
.validatedAssessmentText()
) )
require(status == constraint.status)
item.getString("evidence").validatedAssessmentText()
add(
constraint
) )
} }
} }
@@ -469,6 +490,10 @@ object CandidateReviewBatchJson {
) )
.put("confidence", assessment.confidence) .put("confidence", assessment.confidence)
.put("evidence_sha256", assessment.evidenceSha256) .put("evidence_sha256", assessment.evidenceSha256)
.put(
"specification_evidence_sha256",
assessment.specificationEvidenceSha256
)
.put( .put(
"hard_constraint_results", "hard_constraint_results",
JSONArray().apply { JSONArray().apply {
@@ -1,5 +1,7 @@
package com.roubao.autopilot.vlm package com.roubao.autopilot.vlm
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationEvidence
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationGroupKind
import java.text.Normalizer import java.text.Normalizer
enum class SkuConstraintKind { enum class SkuConstraintKind {
@@ -99,3 +101,136 @@ object SkuHardConstraintExtractor {
"""(?:^|[^0-9])(\d{2}(?:\.\d)?)(?:码|SIZE)(?=$|[^A-Z0-9])""" """(?:^|[^0-9])(\d{2}(?:\.\d)?)(?:码|SIZE)(?=$|[^A-Z0-9])"""
) )
} }
object CandidateSpecificationHardConstraintPolicy {
fun evaluate(
constraints: List<SkuHardConstraint>,
evidence: PinduoduoSpecificationEvidence,
evidenceSha256: String
): List<CandidateHardConstraintResult> =
constraints.map { constraint ->
val kind = when (constraint.kind) {
SkuConstraintKind.COLOR ->
PinduoduoSpecificationGroupKind.COLOR
SkuConstraintKind.SIZE ->
PinduoduoSpecificationGroupKind.SIZE
}
val matchingGroups = evidence.groups.filter { it.kind == kind }
if (matchingGroups.size != 1) {
return@map result(
constraint,
HardConstraintMatchStatus.UNKNOWN,
audit(constraint, "group_missing_or_duplicate", evidenceSha256)
)
}
val group = matchingGroups.single()
val matchingOptions = group.options.filter { option ->
optionMatches(constraint, option.text)
}
when {
matchingOptions.size > 1 ->
result(
constraint,
HardConstraintMatchStatus.UNKNOWN,
audit(constraint, "target_duplicate", evidenceSha256)
)
matchingOptions.singleOrNull()?.enabled == true ->
result(
constraint,
HardConstraintMatchStatus.MATCH,
audit(constraint, "target_enabled", evidenceSha256)
)
matchingOptions.singleOrNull()?.enabled == false ->
result(
constraint,
HardConstraintMatchStatus.MISMATCH,
audit(constraint, "target_disabled", evidenceSha256)
)
group.complete ->
result(
constraint,
HardConstraintMatchStatus.MISMATCH,
audit(constraint, "target_absent_complete", evidenceSha256)
)
else ->
result(
constraint,
HardConstraintMatchStatus.UNKNOWN,
audit(constraint, "group_incomplete", evidenceSha256)
)
}
}
private fun result(
constraint: SkuHardConstraint,
status: HardConstraintMatchStatus,
evidence: String
) = CandidateHardConstraintResult(
kind = constraint.kind,
expected = constraint.expected,
status = status,
evidence = evidence
)
private fun audit(
constraint: SkuHardConstraint,
observation: String,
evidenceSha256: String
): String =
"group=${constraint.kind.name};target=${constraint.expected};" +
"observation=$observation;spec_sha256=$evidenceSha256"
private fun optionMatches(
constraint: SkuHardConstraint,
observed: String
): Boolean {
val normalized = Normalizer.normalize(
observed.trim(),
Normalizer.Form.NFKC
).uppercase()
return when (constraint.kind) {
SkuConstraintKind.COLOR ->
COLOR_ALIASES[constraint.expected].orEmpty()
.any { alias -> normalized.matchesAlias(alias) }
SkuConstraintKind.SIZE ->
sizeAliases(constraint.expected).any { alias ->
Regex(
"""(?:^|[^A-Z0-9])${Regex.escape(alias)}""" +
"""(?=$|[^A-Z0-9])"""
).containsMatchIn(normalized)
}
}
}
private fun String.matchesAlias(alias: String): Boolean =
if (alias.all { it in 'A'..'Z' }) {
Regex(
"""(?:^|[^A-Z])${Regex.escape(alias)}(?=$|[^A-Z])"""
).containsMatchIn(this)
} else {
contains(alias)
}
private fun sizeAliases(expected: String): Set<String> =
when (expected) {
"2XL" -> setOf("2XL", "XXL")
"3XL" -> setOf("3XL", "XXXL")
"FREE" -> setOf("FREE", "FREESIZE", "均码")
else -> setOf(expected)
}
private val COLOR_ALIASES = mapOf(
"BLACK" to listOf("BLACK", "黑色", "亮黑", "纯黑"),
"WHITE" to listOf("WHITE", "白色", "纯白", "米白"),
"GRAY" to listOf("GRAY", "GREY", "灰色", "浅灰", "深灰"),
"RED" to listOf("RED", "红色", "酒红", "玫红"),
"BLUE" to listOf("BLUE", "蓝色", "藏青", "牛仔蓝"),
"GREEN" to listOf("GREEN", "绿色", "军绿"),
"YELLOW" to listOf("YELLOW", "黄色"),
"PINK" to listOf("PINK", "粉色", "粉红"),
"PURPLE" to listOf("PURPLE", "紫色"),
"BROWN" to listOf("BROWN", "棕色", "咖色", "咖啡色"),
"BEIGE" to listOf("BEIGE", "米色", "卡其"),
"ORANGE" to listOf("ORANGE", "橙色")
)
}
@@ -18,14 +18,25 @@ class CandidateEvidenceSourceTest {
assertTrue(result.isSuccess) assertTrue(result.isSuccess)
assertEquals(listOf(1, 2), result.getOrThrow().map { it.ordinal }) assertEquals(listOf(1, 2), result.getOrThrow().map { it.ordinal })
assertTrue(
result.getOrThrow().all {
it.detailPngBytes.isNotEmpty() &&
it.specificationPngBytes.isNotEmpty()
}
)
} }
} }
@Test @Test
fun `rejects non allowlisted filename before reading`() = runTest { fun `rejects non allowlisted filename before reading`() = runTest {
withEvidenceRoot { root -> withEvidenceRoot { root ->
val evidence = writeCandidate(root, ordinal = 1, width = 10, height = 20) val original =
.copy(screenshotFileName = "../candidate-01.png") writeCandidate(root, ordinal = 1, width = 10, height = 20)
val evidence = original.copy(
detailAsset = original.detailAsset.copy(
fileName = "../candidate-01-detail.png"
)
)
val result = CandidateEvidenceSource(root).load(listOf(evidence)) val result = CandidateEvidenceSource(root).load(listOf(evidence))
@@ -56,17 +67,33 @@ class CandidateEvidenceSourceTest {
assertTrue( assertTrue(
source.load( source.load(
listOf(evidence.copy(screenshotByteCount = evidence.screenshotByteCount + 1)) listOf(
evidence.copy(
detailAsset = evidence.detailAsset.copy(
byteCount = evidence.detailAsset.byteCount + 1
)
)
)
).isFailure ).isFailure
) )
assertTrue( assertTrue(
source.load( source.load(
listOf(evidence.copy(screenshotSha256 = "0".repeat(64))) listOf(
evidence.copy(
detailAsset = evidence.detailAsset.copy(
sha256 = "0".repeat(64)
)
)
)
).isFailure ).isFailure
) )
assertTrue( assertTrue(
source.load( source.load(
listOf(evidence.copy(screenshotWidth = 11)) listOf(
evidence.copy(
detailAsset = evidence.detailAsset.copy(width = 11)
)
)
).isFailure ).isFailure
) )
} }
@@ -76,12 +103,18 @@ class CandidateEvidenceSourceTest {
fun `rejects invalid png header and declared size above limit`() = runTest { fun `rejects invalid png header and declared size above limit`() = runTest {
withEvidenceRoot { root -> withEvidenceRoot { root ->
val evidence = writeCandidate(root, ordinal = 1, width = 10, height = 20) val evidence = writeCandidate(root, ordinal = 1, width = 10, height = 20)
File(root, evidence.screenshotFileName).writeBytes(ByteArray(24)) File(root, evidence.detailAsset.fileName).writeBytes(ByteArray(24))
assertTrue(CandidateEvidenceSource(root).load(listOf(evidence)).isFailure) assertTrue(CandidateEvidenceSource(root).load(listOf(evidence)).isFailure)
assertTrue( assertTrue(
CandidateEvidenceSource(root).load( CandidateEvidenceSource(root).load(
listOf(evidence.copy(screenshotByteCount = 8 * 1024 * 1024 + 1)) listOf(
evidence.copy(
detailAsset = evidence.detailAsset.copy(
byteCount = 8 * 1024 * 1024 + 1
)
)
)
).isFailure ).isFailure
) )
} }
@@ -103,22 +136,53 @@ class CandidateEvidenceSourceTest {
height: Int height: Int
): PinduoduoCandidateEvidence { ): PinduoduoCandidateEvidence {
val bytes = pngHeader(width, height) val bytes = pngHeader(width, height)
val fileName = "candidate-%02d.png".format(ordinal) val detailFileName = "candidate-%02d-detail.png".format(ordinal)
File(root, fileName).writeBytes(bytes) val specificationFileName =
"candidate-%02d-specification.png".format(ordinal)
File(root, detailFileName).writeBytes(bytes)
File(root, specificationFileName).writeBytes(bytes)
val asset = PinduoduoEvidenceAsset(
fileName = detailFileName,
sha256 = PinduoduoEvidenceHash.sha256(bytes),
byteCount = bytes.size,
width = width,
height = height
)
return PinduoduoCandidateEvidence( return PinduoduoCandidateEvidence(
ordinal = ordinal, ordinal = ordinal,
cardSignature = "card-$ordinal", cardSignature = "card-$ordinal",
cardSemanticTextCount = 3, cardSemanticTextCount = 3,
detailSignature = "detail-$ordinal", detailSignature = "detail-$ordinal",
detailSemanticTextCount = 4, detailSemanticTextCount = 4,
screenshotFileName = fileName, detailAsset = asset,
screenshotSha256 = PinduoduoEvidenceHash.sha256(bytes), specification = specification(),
screenshotByteCount = bytes.size, specificationAsset = asset.copy(fileName = specificationFileName)
screenshotWidth = width,
screenshotHeight = height
) )
} }
private fun specification() = PinduoduoSpecificationEvidence(
signature = "specification",
semanticTextCount = 6,
groups = listOf(
PinduoduoSpecificationGroup(
kind = PinduoduoSpecificationGroupKind.COLOR,
title = "颜色分类",
options = listOf(
PinduoduoSpecificationOption("黑色", false, true)
),
complete = true
),
PinduoduoSpecificationGroup(
kind = PinduoduoSpecificationGroupKind.SIZE,
title = "尺码",
options = listOf(
PinduoduoSpecificationOption("L", false, true)
),
complete = true
)
)
)
private fun pngHeader(width: Int, height: Int): ByteArray = private fun pngHeader(width: Int, height: Int): ByteArray =
byteArrayOf( byteArrayOf(
0x89.toByte(), 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x89.toByte(), 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
@@ -156,6 +156,31 @@ class PinduoduoCandidateAutomationTest {
assertTrue(automation.evidence.value.isEmpty()) assertTrue(automation.evidence.value.isEmpty())
} }
@Test
fun `missing safe specification entry blocks without purchase fallback`() =
runTest {
val driver = FakeCandidateDriver(
cardPages = listOf(listOf(card("a"))),
failOpenSpecifications = true
)
val automation = PinduoduoCandidateAutomation(
driver = driver,
pagePollIntervalMillis = 1
)
automation.reset()
val result = automation.execute(
PinduoduoCandidateWorkflow.steps().last()
)
assertEquals(
AutomationResult.Blocked(SafetyStopReason.UNKNOWN_PAGE),
result
)
assertEquals(1, driver.openSpecificationCalls)
assertTrue(driver.savedEvidence.isEmpty())
}
private fun card(signature: String) = PinduoduoCandidateCard( private fun card(signature: String) = PinduoduoCandidateCard(
signature = signature, signature = signature,
semanticTextCount = 3, semanticTextCount = 3,
@@ -166,6 +191,7 @@ class PinduoduoCandidateAutomationTest {
private val cardPages: List<List<PinduoduoCandidateCard>>, private val cardPages: List<List<PinduoduoCandidateCard>>,
private val safetyStopReason: SafetyStopReason? = null, private val safetyStopReason: SafetyStopReason? = null,
private val failCapture: Boolean = false, private val failCapture: Boolean = false,
private val failOpenSpecifications: Boolean = false,
private val transitionDelaySnapshots: Int = 0 private val transitionDelaySnapshots: Int = 0
) : PinduoduoCandidateDriver { ) : PinduoduoCandidateDriver {
private var cardPageIndex = 0 private var cardPageIndex = 0
@@ -176,6 +202,7 @@ class PinduoduoCandidateAutomationTest {
val openedSignatures = mutableListOf<String>() val openedSignatures = mutableListOf<String>()
val savedEvidence = mutableListOf<PinduoduoCandidateEvidence>() val savedEvidence = mutableListOf<PinduoduoCandidateEvidence>()
var scrollCalls = 0 var scrollCalls = 0
var openSpecificationCalls = 0
override suspend fun snapshot(): PinduoduoUiSnapshot { override suspend fun snapshot(): PinduoduoUiSnapshot {
if (pendingPage != null) { if (pendingPage != null) {
@@ -204,24 +231,60 @@ class PinduoduoCandidateAutomationTest {
return true return true
} }
override suspend fun captureCandidate( override suspend fun captureDetail(): PinduoduoCandidateCapture? {
ordinal: Int,
card: PinduoduoCandidateCard
): PinduoduoCandidateEvidence? {
if (failCapture) { if (failCapture) {
return null return null
} }
return PinduoduoCandidateCapture(
detail = PinduoduoCandidateDetailEvidence("detail", 10),
screenshot = PinduoduoScreenshotCapture(
pngBytes = byteArrayOf(1),
width = 1080,
height = 2400
)
)
}
override suspend fun openSpecifications(): Boolean {
openSpecificationCalls += 1
if (failOpenSpecifications) {
return false
}
transitionTo(PinduoduoPage.SPECIFICATION_PANEL)
return true
}
override suspend fun captureSpecifications():
PinduoduoSpecificationCapture =
PinduoduoSpecificationCapture(
evidence = specification(),
screenshot = PinduoduoScreenshotCapture(
pngBytes = byteArrayOf(2),
width = 1080,
height = 2400
)
)
override suspend fun closeSpecifications(): Boolean {
transitionTo(PinduoduoPage.PRODUCT_DETAIL)
return true
}
override suspend fun saveCandidate(
ordinal: Int,
card: PinduoduoCandidateCard,
detail: PinduoduoCandidateCapture,
specifications: PinduoduoSpecificationCapture
): PinduoduoCandidateEvidence? {
return PinduoduoCandidateEvidence( return PinduoduoCandidateEvidence(
ordinal = ordinal, ordinal = ordinal,
cardSignature = card.signature, cardSignature = card.signature,
cardSemanticTextCount = card.semanticTextCount, cardSemanticTextCount = card.semanticTextCount,
detailSignature = "detail-${card.signature}", detailSignature = "detail-${card.signature}",
detailSemanticTextCount = 10, detailSemanticTextCount = 10,
screenshotFileName = "candidate-%02d.png".format(ordinal), detailAsset = asset(ordinal, "detail"),
screenshotSha256 = "hash-$ordinal", specification = specifications.evidence,
screenshotByteCount = 100 + ordinal, specificationAsset = asset(ordinal, "specification")
screenshotWidth = 1080,
screenshotHeight = 2400
).also(savedEvidence::add) ).also(savedEvidence::add)
} }
@@ -242,6 +305,7 @@ class PinduoduoCandidateAutomationTest {
openedSignatures.clear() openedSignatures.clear()
savedEvidence.clear() savedEvidence.clear()
scrollCalls = 0 scrollCalls = 0
openSpecificationCalls = 0
cardPageIndex = 0 cardPageIndex = 0
page = PinduoduoPage.SEARCH_RESULTS page = PinduoduoPage.SEARCH_RESULTS
pendingPage = null pendingPage = null
@@ -256,5 +320,39 @@ class PinduoduoCandidateAutomationTest {
delayedSnapshotsRemaining = transitionDelaySnapshots delayedSnapshotsRemaining = transitionDelaySnapshots
} }
} }
private fun asset(
ordinal: Int,
suffix: String
) = PinduoduoEvidenceAsset(
fileName = "candidate-%02d-%s.png".format(ordinal, suffix),
sha256 = "a".repeat(64),
byteCount = 1,
width = 1080,
height = 2400
)
private fun specification() = PinduoduoSpecificationEvidence(
signature = "specification",
semanticTextCount = 6,
groups = listOf(
PinduoduoSpecificationGroup(
kind = PinduoduoSpecificationGroupKind.COLOR,
title = "颜色分类",
options = listOf(
PinduoduoSpecificationOption("黑色", false, true)
),
complete = true
),
PinduoduoSpecificationGroup(
kind = PinduoduoSpecificationGroupKind.SIZE,
title = "尺码",
options = listOf(
PinduoduoSpecificationOption("L", false, true)
),
complete = true
)
)
)
} }
} }
@@ -202,6 +202,41 @@ class PinduoduoPageClassifierTest {
assertNull(snapshot.safetyStopReason) assertNull(snapshot.safetyStopReason)
} }
@Test
fun `specification panel requires color size and options`() {
val snapshot = classify(
element(text = "颜色分类"),
element(text = "黑色", clickable = true),
element(text = "尺码"),
element(text = "L", clickable = true)
)
assertEquals(PinduoduoPage.SPECIFICATION_PANEL, snapshot.page)
assertNull(snapshot.safetyStopReason)
}
@Test
fun `cart and order pages are never classified as product detail`() {
val cart = classify(
element(text = "购物车"),
element(text = "去结算")
)
val order = classify(
element(contentDescription = "返回", clickable = true),
element(text = "颜色分类"),
element(text = "黑色", clickable = true),
element(text = "尺码"),
element(text = "L", clickable = true),
element(text = "确认订单")
)
assertEquals(PinduoduoPage.UNKNOWN, cart.page)
assertEquals(
SafetyStopReason.PAYMENT_BOUNDARY,
order.safetyStopReason
)
}
@Test @Test
fun `other foreground package is unknown and has no inferred blocker`() { fun `other foreground package is unknown and has no inferred blocker`() {
val snapshot = PinduoduoPageClassifier.classify( val snapshot = PinduoduoPageClassifier.classify(
@@ -0,0 +1,98 @@
package com.roubao.autopilot.pinduoduo
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
import org.junit.Test
class PinduoduoSpecificationParserTest {
@Test
fun `parses bounded color and size groups without clicking options`() {
val evidence = PinduoduoSpecificationParser.parse(
listOf(
element("颜色分类", top = 100),
element("黑色", top = 140, clickable = true, selected = true),
element("白色", top = 180, clickable = true),
element("尺码", top = 240),
element("L", top = 280, clickable = true),
element("XL", top = 320, clickable = true, enabled = false)
)
)
requireNotNull(evidence)
assertEquals(
PinduoduoSpecificationGroupKind.entries.toSet(),
evidence.groups.map { it.kind }.toSet()
)
assertTrue(evidence.groups.all { it.complete })
assertTrue(evidence.groups.first().options.first().selected)
assertFalse(evidence.groups.last().options.last().enabled)
}
@Test
fun `scrollable group is marked incomplete`() {
val evidence = PinduoduoSpecificationParser.parse(
listOf(
element("颜色分类", top = 100),
element("黑色", top = 140, clickable = true),
element("", top = 150, scrollable = true),
element("尺码", top = 240),
element("L", top = 280, clickable = true)
)
)
requireNotNull(evidence)
assertFalse(evidence.groups.first().complete)
assertTrue(evidence.groups.last().complete)
}
@Test
fun `missing group and transaction controls are rejected`() {
assertNull(
PinduoduoSpecificationParser.parse(
listOf(
element("颜色分类", top = 100),
element("立即购买", top = 140, clickable = true)
)
)
)
}
@Test
fun `safe entry allowlist excludes purchase controls`() {
assertTrue(
PinduoduoSpecificationParser.isSafeEntryText("请选择规格")
)
assertTrue(
PinduoduoSpecificationParser.isSafeEntryText("已选:黑色,L")
)
assertTrue(
PinduoduoSpecificationParser.isSafeEntryText("颜色\n款式")
)
assertFalse(
PinduoduoSpecificationParser.isSafeEntryText("免拼购买")
)
}
private fun element(
text: String,
top: Int,
clickable: Boolean = false,
enabled: Boolean = true,
selected: Boolean = false,
scrollable: Boolean = false
) = PinduoduoUiElement(
text = text,
contentDescription = null,
className = "android.widget.TextView",
resourceId = null,
clickable = clickable,
editable = false,
enabled = enabled,
visibleToUser = true,
selected = selected,
scrollable = scrollable,
boundsTop = top
)
}
@@ -0,0 +1,64 @@
package com.roubao.autopilot.procurement
import org.junit.Assert.assertEquals
import org.junit.Assert.assertThrows
import org.junit.Test
class ExecutionEvidenceAssociationPolicyTest {
@Test
fun `binds detail and specification assets to each ranked candidate`() {
val grouped = ExecutionEvidenceAssociationPolicy.group(
candidateOrdinals = listOf(1, 2),
evidence = listOf(
draft(2, ExecutionEvidenceKind.SPECIFICATION),
draft(1, ExecutionEvidenceKind.DETAIL),
draft(2, ExecutionEvidenceKind.DETAIL),
draft(1, ExecutionEvidenceKind.SPECIFICATION)
)
)
assertEquals(listOf(1, 2), grouped.keys.toList())
assertEquals(
ExecutionEvidenceKind.entries,
grouped.getValue(1).map { it.kind }
)
assertEquals(
ExecutionEvidenceKind.entries,
grouped.getValue(2).map { it.kind }
)
}
@Test
fun `rejects missing duplicate and wrong candidate assets`() {
listOf(
listOf(
draft(1, ExecutionEvidenceKind.DETAIL)
),
listOf(
draft(1, ExecutionEvidenceKind.DETAIL),
draft(1, ExecutionEvidenceKind.DETAIL)
),
listOf(
draft(1, ExecutionEvidenceKind.DETAIL),
draft(2, ExecutionEvidenceKind.SPECIFICATION)
)
).forEach { evidence ->
assertThrows(IllegalArgumentException::class.java) {
ExecutionEvidenceAssociationPolicy.group(
candidateOrdinals = listOf(1),
evidence = evidence
)
}
}
}
private fun draft(
ordinal: Int,
kind: ExecutionEvidenceKind
) = ExecutionEvidenceDraft(
ordinal = ordinal,
pngBytes = byteArrayOf(ordinal.toByte(), kind.ordinal.toByte()),
sha256 = "a".repeat(64),
kind = kind
)
}
@@ -159,7 +159,12 @@ class CandidateEvaluatorTest {
) )
} }
val result = evaluator.evaluate(input(candidateCount = 2)) val result = evaluator.evaluate(
input(
candidateCount = 2,
hardStatus = HardConstraintMatchStatus.MISMATCH
)
)
as CandidateEvaluationResult.Completed as CandidateEvaluationResult.Completed
assertEquals(CandidateBatchConclusion.NO_MATCH, result.batch.conclusion) assertEquals(CandidateBatchConclusion.NO_MATCH, result.batch.conclusion)
@@ -305,13 +310,40 @@ class CandidateEvaluatorTest {
hardStatus = HardConstraintMatchStatus.UNKNOWN hardStatus = HardConstraintMatchStatus.UNKNOWN
) )
) )
}.evaluate(input(candidateCount = 1)) }.evaluate(
input(
candidateCount = 1,
hardStatus = HardConstraintMatchStatus.UNKNOWN
)
)
as CandidateEvaluationResult.Completed as CandidateEvaluationResult.Completed
assertTrue(CandidateTopFivePolicy.select(result.batch).isEmpty()) assertTrue(CandidateTopFivePolicy.select(result.batch).isEmpty())
assertNull(result.batch.recommendedCandidateOrdinal) assertNull(result.batch.recommendedCandidateOrdinal)
} }
@Test
fun `model cannot override local specification status`() = runTest {
val response = validResponse(
ordinal = 1,
decision = CandidateDecision.REJECT,
score = 0.1,
matched = emptyList(),
rejectionReasons = listOf("模型声称颜色不匹配"),
hardStatus = HardConstraintMatchStatus.MISMATCH
)
val result = evaluator { Result.success(response) }
.evaluate(input(candidateCount = 1))
as CandidateEvaluationResult.Completed
assertEquals(
CandidateDecision.MANUAL_REQUIRED,
result.batch.assessments.single().decision
)
assertNull(result.batch.recommendedCandidateOrdinal)
}
@Test @Test
fun `encoded review batch fixes order submitted to false`() = runTest { fun `encoded review batch fixes order submitted to false`() = runTest {
val result = evaluator { Result.success(validResponse(1)) } val result = evaluator { Result.success(validResponse(1)) }
@@ -323,6 +355,12 @@ class CandidateEvaluatorTest {
assertFalse(json.getBoolean("order_submitted")) assertFalse(json.getBoolean("order_submitted"))
assertTrue(json.getBoolean("manual_review_required")) assertTrue(json.getBoolean("manual_review_required"))
assertEquals(1, json.getJSONArray("candidates").length()) assertEquals(1, json.getJSONArray("candidates").length())
assertEquals(
"e".repeat(64),
json.getJSONArray("candidates")
.getJSONObject(0)
.getString("specification_evidence_sha256")
)
} }
private fun evaluator( private fun evaluator(
@@ -336,18 +374,44 @@ class CandidateEvaluatorTest {
model = "fake-model" model = "fake-model"
) )
private fun input(candidateCount: Int): CandidateEvaluationInput = private fun input(
candidateCount: Int,
hardStatus: HardConstraintMatchStatus =
HardConstraintMatchStatus.MATCH
): CandidateEvaluationInput =
CandidateEvaluationInput( CandidateEvaluationInput(
requirement = requirement(), requirement = requirement(),
candidates = (1..candidateCount).map(::candidate) candidates = (1..candidateCount).map {
candidate(it, hardStatus)
}
) )
private fun candidate(ordinal: Int): CandidateEvaluationImage = private fun candidate(
ordinal: Int,
hardStatus: HardConstraintMatchStatus =
HardConstraintMatchStatus.MATCH
): CandidateEvaluationImage =
CandidateEvaluationImage( CandidateEvaluationImage(
ordinal = ordinal, ordinal = ordinal,
mediaType = "image/png", mediaType = "image/png",
bytes = byteArrayOf(ordinal.toByte()), bytes = byteArrayOf(ordinal.toByte()),
sha256 = ordinal.toString(16).padStart(64, '0') sha256 = ordinal.toString(16).padStart(64, '0'),
specificationBytes = byteArrayOf((ordinal + 10).toByte()),
specificationSha256 = "e".repeat(64),
localHardConstraintResults = listOf(
CandidateHardConstraintResult(
kind = SkuConstraintKind.COLOR,
expected = "BLACK",
status = hardStatus,
evidence = "本地规格颜色证据"
),
CandidateHardConstraintResult(
kind = SkuConstraintKind.SIZE,
expected = "L",
status = hardStatus,
evidence = "本地规格尺码证据"
)
)
) )
private fun requirement(): RequirementExtraction = private fun requirement(): RequirementExtraction =
@@ -0,0 +1,148 @@
package com.roubao.autopilot.vlm
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationEvidence
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationGroup
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationGroupKind
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationOption
import org.junit.Assert.assertEquals
import org.junit.Assert.assertTrue
import org.junit.Test
import java.nio.charset.StandardCharsets
class CandidateSpecificationHardConstraintPolicyTest {
@Test
fun `both explicit enabled targets match`() {
assertEquals(
listOf(
HardConstraintMatchStatus.MATCH,
HardConstraintMatchStatus.MATCH
),
evaluate(
colorOptions = listOf(option("经典黑色")),
sizeOptions = listOf(option("XXL"))
).map { it.status }
)
assertTrue(
evaluate(
colorOptions = listOf(option("经典黑色")),
sizeOptions = listOf(option("XXL"))
).all {
it.evidence.toByteArray(StandardCharsets.UTF_8).size <= 160
}
)
}
@Test
fun `complete missing target is mismatch`() {
assertEquals(
HardConstraintMatchStatus.MISMATCH,
evaluate(
colorOptions = listOf(option("白色")),
sizeOptions = listOf(option("2XL"))
).first().status
)
}
@Test
fun `disabled target is mismatch`() {
assertEquals(
HardConstraintMatchStatus.MISMATCH,
evaluate(
colorOptions = listOf(option("黑色", enabled = false)),
sizeOptions = listOf(option("2XL"))
).first().status
)
}
@Test
fun `truncated group and duplicated matching option are unknown`() {
assertEquals(
HardConstraintMatchStatus.UNKNOWN,
evaluate(
colorOptions = listOf(option("白色")),
sizeOptions = listOf(option("2XL")),
colorComplete = false
).first().status
)
assertEquals(
HardConstraintMatchStatus.UNKNOWN,
evaluate(
colorOptions = listOf(option("黑色"), option("纯黑")),
sizeOptions = listOf(option("2XL"))
).first().status
)
}
@Test
fun `missing group is unknown`() {
val evidence = specification(
colorOptions = listOf(option("黑色")),
sizeOptions = listOf(option("2XL"))
).copy(groups = specification(
colorOptions = listOf(option("黑色")),
sizeOptions = listOf(option("2XL"))
).groups.filter { it.kind == PinduoduoSpecificationGroupKind.COLOR })
val results = CandidateSpecificationHardConstraintPolicy.evaluate(
constraints = constraints(),
evidence = evidence,
evidenceSha256 = "b".repeat(64)
)
assertEquals(
HardConstraintMatchStatus.UNKNOWN,
results.last().status
)
}
private fun evaluate(
colorOptions: List<PinduoduoSpecificationOption>,
sizeOptions: List<PinduoduoSpecificationOption>,
colorComplete: Boolean = true
) = CandidateSpecificationHardConstraintPolicy.evaluate(
constraints = constraints(),
evidence = specification(
colorOptions,
sizeOptions,
colorComplete
),
evidenceSha256 = "b".repeat(64)
)
private fun constraints() = listOf(
SkuHardConstraint(SkuConstraintKind.COLOR, "BLACK"),
SkuHardConstraint(SkuConstraintKind.SIZE, "2XL")
)
private fun specification(
colorOptions: List<PinduoduoSpecificationOption>,
sizeOptions: List<PinduoduoSpecificationOption>,
colorComplete: Boolean = true
) = PinduoduoSpecificationEvidence(
signature = "specification",
semanticTextCount = 8,
groups = listOf(
PinduoduoSpecificationGroup(
PinduoduoSpecificationGroupKind.COLOR,
"颜色分类",
colorOptions,
colorComplete
),
PinduoduoSpecificationGroup(
PinduoduoSpecificationGroupKind.SIZE,
"尺码",
sizeOptions,
true
)
)
)
private fun option(
text: String,
enabled: Boolean = true
) = PinduoduoSpecificationOption(
text = text,
selected = false,
enabled = enabled
)
}
+33 -11
View File
@@ -257,22 +257,44 @@ T-102/T-104 在搜索结果后追加一个有界候选步骤:
-> 最多 2 次滚动预算 -> 最多 2 次滚动预算
-> 最多 5 个去重商品卡 -> 最多 5 个去重商品卡
-> 验证详情页 -> 验证详情页
-> 无障碍截图 -> 详情页无障碍截图
-> App cache/candidate-NN.png + manifest.json -> 唯一、安全、非交易规格入口
-> 规格弹层只读语义和截图
-> App cache/candidate-NN-{detail,specification}.png + manifest.json
-> 关闭规格弹层并复核详情页
-> 一次全局返回并复核固定词结果页 -> 一次全局返回并复核固定词结果页
``` ```
候选卡只保留语义指纹和计数,详情截图保存在 App 内部 cache。manifest 记录匿名文件名、 候选卡只保留语义指纹和计数,详情/规格截图保存在 App 内部 cache。manifest v2 记录
截图 SHA-256、字节数和尺寸,不保存商品标题或页面原文。T-104 使用这些证据时必须通过 匿名文件名、截图 SHA-256、字节数、尺寸和有界规格语义,不保存商品标题或完整页面
受控 evidence 边界读取,不能让 VLM adapter 自行遍历 cache。Android 10/API 29 及 原文。T-104/T-213 使用这些证据时必须通过受控 evidence 边界读取,不能让 VLM adapter
以下不能运行当前截图探针,应在预检时明确不支持,不使用媒体投影或 shell 绕过。 自行遍历 cache。Android 10/API 29 及以下不能运行当前截图探针,应在预检时明确
不支持,不使用媒体投影或 shell 绕过。
`CandidateEvidenceSource` 只接受当前 workflow 内存中的连续 ordinal 元数据,文件名 `CandidateEvidenceSource` 只接受当前 workflow 内存中的连续 ordinal 元数据,文件名
固定为 `candidate-01.png` 至 `candidate-05.png`;每张 PNG 最多 8 MiB、全批最多 固定为 `candidate-01-detail.png`/`candidate-01-specification.png` 至第五组;每张
32 MiB、声明尺寸最长边不超过 10000 px,并复核规范路径、PNG/IHDR、精确字节数、 PNG 最多 8 MiB、全批最多 64 MiB、声明尺寸最长边不超过 10000 px,并复核规范路径、
尺寸和 SHA-256。Android gateway 再解码并把最长边缩至 2048 px。当前需求快照与 PNG/IHDR、精确字节数、尺寸和 SHA-256。每个候选回传时必须恰好绑定
搜索词必须同时匹配候选 session,否则拒绝评估;另一个关键词的结果页只允许重新进入 `DETAIL`/`SPECIFICATION` 两份证据,生成独立 asset ID;先验证全批 ordinal、类型和
搜索框,不能采集候选。 哈希再写入加密 outbox,候选的 asset ID 数组固定按 `DETAIL`、`SPECIFICATION`
排序,重试沿用各自 idempotency key。Android gateway 再解码详情图并把最长边缩至
2048 px。当前需求快照与搜索词必须同时匹配候选 session,否则拒绝评估;另一个
关键词的结果页只允许重新进入搜索框,不能采集候选。
T-213 规格核验只允许精确语义为“选择规格/请选择规格/颜色分类/颜色款式/选择颜色/
选择尺码”或“已选/请选择”前缀的唯一节点及其可点击祖先。弹层内不点击任何规格选项,
只读取最多 160 个语义节点、每组最多 30 个选项及 selected/enabled/scrollable 语义。
颜色和尺码各自必须是唯一分组:目标值明确且可用为 `MATCH`,完整分组明确缺失或目标
禁用为 `MISMATCH`,重复、缺组、滚动截断或无障碍语义不足为 `UNKNOWN`。
候选评估 schema/prompt v3 把本地规格结果作为权威输入;模型返回的颜色/尺码状态必须
逐项与本地结果相同,否则整批降级人工检查。详情和规格证据 SHA-256 同时绑定
assessment、重排身份和 outbox 资产,VLM 不能通过主图推断覆盖本地硬约束。
PKG110 上的拼多多 8.17.0 服装详情只暴露不可点击的“颜色/款式”预览;完整颜色/尺码
弹层只能通过“单独购买/免拼购买”控件进入。自动化不得点击这两个交易语义控件,也
不得用坐标降级,因此当前版本在没有独立可点击规格入口时返回安全阻塞。是否把购买
语义控件重新定义为只读弹层入口,必须单独做产品/安全决策后变更本边界。
评估结束后 automation 不再产生动作,UI 进入 `AWAITING_CONFIRMATION`、 评估结束后 automation 不再产生动作,UI 进入 `AWAITING_CONFIRMATION`、
`MANUAL_REVIEW` 或 `NO_MATCH`。人员可以把本地建议项标记为可用,或拒绝本次候选; `MANUAL_REVIEW` 或 `NO_MATCH`。人员可以把本地建议项标记为可用,或拒绝本次候选;
+20 -15
View File
@@ -5,9 +5,9 @@
## 当前快照 ## 当前快照
- 日期:2026-07-27 - 日期:2026-07-27
- 阶段:T-212 候选重排身份映射完成;下一步 T-213 只读核验拼多多规格弹窗 - 阶段:T-212 已完成;T-213 只读规格核验已实现但被拼多多 8.17 入口语义阻塞
- Git:当前分支为 `main`;T-001 至 T-004、T-101 至 T-104、T-201 至 T-207、T-209、 - Git:当前分支为 `main`;T-001 至 T-004、T-101 至 T-104、T-201 至 T-207、T-209、
T-210、T-211、T-212 均已纳入 Git 历史 T-210、T-211、T-212 均已纳入 Git 历史;T-213 等待安全策略决策
- 生产代码:`android-buyer/` 已接入 Roubao Android 源码 - 生产代码:`android-buyer/` 已接入 Roubao Android 源码
- Android:固定 `main@c8a6d7f03422eb01744b01f3ee77bf7757741f7e`;MIT 许可证已保留 - Android:固定 `main@c8a6d7f03422eb01744b01f3ee77bf7757741f7e`;MIT 许可证已保留
@@ -17,10 +17,10 @@
- 本机 Android 工具:JDK 17.0.13、Command-line Tools 22.0、SDK 34、 - 本机 Android 工具:JDK 17.0.13、Command-line Tools 22.0、SDK 34、
Build Tools 34.0.0、Platform Tools/ADB 37.0.0;用户级 SDK 环境变量已设置 Build Tools 34.0.0、Platform Tools/ADB 37.0.0;用户级 SDK 环境变量已设置
- Android Studio:未安装;`winget` 静默安装卡住后已终止,不阻塞命令行构建 - Android Studio:未安装;`winget` 静默安装卡住后已终止,不阻塞命令行构建
- 测试:T-212 运行 `:app:testDebugUnitTest` 和 `:app:assembleDebug` 通过,21 个 suite - 测试:T-213 运行 `:app:testDebugUnitTest` 和 `:app:assembleDebug` 通过,24 个 suite
共 113 个测试、0 失败;T-211 Debug APK 已安装到 PKG110 共 128 个测试、0 失败;Debug APK `1.4.3 (8)` 已安装到 PKG110
- 后端测试:T-211 运行 `GOTOOLCHAIN=local go test -count=1 ./...` 和 - 后端测试:T-213 运行 `go test ./...` 通过;T-207 的全包 race 与 migration
`go vet ./...` 通过;T-207 的全包 race 与 migration 验证继续有效 验证继续有效
- 原型:4 个管理 Web 页面和 7 个 Android 页面均可离线独立打开;Playwright - 原型:4 个管理 Web 页面和 7 个 Android 页面均可离线独立打开;Playwright
以 1440×900、390×844、360×800 验证 36 个页面/视口组合,无页面横向溢出、 以 1440×900、390×844、360×800 验证 36 个页面/视口组合,无页面横向溢出、
脚本错误或外部请求,Android 可见交互控件均不小于 44px 脚本错误或外部请求,Android 可见交互控件均不小于 44px
@@ -47,9 +47,13 @@
顺序回传,授权到期补报单独审计,所有终态固定 `order_submitted=false`。管理任务 顺序回传,授权到期补报单独审计,所有终态固定 `order_submitted=false`。管理任务
详情展示模型/候选/人工理由/事件/证据摘要,不保存 VLM Key、完整 endpoint 或原始响应。 详情展示模型/候选/人工理由/事件/证据摘要,不保存 VLM Key、完整 endpoint 或原始响应。
- T-211 图片检索:后台任务固定使用经 SHA-256 校验的参考 JPEG,经一次性 MediaStore - T-211 图片检索:后台任务固定使用经 SHA-256 校验的参考 JPEG,经一次性 MediaStore
图片进入拼多多拍照搜索;App 本地从 SKU 唯一提取颜色和尺码,schema v2 逐项返回 图片进入拼多多拍照搜索;App 本地从 SKU 唯一提取颜色和尺码,schema v3 逐项返回
`MATCH/MISMATCH/UNKNOWN`。只有两项均匹配且分数/置信度不低于 `0.75` 的候选按 `MATCH/MISMATCH/UNKNOWN`。只有两项均匹配且分数/置信度不低于 `0.75` 的候选按
分数、置信度和曝光顺序回传 `0..5` 项,弱匹配和未知项不凑数。 分数、置信度和曝光顺序回传 `0..5` 项,弱匹配和未知项不凑数。
- T-213 规格核验:只允许唯一、非交易规格入口;弹层内不点击选项,本地根据有界的
颜色/尺码分组、可用/禁用和完整性语义产生硬约束结果。每个候选绑定详情/规格两张
PNG 及两个 SHA-256/asset ID;模型不能覆盖本地状态。拼多多 8.17 服装详情的完整
规格弹层只挂在“单独购买/免拼购买”后,当前安全边界拒绝该入口,故真机流程阻塞。
- Workflow:纯 Kotlin runner 已支持步骤 timeout、最多 3 次 retry、安全阻塞、 - Workflow:纯 Kotlin runner 已支持步骤 timeout、最多 3 次 retry、安全阻塞、
用户停止和单 runner 并发拒绝;T-102 已接入搜索加有界候选采集五步 用户停止和单 runner 并发拒绝;T-102 已接入搜索加有界候选采集五步
- TaskSource:严格 CLI 已生成并验证真实私有 ProbeTask;默认 APK 不含私有 fixture - TaskSource:严格 CLI 已生成并验证真实私有 ProbeTask;默认 APK 不含私有 fixture
@@ -65,7 +69,7 @@
- 离线执行:默认 30 分钟有限授权和 30 秒 best-effort heartbeat 已实现;PKG110 - 离线执行:默认 30 分钟有限授权和 30 秒 best-effort heartbeat 已实现;PKG110
真机断开临时后端 95 秒后保持同一 execution,重连后滑动续期;到期持久安全停止, 真机断开临时后端 95 秒后保持同一 execution,重连后滑动续期;到期持久安全停止,
RUNNING 不自动重新分配 RUNNING 不自动重新分配
- 测试设备:OnePlus PKG110,Android 16/API 36;肉包 `1.4.2 (7)`;拼多多 - 测试设备:OnePlus PKG110,Android 16/API 36;肉包 `1.4.3 (8)`;拼多多
`8.17.0 (81700)` `8.17.0 (81700)`
- 设备就绪:拼多多首页、文字/图片搜索、双列候选卡、详情截图和返回均已通过 8.17.0 - 设备就绪:拼多多首页、文字/图片搜索、双列候选卡、详情截图和返回均已通过 8.17.0
真机验证;最终 APK 重装/force-stop 后 ColorOS 已关闭肉包采购无障碍,当前需采购员 真机验证;最终 APK 重装/force-stop 后 ColorOS 已关闭肉包采购无障碍,当前需采购员
@@ -80,8 +84,10 @@
已用 CLI 真实导入并逐字段/图片哈希验证,生成物位于被忽略的 `.local/` 已用 CLI 真实导入并逐字段/图片哈希验证,生成物位于被忽略的 `.local/`
- 标准启动路径:`$env:RUN_START_COMMAND="1"; .\init.ps1` - 标准启动路径:`$env:RUN_START_COMMAND="1"; .\init.ps1`
- 标准验证路径:`.\init.ps1` - 标准验证路径:`.\init.ps1`
- 当前 blocker:真实 VLM 服务地址、模型、设备级测试凭证、成本上限和数据留存尚未确认; - 当前 blocker:T-213 需要确认是否允许点击“单独购买/免拼购买”仅用于打开规格弹层,
当前只支持单 SKU/JPEG;候选探针截图要求 Android 11/API 30+ 或提供独立规格入口/受支持数据源;ColorOS 重装后需采购员手动重新启用肉包无障碍。
真实 VLM 服务地址、模型、设备级测试凭证、成本上限和数据留存也尚未确认;当前只
支持单 SKU/JPEG;候选探针截图要求 Android 11/API 30+
## 当前目录 ## 当前目录
@@ -108,7 +114,7 @@
| `docs/tasks/T-210.md` | DONE | 兼容拼多多 8.17 搜索结果页与长词省略显示 | | `docs/tasks/T-210.md` | DONE | 兼容拼多多 8.17 搜索结果页与长词省略显示 |
| `docs/tasks/T-211.md` | DONE | 参考图召回、SKU 颜色尺码硬匹配与 0..5 候选回传 | | `docs/tasks/T-211.md` | DONE | 参考图召回、SKU 颜色尺码硬匹配与 0..5 候选回传 |
| `docs/tasks/T-212.md` | DONE | 修复重排候选、推荐、证据与人工接受的身份映射 | | `docs/tasks/T-212.md` | DONE | 修复重排候选、推荐、证据与人工接受的身份映射 |
| `docs/tasks/T-213.md` | TODO | 只读采集拼多多规格弹窗中的颜色和尺码证据 | | `docs/tasks/T-213.md` | BLOCKED | 实现只读规格核验;等待拼多多购买语义入口安全决策 |
| `docs/design/` | 已确认 | T-202 原型索引、4 个管理页和 7 个 Android 页面 | | `docs/design/` | 已确认 | T-202 原型索引、4 个管理页和 7 个 Android 页面 |
| `deepseek总结.txt` | 已有 | 历史讨论摘要,不是正式需求权威 | | `deepseek总结.txt` | 已有 | 历史讨论摘要,不是正式需求权威 |
| `android-buyer/` | 已有 | Roubao `main` 固定 commit 的 Android 基线 | | `android-buyer/` | 已有 | Roubao `main` 固定 commit 的 Android 基线 |
@@ -121,10 +127,9 @@
- 已完成:T-001 至 T-004、T-101 至 T-104、T-201 至 T-207、T-209、T-210、T-211、 - 已完成:T-001 至 T-004、T-101 至 T-104、T-201 至 T-207、T-209、T-210、T-211、
T-212。 T-212。
- 正在进行:T-213 拼多多规格弹窗只读核验。 - 阻塞中:T-213 拼多多规格弹窗只读核验,等待购买语义入口的安全策略决策。
- 下一个可领取任务:无;T-208 依赖 T-213 完成。 - 下一个可领取任务:无;T-208 依赖 T-213 真机验收完成。
- 后置任务:T-213 拼多多规格弹窗只读核验;完成后再做 T-208 候选决策数据与人工 - 后置任务:解除 T-213 阻塞后再做 T-208 候选决策数据与人工理由闭环。
理由闭环。
## 当前可运行内容 ## 当前可运行内容
+24 -7
View File
@@ -4,11 +4,12 @@ title: 拼多多规格弹窗只读核验
phase: 2 phase: 2
deps: deps:
- T-212 - T-212
status: DOING status: BLOCKED
created: 2026-07-27 created: 2026-07-27
context_ref: edfee2c context_ref: edfee2c
work_branch: null work_branch: null
write_paths: write_paths:
- android-buyer/app/build.gradle.kts
- docs/tasks/T-213.md - docs/tasks/T-213.md
- docs/04-architecture.md - docs/04-architecture.md
- docs/current-state.md - docs/current-state.md
@@ -53,12 +54,12 @@ Top 5 为空。仅凭商品主图推断颜色或尺码不满足硬约束要求
## 验收要点 ## 验收要点
- [ ] 页面分类测试覆盖规格弹窗、普通详情、登录、验证码、风控、购物车、订单和支付。 - [x] 页面分类测试覆盖规格弹窗、普通详情、登录、验证码、风控、购物车、订单和支付。
- [ ] 自动化只点击唯一规格入口,不点击颜色/尺码选项、购买、拼单、购物车或支付控件。 - [x] 自动化只点击唯一规格入口,不点击颜色/尺码选项、购买、拼单、购物车或支付控件。
- [ ] 颜色、尺码分组及选项解析有数量/文本长度边界,截断或歧义不会判为 `MATCH`。 - [x] 颜色、尺码分组及选项解析有数量/文本长度边界,截断或歧义不会判为 `MATCH`。
- [ ] 单元测试覆盖两个目标均匹配、明确不匹配、禁用、缺失和未知。 - [x] 单元测试覆盖两个目标均匹配、明确不匹配、禁用、缺失和未知。
- [ ] 每个候选的详情/规格证据哈希和后端 asset ID 对应正确,离线重试不重复或串位。 - [x] 每个候选的详情/规格证据哈希和后端 asset ID 对应正确,离线重试不重复或串位。
- [ ] Android 单元测试和 Debug 构建通过。 - [x] Android 单元测试和 Debug 构建通过。
- [ ] PKG110、Android 16、拼多多 8.17.0 真机 smoke 完成规格弹窗打开、只读采集、 - [ ] PKG110、Android 16、拼多多 8.17.0 真机 smoke 完成规格弹窗打开、只读采集、
关闭和返回;记录 App/拼多多版本并确认未进入订单或支付。 关闭和返回;记录 App/拼多多版本并确认未进入订单或支付。
@@ -75,3 +76,19 @@ Top 5 为空。仅凭商品主图推断颜色或尺码不满足硬约束要求
关联到错误的重排候选。 关联到错误的重排候选。
- 2026-07-27:T-212 完成提交 `4b795a4` 后领取。先冻结拼多多 8.17 规格弹窗的只读 - 2026-07-27:T-212 完成提交 `4b795a4` 后领取。先冻结拼多多 8.17 规格弹窗的只读
页面/动作边界和候选多证据模型,再进行真机验证。 页面/动作边界和候选多证据模型,再进行真机验证。
- 2026-07-27:实现规格弹层分类、安全入口 allowlist、颜色/尺码有界解析、本地确定性
硬匹配、candidate evaluation schema/prompt v3,以及每个候选详情/规格两份证据的
cache 校验、重排身份绑定和加密 outbox 关联。模型返回状态与本地状态不一致时强制
降级人工检查。
- 2026-07-27:Android `testDebugUnitTest` 24 个 suite、128 个测试全部通过,
`assembleDebug` 通过;后端 `go test ./...` 通过。Debug APK `1.4.3 (8)` 已安装到
PKG110,拼多多版本为 `8.17.0 (81700)`。
- 2026-07-27:真机检查图片结果中的定制商品和文字搜索 `black tshirt L` 的普通服装
商品。服装详情只暴露不可点击的“颜色/款式”预览,完整颜色/尺码弹层只能通过
“单独购买/免拼购买”控件进入;前两个定制商品甚至没有规格预览。全程未点击购买、
拼单、购物车、订单或支付控件。
- 2026-07-27:任务标记 `BLOCKED`。代码在缺少独立可点击规格入口时按
`UNKNOWN_PAGE` 安全停止,符合边界但无法完成“打开/采集/关闭/返回”的真机验收。
外部解除条件是:确认允许把购买语义控件仅用于打开规格弹层,或提供不经过购买控件
的稳定规格入口/受支持数据源。重新安装后 ColorOS 也关闭了肉包无障碍服务,继续
smoke 前需采购员在系统设置手动重新启用。