feat(t213): guard read-only specification verification

This commit is contained in:
QiuSW
2026-07-27 23:06:21 +08:00
parent 1316b9bb32
commit 9a8388fac6
25 changed files with 1540 additions and 207 deletions
+2 -2
View File
@@ -11,8 +11,8 @@ android {
applicationId = "com.roubao.autopilot"
minSdk = 26
targetSdk = 34
versionCode = 7
versionName = "1.4.2"
versionCode = 8
versionName = "1.4.3"
vectorDrawables {
useSupportLibrary = true
@@ -65,6 +65,8 @@ import com.roubao.autopilot.vlm.CandidateEvaluator
import com.roubao.autopilot.vlm.CandidateReviewBatch
import com.roubao.autopilot.vlm.CandidateHumanReviewPolicy
import com.roubao.autopilot.vlm.CandidateTopFivePolicy
import com.roubao.autopilot.vlm.CandidateSpecificationHardConstraintPolicy
import com.roubao.autopilot.vlm.SkuHardConstraintExtractor
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
@@ -98,6 +100,7 @@ import com.roubao.autopilot.procurement.ExecutionCandidateSourceIdentity
import com.roubao.autopilot.procurement.ExecutionHardConstraintEvaluation
import com.roubao.autopilot.procurement.ExecutionRecommendation
import com.roubao.autopilot.procurement.ExecutionEvidenceDraft
import com.roubao.autopilot.procurement.ExecutionEvidenceKind
import com.roubao.autopilot.procurement.ExecutionMode
import com.roubao.autopilot.procurement.ExecutionProvenanceSnapshot
import com.roubao.autopilot.procurement.ProcurementRepository
@@ -951,6 +954,8 @@ class MainActivity : ComponentActivity() {
baseUrl = settings.baseUrl,
model = settings.model
)
val hardConstraints =
SkuHardConstraintExtractor.extract(requirement.sku)
val result = CandidateEvaluator(
gateway = AndroidCandidateEvaluationVlmGateway(client),
providerId = provider.id,
@@ -962,8 +967,23 @@ class MainActivity : ComponentActivity() {
CandidateEvaluationImage(
ordinal = candidate.ordinal,
mediaType = "image/png",
bytes = candidate.pngBytes,
sha256 = candidate.sha256
bytes = candidate.detailPngBytes,
sha256 = candidate.detailSha256,
specificationBytes =
candidate.specificationPngBytes,
specificationSha256 =
candidate.specificationSha256,
localHardConstraintResults =
CandidateSpecificationHardConstraintPolicy
.evaluate(
constraints =
hardConstraints.constraints,
evidence =
candidate.specification,
evidenceSha256 =
candidate
.specificationSha256
)
)
}
)
@@ -1014,20 +1034,46 @@ class MainActivity : ComponentActivity() {
)
)
}
val selectedEvidence = ranked.map { rankedCandidate ->
val selectedEvidence = ranked.flatMap {
rankedCandidate ->
val candidate = requireNotNull(
evidenceByOrdinal[
rankedCandidate.sourceOrdinal
]
)
require(
candidate.sha256 ==
candidate.detailSha256 ==
rankedCandidate.evidenceSha256
)
ExecutionEvidenceDraft(
ordinal = rankedCandidate.rankedOrdinal,
pngBytes = candidate.pngBytes,
sha256 = candidate.sha256
require(
candidate.specificationSha256 ==
rankedCandidate
.specificationEvidenceSha256
)
listOf(
ExecutionEvidenceDraft(
ordinal =
rankedCandidate.rankedOrdinal,
pngBytes =
candidate.detailPngBytes,
sha256 =
candidate.detailSha256,
kind =
ExecutionEvidenceKind.DETAIL
),
ExecutionEvidenceDraft(
ordinal =
rankedCandidate.rankedOrdinal,
pngBytes =
candidate
.specificationPngBytes,
sha256 =
candidate
.specificationSha256,
kind =
ExecutionEvidenceKind
.SPECIFICATION
)
)
}
val queued = procurementRepository.queueCandidateBatch(
@@ -1078,7 +1124,10 @@ class MainActivity : ComponentActivity() {
rankedOrdinal =
rankedCandidate.rankedOrdinal,
evidenceSha256 =
rankedCandidate.evidenceSha256
rankedCandidate.evidenceSha256,
specificationEvidenceSha256 =
rankedCandidate
.specificationEvidenceSha256
)
},
candidates = queued
@@ -1139,11 +1188,20 @@ class MainActivity : ComponentActivity() {
)
}
),
validated.map { candidate ->
ExecutionEvidenceDraft(
ordinal = candidate.ordinal,
pngBytes = candidate.pngBytes,
sha256 = candidate.sha256
validated.flatMap { candidate ->
listOf(
ExecutionEvidenceDraft(
ordinal = candidate.ordinal,
pngBytes = candidate.detailPngBytes,
sha256 = candidate.detailSha256,
kind = ExecutionEvidenceKind.DETAIL
),
ExecutionEvidenceDraft(
ordinal = candidate.ordinal,
pngBytes = candidate.specificationPngBytes,
sha256 = candidate.specificationSha256,
kind = ExecutionEvidenceKind.SPECIFICATION
)
)
}
) ?: return
@@ -1153,7 +1211,9 @@ class MainActivity : ComponentActivity() {
ExecutionCandidateSourceIdentity(
sourceOrdinal = candidate.ordinal,
rankedOrdinal = candidate.ordinal,
evidenceSha256 = candidate.sha256
evidenceSha256 = candidate.detailSha256,
specificationEvidenceSha256 =
candidate.specificationSha256
)
},
candidates = queued
@@ -4,6 +4,7 @@ import com.roubao.autopilot.pinduoduo.PinduoduoPage
import com.roubao.autopilot.pinduoduo.PinduoduoCandidateCard
import com.roubao.autopilot.pinduoduo.PinduoduoCandidateDetailEvidence
import com.roubao.autopilot.pinduoduo.PinduoduoScreenshotCapture
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationEvidence
import com.roubao.autopilot.pinduoduo.PinduoduoUiSnapshot
import com.roubao.autopilot.readiness.DeviceObservationStore
import kotlinx.coroutines.Dispatchers
@@ -76,13 +77,37 @@ object BuyerAccessibilityBridge {
service?.readPinduoduoCandidateDetailEvidence()
}
suspend fun captureScreenshot(): PinduoduoScreenshotCapture? =
suspend fun captureDetailScreenshot(): PinduoduoScreenshotCapture? =
withTimeoutOrNull(SCREENSHOT_TIMEOUT_MILLIS) {
withContext(Dispatchers.Main.immediate) {
service?.capturePinduoduoScreenshot()
service?.capturePinduoduoScreenshot(PinduoduoPage.PRODUCT_DETAIL)
}
}
suspend fun openSpecifications(): Boolean =
withContext(Dispatchers.Main.immediate) {
service?.openPinduoduoSpecifications() == true
}
suspend fun specificationEvidence(): PinduoduoSpecificationEvidence? =
withContext(Dispatchers.Main.immediate) {
service?.readPinduoduoSpecificationEvidence()
}
suspend fun captureSpecificationScreenshot(): PinduoduoScreenshotCapture? =
withTimeoutOrNull(SCREENSHOT_TIMEOUT_MILLIS) {
withContext(Dispatchers.Main.immediate) {
service?.capturePinduoduoScreenshot(
PinduoduoPage.SPECIFICATION_PANEL
)
}
}
suspend fun closeSpecifications(): Boolean =
withContext(Dispatchers.Main.immediate) {
service?.closePinduoduoSpecifications() == true
}
suspend fun returnToResults(): Boolean =
withContext(Dispatchers.Main.immediate) {
service?.returnFromPinduoduoCandidate() == true
@@ -24,6 +24,8 @@ import com.roubao.autopilot.pinduoduo.PinduoduoPage
import com.roubao.autopilot.pinduoduo.isCandidateResultsPage
import com.roubao.autopilot.pinduoduo.SEARCH_PROBE_KEYWORD
import com.roubao.autopilot.pinduoduo.PinduoduoScreenshotCapture
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationEvidence
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationParser
import java.io.ByteArrayOutputStream
import java.util.ArrayDeque
import java.util.concurrent.Executors
@@ -321,6 +323,48 @@ class BuyerAccessibilityService : AccessibilityService() {
)
}
internal fun openPinduoduoSpecifications(): Boolean =
withPinduoduoRoot { root ->
if (!isVerifiedProductDetail(root)) {
return@withPinduoduoRoot false
}
val entries = collectNodes(root).filter { node ->
node.isVisibleToUser &&
node.isEnabled &&
PinduoduoSpecificationParser.isSafeEntryText(
semanticText(node)
)
}
entries.singleOrNull()?.let(::clickNodeOrAncestor) == true
} ?: false
internal fun readPinduoduoSpecificationEvidence():
PinduoduoSpecificationEvidence? =
withPinduoduoRoot { root ->
val snapshot = classifyPinduoduoRoot(root)
if (
snapshot.safetyStopReason != null ||
snapshot.page != PinduoduoPage.SPECIFICATION_PANEL
) {
return@withPinduoduoRoot null
}
PinduoduoSpecificationParser.parse(
collectNodes(root).map(::uiElement)
)
}
internal fun closePinduoduoSpecifications(): Boolean =
withPinduoduoRoot { root ->
val snapshot = classifyPinduoduoRoot(root)
if (
snapshot.safetyStopReason != null ||
snapshot.page != PinduoduoPage.SPECIFICATION_PANEL
) {
return@withPinduoduoRoot false
}
performGlobalAction(GLOBAL_ACTION_BACK)
} ?: false
internal fun returnFromPinduoduoCandidate(): Boolean =
withPinduoduoRoot { root ->
if (!isVerifiedProductDetail(root)) {
@@ -355,20 +399,38 @@ class BuyerAccessibilityService : AccessibilityService() {
) == true
} ?: false
internal suspend fun capturePinduoduoScreenshot(): PinduoduoScreenshotCapture? {
internal suspend fun capturePinduoduoScreenshot(
expectedPage: PinduoduoPage
): PinduoduoScreenshotCapture? {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {
return null
}
return capturePinduoduoScreenshotApi30()
if (
expectedPage !in setOf(
PinduoduoPage.PRODUCT_DETAIL,
PinduoduoPage.SPECIFICATION_PANEL
)
) {
return null
}
return capturePinduoduoScreenshotApi30(expectedPage)
}
@RequiresApi(Build.VERSION_CODES.R)
private suspend fun capturePinduoduoScreenshotApi30():
private suspend fun capturePinduoduoScreenshotApi30(
expectedPage: PinduoduoPage
):
PinduoduoScreenshotCapture? {
val root = rootInActiveWindow
val snapshot = root?.let(::classifyPinduoduoRoot)
if (
root?.packageName?.toString() != PINDUODUO_PACKAGE ||
!isVerifiedProductDetail(root)
snapshot?.safetyStopReason != null ||
snapshot?.page != expectedPage ||
(
expectedPage == PinduoduoPage.PRODUCT_DETAIL &&
!isVerifiedProductDetail(root)
)
) {
return null
}
@@ -447,18 +509,7 @@ class BuyerAccessibilityService : AccessibilityService() {
private fun classifyPinduoduoRoot(
root: AccessibilityNodeInfo
): PinduoduoUiSnapshot {
val elements = collectNodes(root).map { node ->
PinduoduoUiElement(
text = node.text?.toString(),
contentDescription = node.contentDescription?.toString(),
className = node.className?.toString().orEmpty(),
resourceId = node.viewIdResourceName,
clickable = node.isClickable,
editable = node.isEditable,
enabled = node.isEnabled,
visibleToUser = node.isVisibleToUser
)
}
val elements = collectNodes(root).map(::uiElement)
return PinduoduoPageClassifier.classify(
foregroundPackage = root.packageName?.toString(),
elements = elements,
@@ -466,6 +517,24 @@ class BuyerAccessibilityService : AccessibilityService() {
)
}
private fun uiElement(node: AccessibilityNodeInfo): PinduoduoUiElement {
val bounds = Rect().also(node::getBoundsInScreen)
return PinduoduoUiElement(
text = node.text?.toString(),
contentDescription = node.contentDescription?.toString(),
className = node.className?.toString().orEmpty(),
resourceId = node.viewIdResourceName,
clickable = node.isClickable,
editable = node.isEditable,
enabled = node.isEnabled,
visibleToUser = node.isVisibleToUser,
selected = node.isSelected,
scrollable = node.isScrollable,
boundsLeft = bounds.left,
boundsTop = bounds.top
)
}
private fun candidateNodes(
root: AccessibilityNodeInfo,
limit: Int
@@ -668,6 +737,13 @@ class BuyerAccessibilityService : AccessibilityService() {
.replace(Regex("\\s+"), " ")
.take(MAX_EVIDENCE_TEXT_LENGTH)
private fun semanticText(node: AccessibilityNodeInfo): String =
sequenceOf(node.text, node.contentDescription)
.filterNotNull()
.map { it.toString().trim() }
.firstOrNull(String::isNotEmpty)
.orEmpty()
private fun shouldInspect(eventType: Int): Boolean =
eventType == AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED ||
eventType == AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED ||
@@ -25,20 +25,41 @@ class AndroidPinduoduoCandidateDriver(
override suspend fun openCandidate(signature: String): Boolean =
BuyerAccessibilityBridge.openCandidate(signature)
override suspend fun captureCandidate(
ordinal: Int,
card: PinduoduoCandidateCard
): PinduoduoCandidateEvidence? {
override suspend fun captureDetail(): PinduoduoCandidateCapture? {
val detail = BuyerAccessibilityBridge.candidateDetailEvidence()
?: return null
val screenshot = BuyerAccessibilityBridge.captureScreenshot()
val screenshot = BuyerAccessibilityBridge.captureDetailScreenshot()
?: return null
return PinduoduoCandidateCapture(detail, screenshot)
}
override suspend fun openSpecifications(): Boolean =
BuyerAccessibilityBridge.openSpecifications()
override suspend fun captureSpecifications(): PinduoduoSpecificationCapture? {
val evidence = BuyerAccessibilityBridge.specificationEvidence()
?: return null
val screenshot =
BuyerAccessibilityBridge.captureSpecificationScreenshot()
?: return null
return PinduoduoSpecificationCapture(evidence, screenshot)
}
override suspend fun closeSpecifications(): Boolean =
BuyerAccessibilityBridge.closeSpecifications()
override suspend fun saveCandidate(
ordinal: Int,
card: PinduoduoCandidateCard,
detail: PinduoduoCandidateCapture,
specifications: PinduoduoSpecificationCapture
): PinduoduoCandidateEvidence? {
return withContext(Dispatchers.IO) {
evidenceStore.save(
ordinal = ordinal,
card = card,
detail = detail,
screenshot = screenshot
specifications = specifications
)
}
}
@@ -70,8 +91,8 @@ private class CandidateEvidenceStore(context: Context) {
fun save(
ordinal: Int,
card: PinduoduoCandidateCard,
detail: PinduoduoCandidateDetailEvidence,
screenshot: PinduoduoScreenshotCapture
detail: PinduoduoCandidateCapture,
specifications: PinduoduoSpecificationCapture
): PinduoduoCandidateEvidence? {
if (ordinal !in 1..MAX_CANDIDATES_PER_PROBE) {
return null
@@ -80,22 +101,25 @@ private class CandidateEvidenceStore(context: Context) {
if (!root.exists() && !root.mkdirs()) {
return null
}
val fileName = "candidate-%02d.png".format(ordinal)
val screenshotFile = File(root, fileName)
screenshotFile.writeBytes(screenshot.pngBytes)
val detailAsset = writeAsset(
ordinal = ordinal,
suffix = "detail",
screenshot = detail.screenshot
)
val specificationAsset = writeAsset(
ordinal = ordinal,
suffix = "specification",
screenshot = specifications.screenshot
)
val evidence = PinduoduoCandidateEvidence(
ordinal = ordinal,
cardSignature = card.signature,
cardSemanticTextCount = card.semanticTextCount,
detailSignature = detail.signature,
detailSemanticTextCount = detail.semanticTextCount,
screenshotFileName = fileName,
screenshotSha256 = PinduoduoEvidenceHash.sha256(
screenshot.pngBytes
),
screenshotByteCount = screenshot.pngBytes.size,
screenshotWidth = screenshot.width,
screenshotHeight = screenshot.height
detailSignature = detail.detail.signature,
detailSemanticTextCount = detail.detail.semanticTextCount,
detailAsset = detailAsset,
specification = specifications.evidence,
specificationAsset = specificationAsset
)
evidenceByOrdinal[ordinal] = evidence
writeManifest()
@@ -105,6 +129,22 @@ private class CandidateEvidenceStore(context: Context) {
}
}
private fun writeAsset(
ordinal: Int,
suffix: String,
screenshot: PinduoduoScreenshotCapture
): PinduoduoEvidenceAsset {
val fileName = "candidate-%02d-%s.png".format(ordinal, suffix)
File(root, fileName).writeBytes(screenshot.pngBytes)
return PinduoduoEvidenceAsset(
fileName = fileName,
sha256 = PinduoduoEvidenceHash.sha256(screenshot.pngBytes),
byteCount = screenshot.pngBytes.size,
width = screenshot.width,
height = screenshot.height
)
}
private fun writeManifest() {
val candidates = JSONArray()
evidenceByOrdinal.values.forEach { evidence ->
@@ -121,18 +161,23 @@ private class CandidateEvidenceStore(context: Context) {
"detail_semantic_text_count",
evidence.detailSemanticTextCount
)
.put("screenshot_file", evidence.screenshotFileName)
.put("screenshot_sha256", evidence.screenshotSha256)
.put("detail_asset", assetJson(evidence.detailAsset))
.put(
"screenshot_byte_count",
evidence.screenshotByteCount
"specification_signature",
evidence.specification.signature
)
.put(
"specification_semantic_text_count",
evidence.specification.semanticTextCount
)
.put(
"specification_asset",
assetJson(evidence.specificationAsset)
)
.put("screenshot_width", evidence.screenshotWidth)
.put("screenshot_height", evidence.screenshotHeight)
)
}
val manifest = JSONObject()
.put("schema_version", 1)
.put("schema_version", 2)
.put("candidate_count", evidenceByOrdinal.size)
.put("candidates", candidates)
.toString(2)
@@ -147,6 +192,14 @@ private class CandidateEvidenceStore(context: Context) {
}
}
private fun assetJson(asset: PinduoduoEvidenceAsset): JSONObject =
JSONObject()
.put("file", asset.fileName)
.put("sha256", asset.sha256)
.put("byte_count", asset.byteCount)
.put("width", asset.width)
.put("height", asset.height)
private companion object {
const val MANIFEST_FILE = "manifest.json"
}
@@ -9,8 +9,11 @@ import kotlinx.coroutines.withContext
data class ValidatedCandidateEvidence(
val ordinal: Int,
val pngBytes: ByteArray,
val sha256: String
val detailPngBytes: ByteArray,
val detailSha256: String,
val specification: PinduoduoSpecificationEvidence,
val specificationPngBytes: ByteArray,
val specificationSha256: String
)
class CandidateEvidenceSource(
@@ -31,38 +34,63 @@ class CandidateEvidenceSource(
val canonicalRoot = root.canonicalFile
var totalBytes = 0L
sorted.map { metadata ->
val expectedFileName = "candidate-%02d.png".format(metadata.ordinal)
require(metadata.screenshotFileName == expectedFileName)
require(metadata.screenshotByteCount in 1..MAX_PNG_BYTES)
require(metadata.screenshotWidth in 1..MAX_SCREENSHOT_DIMENSION)
require(metadata.screenshotHeight in 1..MAX_SCREENSHOT_DIMENSION)
require(SHA256_PATTERN.matches(metadata.screenshotSha256))
totalBytes += metadata.screenshotByteCount
require(
metadata.specification.groups.map { it.kind }.toSet() ==
PinduoduoSpecificationGroupKind.entries.toSet()
)
val detail = readAsset(
canonicalRoot = canonicalRoot,
asset = metadata.detailAsset,
expectedFileName =
"candidate-%02d-detail.png".format(metadata.ordinal)
)
val specification = readAsset(
canonicalRoot = canonicalRoot,
asset = metadata.specificationAsset,
expectedFileName =
"candidate-%02d-specification.png".format(metadata.ordinal)
)
totalBytes += detail.size + specification.size
require(totalBytes <= MAX_TOTAL_PNG_BYTES)
val file = File(canonicalRoot, expectedFileName).canonicalFile
require(file.parentFile == canonicalRoot)
require(file.isFile && file.length() == metadata.screenshotByteCount.toLong())
val bytes = FileInputStream(file).use {
it.readBytesExact(metadata.screenshotByteCount)
}
require(bytes.hasPngHeader())
val dimensions = bytes.pngDimensions()
require(dimensions.first == metadata.screenshotWidth)
require(dimensions.second == metadata.screenshotHeight)
require(PinduoduoEvidenceHash.sha256(bytes) == metadata.screenshotSha256)
ValidatedCandidateEvidence(
ordinal = metadata.ordinal,
pngBytes = bytes,
sha256 = metadata.screenshotSha256
detailPngBytes = detail,
detailSha256 = metadata.detailAsset.sha256,
specification = metadata.specification,
specificationPngBytes = specification,
specificationSha256 = metadata.specificationAsset.sha256
)
}
}
}
private fun readAsset(
canonicalRoot: File,
asset: PinduoduoEvidenceAsset,
expectedFileName: String
): ByteArray {
require(asset.fileName == expectedFileName)
require(asset.byteCount in 1..MAX_PNG_BYTES)
require(asset.width in 1..MAX_SCREENSHOT_DIMENSION)
require(asset.height in 1..MAX_SCREENSHOT_DIMENSION)
require(SHA256_PATTERN.matches(asset.sha256))
val file = File(canonicalRoot, expectedFileName).canonicalFile
require(file.parentFile == canonicalRoot)
require(file.isFile && file.length() == asset.byteCount.toLong())
val bytes = FileInputStream(file).use {
it.readBytesExact(asset.byteCount)
}
require(bytes.hasPngHeader())
val dimensions = bytes.pngDimensions()
require(dimensions.first == asset.width)
require(dimensions.second == asset.height)
require(PinduoduoEvidenceHash.sha256(bytes) == asset.sha256)
return bytes
}
private companion object {
const val MAX_PNG_BYTES = 8 * 1024 * 1024
const val MAX_TOTAL_PNG_BYTES = 32L * 1024L * 1024L
const val MAX_TOTAL_PNG_BYTES = 64L * 1024L * 1024L
const val MAX_SCREENSHOT_DIMENSION = 10_000
val SHA256_PATTERN = Regex("^[0-9a-f]{64}$")
}
@@ -16,7 +16,10 @@ enum class CandidateBrowsePhase {
READING_RESULTS,
OPENING_CANDIDATE,
WAITING_DETAIL,
CAPTURING_EVIDENCE,
CAPTURING_DETAIL,
OPENING_SPECIFICATIONS,
READING_SPECIFICATIONS,
CLOSING_SPECIFICATIONS,
RETURNING_RESULTS,
SCROLLING_RESULTS,
COMPLETE
@@ -100,11 +103,34 @@ class PinduoduoCandidateAutomation(
mutablePhase.value = CandidateBrowsePhase.WAITING_DETAIL
awaitPage(PinduoduoPage.PRODUCT_DETAIL)?.let { return it }
val ordinal = mutableEvidence.value.size + 1
mutablePhase.value = CandidateBrowsePhase.CAPTURING_EVIDENCE
val captured = driver.captureCandidate(ordinal, nextCard)
mutablePhase.value = CandidateBrowsePhase.CAPTURING_DETAIL
val detail = driver.captureDetail()
?: return AutomationResult.FatalFailure(
WorkflowFailureCode.EVIDENCE_CAPTURE_FAILED
)
mutablePhase.value = CandidateBrowsePhase.OPENING_SPECIFICATIONS
if (!driver.openSpecifications()) {
return AutomationResult.Blocked(SafetyStopReason.UNKNOWN_PAGE)
}
awaitPage(PinduoduoPage.SPECIFICATION_PANEL)?.let { return it }
mutablePhase.value = CandidateBrowsePhase.READING_SPECIFICATIONS
val specifications = driver.captureSpecifications()
?: return AutomationResult.FatalFailure(
WorkflowFailureCode.EVIDENCE_CAPTURE_FAILED
)
mutablePhase.value = CandidateBrowsePhase.CLOSING_SPECIFICATIONS
if (!driver.closeSpecifications()) {
return AutomationResult.Blocked(SafetyStopReason.UNKNOWN_PAGE)
}
awaitPage(PinduoduoPage.PRODUCT_DETAIL)?.let { return it }
val captured = driver.saveCandidate(
ordinal = ordinal,
card = nextCard,
detail = detail,
specifications = specifications
) ?: return AutomationResult.FatalFailure(
WorkflowFailureCode.EVIDENCE_CAPTURE_FAILED
)
mutableEvidence.value = mutableEvidence.value + captured
mutablePhase.value = CandidateBrowsePhase.RETURNING_RESULTS
@@ -123,7 +149,17 @@ class PinduoduoCandidateAutomation(
private suspend fun recoverDetailPageIfNeeded(): AutomationResult? {
val snapshot = driver.snapshot()
safetyResult(snapshot)?.let { return it }
if (snapshot.page != PinduoduoPage.PRODUCT_DETAIL) {
if (snapshot.page == PinduoduoPage.SPECIFICATION_PANEL) {
if (!driver.closeSpecifications()) {
return AutomationResult.RetryableFailure(
WorkflowFailureCode.TRANSIENT_AUTOMATION
)
}
awaitPage(PinduoduoPage.PRODUCT_DETAIL)?.let { return it }
}
val detailSnapshot = driver.snapshot()
safetyResult(detailSnapshot)?.let { return it }
if (detailSnapshot.page != PinduoduoPage.PRODUCT_DETAIL) {
return null
}
if (!driver.returnToResults()) {
@@ -23,17 +23,23 @@ data class PinduoduoScreenshotCapture(
val height: Int
)
data class PinduoduoEvidenceAsset(
val fileName: String,
val sha256: String,
val byteCount: Int,
val width: Int,
val height: Int
)
data class PinduoduoCandidateEvidence(
val ordinal: Int,
val cardSignature: String,
val cardSemanticTextCount: Int,
val detailSignature: String,
val detailSemanticTextCount: Int,
val screenshotFileName: String,
val screenshotSha256: String,
val screenshotByteCount: Int,
val screenshotWidth: Int,
val screenshotHeight: Int
val detailAsset: PinduoduoEvidenceAsset,
val specification: PinduoduoSpecificationEvidence,
val specificationAsset: PinduoduoEvidenceAsset
)
object PinduoduoEvidenceHash {
@@ -50,11 +56,27 @@ interface PinduoduoCandidateDriver {
suspend fun snapshot(): PinduoduoUiSnapshot
suspend fun candidateCards(limit: Int): List<PinduoduoCandidateCard>
suspend fun openCandidate(signature: String): Boolean
suspend fun captureCandidate(
suspend fun captureDetail(): PinduoduoCandidateCapture?
suspend fun openSpecifications(): Boolean
suspend fun captureSpecifications(): PinduoduoSpecificationCapture?
suspend fun closeSpecifications(): Boolean
suspend fun saveCandidate(
ordinal: Int,
card: PinduoduoCandidateCard
card: PinduoduoCandidateCard,
detail: PinduoduoCandidateCapture,
specifications: PinduoduoSpecificationCapture
): PinduoduoCandidateEvidence?
suspend fun returnToResults(): Boolean
suspend fun scrollResults(): Boolean
fun resetEvidence()
}
data class PinduoduoCandidateCapture(
val detail: PinduoduoCandidateDetailEvidence,
val screenshot: PinduoduoScreenshotCapture
)
data class PinduoduoSpecificationCapture(
val evidence: PinduoduoSpecificationEvidence,
val screenshot: PinduoduoScreenshotCapture
)
@@ -13,7 +13,11 @@ data class PinduoduoUiElement(
val clickable: Boolean,
val editable: Boolean,
val enabled: Boolean,
val visibleToUser: Boolean
val visibleToUser: Boolean,
val selected: Boolean = false,
val scrollable: Boolean = false,
val boundsLeft: Int = 0,
val boundsTop: Int = 0
)
enum class PinduoduoPage {
@@ -24,6 +28,7 @@ enum class PinduoduoPage {
IMAGE_SEARCH,
IMAGE_SEARCH_RESULTS,
PRODUCT_DETAIL,
SPECIFICATION_PANEL,
UNKNOWN
}
@@ -142,8 +147,23 @@ object PinduoduoPageClassifier {
}
val detailMarkerCount = setOf("联系客服", "收藏", "店铺")
.count { marker -> normalized.any { it.contains(marker) } }
val specificationGroupCount = setOf("颜色分类", "颜色", "尺码", "尺寸")
.count { marker -> normalized.any { it == marker } }
val specificationOptionCount = visibleElements.count { element ->
val semantic = element.text
?.takeIf(String::isNotBlank)
?: element.contentDescription.orEmpty()
element.clickable &&
semantic.isNotBlank() &&
normalize(semantic) !in specificationEntryMarkers &&
paymentMarkers.none(semantic::contains)
}
val hasSpecificationPanel =
specificationGroupCount >= 2 &&
specificationOptionCount >= 2
val page = when {
hasSpecificationPanel -> PinduoduoPage.SPECIFICATION_PANEL
hasImageResultHeader && legacySortControlCount >= 3 ->
PinduoduoPage.IMAGE_SEARCH_RESULTS
hasImageSearchPage -> PinduoduoPage.IMAGE_SEARCH
@@ -200,6 +220,16 @@ object PinduoduoPageClassifier {
private fun Collection<String>.containsAny(markers: Collection<String>): Boolean =
any { text -> markers.any(text::contains) }
private val specificationEntryMarkers = setOf(
"选择规格",
"请选择规格",
"颜色分类",
"颜色款式",
"选择颜色",
"请选择颜色",
"选择尺码",
"请选择尺码"
)
private const val MINIMUM_QUERY_FRAGMENT_LENGTH = 12
private const val MINIMUM_ELIDED_FRAGMENT_LENGTH = 4
}
@@ -0,0 +1,178 @@
package com.roubao.autopilot.pinduoduo
enum class PinduoduoSpecificationGroupKind {
COLOR,
SIZE
}
data class PinduoduoSpecificationOption(
val text: String,
val selected: Boolean,
val enabled: Boolean
)
data class PinduoduoSpecificationGroup(
val kind: PinduoduoSpecificationGroupKind,
val title: String,
val options: List<PinduoduoSpecificationOption>,
val complete: Boolean
)
data class PinduoduoSpecificationEvidence(
val signature: String,
val semanticTextCount: Int,
val groups: List<PinduoduoSpecificationGroup>
)
object PinduoduoSpecificationParser {
fun parse(
elements: Collection<PinduoduoUiElement>
): PinduoduoSpecificationEvidence? {
val visible = elements
.asSequence()
.filter { it.visibleToUser }
.filter { semanticText(it).isNotEmpty() || it.scrollable }
.take(MAX_ELEMENTS + 1)
.toList()
if (visible.isEmpty() || visible.size > MAX_ELEMENTS) {
return null
}
val ordered = visible.sortedWith(
compareBy<PinduoduoUiElement> { it.boundsTop }
.thenBy { it.boundsLeft }
)
val headers = ordered.mapIndexedNotNull { index, element ->
groupKind(semanticText(element))?.let { kind ->
GroupHeader(index, kind, semanticText(element), element.boundsTop)
}
}
if (
headers.map { it.kind }.toSet() !=
PinduoduoSpecificationGroupKind.entries.toSet()
) {
return null
}
val groups = headers.mapIndexed { headerIndex, header ->
val nextTop = headers.getOrNull(headerIndex + 1)?.top ?: Int.MAX_VALUE
val options = ordered.asSequence()
.filter { element ->
element.boundsTop >= header.top &&
element.boundsTop < nextTop &&
groupKind(semanticText(element)) == null &&
isOption(element)
}
.map { element ->
PinduoduoSpecificationOption(
text = semanticText(element).take(MAX_OPTION_TEXT_LENGTH),
selected = element.selected,
enabled = element.enabled
)
}
.distinctBy { normalize(it.text) }
.take(MAX_OPTIONS_PER_GROUP + 1)
.toList()
val clipped =
options.size > MAX_OPTIONS_PER_GROUP ||
ordered.any { element ->
element.scrollable &&
element.boundsTop >= header.top &&
element.boundsTop < nextTop
}
PinduoduoSpecificationGroup(
kind = header.kind,
title = header.title.take(MAX_OPTION_TEXT_LENGTH),
options = options.take(MAX_OPTIONS_PER_GROUP),
complete = !clipped
)
}
if (groups.any { it.options.isEmpty() }) {
return null
}
val semantics = buildList {
groups.forEach { group ->
add("${group.kind.name}:${normalize(group.title)}:${group.complete}")
group.options.forEach { option ->
add(
"${normalize(option.text)}:" +
"${option.selected}:${option.enabled}"
)
}
}
}
return PinduoduoSpecificationEvidence(
signature = PinduoduoEvidenceHash.sha256(
semantics.joinToString("\u001f")
),
semanticTextCount = semantics.size,
groups = groups
)
}
fun isSafeEntryText(value: String): Boolean {
val normalized = normalize(value)
return normalized in SAFE_ENTRY_TEXTS ||
SAFE_ENTRY_PREFIXES.any(normalized::startsWith)
}
private fun isOption(element: PinduoduoUiElement): Boolean {
val text = semanticText(element)
val normalized = normalize(text)
return text.length <= MAX_OPTION_TEXT_LENGTH &&
normalized !in SAFE_ENTRY_TEXTS &&
SAFE_ENTRY_PREFIXES.none(normalized::startsWith) &&
TRANSACTION_MARKERS.none(normalized::contains) &&
(element.clickable || element.selected || !element.enabled)
}
private fun groupKind(value: String): PinduoduoSpecificationGroupKind? =
when (normalize(value).removeSuffix(":").removeSuffix(":")) {
"颜色", "颜色分类", "颜色选择" ->
PinduoduoSpecificationGroupKind.COLOR
"尺码", "尺寸", "大小", "尺码选择" ->
PinduoduoSpecificationGroupKind.SIZE
else -> null
}
private fun semanticText(element: PinduoduoUiElement): String =
sequenceOf(element.text, element.contentDescription)
.filterNotNull()
.map(String::trim)
.firstOrNull(String::isNotEmpty)
.orEmpty()
private fun normalize(value: String): String =
value.trim().lowercase().replace(Regex("\\s+"), "")
private data class GroupHeader(
val index: Int,
val kind: PinduoduoSpecificationGroupKind,
val title: String,
val top: Int
)
private val SAFE_ENTRY_TEXTS = setOf(
"选择规格",
"请选择规格",
"颜色分类",
"颜色款式",
"选择颜色",
"请选择颜色",
"选择尺码",
"请选择尺码"
)
private val SAFE_ENTRY_PREFIXES = setOf("请选择:", "请选择:", "已选:", "已选:")
private val TRANSACTION_MARKERS = setOf(
"购买",
"拼单",
"购物车",
"下单",
"支付",
"结算"
)
private const val MAX_ELEMENTS = 160
private const val MAX_OPTIONS_PER_GROUP = 30
private const val MAX_OPTION_TEXT_LENGTH = 80
}
@@ -5,9 +5,37 @@ import java.security.MessageDigest
data class ExecutionEvidenceDraft(
val ordinal: Int,
val pngBytes: ByteArray,
val sha256: String
val sha256: String,
val kind: ExecutionEvidenceKind = ExecutionEvidenceKind.DETAIL
)
enum class ExecutionEvidenceKind {
DETAIL,
SPECIFICATION
}
object ExecutionEvidenceAssociationPolicy {
fun group(
candidateOrdinals: List<Int>,
evidence: List<ExecutionEvidenceDraft>
): Map<Int, List<ExecutionEvidenceDraft>> {
require(candidateOrdinals == (1..candidateOrdinals.size).toList())
val grouped = evidence.groupBy { it.ordinal }
require(grouped.keys == candidateOrdinals.toSet())
require(
candidateOrdinals.all { ordinal ->
val assets = requireNotNull(grouped[ordinal])
assets.size == 2 &&
assets.map { it.kind }.toSet() ==
ExecutionEvidenceKind.entries.toSet()
}
)
return candidateOrdinals.associateWith { ordinal ->
requireNotNull(grouped[ordinal]).sortedBy { it.kind.ordinal }
}
}
}
data class ExecutionCandidateDraft(
val ordinal: Int,
val title: String,
@@ -22,7 +50,8 @@ data class ExecutionCandidateDraft(
data class ExecutionCandidateSourceIdentity(
val sourceOrdinal: Int,
val rankedOrdinal: Int,
val evidenceSha256: String
val evidenceSha256: String,
val specificationEvidenceSha256: String = evidenceSha256
)
data class RankedExecutionCandidateDraft(
@@ -45,6 +74,11 @@ object ExecutionCandidateIdentityPolicy {
require(identities.all { it.sourceOrdinal > 0 })
require(identities.map { it.sourceOrdinal }.distinct().size == identities.size)
require(identities.all { sha256Pattern.matches(it.evidenceSha256) })
require(
identities.all {
sha256Pattern.matches(it.specificationEvidenceSha256)
}
)
val candidatesByOrdinal = candidates.associateBy { it.ordinal }
require(candidatesByOrdinal.size == candidates.size)
return identities.map { identity ->
@@ -343,11 +343,8 @@ class ProcurementRepository(
require(!execution.safetyStopped && !execution.isExpired()) {
"执行授权已到期,不能继续采集"
}
require(evidence.size in 0..5) { "候选证据必须为 0 至 5 张" }
require(evidence.size in 0..25) { "候选证据必须为 0 至 25 张" }
require(batch.candidates.size in 0..5) { "候选数量必须为 0 至 5 个" }
require(evidence.size == batch.candidates.size) {
"候选和证据数量必须一致"
}
require(batch.candidates.map { it.ordinal } == (1..batch.candidates.size).toList()) {
"候选编号必须连续"
}
@@ -359,27 +356,34 @@ class ProcurementRepository(
} else {
require(batch.provenance == null) { "手工优先模式不能附带模型出处" }
}
val evidenceIDs = evidence.sortedBy { it.ordinal }.associate { draft ->
require(draft.ordinal in 1..batch.candidates.size)
val evidenceByOrdinal = ExecutionEvidenceAssociationPolicy.group(
candidateOrdinals = batch.candidates.map { it.ordinal },
evidence = evidence
)
evidence.forEach { draft ->
require(sha256(draft.pngBytes) == draft.sha256)
val localID = UUID.randomUUID().toString()
persistEvidenceLocked(localID, draft.pngBytes)
appendOutboxLocked(
ExecutionOutboxItem(
id = localID,
type = ExecutionOutboxType.EVIDENCE,
idempotencyKey = newOpaqueSecret(),
payload = "{}",
evidenceRelativePath = "$OUTBOX_DIRECTORY/$localID.png"
}
val evidenceIDs = evidenceByOrdinal.mapValues { (_, drafts) ->
drafts.sortedBy { it.kind.ordinal }.map { draft ->
val localID = UUID.randomUUID().toString()
persistEvidenceLocked(localID, draft.pngBytes)
appendOutboxLocked(
ExecutionOutboxItem(
id = localID,
type = ExecutionOutboxType.EVIDENCE,
idempotencyKey = newOpaqueSecret(),
payload = "{}",
evidenceRelativePath =
"$OUTBOX_DIRECTORY/$localID.png"
)
)
)
draft.ordinal to localID
localID
}
}
val candidates = batch.candidates.map { candidate ->
candidate.copy(
evidenceLocalIDs = listOf(
evidenceLocalIDs =
requireNotNull(evidenceIDs[candidate.ordinal])
)
)
}
val payload = JSONObject()
@@ -1,14 +1,18 @@
package com.roubao.autopilot.vlm
const val CANDIDATE_EVALUATION_SCHEMA_VERSION = 2
const val CANDIDATE_EVALUATION_PROMPT_VERSION = "candidate-evaluation-v2"
const val CANDIDATE_EVALUATION_SCHEMA_VERSION = 3
const val CANDIDATE_EVALUATION_PROMPT_VERSION = "candidate-evaluation-v3"
const val CANDIDATE_RECOMMENDATION_THRESHOLD = 0.75
data class CandidateEvaluationImage(
val ordinal: Int,
val mediaType: String,
val bytes: ByteArray,
val sha256: String
val sha256: String,
val specificationBytes: ByteArray = bytes,
val specificationSha256: String = sha256,
val localHardConstraintResults: List<CandidateHardConstraintResult> =
emptyList()
)
data class CandidateEvaluationInput(
@@ -56,6 +60,7 @@ data class CandidateAssessment(
val rejectionReasons: List<String>,
val confidence: Double,
val evidenceSha256: String,
val specificationEvidenceSha256: String = evidenceSha256,
val hardConstraintResults: List<CandidateHardConstraintResult> = emptyList()
)
@@ -63,12 +68,17 @@ data class RankedCandidateAssessment(
val sourceOrdinal: Int,
val rankedOrdinal: Int,
val evidenceSha256: String,
val specificationEvidenceSha256: String,
val assessment: CandidateAssessment
) {
init {
require(sourceOrdinal == assessment.ordinal)
require(rankedOrdinal > 0)
require(evidenceSha256 == assessment.evidenceSha256)
require(
specificationEvidenceSha256 ==
assessment.specificationEvidenceSha256
)
}
}
@@ -206,6 +216,8 @@ object CandidateTopFivePolicy {
sourceOrdinal = assessment.ordinal,
rankedOrdinal = index + 1,
evidenceSha256 = assessment.evidenceSha256,
specificationEvidenceSha256 =
assessment.specificationEvidenceSha256,
assessment = assessment
)
}
@@ -36,7 +36,7 @@ class CandidateEvaluator(
retryable = false
)
}
if (!input.isValid()) {
if (!input.isValid(hardConstraints.constraints)) {
return CandidateEvaluationResult.Failed(
code = CandidateEvaluationFailureCode.EVIDENCE_INVALID,
retryable = false
@@ -59,7 +59,8 @@ class CandidateEvaluator(
prompt = CandidateEvaluationPrompt.build(
requirement = input.requirement,
candidateOrdinal = candidate.ordinal,
hardConstraints = hardConstraints.constraints
hardConstraintResults =
candidate.localHardConstraintResults
),
imageMediaType = candidate.mediaType,
imageBytes = candidate.bytes
@@ -95,7 +96,10 @@ class CandidateEvaluator(
rawResponse = rawResponse,
expectedOrdinal = candidate.ordinal,
evidenceSha256 = candidate.sha256,
expectedHardConstraints = hardConstraints.constraints
specificationEvidenceSha256 =
candidate.specificationSha256,
expectedHardConstraintResults =
candidate.localHardConstraintResults
)
}
if (parsed == null) {
@@ -165,7 +169,9 @@ class CandidateEvaluator(
)
}
private fun CandidateEvaluationInput.isValid(): Boolean {
private fun CandidateEvaluationInput.isValid(
expectedConstraints: List<SkuHardConstraint>
): Boolean {
if (candidates.size !in 1..MAX_CANDIDATES) {
return false
}
@@ -175,7 +181,18 @@ class CandidateEvaluator(
candidates.all { candidate ->
candidate.mediaType == SUPPORTED_MEDIA_TYPE &&
candidate.bytes.isNotEmpty() &&
SHA256_PATTERN.matches(candidate.sha256)
candidate.specificationBytes.isNotEmpty() &&
SHA256_PATTERN.matches(candidate.sha256) &&
SHA256_PATTERN.matches(candidate.specificationSha256) &&
candidate.localHardConstraintResults.size ==
expectedConstraints.size &&
candidate.localHardConstraintResults.zip(
expectedConstraints
).all { (result, expected) ->
result.kind == expected.kind &&
result.expected == expected.expected &&
result.evidence.isNotBlank()
}
}
}
@@ -191,7 +208,9 @@ class CandidateEvaluator(
rejectionReasons = emptyList(),
confidence = 0.0,
evidenceSha256 = candidate.sha256,
hardConstraintResults = emptyList()
specificationEvidenceSha256 =
candidate.specificationSha256,
hardConstraintResults = candidate.localHardConstraintResults
)
private companion object {
@@ -205,7 +224,7 @@ object CandidateEvaluationPrompt {
fun build(
requirement: RequirementExtraction,
candidateOrdinal: Int,
hardConstraints: List<SkuHardConstraint>
hardConstraintResults: List<CandidateHardConstraintResult>
): String {
val requirementJson = JSONObject()
.put("search_query", requirement.searchQuery)
@@ -226,11 +245,13 @@ object CandidateEvaluationPrompt {
.put(
"hard_constraints",
JSONArray().apply {
hardConstraints.forEach { constraint ->
hardConstraintResults.forEach { constraint ->
put(
JSONObject()
.put("kind", constraint.kind.name)
.put("expected", constraint.expected)
.put("status", constraint.status.name)
.put("evidence", constraint.evidence)
)
}
}
@@ -244,13 +265,13 @@ object CandidateEvaluationPrompt {
Do not return coordinates, UI actions, state transitions, purchase decisions, order submission, or payment authorization.
decision must be REVIEW, REJECT, or MANUAL_REQUIRED.
REVIEW means potentially suitable for a person to inspect, never permission to buy.
Evaluate every hard constraint using only visible screenshot evidence.
hard constraint status must be MATCH, MISMATCH, or UNKNOWN.
Local read-only specification parsing already determined every hard constraint status.
Copy each provided local status exactly; never override it from visual inference.
REVIEW requires every hard constraint to be MATCH. UNKNOWN requires MANUAL_REQUIRED.
A hard constraint MISMATCH requires REJECT. Never infer a hidden color or size.
Required schema:
{
"schema_version": 2,
"schema_version": 3,
"candidate_index": $candidateOrdinal,
"decision": "REVIEW",
"score": 0.0,
@@ -295,7 +316,8 @@ private object CandidateAssessmentParser {
rawResponse: String,
expectedOrdinal: Int,
evidenceSha256: String,
expectedHardConstraints: List<SkuHardConstraint>
specificationEvidenceSha256: String,
expectedHardConstraintResults: List<CandidateHardConstraintResult>
): CandidateAssessment? =
runCatching {
val root = JSONObject(rawResponse.trim())
@@ -313,7 +335,7 @@ private object CandidateAssessmentParser {
val confidence = root.strictUnitDouble("confidence")
val hardConstraintResults = root
.getJSONArray("hard_constraint_results")
.strictHardConstraintResults(expectedHardConstraints)
.strictHardConstraintResults(expectedHardConstraintResults)
when (decision) {
CandidateDecision.REVIEW -> {
require(matched.isNotEmpty())
@@ -351,12 +373,14 @@ private object CandidateAssessmentParser {
rejectionReasons = rejectionReasons,
confidence = confidence,
evidenceSha256 = evidenceSha256,
specificationEvidenceSha256 =
specificationEvidenceSha256,
hardConstraintResults = hardConstraintResults
)
}.getOrNull()
private fun JSONArray.strictHardConstraintResults(
expected: List<SkuHardConstraint>
expected: List<CandidateHardConstraintResult>
): List<CandidateHardConstraintResult> {
require(length() == expected.size)
return buildList(length()) {
@@ -375,16 +399,13 @@ private object CandidateAssessmentParser {
val expectedValue = item.getString("expected").trim()
require(kind == constraint.kind)
require(expectedValue == constraint.expected)
val status = HardConstraintMatchStatus.valueOf(
item.getString("status")
)
require(status == constraint.status)
item.getString("evidence").validatedAssessmentText()
add(
CandidateHardConstraintResult(
kind = kind,
expected = expectedValue,
status = HardConstraintMatchStatus.valueOf(
item.getString("status")
),
evidence = item.getString("evidence")
.validatedAssessmentText()
)
constraint
)
}
}
@@ -469,6 +490,10 @@ object CandidateReviewBatchJson {
)
.put("confidence", assessment.confidence)
.put("evidence_sha256", assessment.evidenceSha256)
.put(
"specification_evidence_sha256",
assessment.specificationEvidenceSha256
)
.put(
"hard_constraint_results",
JSONArray().apply {
@@ -1,5 +1,7 @@
package com.roubao.autopilot.vlm
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationEvidence
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationGroupKind
import java.text.Normalizer
enum class SkuConstraintKind {
@@ -99,3 +101,136 @@ object SkuHardConstraintExtractor {
"""(?:^|[^0-9])(\d{2}(?:\.\d)?)(?:码|SIZE)(?=$|[^A-Z0-9])"""
)
}
object CandidateSpecificationHardConstraintPolicy {
fun evaluate(
constraints: List<SkuHardConstraint>,
evidence: PinduoduoSpecificationEvidence,
evidenceSha256: String
): List<CandidateHardConstraintResult> =
constraints.map { constraint ->
val kind = when (constraint.kind) {
SkuConstraintKind.COLOR ->
PinduoduoSpecificationGroupKind.COLOR
SkuConstraintKind.SIZE ->
PinduoduoSpecificationGroupKind.SIZE
}
val matchingGroups = evidence.groups.filter { it.kind == kind }
if (matchingGroups.size != 1) {
return@map result(
constraint,
HardConstraintMatchStatus.UNKNOWN,
audit(constraint, "group_missing_or_duplicate", evidenceSha256)
)
}
val group = matchingGroups.single()
val matchingOptions = group.options.filter { option ->
optionMatches(constraint, option.text)
}
when {
matchingOptions.size > 1 ->
result(
constraint,
HardConstraintMatchStatus.UNKNOWN,
audit(constraint, "target_duplicate", evidenceSha256)
)
matchingOptions.singleOrNull()?.enabled == true ->
result(
constraint,
HardConstraintMatchStatus.MATCH,
audit(constraint, "target_enabled", evidenceSha256)
)
matchingOptions.singleOrNull()?.enabled == false ->
result(
constraint,
HardConstraintMatchStatus.MISMATCH,
audit(constraint, "target_disabled", evidenceSha256)
)
group.complete ->
result(
constraint,
HardConstraintMatchStatus.MISMATCH,
audit(constraint, "target_absent_complete", evidenceSha256)
)
else ->
result(
constraint,
HardConstraintMatchStatus.UNKNOWN,
audit(constraint, "group_incomplete", evidenceSha256)
)
}
}
private fun result(
constraint: SkuHardConstraint,
status: HardConstraintMatchStatus,
evidence: String
) = CandidateHardConstraintResult(
kind = constraint.kind,
expected = constraint.expected,
status = status,
evidence = evidence
)
private fun audit(
constraint: SkuHardConstraint,
observation: String,
evidenceSha256: String
): String =
"group=${constraint.kind.name};target=${constraint.expected};" +
"observation=$observation;spec_sha256=$evidenceSha256"
private fun optionMatches(
constraint: SkuHardConstraint,
observed: String
): Boolean {
val normalized = Normalizer.normalize(
observed.trim(),
Normalizer.Form.NFKC
).uppercase()
return when (constraint.kind) {
SkuConstraintKind.COLOR ->
COLOR_ALIASES[constraint.expected].orEmpty()
.any { alias -> normalized.matchesAlias(alias) }
SkuConstraintKind.SIZE ->
sizeAliases(constraint.expected).any { alias ->
Regex(
"""(?:^|[^A-Z0-9])${Regex.escape(alias)}""" +
"""(?=$|[^A-Z0-9])"""
).containsMatchIn(normalized)
}
}
}
private fun String.matchesAlias(alias: String): Boolean =
if (alias.all { it in 'A'..'Z' }) {
Regex(
"""(?:^|[^A-Z])${Regex.escape(alias)}(?=$|[^A-Z])"""
).containsMatchIn(this)
} else {
contains(alias)
}
private fun sizeAliases(expected: String): Set<String> =
when (expected) {
"2XL" -> setOf("2XL", "XXL")
"3XL" -> setOf("3XL", "XXXL")
"FREE" -> setOf("FREE", "FREESIZE", "均码")
else -> setOf(expected)
}
private val COLOR_ALIASES = mapOf(
"BLACK" to listOf("BLACK", "黑色", "亮黑", "纯黑"),
"WHITE" to listOf("WHITE", "白色", "纯白", "米白"),
"GRAY" to listOf("GRAY", "GREY", "灰色", "浅灰", "深灰"),
"RED" to listOf("RED", "红色", "酒红", "玫红"),
"BLUE" to listOf("BLUE", "蓝色", "藏青", "牛仔蓝"),
"GREEN" to listOf("GREEN", "绿色", "军绿"),
"YELLOW" to listOf("YELLOW", "黄色"),
"PINK" to listOf("PINK", "粉色", "粉红"),
"PURPLE" to listOf("PURPLE", "紫色"),
"BROWN" to listOf("BROWN", "棕色", "咖色", "咖啡色"),
"BEIGE" to listOf("BEIGE", "米色", "卡其"),
"ORANGE" to listOf("ORANGE", "橙色")
)
}
@@ -18,14 +18,25 @@ class CandidateEvidenceSourceTest {
assertTrue(result.isSuccess)
assertEquals(listOf(1, 2), result.getOrThrow().map { it.ordinal })
assertTrue(
result.getOrThrow().all {
it.detailPngBytes.isNotEmpty() &&
it.specificationPngBytes.isNotEmpty()
}
)
}
}
@Test
fun `rejects non allowlisted filename before reading`() = runTest {
withEvidenceRoot { root ->
val evidence = writeCandidate(root, ordinal = 1, width = 10, height = 20)
.copy(screenshotFileName = "../candidate-01.png")
val original =
writeCandidate(root, ordinal = 1, width = 10, height = 20)
val evidence = original.copy(
detailAsset = original.detailAsset.copy(
fileName = "../candidate-01-detail.png"
)
)
val result = CandidateEvidenceSource(root).load(listOf(evidence))
@@ -56,17 +67,33 @@ class CandidateEvidenceSourceTest {
assertTrue(
source.load(
listOf(evidence.copy(screenshotByteCount = evidence.screenshotByteCount + 1))
listOf(
evidence.copy(
detailAsset = evidence.detailAsset.copy(
byteCount = evidence.detailAsset.byteCount + 1
)
)
)
).isFailure
)
assertTrue(
source.load(
listOf(evidence.copy(screenshotSha256 = "0".repeat(64)))
listOf(
evidence.copy(
detailAsset = evidence.detailAsset.copy(
sha256 = "0".repeat(64)
)
)
)
).isFailure
)
assertTrue(
source.load(
listOf(evidence.copy(screenshotWidth = 11))
listOf(
evidence.copy(
detailAsset = evidence.detailAsset.copy(width = 11)
)
)
).isFailure
)
}
@@ -76,12 +103,18 @@ class CandidateEvidenceSourceTest {
fun `rejects invalid png header and declared size above limit`() = runTest {
withEvidenceRoot { root ->
val evidence = writeCandidate(root, ordinal = 1, width = 10, height = 20)
File(root, evidence.screenshotFileName).writeBytes(ByteArray(24))
File(root, evidence.detailAsset.fileName).writeBytes(ByteArray(24))
assertTrue(CandidateEvidenceSource(root).load(listOf(evidence)).isFailure)
assertTrue(
CandidateEvidenceSource(root).load(
listOf(evidence.copy(screenshotByteCount = 8 * 1024 * 1024 + 1))
listOf(
evidence.copy(
detailAsset = evidence.detailAsset.copy(
byteCount = 8 * 1024 * 1024 + 1
)
)
)
).isFailure
)
}
@@ -103,22 +136,53 @@ class CandidateEvidenceSourceTest {
height: Int
): PinduoduoCandidateEvidence {
val bytes = pngHeader(width, height)
val fileName = "candidate-%02d.png".format(ordinal)
File(root, fileName).writeBytes(bytes)
val detailFileName = "candidate-%02d-detail.png".format(ordinal)
val specificationFileName =
"candidate-%02d-specification.png".format(ordinal)
File(root, detailFileName).writeBytes(bytes)
File(root, specificationFileName).writeBytes(bytes)
val asset = PinduoduoEvidenceAsset(
fileName = detailFileName,
sha256 = PinduoduoEvidenceHash.sha256(bytes),
byteCount = bytes.size,
width = width,
height = height
)
return PinduoduoCandidateEvidence(
ordinal = ordinal,
cardSignature = "card-$ordinal",
cardSemanticTextCount = 3,
detailSignature = "detail-$ordinal",
detailSemanticTextCount = 4,
screenshotFileName = fileName,
screenshotSha256 = PinduoduoEvidenceHash.sha256(bytes),
screenshotByteCount = bytes.size,
screenshotWidth = width,
screenshotHeight = height
detailAsset = asset,
specification = specification(),
specificationAsset = asset.copy(fileName = specificationFileName)
)
}
private fun specification() = PinduoduoSpecificationEvidence(
signature = "specification",
semanticTextCount = 6,
groups = listOf(
PinduoduoSpecificationGroup(
kind = PinduoduoSpecificationGroupKind.COLOR,
title = "颜色分类",
options = listOf(
PinduoduoSpecificationOption("黑色", false, true)
),
complete = true
),
PinduoduoSpecificationGroup(
kind = PinduoduoSpecificationGroupKind.SIZE,
title = "尺码",
options = listOf(
PinduoduoSpecificationOption("L", false, true)
),
complete = true
)
)
)
private fun pngHeader(width: Int, height: Int): ByteArray =
byteArrayOf(
0x89.toByte(), 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
@@ -156,6 +156,31 @@ class PinduoduoCandidateAutomationTest {
assertTrue(automation.evidence.value.isEmpty())
}
@Test
fun `missing safe specification entry blocks without purchase fallback`() =
runTest {
val driver = FakeCandidateDriver(
cardPages = listOf(listOf(card("a"))),
failOpenSpecifications = true
)
val automation = PinduoduoCandidateAutomation(
driver = driver,
pagePollIntervalMillis = 1
)
automation.reset()
val result = automation.execute(
PinduoduoCandidateWorkflow.steps().last()
)
assertEquals(
AutomationResult.Blocked(SafetyStopReason.UNKNOWN_PAGE),
result
)
assertEquals(1, driver.openSpecificationCalls)
assertTrue(driver.savedEvidence.isEmpty())
}
private fun card(signature: String) = PinduoduoCandidateCard(
signature = signature,
semanticTextCount = 3,
@@ -166,6 +191,7 @@ class PinduoduoCandidateAutomationTest {
private val cardPages: List<List<PinduoduoCandidateCard>>,
private val safetyStopReason: SafetyStopReason? = null,
private val failCapture: Boolean = false,
private val failOpenSpecifications: Boolean = false,
private val transitionDelaySnapshots: Int = 0
) : PinduoduoCandidateDriver {
private var cardPageIndex = 0
@@ -176,6 +202,7 @@ class PinduoduoCandidateAutomationTest {
val openedSignatures = mutableListOf<String>()
val savedEvidence = mutableListOf<PinduoduoCandidateEvidence>()
var scrollCalls = 0
var openSpecificationCalls = 0
override suspend fun snapshot(): PinduoduoUiSnapshot {
if (pendingPage != null) {
@@ -204,24 +231,60 @@ class PinduoduoCandidateAutomationTest {
return true
}
override suspend fun captureCandidate(
ordinal: Int,
card: PinduoduoCandidateCard
): PinduoduoCandidateEvidence? {
override suspend fun captureDetail(): PinduoduoCandidateCapture? {
if (failCapture) {
return null
}
return PinduoduoCandidateCapture(
detail = PinduoduoCandidateDetailEvidence("detail", 10),
screenshot = PinduoduoScreenshotCapture(
pngBytes = byteArrayOf(1),
width = 1080,
height = 2400
)
)
}
override suspend fun openSpecifications(): Boolean {
openSpecificationCalls += 1
if (failOpenSpecifications) {
return false
}
transitionTo(PinduoduoPage.SPECIFICATION_PANEL)
return true
}
override suspend fun captureSpecifications():
PinduoduoSpecificationCapture =
PinduoduoSpecificationCapture(
evidence = specification(),
screenshot = PinduoduoScreenshotCapture(
pngBytes = byteArrayOf(2),
width = 1080,
height = 2400
)
)
override suspend fun closeSpecifications(): Boolean {
transitionTo(PinduoduoPage.PRODUCT_DETAIL)
return true
}
override suspend fun saveCandidate(
ordinal: Int,
card: PinduoduoCandidateCard,
detail: PinduoduoCandidateCapture,
specifications: PinduoduoSpecificationCapture
): PinduoduoCandidateEvidence? {
return PinduoduoCandidateEvidence(
ordinal = ordinal,
cardSignature = card.signature,
cardSemanticTextCount = card.semanticTextCount,
detailSignature = "detail-${card.signature}",
detailSemanticTextCount = 10,
screenshotFileName = "candidate-%02d.png".format(ordinal),
screenshotSha256 = "hash-$ordinal",
screenshotByteCount = 100 + ordinal,
screenshotWidth = 1080,
screenshotHeight = 2400
detailAsset = asset(ordinal, "detail"),
specification = specifications.evidence,
specificationAsset = asset(ordinal, "specification")
).also(savedEvidence::add)
}
@@ -242,6 +305,7 @@ class PinduoduoCandidateAutomationTest {
openedSignatures.clear()
savedEvidence.clear()
scrollCalls = 0
openSpecificationCalls = 0
cardPageIndex = 0
page = PinduoduoPage.SEARCH_RESULTS
pendingPage = null
@@ -256,5 +320,39 @@ class PinduoduoCandidateAutomationTest {
delayedSnapshotsRemaining = transitionDelaySnapshots
}
}
private fun asset(
ordinal: Int,
suffix: String
) = PinduoduoEvidenceAsset(
fileName = "candidate-%02d-%s.png".format(ordinal, suffix),
sha256 = "a".repeat(64),
byteCount = 1,
width = 1080,
height = 2400
)
private fun specification() = PinduoduoSpecificationEvidence(
signature = "specification",
semanticTextCount = 6,
groups = listOf(
PinduoduoSpecificationGroup(
kind = PinduoduoSpecificationGroupKind.COLOR,
title = "颜色分类",
options = listOf(
PinduoduoSpecificationOption("黑色", false, true)
),
complete = true
),
PinduoduoSpecificationGroup(
kind = PinduoduoSpecificationGroupKind.SIZE,
title = "尺码",
options = listOf(
PinduoduoSpecificationOption("L", false, true)
),
complete = true
)
)
)
}
}
@@ -202,6 +202,41 @@ class PinduoduoPageClassifierTest {
assertNull(snapshot.safetyStopReason)
}
@Test
fun `specification panel requires color size and options`() {
val snapshot = classify(
element(text = "颜色分类"),
element(text = "黑色", clickable = true),
element(text = "尺码"),
element(text = "L", clickable = true)
)
assertEquals(PinduoduoPage.SPECIFICATION_PANEL, snapshot.page)
assertNull(snapshot.safetyStopReason)
}
@Test
fun `cart and order pages are never classified as product detail`() {
val cart = classify(
element(text = "购物车"),
element(text = "去结算")
)
val order = classify(
element(contentDescription = "返回", clickable = true),
element(text = "颜色分类"),
element(text = "黑色", clickable = true),
element(text = "尺码"),
element(text = "L", clickable = true),
element(text = "确认订单")
)
assertEquals(PinduoduoPage.UNKNOWN, cart.page)
assertEquals(
SafetyStopReason.PAYMENT_BOUNDARY,
order.safetyStopReason
)
}
@Test
fun `other foreground package is unknown and has no inferred blocker`() {
val snapshot = PinduoduoPageClassifier.classify(
@@ -0,0 +1,98 @@
package com.roubao.autopilot.pinduoduo
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
import org.junit.Test
class PinduoduoSpecificationParserTest {
@Test
fun `parses bounded color and size groups without clicking options`() {
val evidence = PinduoduoSpecificationParser.parse(
listOf(
element("颜色分类", top = 100),
element("黑色", top = 140, clickable = true, selected = true),
element("白色", top = 180, clickable = true),
element("尺码", top = 240),
element("L", top = 280, clickable = true),
element("XL", top = 320, clickable = true, enabled = false)
)
)
requireNotNull(evidence)
assertEquals(
PinduoduoSpecificationGroupKind.entries.toSet(),
evidence.groups.map { it.kind }.toSet()
)
assertTrue(evidence.groups.all { it.complete })
assertTrue(evidence.groups.first().options.first().selected)
assertFalse(evidence.groups.last().options.last().enabled)
}
@Test
fun `scrollable group is marked incomplete`() {
val evidence = PinduoduoSpecificationParser.parse(
listOf(
element("颜色分类", top = 100),
element("黑色", top = 140, clickable = true),
element("", top = 150, scrollable = true),
element("尺码", top = 240),
element("L", top = 280, clickable = true)
)
)
requireNotNull(evidence)
assertFalse(evidence.groups.first().complete)
assertTrue(evidence.groups.last().complete)
}
@Test
fun `missing group and transaction controls are rejected`() {
assertNull(
PinduoduoSpecificationParser.parse(
listOf(
element("颜色分类", top = 100),
element("立即购买", top = 140, clickable = true)
)
)
)
}
@Test
fun `safe entry allowlist excludes purchase controls`() {
assertTrue(
PinduoduoSpecificationParser.isSafeEntryText("请选择规格")
)
assertTrue(
PinduoduoSpecificationParser.isSafeEntryText("已选:黑色,L")
)
assertTrue(
PinduoduoSpecificationParser.isSafeEntryText("颜色\n款式")
)
assertFalse(
PinduoduoSpecificationParser.isSafeEntryText("免拼购买")
)
}
private fun element(
text: String,
top: Int,
clickable: Boolean = false,
enabled: Boolean = true,
selected: Boolean = false,
scrollable: Boolean = false
) = PinduoduoUiElement(
text = text,
contentDescription = null,
className = "android.widget.TextView",
resourceId = null,
clickable = clickable,
editable = false,
enabled = enabled,
visibleToUser = true,
selected = selected,
scrollable = scrollable,
boundsTop = top
)
}
@@ -0,0 +1,64 @@
package com.roubao.autopilot.procurement
import org.junit.Assert.assertEquals
import org.junit.Assert.assertThrows
import org.junit.Test
class ExecutionEvidenceAssociationPolicyTest {
@Test
fun `binds detail and specification assets to each ranked candidate`() {
val grouped = ExecutionEvidenceAssociationPolicy.group(
candidateOrdinals = listOf(1, 2),
evidence = listOf(
draft(2, ExecutionEvidenceKind.SPECIFICATION),
draft(1, ExecutionEvidenceKind.DETAIL),
draft(2, ExecutionEvidenceKind.DETAIL),
draft(1, ExecutionEvidenceKind.SPECIFICATION)
)
)
assertEquals(listOf(1, 2), grouped.keys.toList())
assertEquals(
ExecutionEvidenceKind.entries,
grouped.getValue(1).map { it.kind }
)
assertEquals(
ExecutionEvidenceKind.entries,
grouped.getValue(2).map { it.kind }
)
}
@Test
fun `rejects missing duplicate and wrong candidate assets`() {
listOf(
listOf(
draft(1, ExecutionEvidenceKind.DETAIL)
),
listOf(
draft(1, ExecutionEvidenceKind.DETAIL),
draft(1, ExecutionEvidenceKind.DETAIL)
),
listOf(
draft(1, ExecutionEvidenceKind.DETAIL),
draft(2, ExecutionEvidenceKind.SPECIFICATION)
)
).forEach { evidence ->
assertThrows(IllegalArgumentException::class.java) {
ExecutionEvidenceAssociationPolicy.group(
candidateOrdinals = listOf(1),
evidence = evidence
)
}
}
}
private fun draft(
ordinal: Int,
kind: ExecutionEvidenceKind
) = ExecutionEvidenceDraft(
ordinal = ordinal,
pngBytes = byteArrayOf(ordinal.toByte(), kind.ordinal.toByte()),
sha256 = "a".repeat(64),
kind = kind
)
}
@@ -159,7 +159,12 @@ class CandidateEvaluatorTest {
)
}
val result = evaluator.evaluate(input(candidateCount = 2))
val result = evaluator.evaluate(
input(
candidateCount = 2,
hardStatus = HardConstraintMatchStatus.MISMATCH
)
)
as CandidateEvaluationResult.Completed
assertEquals(CandidateBatchConclusion.NO_MATCH, result.batch.conclusion)
@@ -305,13 +310,40 @@ class CandidateEvaluatorTest {
hardStatus = HardConstraintMatchStatus.UNKNOWN
)
)
}.evaluate(input(candidateCount = 1))
}.evaluate(
input(
candidateCount = 1,
hardStatus = HardConstraintMatchStatus.UNKNOWN
)
)
as CandidateEvaluationResult.Completed
assertTrue(CandidateTopFivePolicy.select(result.batch).isEmpty())
assertNull(result.batch.recommendedCandidateOrdinal)
}
@Test
fun `model cannot override local specification status`() = runTest {
val response = validResponse(
ordinal = 1,
decision = CandidateDecision.REJECT,
score = 0.1,
matched = emptyList(),
rejectionReasons = listOf("模型声称颜色不匹配"),
hardStatus = HardConstraintMatchStatus.MISMATCH
)
val result = evaluator { Result.success(response) }
.evaluate(input(candidateCount = 1))
as CandidateEvaluationResult.Completed
assertEquals(
CandidateDecision.MANUAL_REQUIRED,
result.batch.assessments.single().decision
)
assertNull(result.batch.recommendedCandidateOrdinal)
}
@Test
fun `encoded review batch fixes order submitted to false`() = runTest {
val result = evaluator { Result.success(validResponse(1)) }
@@ -323,6 +355,12 @@ class CandidateEvaluatorTest {
assertFalse(json.getBoolean("order_submitted"))
assertTrue(json.getBoolean("manual_review_required"))
assertEquals(1, json.getJSONArray("candidates").length())
assertEquals(
"e".repeat(64),
json.getJSONArray("candidates")
.getJSONObject(0)
.getString("specification_evidence_sha256")
)
}
private fun evaluator(
@@ -336,18 +374,44 @@ class CandidateEvaluatorTest {
model = "fake-model"
)
private fun input(candidateCount: Int): CandidateEvaluationInput =
private fun input(
candidateCount: Int,
hardStatus: HardConstraintMatchStatus =
HardConstraintMatchStatus.MATCH
): CandidateEvaluationInput =
CandidateEvaluationInput(
requirement = requirement(),
candidates = (1..candidateCount).map(::candidate)
candidates = (1..candidateCount).map {
candidate(it, hardStatus)
}
)
private fun candidate(ordinal: Int): CandidateEvaluationImage =
private fun candidate(
ordinal: Int,
hardStatus: HardConstraintMatchStatus =
HardConstraintMatchStatus.MATCH
): CandidateEvaluationImage =
CandidateEvaluationImage(
ordinal = ordinal,
mediaType = "image/png",
bytes = byteArrayOf(ordinal.toByte()),
sha256 = ordinal.toString(16).padStart(64, '0')
sha256 = ordinal.toString(16).padStart(64, '0'),
specificationBytes = byteArrayOf((ordinal + 10).toByte()),
specificationSha256 = "e".repeat(64),
localHardConstraintResults = listOf(
CandidateHardConstraintResult(
kind = SkuConstraintKind.COLOR,
expected = "BLACK",
status = hardStatus,
evidence = "本地规格颜色证据"
),
CandidateHardConstraintResult(
kind = SkuConstraintKind.SIZE,
expected = "L",
status = hardStatus,
evidence = "本地规格尺码证据"
)
)
)
private fun requirement(): RequirementExtraction =
@@ -0,0 +1,148 @@
package com.roubao.autopilot.vlm
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationEvidence
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationGroup
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationGroupKind
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationOption
import org.junit.Assert.assertEquals
import org.junit.Assert.assertTrue
import org.junit.Test
import java.nio.charset.StandardCharsets
class CandidateSpecificationHardConstraintPolicyTest {
@Test
fun `both explicit enabled targets match`() {
assertEquals(
listOf(
HardConstraintMatchStatus.MATCH,
HardConstraintMatchStatus.MATCH
),
evaluate(
colorOptions = listOf(option("经典黑色")),
sizeOptions = listOf(option("XXL"))
).map { it.status }
)
assertTrue(
evaluate(
colorOptions = listOf(option("经典黑色")),
sizeOptions = listOf(option("XXL"))
).all {
it.evidence.toByteArray(StandardCharsets.UTF_8).size <= 160
}
)
}
@Test
fun `complete missing target is mismatch`() {
assertEquals(
HardConstraintMatchStatus.MISMATCH,
evaluate(
colorOptions = listOf(option("白色")),
sizeOptions = listOf(option("2XL"))
).first().status
)
}
@Test
fun `disabled target is mismatch`() {
assertEquals(
HardConstraintMatchStatus.MISMATCH,
evaluate(
colorOptions = listOf(option("黑色", enabled = false)),
sizeOptions = listOf(option("2XL"))
).first().status
)
}
@Test
fun `truncated group and duplicated matching option are unknown`() {
assertEquals(
HardConstraintMatchStatus.UNKNOWN,
evaluate(
colorOptions = listOf(option("白色")),
sizeOptions = listOf(option("2XL")),
colorComplete = false
).first().status
)
assertEquals(
HardConstraintMatchStatus.UNKNOWN,
evaluate(
colorOptions = listOf(option("黑色"), option("纯黑")),
sizeOptions = listOf(option("2XL"))
).first().status
)
}
@Test
fun `missing group is unknown`() {
val evidence = specification(
colorOptions = listOf(option("黑色")),
sizeOptions = listOf(option("2XL"))
).copy(groups = specification(
colorOptions = listOf(option("黑色")),
sizeOptions = listOf(option("2XL"))
).groups.filter { it.kind == PinduoduoSpecificationGroupKind.COLOR })
val results = CandidateSpecificationHardConstraintPolicy.evaluate(
constraints = constraints(),
evidence = evidence,
evidenceSha256 = "b".repeat(64)
)
assertEquals(
HardConstraintMatchStatus.UNKNOWN,
results.last().status
)
}
private fun evaluate(
colorOptions: List<PinduoduoSpecificationOption>,
sizeOptions: List<PinduoduoSpecificationOption>,
colorComplete: Boolean = true
) = CandidateSpecificationHardConstraintPolicy.evaluate(
constraints = constraints(),
evidence = specification(
colorOptions,
sizeOptions,
colorComplete
),
evidenceSha256 = "b".repeat(64)
)
private fun constraints() = listOf(
SkuHardConstraint(SkuConstraintKind.COLOR, "BLACK"),
SkuHardConstraint(SkuConstraintKind.SIZE, "2XL")
)
private fun specification(
colorOptions: List<PinduoduoSpecificationOption>,
sizeOptions: List<PinduoduoSpecificationOption>,
colorComplete: Boolean = true
) = PinduoduoSpecificationEvidence(
signature = "specification",
semanticTextCount = 8,
groups = listOf(
PinduoduoSpecificationGroup(
PinduoduoSpecificationGroupKind.COLOR,
"颜色分类",
colorOptions,
colorComplete
),
PinduoduoSpecificationGroup(
PinduoduoSpecificationGroupKind.SIZE,
"尺码",
sizeOptions,
true
)
)
)
private fun option(
text: String,
enabled: Boolean = true
) = PinduoduoSpecificationOption(
text = text,
selected = false,
enabled = enabled
)
}
+33 -11
View File
@@ -257,22 +257,44 @@ T-102/T-104 在搜索结果后追加一个有界候选步骤:
-> 最多 2 次滚动预算
-> 最多 5 个去重商品卡
-> 验证详情页
-> 无障碍截图
-> App cache/candidate-NN.png + manifest.json
-> 详情页无障碍截图
-> 唯一、安全、非交易规格入口
-> 规格弹层只读语义和截图
-> App cache/candidate-NN-{detail,specification}.png + manifest.json
-> 关闭规格弹层并复核详情页
-> 一次全局返回并复核固定词结果页
```
候选卡只保留语义指纹和计数,详情截图保存在 App 内部 cache。manifest 记录匿名文件名、
截图 SHA-256、字节数和尺寸,不保存商品标题或页面原文。T-104 使用这些证据时必须通过
受控 evidence 边界读取,不能让 VLM adapter 自行遍历 cache。Android 10/API 29 及
以下不能运行当前截图探针,应在预检时明确不支持,不使用媒体投影或 shell 绕过。
候选卡只保留语义指纹和计数,详情/规格截图保存在 App 内部 cache。manifest v2 记录
匿名文件名、截图 SHA-256、字节数、尺寸和有界规格语义,不保存商品标题或完整页面
原文。T-104/T-213 使用这些证据时必须通过受控 evidence 边界读取,不能让 VLM adapter
自行遍历 cache。Android 10/API 29 及以下不能运行当前截图探针,应在预检时明确
不支持,不使用媒体投影或 shell 绕过。
`CandidateEvidenceSource` 只接受当前 workflow 内存中的连续 ordinal 元数据,文件名
固定为 `candidate-01.png` 至 `candidate-05.png`;每张 PNG 最多 8 MiB、全批最多
32 MiB、声明尺寸最长边不超过 10000 px,并复核规范路径、PNG/IHDR、精确字节数、
尺寸和 SHA-256。Android gateway 再解码并把最长边缩至 2048 px。当前需求快照与
搜索词必须同时匹配候选 session,否则拒绝评估;另一个关键词的结果页只允许重新进入
搜索框,不能采集候选。
固定为 `candidate-01-detail.png`/`candidate-01-specification.png` 至第五组;每张
PNG 最多 8 MiB、全批最多 64 MiB、声明尺寸最长边不超过 10000 px,并复核规范路径、
PNG/IHDR、精确字节数、尺寸和 SHA-256。每个候选回传时必须恰好绑定
`DETAIL`/`SPECIFICATION` 两份证据,生成独立 asset ID;先验证全批 ordinal、类型和
哈希再写入加密 outbox,候选的 asset ID 数组固定按 `DETAIL`、`SPECIFICATION`
排序,重试沿用各自 idempotency key。Android gateway 再解码详情图并把最长边缩至
2048 px。当前需求快照与搜索词必须同时匹配候选 session,否则拒绝评估;另一个
关键词的结果页只允许重新进入搜索框,不能采集候选。
T-213 规格核验只允许精确语义为“选择规格/请选择规格/颜色分类/颜色款式/选择颜色/
选择尺码”或“已选/请选择”前缀的唯一节点及其可点击祖先。弹层内不点击任何规格选项,
只读取最多 160 个语义节点、每组最多 30 个选项及 selected/enabled/scrollable 语义。
颜色和尺码各自必须是唯一分组:目标值明确且可用为 `MATCH`,完整分组明确缺失或目标
禁用为 `MISMATCH`,重复、缺组、滚动截断或无障碍语义不足为 `UNKNOWN`。
候选评估 schema/prompt v3 把本地规格结果作为权威输入;模型返回的颜色/尺码状态必须
逐项与本地结果相同,否则整批降级人工检查。详情和规格证据 SHA-256 同时绑定
assessment、重排身份和 outbox 资产,VLM 不能通过主图推断覆盖本地硬约束。
PKG110 上的拼多多 8.17.0 服装详情只暴露不可点击的“颜色/款式”预览;完整颜色/尺码
弹层只能通过“单独购买/免拼购买”控件进入。自动化不得点击这两个交易语义控件,也
不得用坐标降级,因此当前版本在没有独立可点击规格入口时返回安全阻塞。是否把购买
语义控件重新定义为只读弹层入口,必须单独做产品/安全决策后变更本边界。
评估结束后 automation 不再产生动作,UI 进入 `AWAITING_CONFIRMATION`、
`MANUAL_REVIEW` 或 `NO_MATCH`。人员可以把本地建议项标记为可用,或拒绝本次候选;
+20 -15
View File
@@ -5,9 +5,9 @@
## 当前快照
- 日期:2026-07-27
- 阶段:T-212 候选重排身份映射完成;下一步 T-213 只读核验拼多多规格弹窗
- 阶段:T-212 已完成;T-213 只读规格核验已实现但被拼多多 8.17 入口语义阻塞
- Git:当前分支为 `main`;T-001 至 T-004、T-101 至 T-104、T-201 至 T-207、T-209、
T-210、T-211、T-212 均已纳入 Git 历史
T-210、T-211、T-212 均已纳入 Git 历史;T-213 等待安全策略决策
- 生产代码:`android-buyer/` 已接入 Roubao Android 源码
- Android:固定 `main@c8a6d7f03422eb01744b01f3ee77bf7757741f7e`;MIT 许可证已保留
@@ -17,10 +17,10 @@
- 本机 Android 工具:JDK 17.0.13、Command-line Tools 22.0、SDK 34、
Build Tools 34.0.0、Platform Tools/ADB 37.0.0;用户级 SDK 环境变量已设置
- Android Studio:未安装;`winget` 静默安装卡住后已终止,不阻塞命令行构建
- 测试:T-212 运行 `:app:testDebugUnitTest` 和 `:app:assembleDebug` 通过,21 个 suite
共 113 个测试、0 失败;T-211 Debug APK 已安装到 PKG110
- 后端测试:T-211 运行 `GOTOOLCHAIN=local go test -count=1 ./...` 和
`go vet ./...` 通过;T-207 的全包 race 与 migration 验证继续有效
- 测试:T-213 运行 `:app:testDebugUnitTest` 和 `:app:assembleDebug` 通过,24 个 suite
共 128 个测试、0 失败;Debug APK `1.4.3 (8)` 已安装到 PKG110
- 后端测试:T-213 运行 `go test ./...` 通过;T-207 的全包 race 与 migration
验证继续有效
- 原型:4 个管理 Web 页面和 7 个 Android 页面均可离线独立打开;Playwright
以 1440×900、390×844、360×800 验证 36 个页面/视口组合,无页面横向溢出、
脚本错误或外部请求,Android 可见交互控件均不小于 44px
@@ -47,9 +47,13 @@
顺序回传,授权到期补报单独审计,所有终态固定 `order_submitted=false`。管理任务
详情展示模型/候选/人工理由/事件/证据摘要,不保存 VLM Key、完整 endpoint 或原始响应。
- T-211 图片检索:后台任务固定使用经 SHA-256 校验的参考 JPEG,经一次性 MediaStore
图片进入拼多多拍照搜索;App 本地从 SKU 唯一提取颜色和尺码,schema v2 逐项返回
图片进入拼多多拍照搜索;App 本地从 SKU 唯一提取颜色和尺码,schema v3 逐项返回
`MATCH/MISMATCH/UNKNOWN`。只有两项均匹配且分数/置信度不低于 `0.75` 的候选按
分数、置信度和曝光顺序回传 `0..5` 项,弱匹配和未知项不凑数。
- T-213 规格核验:只允许唯一、非交易规格入口;弹层内不点击选项,本地根据有界的
颜色/尺码分组、可用/禁用和完整性语义产生硬约束结果。每个候选绑定详情/规格两张
PNG 及两个 SHA-256/asset ID;模型不能覆盖本地状态。拼多多 8.17 服装详情的完整
规格弹层只挂在“单独购买/免拼购买”后,当前安全边界拒绝该入口,故真机流程阻塞。
- Workflow:纯 Kotlin runner 已支持步骤 timeout、最多 3 次 retry、安全阻塞、
用户停止和单 runner 并发拒绝;T-102 已接入搜索加有界候选采集五步
- TaskSource:严格 CLI 已生成并验证真实私有 ProbeTask;默认 APK 不含私有 fixture
@@ -65,7 +69,7 @@
- 离线执行:默认 30 分钟有限授权和 30 秒 best-effort heartbeat 已实现;PKG110
真机断开临时后端 95 秒后保持同一 execution,重连后滑动续期;到期持久安全停止,
RUNNING 不自动重新分配
- 测试设备:OnePlus PKG110,Android 16/API 36;肉包 `1.4.2 (7)`;拼多多
- 测试设备:OnePlus PKG110,Android 16/API 36;肉包 `1.4.3 (8)`;拼多多
`8.17.0 (81700)`
- 设备就绪:拼多多首页、文字/图片搜索、双列候选卡、详情截图和返回均已通过 8.17.0
真机验证;最终 APK 重装/force-stop 后 ColorOS 已关闭肉包采购无障碍,当前需采购员
@@ -80,8 +84,10 @@
已用 CLI 真实导入并逐字段/图片哈希验证,生成物位于被忽略的 `.local/`
- 标准启动路径:`$env:RUN_START_COMMAND="1"; .\init.ps1`
- 标准验证路径:`.\init.ps1`
- 当前 blocker:真实 VLM 服务地址、模型、设备级测试凭证、成本上限和数据留存尚未确认;
当前只支持单 SKU/JPEG;候选探针截图要求 Android 11/API 30+
- 当前 blocker:T-213 需要确认是否允许点击“单独购买/免拼购买”仅用于打开规格弹层,
或提供独立规格入口/受支持数据源;ColorOS 重装后需采购员手动重新启用肉包无障碍。
真实 VLM 服务地址、模型、设备级测试凭证、成本上限和数据留存也尚未确认;当前只
支持单 SKU/JPEG;候选探针截图要求 Android 11/API 30+
## 当前目录
@@ -108,7 +114,7 @@
| `docs/tasks/T-210.md` | DONE | 兼容拼多多 8.17 搜索结果页与长词省略显示 |
| `docs/tasks/T-211.md` | DONE | 参考图召回、SKU 颜色尺码硬匹配与 0..5 候选回传 |
| `docs/tasks/T-212.md` | DONE | 修复重排候选、推荐、证据与人工接受的身份映射 |
| `docs/tasks/T-213.md` | TODO | 只读采集拼多多规格弹窗中的颜色和尺码证据 |
| `docs/tasks/T-213.md` | BLOCKED | 实现只读规格核验;等待拼多多购买语义入口安全决策 |
| `docs/design/` | 已确认 | T-202 原型索引、4 个管理页和 7 个 Android 页面 |
| `deepseek总结.txt` | 已有 | 历史讨论摘要,不是正式需求权威 |
| `android-buyer/` | 已有 | Roubao `main` 固定 commit 的 Android 基线 |
@@ -121,10 +127,9 @@
- 已完成:T-001 至 T-004、T-101 至 T-104、T-201 至 T-207、T-209、T-210、T-211、
T-212。
- 正在进行:T-213 拼多多规格弹窗只读核验。
- 下一个可领取任务:无;T-208 依赖 T-213 完成。
- 后置任务:T-213 拼多多规格弹窗只读核验;完成后再做 T-208 候选决策数据与人工
理由闭环。
- 阻塞中:T-213 拼多多规格弹窗只读核验,等待购买语义入口的安全策略决策。
- 下一个可领取任务:无;T-208 依赖 T-213 真机验收完成。
- 后置任务:解除 T-213 阻塞后再做 T-208 候选决策数据与人工理由闭环。
## 当前可运行内容
+24 -7
View File
@@ -4,11 +4,12 @@ title: 拼多多规格弹窗只读核验
phase: 2
deps:
- T-212
status: DOING
status: BLOCKED
created: 2026-07-27
context_ref: edfee2c
work_branch: null
write_paths:
- android-buyer/app/build.gradle.kts
- docs/tasks/T-213.md
- docs/04-architecture.md
- docs/current-state.md
@@ -53,12 +54,12 @@ Top 5 为空。仅凭商品主图推断颜色或尺码不满足硬约束要求
## 验收要点
- [ ] 页面分类测试覆盖规格弹窗、普通详情、登录、验证码、风控、购物车、订单和支付。
- [ ] 自动化只点击唯一规格入口,不点击颜色/尺码选项、购买、拼单、购物车或支付控件。
- [ ] 颜色、尺码分组及选项解析有数量/文本长度边界,截断或歧义不会判为 `MATCH`。
- [ ] 单元测试覆盖两个目标均匹配、明确不匹配、禁用、缺失和未知。
- [ ] 每个候选的详情/规格证据哈希和后端 asset ID 对应正确,离线重试不重复或串位。
- [ ] Android 单元测试和 Debug 构建通过。
- [x] 页面分类测试覆盖规格弹窗、普通详情、登录、验证码、风控、购物车、订单和支付。
- [x] 自动化只点击唯一规格入口,不点击颜色/尺码选项、购买、拼单、购物车或支付控件。
- [x] 颜色、尺码分组及选项解析有数量/文本长度边界,截断或歧义不会判为 `MATCH`。
- [x] 单元测试覆盖两个目标均匹配、明确不匹配、禁用、缺失和未知。
- [x] 每个候选的详情/规格证据哈希和后端 asset ID 对应正确,离线重试不重复或串位。
- [x] Android 单元测试和 Debug 构建通过。
- [ ] PKG110、Android 16、拼多多 8.17.0 真机 smoke 完成规格弹窗打开、只读采集、
关闭和返回;记录 App/拼多多版本并确认未进入订单或支付。
@@ -75,3 +76,19 @@ Top 5 为空。仅凭商品主图推断颜色或尺码不满足硬约束要求
关联到错误的重排候选。
- 2026-07-27:T-212 完成提交 `4b795a4` 后领取。先冻结拼多多 8.17 规格弹窗的只读
页面/动作边界和候选多证据模型,再进行真机验证。
- 2026-07-27:实现规格弹层分类、安全入口 allowlist、颜色/尺码有界解析、本地确定性
硬匹配、candidate evaluation schema/prompt v3,以及每个候选详情/规格两份证据的
cache 校验、重排身份绑定和加密 outbox 关联。模型返回状态与本地状态不一致时强制
降级人工检查。
- 2026-07-27:Android `testDebugUnitTest` 24 个 suite、128 个测试全部通过,
`assembleDebug` 通过;后端 `go test ./...` 通过。Debug APK `1.4.3 (8)` 已安装到
PKG110,拼多多版本为 `8.17.0 (81700)`。
- 2026-07-27:真机检查图片结果中的定制商品和文字搜索 `black tshirt L` 的普通服装
商品。服装详情只暴露不可点击的“颜色/款式”预览,完整颜色/尺码弹层只能通过
“单独购买/免拼购买”控件进入;前两个定制商品甚至没有规格预览。全程未点击购买、
拼单、购物车、订单或支付控件。
- 2026-07-27:任务标记 `BLOCKED`。代码在缺少独立可点击规格入口时按
`UNKNOWN_PAGE` 安全停止,符合边界但无法完成“打开/采集/关闭/返回”的真机验收。
外部解除条件是:确认允许把购买语义控件仅用于打开规格弹层,或提供不经过购买控件
的稳定规格入口/受支持数据源。重新安装后 ColorOS 也关闭了肉包无障碍服务,继续
smoke 前需采购员在系统设置手动重新启用。