feat(t213): guard read-only specification verification
This commit is contained in:
@@ -11,8 +11,8 @@ android {
|
||||
applicationId = "com.roubao.autopilot"
|
||||
minSdk = 26
|
||||
targetSdk = 34
|
||||
versionCode = 7
|
||||
versionName = "1.4.2"
|
||||
versionCode = 8
|
||||
versionName = "1.4.3"
|
||||
|
||||
vectorDrawables {
|
||||
useSupportLibrary = true
|
||||
|
||||
@@ -65,6 +65,8 @@ import com.roubao.autopilot.vlm.CandidateEvaluator
|
||||
import com.roubao.autopilot.vlm.CandidateReviewBatch
|
||||
import com.roubao.autopilot.vlm.CandidateHumanReviewPolicy
|
||||
import com.roubao.autopilot.vlm.CandidateTopFivePolicy
|
||||
import com.roubao.autopilot.vlm.CandidateSpecificationHardConstraintPolicy
|
||||
import com.roubao.autopilot.vlm.SkuHardConstraintExtractor
|
||||
import kotlinx.coroutines.CancellationException
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.Job
|
||||
@@ -98,6 +100,7 @@ import com.roubao.autopilot.procurement.ExecutionCandidateSourceIdentity
|
||||
import com.roubao.autopilot.procurement.ExecutionHardConstraintEvaluation
|
||||
import com.roubao.autopilot.procurement.ExecutionRecommendation
|
||||
import com.roubao.autopilot.procurement.ExecutionEvidenceDraft
|
||||
import com.roubao.autopilot.procurement.ExecutionEvidenceKind
|
||||
import com.roubao.autopilot.procurement.ExecutionMode
|
||||
import com.roubao.autopilot.procurement.ExecutionProvenanceSnapshot
|
||||
import com.roubao.autopilot.procurement.ProcurementRepository
|
||||
@@ -951,6 +954,8 @@ class MainActivity : ComponentActivity() {
|
||||
baseUrl = settings.baseUrl,
|
||||
model = settings.model
|
||||
)
|
||||
val hardConstraints =
|
||||
SkuHardConstraintExtractor.extract(requirement.sku)
|
||||
val result = CandidateEvaluator(
|
||||
gateway = AndroidCandidateEvaluationVlmGateway(client),
|
||||
providerId = provider.id,
|
||||
@@ -962,8 +967,23 @@ class MainActivity : ComponentActivity() {
|
||||
CandidateEvaluationImage(
|
||||
ordinal = candidate.ordinal,
|
||||
mediaType = "image/png",
|
||||
bytes = candidate.pngBytes,
|
||||
sha256 = candidate.sha256
|
||||
bytes = candidate.detailPngBytes,
|
||||
sha256 = candidate.detailSha256,
|
||||
specificationBytes =
|
||||
candidate.specificationPngBytes,
|
||||
specificationSha256 =
|
||||
candidate.specificationSha256,
|
||||
localHardConstraintResults =
|
||||
CandidateSpecificationHardConstraintPolicy
|
||||
.evaluate(
|
||||
constraints =
|
||||
hardConstraints.constraints,
|
||||
evidence =
|
||||
candidate.specification,
|
||||
evidenceSha256 =
|
||||
candidate
|
||||
.specificationSha256
|
||||
)
|
||||
)
|
||||
}
|
||||
)
|
||||
@@ -1014,20 +1034,46 @@ class MainActivity : ComponentActivity() {
|
||||
)
|
||||
)
|
||||
}
|
||||
val selectedEvidence = ranked.map { rankedCandidate ->
|
||||
val selectedEvidence = ranked.flatMap {
|
||||
rankedCandidate ->
|
||||
val candidate = requireNotNull(
|
||||
evidenceByOrdinal[
|
||||
rankedCandidate.sourceOrdinal
|
||||
]
|
||||
)
|
||||
require(
|
||||
candidate.sha256 ==
|
||||
candidate.detailSha256 ==
|
||||
rankedCandidate.evidenceSha256
|
||||
)
|
||||
ExecutionEvidenceDraft(
|
||||
ordinal = rankedCandidate.rankedOrdinal,
|
||||
pngBytes = candidate.pngBytes,
|
||||
sha256 = candidate.sha256
|
||||
require(
|
||||
candidate.specificationSha256 ==
|
||||
rankedCandidate
|
||||
.specificationEvidenceSha256
|
||||
)
|
||||
listOf(
|
||||
ExecutionEvidenceDraft(
|
||||
ordinal =
|
||||
rankedCandidate.rankedOrdinal,
|
||||
pngBytes =
|
||||
candidate.detailPngBytes,
|
||||
sha256 =
|
||||
candidate.detailSha256,
|
||||
kind =
|
||||
ExecutionEvidenceKind.DETAIL
|
||||
),
|
||||
ExecutionEvidenceDraft(
|
||||
ordinal =
|
||||
rankedCandidate.rankedOrdinal,
|
||||
pngBytes =
|
||||
candidate
|
||||
.specificationPngBytes,
|
||||
sha256 =
|
||||
candidate
|
||||
.specificationSha256,
|
||||
kind =
|
||||
ExecutionEvidenceKind
|
||||
.SPECIFICATION
|
||||
)
|
||||
)
|
||||
}
|
||||
val queued = procurementRepository.queueCandidateBatch(
|
||||
@@ -1078,7 +1124,10 @@ class MainActivity : ComponentActivity() {
|
||||
rankedOrdinal =
|
||||
rankedCandidate.rankedOrdinal,
|
||||
evidenceSha256 =
|
||||
rankedCandidate.evidenceSha256
|
||||
rankedCandidate.evidenceSha256,
|
||||
specificationEvidenceSha256 =
|
||||
rankedCandidate
|
||||
.specificationEvidenceSha256
|
||||
)
|
||||
},
|
||||
candidates = queued
|
||||
@@ -1139,11 +1188,20 @@ class MainActivity : ComponentActivity() {
|
||||
)
|
||||
}
|
||||
),
|
||||
validated.map { candidate ->
|
||||
ExecutionEvidenceDraft(
|
||||
ordinal = candidate.ordinal,
|
||||
pngBytes = candidate.pngBytes,
|
||||
sha256 = candidate.sha256
|
||||
validated.flatMap { candidate ->
|
||||
listOf(
|
||||
ExecutionEvidenceDraft(
|
||||
ordinal = candidate.ordinal,
|
||||
pngBytes = candidate.detailPngBytes,
|
||||
sha256 = candidate.detailSha256,
|
||||
kind = ExecutionEvidenceKind.DETAIL
|
||||
),
|
||||
ExecutionEvidenceDraft(
|
||||
ordinal = candidate.ordinal,
|
||||
pngBytes = candidate.specificationPngBytes,
|
||||
sha256 = candidate.specificationSha256,
|
||||
kind = ExecutionEvidenceKind.SPECIFICATION
|
||||
)
|
||||
)
|
||||
}
|
||||
) ?: return
|
||||
@@ -1153,7 +1211,9 @@ class MainActivity : ComponentActivity() {
|
||||
ExecutionCandidateSourceIdentity(
|
||||
sourceOrdinal = candidate.ordinal,
|
||||
rankedOrdinal = candidate.ordinal,
|
||||
evidenceSha256 = candidate.sha256
|
||||
evidenceSha256 = candidate.detailSha256,
|
||||
specificationEvidenceSha256 =
|
||||
candidate.specificationSha256
|
||||
)
|
||||
},
|
||||
candidates = queued
|
||||
|
||||
+27
-2
@@ -4,6 +4,7 @@ import com.roubao.autopilot.pinduoduo.PinduoduoPage
|
||||
import com.roubao.autopilot.pinduoduo.PinduoduoCandidateCard
|
||||
import com.roubao.autopilot.pinduoduo.PinduoduoCandidateDetailEvidence
|
||||
import com.roubao.autopilot.pinduoduo.PinduoduoScreenshotCapture
|
||||
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationEvidence
|
||||
import com.roubao.autopilot.pinduoduo.PinduoduoUiSnapshot
|
||||
import com.roubao.autopilot.readiness.DeviceObservationStore
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
@@ -76,13 +77,37 @@ object BuyerAccessibilityBridge {
|
||||
service?.readPinduoduoCandidateDetailEvidence()
|
||||
}
|
||||
|
||||
suspend fun captureScreenshot(): PinduoduoScreenshotCapture? =
|
||||
suspend fun captureDetailScreenshot(): PinduoduoScreenshotCapture? =
|
||||
withTimeoutOrNull(SCREENSHOT_TIMEOUT_MILLIS) {
|
||||
withContext(Dispatchers.Main.immediate) {
|
||||
service?.capturePinduoduoScreenshot()
|
||||
service?.capturePinduoduoScreenshot(PinduoduoPage.PRODUCT_DETAIL)
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun openSpecifications(): Boolean =
|
||||
withContext(Dispatchers.Main.immediate) {
|
||||
service?.openPinduoduoSpecifications() == true
|
||||
}
|
||||
|
||||
suspend fun specificationEvidence(): PinduoduoSpecificationEvidence? =
|
||||
withContext(Dispatchers.Main.immediate) {
|
||||
service?.readPinduoduoSpecificationEvidence()
|
||||
}
|
||||
|
||||
suspend fun captureSpecificationScreenshot(): PinduoduoScreenshotCapture? =
|
||||
withTimeoutOrNull(SCREENSHOT_TIMEOUT_MILLIS) {
|
||||
withContext(Dispatchers.Main.immediate) {
|
||||
service?.capturePinduoduoScreenshot(
|
||||
PinduoduoPage.SPECIFICATION_PANEL
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun closeSpecifications(): Boolean =
|
||||
withContext(Dispatchers.Main.immediate) {
|
||||
service?.closePinduoduoSpecifications() == true
|
||||
}
|
||||
|
||||
suspend fun returnToResults(): Boolean =
|
||||
withContext(Dispatchers.Main.immediate) {
|
||||
service?.returnFromPinduoduoCandidate() == true
|
||||
|
||||
+92
-16
@@ -24,6 +24,8 @@ import com.roubao.autopilot.pinduoduo.PinduoduoPage
|
||||
import com.roubao.autopilot.pinduoduo.isCandidateResultsPage
|
||||
import com.roubao.autopilot.pinduoduo.SEARCH_PROBE_KEYWORD
|
||||
import com.roubao.autopilot.pinduoduo.PinduoduoScreenshotCapture
|
||||
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationEvidence
|
||||
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationParser
|
||||
import java.io.ByteArrayOutputStream
|
||||
import java.util.ArrayDeque
|
||||
import java.util.concurrent.Executors
|
||||
@@ -321,6 +323,48 @@ class BuyerAccessibilityService : AccessibilityService() {
|
||||
)
|
||||
}
|
||||
|
||||
internal fun openPinduoduoSpecifications(): Boolean =
|
||||
withPinduoduoRoot { root ->
|
||||
if (!isVerifiedProductDetail(root)) {
|
||||
return@withPinduoduoRoot false
|
||||
}
|
||||
val entries = collectNodes(root).filter { node ->
|
||||
node.isVisibleToUser &&
|
||||
node.isEnabled &&
|
||||
PinduoduoSpecificationParser.isSafeEntryText(
|
||||
semanticText(node)
|
||||
)
|
||||
}
|
||||
entries.singleOrNull()?.let(::clickNodeOrAncestor) == true
|
||||
} ?: false
|
||||
|
||||
internal fun readPinduoduoSpecificationEvidence():
|
||||
PinduoduoSpecificationEvidence? =
|
||||
withPinduoduoRoot { root ->
|
||||
val snapshot = classifyPinduoduoRoot(root)
|
||||
if (
|
||||
snapshot.safetyStopReason != null ||
|
||||
snapshot.page != PinduoduoPage.SPECIFICATION_PANEL
|
||||
) {
|
||||
return@withPinduoduoRoot null
|
||||
}
|
||||
PinduoduoSpecificationParser.parse(
|
||||
collectNodes(root).map(::uiElement)
|
||||
)
|
||||
}
|
||||
|
||||
internal fun closePinduoduoSpecifications(): Boolean =
|
||||
withPinduoduoRoot { root ->
|
||||
val snapshot = classifyPinduoduoRoot(root)
|
||||
if (
|
||||
snapshot.safetyStopReason != null ||
|
||||
snapshot.page != PinduoduoPage.SPECIFICATION_PANEL
|
||||
) {
|
||||
return@withPinduoduoRoot false
|
||||
}
|
||||
performGlobalAction(GLOBAL_ACTION_BACK)
|
||||
} ?: false
|
||||
|
||||
internal fun returnFromPinduoduoCandidate(): Boolean =
|
||||
withPinduoduoRoot { root ->
|
||||
if (!isVerifiedProductDetail(root)) {
|
||||
@@ -355,20 +399,38 @@ class BuyerAccessibilityService : AccessibilityService() {
|
||||
) == true
|
||||
} ?: false
|
||||
|
||||
internal suspend fun capturePinduoduoScreenshot(): PinduoduoScreenshotCapture? {
|
||||
internal suspend fun capturePinduoduoScreenshot(
|
||||
expectedPage: PinduoduoPage
|
||||
): PinduoduoScreenshotCapture? {
|
||||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {
|
||||
return null
|
||||
}
|
||||
return capturePinduoduoScreenshotApi30()
|
||||
if (
|
||||
expectedPage !in setOf(
|
||||
PinduoduoPage.PRODUCT_DETAIL,
|
||||
PinduoduoPage.SPECIFICATION_PANEL
|
||||
)
|
||||
) {
|
||||
return null
|
||||
}
|
||||
return capturePinduoduoScreenshotApi30(expectedPage)
|
||||
}
|
||||
|
||||
@RequiresApi(Build.VERSION_CODES.R)
|
||||
private suspend fun capturePinduoduoScreenshotApi30():
|
||||
private suspend fun capturePinduoduoScreenshotApi30(
|
||||
expectedPage: PinduoduoPage
|
||||
):
|
||||
PinduoduoScreenshotCapture? {
|
||||
val root = rootInActiveWindow
|
||||
val snapshot = root?.let(::classifyPinduoduoRoot)
|
||||
if (
|
||||
root?.packageName?.toString() != PINDUODUO_PACKAGE ||
|
||||
!isVerifiedProductDetail(root)
|
||||
snapshot?.safetyStopReason != null ||
|
||||
snapshot?.page != expectedPage ||
|
||||
(
|
||||
expectedPage == PinduoduoPage.PRODUCT_DETAIL &&
|
||||
!isVerifiedProductDetail(root)
|
||||
)
|
||||
) {
|
||||
return null
|
||||
}
|
||||
@@ -447,18 +509,7 @@ class BuyerAccessibilityService : AccessibilityService() {
|
||||
private fun classifyPinduoduoRoot(
|
||||
root: AccessibilityNodeInfo
|
||||
): PinduoduoUiSnapshot {
|
||||
val elements = collectNodes(root).map { node ->
|
||||
PinduoduoUiElement(
|
||||
text = node.text?.toString(),
|
||||
contentDescription = node.contentDescription?.toString(),
|
||||
className = node.className?.toString().orEmpty(),
|
||||
resourceId = node.viewIdResourceName,
|
||||
clickable = node.isClickable,
|
||||
editable = node.isEditable,
|
||||
enabled = node.isEnabled,
|
||||
visibleToUser = node.isVisibleToUser
|
||||
)
|
||||
}
|
||||
val elements = collectNodes(root).map(::uiElement)
|
||||
return PinduoduoPageClassifier.classify(
|
||||
foregroundPackage = root.packageName?.toString(),
|
||||
elements = elements,
|
||||
@@ -466,6 +517,24 @@ class BuyerAccessibilityService : AccessibilityService() {
|
||||
)
|
||||
}
|
||||
|
||||
private fun uiElement(node: AccessibilityNodeInfo): PinduoduoUiElement {
|
||||
val bounds = Rect().also(node::getBoundsInScreen)
|
||||
return PinduoduoUiElement(
|
||||
text = node.text?.toString(),
|
||||
contentDescription = node.contentDescription?.toString(),
|
||||
className = node.className?.toString().orEmpty(),
|
||||
resourceId = node.viewIdResourceName,
|
||||
clickable = node.isClickable,
|
||||
editable = node.isEditable,
|
||||
enabled = node.isEnabled,
|
||||
visibleToUser = node.isVisibleToUser,
|
||||
selected = node.isSelected,
|
||||
scrollable = node.isScrollable,
|
||||
boundsLeft = bounds.left,
|
||||
boundsTop = bounds.top
|
||||
)
|
||||
}
|
||||
|
||||
private fun candidateNodes(
|
||||
root: AccessibilityNodeInfo,
|
||||
limit: Int
|
||||
@@ -668,6 +737,13 @@ class BuyerAccessibilityService : AccessibilityService() {
|
||||
.replace(Regex("\\s+"), " ")
|
||||
.take(MAX_EVIDENCE_TEXT_LENGTH)
|
||||
|
||||
private fun semanticText(node: AccessibilityNodeInfo): String =
|
||||
sequenceOf(node.text, node.contentDescription)
|
||||
.filterNotNull()
|
||||
.map { it.toString().trim() }
|
||||
.firstOrNull(String::isNotEmpty)
|
||||
.orEmpty()
|
||||
|
||||
private fun shouldInspect(eventType: Int): Boolean =
|
||||
eventType == AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED ||
|
||||
eventType == AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED ||
|
||||
|
||||
+80
-27
@@ -25,20 +25,41 @@ class AndroidPinduoduoCandidateDriver(
|
||||
override suspend fun openCandidate(signature: String): Boolean =
|
||||
BuyerAccessibilityBridge.openCandidate(signature)
|
||||
|
||||
override suspend fun captureCandidate(
|
||||
ordinal: Int,
|
||||
card: PinduoduoCandidateCard
|
||||
): PinduoduoCandidateEvidence? {
|
||||
override suspend fun captureDetail(): PinduoduoCandidateCapture? {
|
||||
val detail = BuyerAccessibilityBridge.candidateDetailEvidence()
|
||||
?: return null
|
||||
val screenshot = BuyerAccessibilityBridge.captureScreenshot()
|
||||
val screenshot = BuyerAccessibilityBridge.captureDetailScreenshot()
|
||||
?: return null
|
||||
return PinduoduoCandidateCapture(detail, screenshot)
|
||||
}
|
||||
|
||||
override suspend fun openSpecifications(): Boolean =
|
||||
BuyerAccessibilityBridge.openSpecifications()
|
||||
|
||||
override suspend fun captureSpecifications(): PinduoduoSpecificationCapture? {
|
||||
val evidence = BuyerAccessibilityBridge.specificationEvidence()
|
||||
?: return null
|
||||
val screenshot =
|
||||
BuyerAccessibilityBridge.captureSpecificationScreenshot()
|
||||
?: return null
|
||||
return PinduoduoSpecificationCapture(evidence, screenshot)
|
||||
}
|
||||
|
||||
override suspend fun closeSpecifications(): Boolean =
|
||||
BuyerAccessibilityBridge.closeSpecifications()
|
||||
|
||||
override suspend fun saveCandidate(
|
||||
ordinal: Int,
|
||||
card: PinduoduoCandidateCard,
|
||||
detail: PinduoduoCandidateCapture,
|
||||
specifications: PinduoduoSpecificationCapture
|
||||
): PinduoduoCandidateEvidence? {
|
||||
return withContext(Dispatchers.IO) {
|
||||
evidenceStore.save(
|
||||
ordinal = ordinal,
|
||||
card = card,
|
||||
detail = detail,
|
||||
screenshot = screenshot
|
||||
specifications = specifications
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -70,8 +91,8 @@ private class CandidateEvidenceStore(context: Context) {
|
||||
fun save(
|
||||
ordinal: Int,
|
||||
card: PinduoduoCandidateCard,
|
||||
detail: PinduoduoCandidateDetailEvidence,
|
||||
screenshot: PinduoduoScreenshotCapture
|
||||
detail: PinduoduoCandidateCapture,
|
||||
specifications: PinduoduoSpecificationCapture
|
||||
): PinduoduoCandidateEvidence? {
|
||||
if (ordinal !in 1..MAX_CANDIDATES_PER_PROBE) {
|
||||
return null
|
||||
@@ -80,22 +101,25 @@ private class CandidateEvidenceStore(context: Context) {
|
||||
if (!root.exists() && !root.mkdirs()) {
|
||||
return null
|
||||
}
|
||||
val fileName = "candidate-%02d.png".format(ordinal)
|
||||
val screenshotFile = File(root, fileName)
|
||||
screenshotFile.writeBytes(screenshot.pngBytes)
|
||||
val detailAsset = writeAsset(
|
||||
ordinal = ordinal,
|
||||
suffix = "detail",
|
||||
screenshot = detail.screenshot
|
||||
)
|
||||
val specificationAsset = writeAsset(
|
||||
ordinal = ordinal,
|
||||
suffix = "specification",
|
||||
screenshot = specifications.screenshot
|
||||
)
|
||||
val evidence = PinduoduoCandidateEvidence(
|
||||
ordinal = ordinal,
|
||||
cardSignature = card.signature,
|
||||
cardSemanticTextCount = card.semanticTextCount,
|
||||
detailSignature = detail.signature,
|
||||
detailSemanticTextCount = detail.semanticTextCount,
|
||||
screenshotFileName = fileName,
|
||||
screenshotSha256 = PinduoduoEvidenceHash.sha256(
|
||||
screenshot.pngBytes
|
||||
),
|
||||
screenshotByteCount = screenshot.pngBytes.size,
|
||||
screenshotWidth = screenshot.width,
|
||||
screenshotHeight = screenshot.height
|
||||
detailSignature = detail.detail.signature,
|
||||
detailSemanticTextCount = detail.detail.semanticTextCount,
|
||||
detailAsset = detailAsset,
|
||||
specification = specifications.evidence,
|
||||
specificationAsset = specificationAsset
|
||||
)
|
||||
evidenceByOrdinal[ordinal] = evidence
|
||||
writeManifest()
|
||||
@@ -105,6 +129,22 @@ private class CandidateEvidenceStore(context: Context) {
|
||||
}
|
||||
}
|
||||
|
||||
private fun writeAsset(
|
||||
ordinal: Int,
|
||||
suffix: String,
|
||||
screenshot: PinduoduoScreenshotCapture
|
||||
): PinduoduoEvidenceAsset {
|
||||
val fileName = "candidate-%02d-%s.png".format(ordinal, suffix)
|
||||
File(root, fileName).writeBytes(screenshot.pngBytes)
|
||||
return PinduoduoEvidenceAsset(
|
||||
fileName = fileName,
|
||||
sha256 = PinduoduoEvidenceHash.sha256(screenshot.pngBytes),
|
||||
byteCount = screenshot.pngBytes.size,
|
||||
width = screenshot.width,
|
||||
height = screenshot.height
|
||||
)
|
||||
}
|
||||
|
||||
private fun writeManifest() {
|
||||
val candidates = JSONArray()
|
||||
evidenceByOrdinal.values.forEach { evidence ->
|
||||
@@ -121,18 +161,23 @@ private class CandidateEvidenceStore(context: Context) {
|
||||
"detail_semantic_text_count",
|
||||
evidence.detailSemanticTextCount
|
||||
)
|
||||
.put("screenshot_file", evidence.screenshotFileName)
|
||||
.put("screenshot_sha256", evidence.screenshotSha256)
|
||||
.put("detail_asset", assetJson(evidence.detailAsset))
|
||||
.put(
|
||||
"screenshot_byte_count",
|
||||
evidence.screenshotByteCount
|
||||
"specification_signature",
|
||||
evidence.specification.signature
|
||||
)
|
||||
.put(
|
||||
"specification_semantic_text_count",
|
||||
evidence.specification.semanticTextCount
|
||||
)
|
||||
.put(
|
||||
"specification_asset",
|
||||
assetJson(evidence.specificationAsset)
|
||||
)
|
||||
.put("screenshot_width", evidence.screenshotWidth)
|
||||
.put("screenshot_height", evidence.screenshotHeight)
|
||||
)
|
||||
}
|
||||
val manifest = JSONObject()
|
||||
.put("schema_version", 1)
|
||||
.put("schema_version", 2)
|
||||
.put("candidate_count", evidenceByOrdinal.size)
|
||||
.put("candidates", candidates)
|
||||
.toString(2)
|
||||
@@ -147,6 +192,14 @@ private class CandidateEvidenceStore(context: Context) {
|
||||
}
|
||||
}
|
||||
|
||||
private fun assetJson(asset: PinduoduoEvidenceAsset): JSONObject =
|
||||
JSONObject()
|
||||
.put("file", asset.fileName)
|
||||
.put("sha256", asset.sha256)
|
||||
.put("byte_count", asset.byteCount)
|
||||
.put("width", asset.width)
|
||||
.put("height", asset.height)
|
||||
|
||||
private companion object {
|
||||
const val MANIFEST_FILE = "manifest.json"
|
||||
}
|
||||
|
||||
+52
-24
@@ -9,8 +9,11 @@ import kotlinx.coroutines.withContext
|
||||
|
||||
data class ValidatedCandidateEvidence(
|
||||
val ordinal: Int,
|
||||
val pngBytes: ByteArray,
|
||||
val sha256: String
|
||||
val detailPngBytes: ByteArray,
|
||||
val detailSha256: String,
|
||||
val specification: PinduoduoSpecificationEvidence,
|
||||
val specificationPngBytes: ByteArray,
|
||||
val specificationSha256: String
|
||||
)
|
||||
|
||||
class CandidateEvidenceSource(
|
||||
@@ -31,38 +34,63 @@ class CandidateEvidenceSource(
|
||||
val canonicalRoot = root.canonicalFile
|
||||
var totalBytes = 0L
|
||||
sorted.map { metadata ->
|
||||
val expectedFileName = "candidate-%02d.png".format(metadata.ordinal)
|
||||
require(metadata.screenshotFileName == expectedFileName)
|
||||
require(metadata.screenshotByteCount in 1..MAX_PNG_BYTES)
|
||||
require(metadata.screenshotWidth in 1..MAX_SCREENSHOT_DIMENSION)
|
||||
require(metadata.screenshotHeight in 1..MAX_SCREENSHOT_DIMENSION)
|
||||
require(SHA256_PATTERN.matches(metadata.screenshotSha256))
|
||||
totalBytes += metadata.screenshotByteCount
|
||||
require(
|
||||
metadata.specification.groups.map { it.kind }.toSet() ==
|
||||
PinduoduoSpecificationGroupKind.entries.toSet()
|
||||
)
|
||||
val detail = readAsset(
|
||||
canonicalRoot = canonicalRoot,
|
||||
asset = metadata.detailAsset,
|
||||
expectedFileName =
|
||||
"candidate-%02d-detail.png".format(metadata.ordinal)
|
||||
)
|
||||
val specification = readAsset(
|
||||
canonicalRoot = canonicalRoot,
|
||||
asset = metadata.specificationAsset,
|
||||
expectedFileName =
|
||||
"candidate-%02d-specification.png".format(metadata.ordinal)
|
||||
)
|
||||
totalBytes += detail.size + specification.size
|
||||
require(totalBytes <= MAX_TOTAL_PNG_BYTES)
|
||||
|
||||
val file = File(canonicalRoot, expectedFileName).canonicalFile
|
||||
require(file.parentFile == canonicalRoot)
|
||||
require(file.isFile && file.length() == metadata.screenshotByteCount.toLong())
|
||||
val bytes = FileInputStream(file).use {
|
||||
it.readBytesExact(metadata.screenshotByteCount)
|
||||
}
|
||||
require(bytes.hasPngHeader())
|
||||
val dimensions = bytes.pngDimensions()
|
||||
require(dimensions.first == metadata.screenshotWidth)
|
||||
require(dimensions.second == metadata.screenshotHeight)
|
||||
require(PinduoduoEvidenceHash.sha256(bytes) == metadata.screenshotSha256)
|
||||
ValidatedCandidateEvidence(
|
||||
ordinal = metadata.ordinal,
|
||||
pngBytes = bytes,
|
||||
sha256 = metadata.screenshotSha256
|
||||
detailPngBytes = detail,
|
||||
detailSha256 = metadata.detailAsset.sha256,
|
||||
specification = metadata.specification,
|
||||
specificationPngBytes = specification,
|
||||
specificationSha256 = metadata.specificationAsset.sha256
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun readAsset(
|
||||
canonicalRoot: File,
|
||||
asset: PinduoduoEvidenceAsset,
|
||||
expectedFileName: String
|
||||
): ByteArray {
|
||||
require(asset.fileName == expectedFileName)
|
||||
require(asset.byteCount in 1..MAX_PNG_BYTES)
|
||||
require(asset.width in 1..MAX_SCREENSHOT_DIMENSION)
|
||||
require(asset.height in 1..MAX_SCREENSHOT_DIMENSION)
|
||||
require(SHA256_PATTERN.matches(asset.sha256))
|
||||
val file = File(canonicalRoot, expectedFileName).canonicalFile
|
||||
require(file.parentFile == canonicalRoot)
|
||||
require(file.isFile && file.length() == asset.byteCount.toLong())
|
||||
val bytes = FileInputStream(file).use {
|
||||
it.readBytesExact(asset.byteCount)
|
||||
}
|
||||
require(bytes.hasPngHeader())
|
||||
val dimensions = bytes.pngDimensions()
|
||||
require(dimensions.first == asset.width)
|
||||
require(dimensions.second == asset.height)
|
||||
require(PinduoduoEvidenceHash.sha256(bytes) == asset.sha256)
|
||||
return bytes
|
||||
}
|
||||
|
||||
private companion object {
|
||||
const val MAX_PNG_BYTES = 8 * 1024 * 1024
|
||||
const val MAX_TOTAL_PNG_BYTES = 32L * 1024L * 1024L
|
||||
const val MAX_TOTAL_PNG_BYTES = 64L * 1024L * 1024L
|
||||
const val MAX_SCREENSHOT_DIMENSION = 10_000
|
||||
val SHA256_PATTERN = Regex("^[0-9a-f]{64}$")
|
||||
}
|
||||
|
||||
+40
-4
@@ -16,7 +16,10 @@ enum class CandidateBrowsePhase {
|
||||
READING_RESULTS,
|
||||
OPENING_CANDIDATE,
|
||||
WAITING_DETAIL,
|
||||
CAPTURING_EVIDENCE,
|
||||
CAPTURING_DETAIL,
|
||||
OPENING_SPECIFICATIONS,
|
||||
READING_SPECIFICATIONS,
|
||||
CLOSING_SPECIFICATIONS,
|
||||
RETURNING_RESULTS,
|
||||
SCROLLING_RESULTS,
|
||||
COMPLETE
|
||||
@@ -100,11 +103,34 @@ class PinduoduoCandidateAutomation(
|
||||
mutablePhase.value = CandidateBrowsePhase.WAITING_DETAIL
|
||||
awaitPage(PinduoduoPage.PRODUCT_DETAIL)?.let { return it }
|
||||
val ordinal = mutableEvidence.value.size + 1
|
||||
mutablePhase.value = CandidateBrowsePhase.CAPTURING_EVIDENCE
|
||||
val captured = driver.captureCandidate(ordinal, nextCard)
|
||||
mutablePhase.value = CandidateBrowsePhase.CAPTURING_DETAIL
|
||||
val detail = driver.captureDetail()
|
||||
?: return AutomationResult.FatalFailure(
|
||||
WorkflowFailureCode.EVIDENCE_CAPTURE_FAILED
|
||||
)
|
||||
mutablePhase.value = CandidateBrowsePhase.OPENING_SPECIFICATIONS
|
||||
if (!driver.openSpecifications()) {
|
||||
return AutomationResult.Blocked(SafetyStopReason.UNKNOWN_PAGE)
|
||||
}
|
||||
awaitPage(PinduoduoPage.SPECIFICATION_PANEL)?.let { return it }
|
||||
mutablePhase.value = CandidateBrowsePhase.READING_SPECIFICATIONS
|
||||
val specifications = driver.captureSpecifications()
|
||||
?: return AutomationResult.FatalFailure(
|
||||
WorkflowFailureCode.EVIDENCE_CAPTURE_FAILED
|
||||
)
|
||||
mutablePhase.value = CandidateBrowsePhase.CLOSING_SPECIFICATIONS
|
||||
if (!driver.closeSpecifications()) {
|
||||
return AutomationResult.Blocked(SafetyStopReason.UNKNOWN_PAGE)
|
||||
}
|
||||
awaitPage(PinduoduoPage.PRODUCT_DETAIL)?.let { return it }
|
||||
val captured = driver.saveCandidate(
|
||||
ordinal = ordinal,
|
||||
card = nextCard,
|
||||
detail = detail,
|
||||
specifications = specifications
|
||||
) ?: return AutomationResult.FatalFailure(
|
||||
WorkflowFailureCode.EVIDENCE_CAPTURE_FAILED
|
||||
)
|
||||
mutableEvidence.value = mutableEvidence.value + captured
|
||||
|
||||
mutablePhase.value = CandidateBrowsePhase.RETURNING_RESULTS
|
||||
@@ -123,7 +149,17 @@ class PinduoduoCandidateAutomation(
|
||||
private suspend fun recoverDetailPageIfNeeded(): AutomationResult? {
|
||||
val snapshot = driver.snapshot()
|
||||
safetyResult(snapshot)?.let { return it }
|
||||
if (snapshot.page != PinduoduoPage.PRODUCT_DETAIL) {
|
||||
if (snapshot.page == PinduoduoPage.SPECIFICATION_PANEL) {
|
||||
if (!driver.closeSpecifications()) {
|
||||
return AutomationResult.RetryableFailure(
|
||||
WorkflowFailureCode.TRANSIENT_AUTOMATION
|
||||
)
|
||||
}
|
||||
awaitPage(PinduoduoPage.PRODUCT_DETAIL)?.let { return it }
|
||||
}
|
||||
val detailSnapshot = driver.snapshot()
|
||||
safetyResult(detailSnapshot)?.let { return it }
|
||||
if (detailSnapshot.page != PinduoduoPage.PRODUCT_DETAIL) {
|
||||
return null
|
||||
}
|
||||
if (!driver.returnToResults()) {
|
||||
|
||||
+29
-7
@@ -23,17 +23,23 @@ data class PinduoduoScreenshotCapture(
|
||||
val height: Int
|
||||
)
|
||||
|
||||
data class PinduoduoEvidenceAsset(
|
||||
val fileName: String,
|
||||
val sha256: String,
|
||||
val byteCount: Int,
|
||||
val width: Int,
|
||||
val height: Int
|
||||
)
|
||||
|
||||
data class PinduoduoCandidateEvidence(
|
||||
val ordinal: Int,
|
||||
val cardSignature: String,
|
||||
val cardSemanticTextCount: Int,
|
||||
val detailSignature: String,
|
||||
val detailSemanticTextCount: Int,
|
||||
val screenshotFileName: String,
|
||||
val screenshotSha256: String,
|
||||
val screenshotByteCount: Int,
|
||||
val screenshotWidth: Int,
|
||||
val screenshotHeight: Int
|
||||
val detailAsset: PinduoduoEvidenceAsset,
|
||||
val specification: PinduoduoSpecificationEvidence,
|
||||
val specificationAsset: PinduoduoEvidenceAsset
|
||||
)
|
||||
|
||||
object PinduoduoEvidenceHash {
|
||||
@@ -50,11 +56,27 @@ interface PinduoduoCandidateDriver {
|
||||
suspend fun snapshot(): PinduoduoUiSnapshot
|
||||
suspend fun candidateCards(limit: Int): List<PinduoduoCandidateCard>
|
||||
suspend fun openCandidate(signature: String): Boolean
|
||||
suspend fun captureCandidate(
|
||||
suspend fun captureDetail(): PinduoduoCandidateCapture?
|
||||
suspend fun openSpecifications(): Boolean
|
||||
suspend fun captureSpecifications(): PinduoduoSpecificationCapture?
|
||||
suspend fun closeSpecifications(): Boolean
|
||||
suspend fun saveCandidate(
|
||||
ordinal: Int,
|
||||
card: PinduoduoCandidateCard
|
||||
card: PinduoduoCandidateCard,
|
||||
detail: PinduoduoCandidateCapture,
|
||||
specifications: PinduoduoSpecificationCapture
|
||||
): PinduoduoCandidateEvidence?
|
||||
suspend fun returnToResults(): Boolean
|
||||
suspend fun scrollResults(): Boolean
|
||||
fun resetEvidence()
|
||||
}
|
||||
|
||||
data class PinduoduoCandidateCapture(
|
||||
val detail: PinduoduoCandidateDetailEvidence,
|
||||
val screenshot: PinduoduoScreenshotCapture
|
||||
)
|
||||
|
||||
data class PinduoduoSpecificationCapture(
|
||||
val evidence: PinduoduoSpecificationEvidence,
|
||||
val screenshot: PinduoduoScreenshotCapture
|
||||
)
|
||||
|
||||
+31
-1
@@ -13,7 +13,11 @@ data class PinduoduoUiElement(
|
||||
val clickable: Boolean,
|
||||
val editable: Boolean,
|
||||
val enabled: Boolean,
|
||||
val visibleToUser: Boolean
|
||||
val visibleToUser: Boolean,
|
||||
val selected: Boolean = false,
|
||||
val scrollable: Boolean = false,
|
||||
val boundsLeft: Int = 0,
|
||||
val boundsTop: Int = 0
|
||||
)
|
||||
|
||||
enum class PinduoduoPage {
|
||||
@@ -24,6 +28,7 @@ enum class PinduoduoPage {
|
||||
IMAGE_SEARCH,
|
||||
IMAGE_SEARCH_RESULTS,
|
||||
PRODUCT_DETAIL,
|
||||
SPECIFICATION_PANEL,
|
||||
UNKNOWN
|
||||
}
|
||||
|
||||
@@ -142,8 +147,23 @@ object PinduoduoPageClassifier {
|
||||
}
|
||||
val detailMarkerCount = setOf("联系客服", "收藏", "店铺")
|
||||
.count { marker -> normalized.any { it.contains(marker) } }
|
||||
val specificationGroupCount = setOf("颜色分类", "颜色", "尺码", "尺寸")
|
||||
.count { marker -> normalized.any { it == marker } }
|
||||
val specificationOptionCount = visibleElements.count { element ->
|
||||
val semantic = element.text
|
||||
?.takeIf(String::isNotBlank)
|
||||
?: element.contentDescription.orEmpty()
|
||||
element.clickable &&
|
||||
semantic.isNotBlank() &&
|
||||
normalize(semantic) !in specificationEntryMarkers &&
|
||||
paymentMarkers.none(semantic::contains)
|
||||
}
|
||||
val hasSpecificationPanel =
|
||||
specificationGroupCount >= 2 &&
|
||||
specificationOptionCount >= 2
|
||||
|
||||
val page = when {
|
||||
hasSpecificationPanel -> PinduoduoPage.SPECIFICATION_PANEL
|
||||
hasImageResultHeader && legacySortControlCount >= 3 ->
|
||||
PinduoduoPage.IMAGE_SEARCH_RESULTS
|
||||
hasImageSearchPage -> PinduoduoPage.IMAGE_SEARCH
|
||||
@@ -200,6 +220,16 @@ object PinduoduoPageClassifier {
|
||||
private fun Collection<String>.containsAny(markers: Collection<String>): Boolean =
|
||||
any { text -> markers.any(text::contains) }
|
||||
|
||||
private val specificationEntryMarkers = setOf(
|
||||
"选择规格",
|
||||
"请选择规格",
|
||||
"颜色分类",
|
||||
"颜色款式",
|
||||
"选择颜色",
|
||||
"请选择颜色",
|
||||
"选择尺码",
|
||||
"请选择尺码"
|
||||
)
|
||||
private const val MINIMUM_QUERY_FRAGMENT_LENGTH = 12
|
||||
private const val MINIMUM_ELIDED_FRAGMENT_LENGTH = 4
|
||||
}
|
||||
|
||||
+178
@@ -0,0 +1,178 @@
|
||||
package com.roubao.autopilot.pinduoduo
|
||||
|
||||
enum class PinduoduoSpecificationGroupKind {
|
||||
COLOR,
|
||||
SIZE
|
||||
}
|
||||
|
||||
data class PinduoduoSpecificationOption(
|
||||
val text: String,
|
||||
val selected: Boolean,
|
||||
val enabled: Boolean
|
||||
)
|
||||
|
||||
data class PinduoduoSpecificationGroup(
|
||||
val kind: PinduoduoSpecificationGroupKind,
|
||||
val title: String,
|
||||
val options: List<PinduoduoSpecificationOption>,
|
||||
val complete: Boolean
|
||||
)
|
||||
|
||||
data class PinduoduoSpecificationEvidence(
|
||||
val signature: String,
|
||||
val semanticTextCount: Int,
|
||||
val groups: List<PinduoduoSpecificationGroup>
|
||||
)
|
||||
|
||||
object PinduoduoSpecificationParser {
|
||||
fun parse(
|
||||
elements: Collection<PinduoduoUiElement>
|
||||
): PinduoduoSpecificationEvidence? {
|
||||
val visible = elements
|
||||
.asSequence()
|
||||
.filter { it.visibleToUser }
|
||||
.filter { semanticText(it).isNotEmpty() || it.scrollable }
|
||||
.take(MAX_ELEMENTS + 1)
|
||||
.toList()
|
||||
if (visible.isEmpty() || visible.size > MAX_ELEMENTS) {
|
||||
return null
|
||||
}
|
||||
|
||||
val ordered = visible.sortedWith(
|
||||
compareBy<PinduoduoUiElement> { it.boundsTop }
|
||||
.thenBy { it.boundsLeft }
|
||||
)
|
||||
val headers = ordered.mapIndexedNotNull { index, element ->
|
||||
groupKind(semanticText(element))?.let { kind ->
|
||||
GroupHeader(index, kind, semanticText(element), element.boundsTop)
|
||||
}
|
||||
}
|
||||
if (
|
||||
headers.map { it.kind }.toSet() !=
|
||||
PinduoduoSpecificationGroupKind.entries.toSet()
|
||||
) {
|
||||
return null
|
||||
}
|
||||
|
||||
val groups = headers.mapIndexed { headerIndex, header ->
|
||||
val nextTop = headers.getOrNull(headerIndex + 1)?.top ?: Int.MAX_VALUE
|
||||
val options = ordered.asSequence()
|
||||
.filter { element ->
|
||||
element.boundsTop >= header.top &&
|
||||
element.boundsTop < nextTop &&
|
||||
groupKind(semanticText(element)) == null &&
|
||||
isOption(element)
|
||||
}
|
||||
.map { element ->
|
||||
PinduoduoSpecificationOption(
|
||||
text = semanticText(element).take(MAX_OPTION_TEXT_LENGTH),
|
||||
selected = element.selected,
|
||||
enabled = element.enabled
|
||||
)
|
||||
}
|
||||
.distinctBy { normalize(it.text) }
|
||||
.take(MAX_OPTIONS_PER_GROUP + 1)
|
||||
.toList()
|
||||
val clipped =
|
||||
options.size > MAX_OPTIONS_PER_GROUP ||
|
||||
ordered.any { element ->
|
||||
element.scrollable &&
|
||||
element.boundsTop >= header.top &&
|
||||
element.boundsTop < nextTop
|
||||
}
|
||||
PinduoduoSpecificationGroup(
|
||||
kind = header.kind,
|
||||
title = header.title.take(MAX_OPTION_TEXT_LENGTH),
|
||||
options = options.take(MAX_OPTIONS_PER_GROUP),
|
||||
complete = !clipped
|
||||
)
|
||||
}
|
||||
if (groups.any { it.options.isEmpty() }) {
|
||||
return null
|
||||
}
|
||||
|
||||
val semantics = buildList {
|
||||
groups.forEach { group ->
|
||||
add("${group.kind.name}:${normalize(group.title)}:${group.complete}")
|
||||
group.options.forEach { option ->
|
||||
add(
|
||||
"${normalize(option.text)}:" +
|
||||
"${option.selected}:${option.enabled}"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
return PinduoduoSpecificationEvidence(
|
||||
signature = PinduoduoEvidenceHash.sha256(
|
||||
semantics.joinToString("\u001f")
|
||||
),
|
||||
semanticTextCount = semantics.size,
|
||||
groups = groups
|
||||
)
|
||||
}
|
||||
|
||||
fun isSafeEntryText(value: String): Boolean {
|
||||
val normalized = normalize(value)
|
||||
return normalized in SAFE_ENTRY_TEXTS ||
|
||||
SAFE_ENTRY_PREFIXES.any(normalized::startsWith)
|
||||
}
|
||||
|
||||
private fun isOption(element: PinduoduoUiElement): Boolean {
|
||||
val text = semanticText(element)
|
||||
val normalized = normalize(text)
|
||||
return text.length <= MAX_OPTION_TEXT_LENGTH &&
|
||||
normalized !in SAFE_ENTRY_TEXTS &&
|
||||
SAFE_ENTRY_PREFIXES.none(normalized::startsWith) &&
|
||||
TRANSACTION_MARKERS.none(normalized::contains) &&
|
||||
(element.clickable || element.selected || !element.enabled)
|
||||
}
|
||||
|
||||
private fun groupKind(value: String): PinduoduoSpecificationGroupKind? =
|
||||
when (normalize(value).removeSuffix(":").removeSuffix(":")) {
|
||||
"颜色", "颜色分类", "颜色选择" ->
|
||||
PinduoduoSpecificationGroupKind.COLOR
|
||||
"尺码", "尺寸", "大小", "尺码选择" ->
|
||||
PinduoduoSpecificationGroupKind.SIZE
|
||||
else -> null
|
||||
}
|
||||
|
||||
private fun semanticText(element: PinduoduoUiElement): String =
|
||||
sequenceOf(element.text, element.contentDescription)
|
||||
.filterNotNull()
|
||||
.map(String::trim)
|
||||
.firstOrNull(String::isNotEmpty)
|
||||
.orEmpty()
|
||||
|
||||
private fun normalize(value: String): String =
|
||||
value.trim().lowercase().replace(Regex("\\s+"), "")
|
||||
|
||||
private data class GroupHeader(
|
||||
val index: Int,
|
||||
val kind: PinduoduoSpecificationGroupKind,
|
||||
val title: String,
|
||||
val top: Int
|
||||
)
|
||||
|
||||
private val SAFE_ENTRY_TEXTS = setOf(
|
||||
"选择规格",
|
||||
"请选择规格",
|
||||
"颜色分类",
|
||||
"颜色款式",
|
||||
"选择颜色",
|
||||
"请选择颜色",
|
||||
"选择尺码",
|
||||
"请选择尺码"
|
||||
)
|
||||
private val SAFE_ENTRY_PREFIXES = setOf("请选择:", "请选择:", "已选:", "已选:")
|
||||
private val TRANSACTION_MARKERS = setOf(
|
||||
"购买",
|
||||
"拼单",
|
||||
"购物车",
|
||||
"下单",
|
||||
"支付",
|
||||
"结算"
|
||||
)
|
||||
private const val MAX_ELEMENTS = 160
|
||||
private const val MAX_OPTIONS_PER_GROUP = 30
|
||||
private const val MAX_OPTION_TEXT_LENGTH = 80
|
||||
}
|
||||
+36
-2
@@ -5,9 +5,37 @@ import java.security.MessageDigest
|
||||
data class ExecutionEvidenceDraft(
|
||||
val ordinal: Int,
|
||||
val pngBytes: ByteArray,
|
||||
val sha256: String
|
||||
val sha256: String,
|
||||
val kind: ExecutionEvidenceKind = ExecutionEvidenceKind.DETAIL
|
||||
)
|
||||
|
||||
enum class ExecutionEvidenceKind {
|
||||
DETAIL,
|
||||
SPECIFICATION
|
||||
}
|
||||
|
||||
object ExecutionEvidenceAssociationPolicy {
|
||||
fun group(
|
||||
candidateOrdinals: List<Int>,
|
||||
evidence: List<ExecutionEvidenceDraft>
|
||||
): Map<Int, List<ExecutionEvidenceDraft>> {
|
||||
require(candidateOrdinals == (1..candidateOrdinals.size).toList())
|
||||
val grouped = evidence.groupBy { it.ordinal }
|
||||
require(grouped.keys == candidateOrdinals.toSet())
|
||||
require(
|
||||
candidateOrdinals.all { ordinal ->
|
||||
val assets = requireNotNull(grouped[ordinal])
|
||||
assets.size == 2 &&
|
||||
assets.map { it.kind }.toSet() ==
|
||||
ExecutionEvidenceKind.entries.toSet()
|
||||
}
|
||||
)
|
||||
return candidateOrdinals.associateWith { ordinal ->
|
||||
requireNotNull(grouped[ordinal]).sortedBy { it.kind.ordinal }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
data class ExecutionCandidateDraft(
|
||||
val ordinal: Int,
|
||||
val title: String,
|
||||
@@ -22,7 +50,8 @@ data class ExecutionCandidateDraft(
|
||||
data class ExecutionCandidateSourceIdentity(
|
||||
val sourceOrdinal: Int,
|
||||
val rankedOrdinal: Int,
|
||||
val evidenceSha256: String
|
||||
val evidenceSha256: String,
|
||||
val specificationEvidenceSha256: String = evidenceSha256
|
||||
)
|
||||
|
||||
data class RankedExecutionCandidateDraft(
|
||||
@@ -45,6 +74,11 @@ object ExecutionCandidateIdentityPolicy {
|
||||
require(identities.all { it.sourceOrdinal > 0 })
|
||||
require(identities.map { it.sourceOrdinal }.distinct().size == identities.size)
|
||||
require(identities.all { sha256Pattern.matches(it.evidenceSha256) })
|
||||
require(
|
||||
identities.all {
|
||||
sha256Pattern.matches(it.specificationEvidenceSha256)
|
||||
}
|
||||
)
|
||||
val candidatesByOrdinal = candidates.associateBy { it.ordinal }
|
||||
require(candidatesByOrdinal.size == candidates.size)
|
||||
return identities.map { identity ->
|
||||
|
||||
+23
-19
@@ -343,11 +343,8 @@ class ProcurementRepository(
|
||||
require(!execution.safetyStopped && !execution.isExpired()) {
|
||||
"执行授权已到期,不能继续采集"
|
||||
}
|
||||
require(evidence.size in 0..5) { "候选证据必须为 0 至 5 张" }
|
||||
require(evidence.size in 0..25) { "候选证据必须为 0 至 25 张" }
|
||||
require(batch.candidates.size in 0..5) { "候选数量必须为 0 至 5 个" }
|
||||
require(evidence.size == batch.candidates.size) {
|
||||
"候选和证据数量必须一致"
|
||||
}
|
||||
require(batch.candidates.map { it.ordinal } == (1..batch.candidates.size).toList()) {
|
||||
"候选编号必须连续"
|
||||
}
|
||||
@@ -359,27 +356,34 @@ class ProcurementRepository(
|
||||
} else {
|
||||
require(batch.provenance == null) { "手工优先模式不能附带模型出处" }
|
||||
}
|
||||
val evidenceIDs = evidence.sortedBy { it.ordinal }.associate { draft ->
|
||||
require(draft.ordinal in 1..batch.candidates.size)
|
||||
val evidenceByOrdinal = ExecutionEvidenceAssociationPolicy.group(
|
||||
candidateOrdinals = batch.candidates.map { it.ordinal },
|
||||
evidence = evidence
|
||||
)
|
||||
evidence.forEach { draft ->
|
||||
require(sha256(draft.pngBytes) == draft.sha256)
|
||||
val localID = UUID.randomUUID().toString()
|
||||
persistEvidenceLocked(localID, draft.pngBytes)
|
||||
appendOutboxLocked(
|
||||
ExecutionOutboxItem(
|
||||
id = localID,
|
||||
type = ExecutionOutboxType.EVIDENCE,
|
||||
idempotencyKey = newOpaqueSecret(),
|
||||
payload = "{}",
|
||||
evidenceRelativePath = "$OUTBOX_DIRECTORY/$localID.png"
|
||||
}
|
||||
val evidenceIDs = evidenceByOrdinal.mapValues { (_, drafts) ->
|
||||
drafts.sortedBy { it.kind.ordinal }.map { draft ->
|
||||
val localID = UUID.randomUUID().toString()
|
||||
persistEvidenceLocked(localID, draft.pngBytes)
|
||||
appendOutboxLocked(
|
||||
ExecutionOutboxItem(
|
||||
id = localID,
|
||||
type = ExecutionOutboxType.EVIDENCE,
|
||||
idempotencyKey = newOpaqueSecret(),
|
||||
payload = "{}",
|
||||
evidenceRelativePath =
|
||||
"$OUTBOX_DIRECTORY/$localID.png"
|
||||
)
|
||||
)
|
||||
)
|
||||
draft.ordinal to localID
|
||||
localID
|
||||
}
|
||||
}
|
||||
val candidates = batch.candidates.map { candidate ->
|
||||
candidate.copy(
|
||||
evidenceLocalIDs = listOf(
|
||||
evidenceLocalIDs =
|
||||
requireNotNull(evidenceIDs[candidate.ordinal])
|
||||
)
|
||||
)
|
||||
}
|
||||
val payload = JSONObject()
|
||||
|
||||
+15
-3
@@ -1,14 +1,18 @@
|
||||
package com.roubao.autopilot.vlm
|
||||
|
||||
const val CANDIDATE_EVALUATION_SCHEMA_VERSION = 2
|
||||
const val CANDIDATE_EVALUATION_PROMPT_VERSION = "candidate-evaluation-v2"
|
||||
const val CANDIDATE_EVALUATION_SCHEMA_VERSION = 3
|
||||
const val CANDIDATE_EVALUATION_PROMPT_VERSION = "candidate-evaluation-v3"
|
||||
const val CANDIDATE_RECOMMENDATION_THRESHOLD = 0.75
|
||||
|
||||
data class CandidateEvaluationImage(
|
||||
val ordinal: Int,
|
||||
val mediaType: String,
|
||||
val bytes: ByteArray,
|
||||
val sha256: String
|
||||
val sha256: String,
|
||||
val specificationBytes: ByteArray = bytes,
|
||||
val specificationSha256: String = sha256,
|
||||
val localHardConstraintResults: List<CandidateHardConstraintResult> =
|
||||
emptyList()
|
||||
)
|
||||
|
||||
data class CandidateEvaluationInput(
|
||||
@@ -56,6 +60,7 @@ data class CandidateAssessment(
|
||||
val rejectionReasons: List<String>,
|
||||
val confidence: Double,
|
||||
val evidenceSha256: String,
|
||||
val specificationEvidenceSha256: String = evidenceSha256,
|
||||
val hardConstraintResults: List<CandidateHardConstraintResult> = emptyList()
|
||||
)
|
||||
|
||||
@@ -63,12 +68,17 @@ data class RankedCandidateAssessment(
|
||||
val sourceOrdinal: Int,
|
||||
val rankedOrdinal: Int,
|
||||
val evidenceSha256: String,
|
||||
val specificationEvidenceSha256: String,
|
||||
val assessment: CandidateAssessment
|
||||
) {
|
||||
init {
|
||||
require(sourceOrdinal == assessment.ordinal)
|
||||
require(rankedOrdinal > 0)
|
||||
require(evidenceSha256 == assessment.evidenceSha256)
|
||||
require(
|
||||
specificationEvidenceSha256 ==
|
||||
assessment.specificationEvidenceSha256
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -206,6 +216,8 @@ object CandidateTopFivePolicy {
|
||||
sourceOrdinal = assessment.ordinal,
|
||||
rankedOrdinal = index + 1,
|
||||
evidenceSha256 = assessment.evidenceSha256,
|
||||
specificationEvidenceSha256 =
|
||||
assessment.specificationEvidenceSha256,
|
||||
assessment = assessment
|
||||
)
|
||||
}
|
||||
|
||||
@@ -36,7 +36,7 @@ class CandidateEvaluator(
|
||||
retryable = false
|
||||
)
|
||||
}
|
||||
if (!input.isValid()) {
|
||||
if (!input.isValid(hardConstraints.constraints)) {
|
||||
return CandidateEvaluationResult.Failed(
|
||||
code = CandidateEvaluationFailureCode.EVIDENCE_INVALID,
|
||||
retryable = false
|
||||
@@ -59,7 +59,8 @@ class CandidateEvaluator(
|
||||
prompt = CandidateEvaluationPrompt.build(
|
||||
requirement = input.requirement,
|
||||
candidateOrdinal = candidate.ordinal,
|
||||
hardConstraints = hardConstraints.constraints
|
||||
hardConstraintResults =
|
||||
candidate.localHardConstraintResults
|
||||
),
|
||||
imageMediaType = candidate.mediaType,
|
||||
imageBytes = candidate.bytes
|
||||
@@ -95,7 +96,10 @@ class CandidateEvaluator(
|
||||
rawResponse = rawResponse,
|
||||
expectedOrdinal = candidate.ordinal,
|
||||
evidenceSha256 = candidate.sha256,
|
||||
expectedHardConstraints = hardConstraints.constraints
|
||||
specificationEvidenceSha256 =
|
||||
candidate.specificationSha256,
|
||||
expectedHardConstraintResults =
|
||||
candidate.localHardConstraintResults
|
||||
)
|
||||
}
|
||||
if (parsed == null) {
|
||||
@@ -165,7 +169,9 @@ class CandidateEvaluator(
|
||||
)
|
||||
}
|
||||
|
||||
private fun CandidateEvaluationInput.isValid(): Boolean {
|
||||
private fun CandidateEvaluationInput.isValid(
|
||||
expectedConstraints: List<SkuHardConstraint>
|
||||
): Boolean {
|
||||
if (candidates.size !in 1..MAX_CANDIDATES) {
|
||||
return false
|
||||
}
|
||||
@@ -175,7 +181,18 @@ class CandidateEvaluator(
|
||||
candidates.all { candidate ->
|
||||
candidate.mediaType == SUPPORTED_MEDIA_TYPE &&
|
||||
candidate.bytes.isNotEmpty() &&
|
||||
SHA256_PATTERN.matches(candidate.sha256)
|
||||
candidate.specificationBytes.isNotEmpty() &&
|
||||
SHA256_PATTERN.matches(candidate.sha256) &&
|
||||
SHA256_PATTERN.matches(candidate.specificationSha256) &&
|
||||
candidate.localHardConstraintResults.size ==
|
||||
expectedConstraints.size &&
|
||||
candidate.localHardConstraintResults.zip(
|
||||
expectedConstraints
|
||||
).all { (result, expected) ->
|
||||
result.kind == expected.kind &&
|
||||
result.expected == expected.expected &&
|
||||
result.evidence.isNotBlank()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -191,7 +208,9 @@ class CandidateEvaluator(
|
||||
rejectionReasons = emptyList(),
|
||||
confidence = 0.0,
|
||||
evidenceSha256 = candidate.sha256,
|
||||
hardConstraintResults = emptyList()
|
||||
specificationEvidenceSha256 =
|
||||
candidate.specificationSha256,
|
||||
hardConstraintResults = candidate.localHardConstraintResults
|
||||
)
|
||||
|
||||
private companion object {
|
||||
@@ -205,7 +224,7 @@ object CandidateEvaluationPrompt {
|
||||
fun build(
|
||||
requirement: RequirementExtraction,
|
||||
candidateOrdinal: Int,
|
||||
hardConstraints: List<SkuHardConstraint>
|
||||
hardConstraintResults: List<CandidateHardConstraintResult>
|
||||
): String {
|
||||
val requirementJson = JSONObject()
|
||||
.put("search_query", requirement.searchQuery)
|
||||
@@ -226,11 +245,13 @@ object CandidateEvaluationPrompt {
|
||||
.put(
|
||||
"hard_constraints",
|
||||
JSONArray().apply {
|
||||
hardConstraints.forEach { constraint ->
|
||||
hardConstraintResults.forEach { constraint ->
|
||||
put(
|
||||
JSONObject()
|
||||
.put("kind", constraint.kind.name)
|
||||
.put("expected", constraint.expected)
|
||||
.put("status", constraint.status.name)
|
||||
.put("evidence", constraint.evidence)
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -244,13 +265,13 @@ object CandidateEvaluationPrompt {
|
||||
Do not return coordinates, UI actions, state transitions, purchase decisions, order submission, or payment authorization.
|
||||
decision must be REVIEW, REJECT, or MANUAL_REQUIRED.
|
||||
REVIEW means potentially suitable for a person to inspect, never permission to buy.
|
||||
Evaluate every hard constraint using only visible screenshot evidence.
|
||||
hard constraint status must be MATCH, MISMATCH, or UNKNOWN.
|
||||
Local read-only specification parsing already determined every hard constraint status.
|
||||
Copy each provided local status exactly; never override it from visual inference.
|
||||
REVIEW requires every hard constraint to be MATCH. UNKNOWN requires MANUAL_REQUIRED.
|
||||
A hard constraint MISMATCH requires REJECT. Never infer a hidden color or size.
|
||||
Required schema:
|
||||
{
|
||||
"schema_version": 2,
|
||||
"schema_version": 3,
|
||||
"candidate_index": $candidateOrdinal,
|
||||
"decision": "REVIEW",
|
||||
"score": 0.0,
|
||||
@@ -295,7 +316,8 @@ private object CandidateAssessmentParser {
|
||||
rawResponse: String,
|
||||
expectedOrdinal: Int,
|
||||
evidenceSha256: String,
|
||||
expectedHardConstraints: List<SkuHardConstraint>
|
||||
specificationEvidenceSha256: String,
|
||||
expectedHardConstraintResults: List<CandidateHardConstraintResult>
|
||||
): CandidateAssessment? =
|
||||
runCatching {
|
||||
val root = JSONObject(rawResponse.trim())
|
||||
@@ -313,7 +335,7 @@ private object CandidateAssessmentParser {
|
||||
val confidence = root.strictUnitDouble("confidence")
|
||||
val hardConstraintResults = root
|
||||
.getJSONArray("hard_constraint_results")
|
||||
.strictHardConstraintResults(expectedHardConstraints)
|
||||
.strictHardConstraintResults(expectedHardConstraintResults)
|
||||
when (decision) {
|
||||
CandidateDecision.REVIEW -> {
|
||||
require(matched.isNotEmpty())
|
||||
@@ -351,12 +373,14 @@ private object CandidateAssessmentParser {
|
||||
rejectionReasons = rejectionReasons,
|
||||
confidence = confidence,
|
||||
evidenceSha256 = evidenceSha256,
|
||||
specificationEvidenceSha256 =
|
||||
specificationEvidenceSha256,
|
||||
hardConstraintResults = hardConstraintResults
|
||||
)
|
||||
}.getOrNull()
|
||||
|
||||
private fun JSONArray.strictHardConstraintResults(
|
||||
expected: List<SkuHardConstraint>
|
||||
expected: List<CandidateHardConstraintResult>
|
||||
): List<CandidateHardConstraintResult> {
|
||||
require(length() == expected.size)
|
||||
return buildList(length()) {
|
||||
@@ -375,16 +399,13 @@ private object CandidateAssessmentParser {
|
||||
val expectedValue = item.getString("expected").trim()
|
||||
require(kind == constraint.kind)
|
||||
require(expectedValue == constraint.expected)
|
||||
val status = HardConstraintMatchStatus.valueOf(
|
||||
item.getString("status")
|
||||
)
|
||||
require(status == constraint.status)
|
||||
item.getString("evidence").validatedAssessmentText()
|
||||
add(
|
||||
CandidateHardConstraintResult(
|
||||
kind = kind,
|
||||
expected = expectedValue,
|
||||
status = HardConstraintMatchStatus.valueOf(
|
||||
item.getString("status")
|
||||
),
|
||||
evidence = item.getString("evidence")
|
||||
.validatedAssessmentText()
|
||||
)
|
||||
constraint
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -469,6 +490,10 @@ object CandidateReviewBatchJson {
|
||||
)
|
||||
.put("confidence", assessment.confidence)
|
||||
.put("evidence_sha256", assessment.evidenceSha256)
|
||||
.put(
|
||||
"specification_evidence_sha256",
|
||||
assessment.specificationEvidenceSha256
|
||||
)
|
||||
.put(
|
||||
"hard_constraint_results",
|
||||
JSONArray().apply {
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package com.roubao.autopilot.vlm
|
||||
|
||||
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationEvidence
|
||||
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationGroupKind
|
||||
import java.text.Normalizer
|
||||
|
||||
enum class SkuConstraintKind {
|
||||
@@ -99,3 +101,136 @@ object SkuHardConstraintExtractor {
|
||||
"""(?:^|[^0-9])(\d{2}(?:\.\d)?)(?:码|SIZE)(?=$|[^A-Z0-9])"""
|
||||
)
|
||||
}
|
||||
|
||||
object CandidateSpecificationHardConstraintPolicy {
|
||||
fun evaluate(
|
||||
constraints: List<SkuHardConstraint>,
|
||||
evidence: PinduoduoSpecificationEvidence,
|
||||
evidenceSha256: String
|
||||
): List<CandidateHardConstraintResult> =
|
||||
constraints.map { constraint ->
|
||||
val kind = when (constraint.kind) {
|
||||
SkuConstraintKind.COLOR ->
|
||||
PinduoduoSpecificationGroupKind.COLOR
|
||||
SkuConstraintKind.SIZE ->
|
||||
PinduoduoSpecificationGroupKind.SIZE
|
||||
}
|
||||
val matchingGroups = evidence.groups.filter { it.kind == kind }
|
||||
if (matchingGroups.size != 1) {
|
||||
return@map result(
|
||||
constraint,
|
||||
HardConstraintMatchStatus.UNKNOWN,
|
||||
audit(constraint, "group_missing_or_duplicate", evidenceSha256)
|
||||
)
|
||||
}
|
||||
val group = matchingGroups.single()
|
||||
val matchingOptions = group.options.filter { option ->
|
||||
optionMatches(constraint, option.text)
|
||||
}
|
||||
when {
|
||||
matchingOptions.size > 1 ->
|
||||
result(
|
||||
constraint,
|
||||
HardConstraintMatchStatus.UNKNOWN,
|
||||
audit(constraint, "target_duplicate", evidenceSha256)
|
||||
)
|
||||
matchingOptions.singleOrNull()?.enabled == true ->
|
||||
result(
|
||||
constraint,
|
||||
HardConstraintMatchStatus.MATCH,
|
||||
audit(constraint, "target_enabled", evidenceSha256)
|
||||
)
|
||||
matchingOptions.singleOrNull()?.enabled == false ->
|
||||
result(
|
||||
constraint,
|
||||
HardConstraintMatchStatus.MISMATCH,
|
||||
audit(constraint, "target_disabled", evidenceSha256)
|
||||
)
|
||||
group.complete ->
|
||||
result(
|
||||
constraint,
|
||||
HardConstraintMatchStatus.MISMATCH,
|
||||
audit(constraint, "target_absent_complete", evidenceSha256)
|
||||
)
|
||||
else ->
|
||||
result(
|
||||
constraint,
|
||||
HardConstraintMatchStatus.UNKNOWN,
|
||||
audit(constraint, "group_incomplete", evidenceSha256)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun result(
|
||||
constraint: SkuHardConstraint,
|
||||
status: HardConstraintMatchStatus,
|
||||
evidence: String
|
||||
) = CandidateHardConstraintResult(
|
||||
kind = constraint.kind,
|
||||
expected = constraint.expected,
|
||||
status = status,
|
||||
evidence = evidence
|
||||
)
|
||||
|
||||
private fun audit(
|
||||
constraint: SkuHardConstraint,
|
||||
observation: String,
|
||||
evidenceSha256: String
|
||||
): String =
|
||||
"group=${constraint.kind.name};target=${constraint.expected};" +
|
||||
"observation=$observation;spec_sha256=$evidenceSha256"
|
||||
|
||||
private fun optionMatches(
|
||||
constraint: SkuHardConstraint,
|
||||
observed: String
|
||||
): Boolean {
|
||||
val normalized = Normalizer.normalize(
|
||||
observed.trim(),
|
||||
Normalizer.Form.NFKC
|
||||
).uppercase()
|
||||
return when (constraint.kind) {
|
||||
SkuConstraintKind.COLOR ->
|
||||
COLOR_ALIASES[constraint.expected].orEmpty()
|
||||
.any { alias -> normalized.matchesAlias(alias) }
|
||||
SkuConstraintKind.SIZE ->
|
||||
sizeAliases(constraint.expected).any { alias ->
|
||||
Regex(
|
||||
"""(?:^|[^A-Z0-9])${Regex.escape(alias)}""" +
|
||||
"""(?=$|[^A-Z0-9])"""
|
||||
).containsMatchIn(normalized)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun String.matchesAlias(alias: String): Boolean =
|
||||
if (alias.all { it in 'A'..'Z' }) {
|
||||
Regex(
|
||||
"""(?:^|[^A-Z])${Regex.escape(alias)}(?=$|[^A-Z])"""
|
||||
).containsMatchIn(this)
|
||||
} else {
|
||||
contains(alias)
|
||||
}
|
||||
|
||||
private fun sizeAliases(expected: String): Set<String> =
|
||||
when (expected) {
|
||||
"2XL" -> setOf("2XL", "XXL")
|
||||
"3XL" -> setOf("3XL", "XXXL")
|
||||
"FREE" -> setOf("FREE", "FREESIZE", "均码")
|
||||
else -> setOf(expected)
|
||||
}
|
||||
|
||||
private val COLOR_ALIASES = mapOf(
|
||||
"BLACK" to listOf("BLACK", "黑色", "亮黑", "纯黑"),
|
||||
"WHITE" to listOf("WHITE", "白色", "纯白", "米白"),
|
||||
"GRAY" to listOf("GRAY", "GREY", "灰色", "浅灰", "深灰"),
|
||||
"RED" to listOf("RED", "红色", "酒红", "玫红"),
|
||||
"BLUE" to listOf("BLUE", "蓝色", "藏青", "牛仔蓝"),
|
||||
"GREEN" to listOf("GREEN", "绿色", "军绿"),
|
||||
"YELLOW" to listOf("YELLOW", "黄色"),
|
||||
"PINK" to listOf("PINK", "粉色", "粉红"),
|
||||
"PURPLE" to listOf("PURPLE", "紫色"),
|
||||
"BROWN" to listOf("BROWN", "棕色", "咖色", "咖啡色"),
|
||||
"BEIGE" to listOf("BEIGE", "米色", "卡其"),
|
||||
"ORANGE" to listOf("ORANGE", "橙色")
|
||||
)
|
||||
}
|
||||
|
||||
+78
-14
@@ -18,14 +18,25 @@ class CandidateEvidenceSourceTest {
|
||||
|
||||
assertTrue(result.isSuccess)
|
||||
assertEquals(listOf(1, 2), result.getOrThrow().map { it.ordinal })
|
||||
assertTrue(
|
||||
result.getOrThrow().all {
|
||||
it.detailPngBytes.isNotEmpty() &&
|
||||
it.specificationPngBytes.isNotEmpty()
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `rejects non allowlisted filename before reading`() = runTest {
|
||||
withEvidenceRoot { root ->
|
||||
val evidence = writeCandidate(root, ordinal = 1, width = 10, height = 20)
|
||||
.copy(screenshotFileName = "../candidate-01.png")
|
||||
val original =
|
||||
writeCandidate(root, ordinal = 1, width = 10, height = 20)
|
||||
val evidence = original.copy(
|
||||
detailAsset = original.detailAsset.copy(
|
||||
fileName = "../candidate-01-detail.png"
|
||||
)
|
||||
)
|
||||
|
||||
val result = CandidateEvidenceSource(root).load(listOf(evidence))
|
||||
|
||||
@@ -56,17 +67,33 @@ class CandidateEvidenceSourceTest {
|
||||
|
||||
assertTrue(
|
||||
source.load(
|
||||
listOf(evidence.copy(screenshotByteCount = evidence.screenshotByteCount + 1))
|
||||
listOf(
|
||||
evidence.copy(
|
||||
detailAsset = evidence.detailAsset.copy(
|
||||
byteCount = evidence.detailAsset.byteCount + 1
|
||||
)
|
||||
)
|
||||
)
|
||||
).isFailure
|
||||
)
|
||||
assertTrue(
|
||||
source.load(
|
||||
listOf(evidence.copy(screenshotSha256 = "0".repeat(64)))
|
||||
listOf(
|
||||
evidence.copy(
|
||||
detailAsset = evidence.detailAsset.copy(
|
||||
sha256 = "0".repeat(64)
|
||||
)
|
||||
)
|
||||
)
|
||||
).isFailure
|
||||
)
|
||||
assertTrue(
|
||||
source.load(
|
||||
listOf(evidence.copy(screenshotWidth = 11))
|
||||
listOf(
|
||||
evidence.copy(
|
||||
detailAsset = evidence.detailAsset.copy(width = 11)
|
||||
)
|
||||
)
|
||||
).isFailure
|
||||
)
|
||||
}
|
||||
@@ -76,12 +103,18 @@ class CandidateEvidenceSourceTest {
|
||||
fun `rejects invalid png header and declared size above limit`() = runTest {
|
||||
withEvidenceRoot { root ->
|
||||
val evidence = writeCandidate(root, ordinal = 1, width = 10, height = 20)
|
||||
File(root, evidence.screenshotFileName).writeBytes(ByteArray(24))
|
||||
File(root, evidence.detailAsset.fileName).writeBytes(ByteArray(24))
|
||||
|
||||
assertTrue(CandidateEvidenceSource(root).load(listOf(evidence)).isFailure)
|
||||
assertTrue(
|
||||
CandidateEvidenceSource(root).load(
|
||||
listOf(evidence.copy(screenshotByteCount = 8 * 1024 * 1024 + 1))
|
||||
listOf(
|
||||
evidence.copy(
|
||||
detailAsset = evidence.detailAsset.copy(
|
||||
byteCount = 8 * 1024 * 1024 + 1
|
||||
)
|
||||
)
|
||||
)
|
||||
).isFailure
|
||||
)
|
||||
}
|
||||
@@ -103,22 +136,53 @@ class CandidateEvidenceSourceTest {
|
||||
height: Int
|
||||
): PinduoduoCandidateEvidence {
|
||||
val bytes = pngHeader(width, height)
|
||||
val fileName = "candidate-%02d.png".format(ordinal)
|
||||
File(root, fileName).writeBytes(bytes)
|
||||
val detailFileName = "candidate-%02d-detail.png".format(ordinal)
|
||||
val specificationFileName =
|
||||
"candidate-%02d-specification.png".format(ordinal)
|
||||
File(root, detailFileName).writeBytes(bytes)
|
||||
File(root, specificationFileName).writeBytes(bytes)
|
||||
val asset = PinduoduoEvidenceAsset(
|
||||
fileName = detailFileName,
|
||||
sha256 = PinduoduoEvidenceHash.sha256(bytes),
|
||||
byteCount = bytes.size,
|
||||
width = width,
|
||||
height = height
|
||||
)
|
||||
return PinduoduoCandidateEvidence(
|
||||
ordinal = ordinal,
|
||||
cardSignature = "card-$ordinal",
|
||||
cardSemanticTextCount = 3,
|
||||
detailSignature = "detail-$ordinal",
|
||||
detailSemanticTextCount = 4,
|
||||
screenshotFileName = fileName,
|
||||
screenshotSha256 = PinduoduoEvidenceHash.sha256(bytes),
|
||||
screenshotByteCount = bytes.size,
|
||||
screenshotWidth = width,
|
||||
screenshotHeight = height
|
||||
detailAsset = asset,
|
||||
specification = specification(),
|
||||
specificationAsset = asset.copy(fileName = specificationFileName)
|
||||
)
|
||||
}
|
||||
|
||||
private fun specification() = PinduoduoSpecificationEvidence(
|
||||
signature = "specification",
|
||||
semanticTextCount = 6,
|
||||
groups = listOf(
|
||||
PinduoduoSpecificationGroup(
|
||||
kind = PinduoduoSpecificationGroupKind.COLOR,
|
||||
title = "颜色分类",
|
||||
options = listOf(
|
||||
PinduoduoSpecificationOption("黑色", false, true)
|
||||
),
|
||||
complete = true
|
||||
),
|
||||
PinduoduoSpecificationGroup(
|
||||
kind = PinduoduoSpecificationGroupKind.SIZE,
|
||||
title = "尺码",
|
||||
options = listOf(
|
||||
PinduoduoSpecificationOption("L", false, true)
|
||||
),
|
||||
complete = true
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
private fun pngHeader(width: Int, height: Int): ByteArray =
|
||||
byteArrayOf(
|
||||
0x89.toByte(), 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
|
||||
|
||||
+107
-9
@@ -156,6 +156,31 @@ class PinduoduoCandidateAutomationTest {
|
||||
assertTrue(automation.evidence.value.isEmpty())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `missing safe specification entry blocks without purchase fallback`() =
|
||||
runTest {
|
||||
val driver = FakeCandidateDriver(
|
||||
cardPages = listOf(listOf(card("a"))),
|
||||
failOpenSpecifications = true
|
||||
)
|
||||
val automation = PinduoduoCandidateAutomation(
|
||||
driver = driver,
|
||||
pagePollIntervalMillis = 1
|
||||
)
|
||||
automation.reset()
|
||||
|
||||
val result = automation.execute(
|
||||
PinduoduoCandidateWorkflow.steps().last()
|
||||
)
|
||||
|
||||
assertEquals(
|
||||
AutomationResult.Blocked(SafetyStopReason.UNKNOWN_PAGE),
|
||||
result
|
||||
)
|
||||
assertEquals(1, driver.openSpecificationCalls)
|
||||
assertTrue(driver.savedEvidence.isEmpty())
|
||||
}
|
||||
|
||||
private fun card(signature: String) = PinduoduoCandidateCard(
|
||||
signature = signature,
|
||||
semanticTextCount = 3,
|
||||
@@ -166,6 +191,7 @@ class PinduoduoCandidateAutomationTest {
|
||||
private val cardPages: List<List<PinduoduoCandidateCard>>,
|
||||
private val safetyStopReason: SafetyStopReason? = null,
|
||||
private val failCapture: Boolean = false,
|
||||
private val failOpenSpecifications: Boolean = false,
|
||||
private val transitionDelaySnapshots: Int = 0
|
||||
) : PinduoduoCandidateDriver {
|
||||
private var cardPageIndex = 0
|
||||
@@ -176,6 +202,7 @@ class PinduoduoCandidateAutomationTest {
|
||||
val openedSignatures = mutableListOf<String>()
|
||||
val savedEvidence = mutableListOf<PinduoduoCandidateEvidence>()
|
||||
var scrollCalls = 0
|
||||
var openSpecificationCalls = 0
|
||||
|
||||
override suspend fun snapshot(): PinduoduoUiSnapshot {
|
||||
if (pendingPage != null) {
|
||||
@@ -204,24 +231,60 @@ class PinduoduoCandidateAutomationTest {
|
||||
return true
|
||||
}
|
||||
|
||||
override suspend fun captureCandidate(
|
||||
ordinal: Int,
|
||||
card: PinduoduoCandidateCard
|
||||
): PinduoduoCandidateEvidence? {
|
||||
override suspend fun captureDetail(): PinduoduoCandidateCapture? {
|
||||
if (failCapture) {
|
||||
return null
|
||||
}
|
||||
return PinduoduoCandidateCapture(
|
||||
detail = PinduoduoCandidateDetailEvidence("detail", 10),
|
||||
screenshot = PinduoduoScreenshotCapture(
|
||||
pngBytes = byteArrayOf(1),
|
||||
width = 1080,
|
||||
height = 2400
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
override suspend fun openSpecifications(): Boolean {
|
||||
openSpecificationCalls += 1
|
||||
if (failOpenSpecifications) {
|
||||
return false
|
||||
}
|
||||
transitionTo(PinduoduoPage.SPECIFICATION_PANEL)
|
||||
return true
|
||||
}
|
||||
|
||||
override suspend fun captureSpecifications():
|
||||
PinduoduoSpecificationCapture =
|
||||
PinduoduoSpecificationCapture(
|
||||
evidence = specification(),
|
||||
screenshot = PinduoduoScreenshotCapture(
|
||||
pngBytes = byteArrayOf(2),
|
||||
width = 1080,
|
||||
height = 2400
|
||||
)
|
||||
)
|
||||
|
||||
override suspend fun closeSpecifications(): Boolean {
|
||||
transitionTo(PinduoduoPage.PRODUCT_DETAIL)
|
||||
return true
|
||||
}
|
||||
|
||||
override suspend fun saveCandidate(
|
||||
ordinal: Int,
|
||||
card: PinduoduoCandidateCard,
|
||||
detail: PinduoduoCandidateCapture,
|
||||
specifications: PinduoduoSpecificationCapture
|
||||
): PinduoduoCandidateEvidence? {
|
||||
return PinduoduoCandidateEvidence(
|
||||
ordinal = ordinal,
|
||||
cardSignature = card.signature,
|
||||
cardSemanticTextCount = card.semanticTextCount,
|
||||
detailSignature = "detail-${card.signature}",
|
||||
detailSemanticTextCount = 10,
|
||||
screenshotFileName = "candidate-%02d.png".format(ordinal),
|
||||
screenshotSha256 = "hash-$ordinal",
|
||||
screenshotByteCount = 100 + ordinal,
|
||||
screenshotWidth = 1080,
|
||||
screenshotHeight = 2400
|
||||
detailAsset = asset(ordinal, "detail"),
|
||||
specification = specifications.evidence,
|
||||
specificationAsset = asset(ordinal, "specification")
|
||||
).also(savedEvidence::add)
|
||||
}
|
||||
|
||||
@@ -242,6 +305,7 @@ class PinduoduoCandidateAutomationTest {
|
||||
openedSignatures.clear()
|
||||
savedEvidence.clear()
|
||||
scrollCalls = 0
|
||||
openSpecificationCalls = 0
|
||||
cardPageIndex = 0
|
||||
page = PinduoduoPage.SEARCH_RESULTS
|
||||
pendingPage = null
|
||||
@@ -256,5 +320,39 @@ class PinduoduoCandidateAutomationTest {
|
||||
delayedSnapshotsRemaining = transitionDelaySnapshots
|
||||
}
|
||||
}
|
||||
|
||||
private fun asset(
|
||||
ordinal: Int,
|
||||
suffix: String
|
||||
) = PinduoduoEvidenceAsset(
|
||||
fileName = "candidate-%02d-%s.png".format(ordinal, suffix),
|
||||
sha256 = "a".repeat(64),
|
||||
byteCount = 1,
|
||||
width = 1080,
|
||||
height = 2400
|
||||
)
|
||||
|
||||
private fun specification() = PinduoduoSpecificationEvidence(
|
||||
signature = "specification",
|
||||
semanticTextCount = 6,
|
||||
groups = listOf(
|
||||
PinduoduoSpecificationGroup(
|
||||
kind = PinduoduoSpecificationGroupKind.COLOR,
|
||||
title = "颜色分类",
|
||||
options = listOf(
|
||||
PinduoduoSpecificationOption("黑色", false, true)
|
||||
),
|
||||
complete = true
|
||||
),
|
||||
PinduoduoSpecificationGroup(
|
||||
kind = PinduoduoSpecificationGroupKind.SIZE,
|
||||
title = "尺码",
|
||||
options = listOf(
|
||||
PinduoduoSpecificationOption("L", false, true)
|
||||
),
|
||||
complete = true
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
+35
@@ -202,6 +202,41 @@ class PinduoduoPageClassifierTest {
|
||||
assertNull(snapshot.safetyStopReason)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `specification panel requires color size and options`() {
|
||||
val snapshot = classify(
|
||||
element(text = "颜色分类"),
|
||||
element(text = "黑色", clickable = true),
|
||||
element(text = "尺码"),
|
||||
element(text = "L", clickable = true)
|
||||
)
|
||||
|
||||
assertEquals(PinduoduoPage.SPECIFICATION_PANEL, snapshot.page)
|
||||
assertNull(snapshot.safetyStopReason)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `cart and order pages are never classified as product detail`() {
|
||||
val cart = classify(
|
||||
element(text = "购物车"),
|
||||
element(text = "去结算")
|
||||
)
|
||||
val order = classify(
|
||||
element(contentDescription = "返回", clickable = true),
|
||||
element(text = "颜色分类"),
|
||||
element(text = "黑色", clickable = true),
|
||||
element(text = "尺码"),
|
||||
element(text = "L", clickable = true),
|
||||
element(text = "确认订单")
|
||||
)
|
||||
|
||||
assertEquals(PinduoduoPage.UNKNOWN, cart.page)
|
||||
assertEquals(
|
||||
SafetyStopReason.PAYMENT_BOUNDARY,
|
||||
order.safetyStopReason
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `other foreground package is unknown and has no inferred blocker`() {
|
||||
val snapshot = PinduoduoPageClassifier.classify(
|
||||
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
package com.roubao.autopilot.pinduoduo
|
||||
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertFalse
|
||||
import org.junit.Assert.assertNull
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
|
||||
class PinduoduoSpecificationParserTest {
|
||||
@Test
|
||||
fun `parses bounded color and size groups without clicking options`() {
|
||||
val evidence = PinduoduoSpecificationParser.parse(
|
||||
listOf(
|
||||
element("颜色分类", top = 100),
|
||||
element("黑色", top = 140, clickable = true, selected = true),
|
||||
element("白色", top = 180, clickable = true),
|
||||
element("尺码", top = 240),
|
||||
element("L", top = 280, clickable = true),
|
||||
element("XL", top = 320, clickable = true, enabled = false)
|
||||
)
|
||||
)
|
||||
|
||||
requireNotNull(evidence)
|
||||
assertEquals(
|
||||
PinduoduoSpecificationGroupKind.entries.toSet(),
|
||||
evidence.groups.map { it.kind }.toSet()
|
||||
)
|
||||
assertTrue(evidence.groups.all { it.complete })
|
||||
assertTrue(evidence.groups.first().options.first().selected)
|
||||
assertFalse(evidence.groups.last().options.last().enabled)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `scrollable group is marked incomplete`() {
|
||||
val evidence = PinduoduoSpecificationParser.parse(
|
||||
listOf(
|
||||
element("颜色分类", top = 100),
|
||||
element("黑色", top = 140, clickable = true),
|
||||
element("", top = 150, scrollable = true),
|
||||
element("尺码", top = 240),
|
||||
element("L", top = 280, clickable = true)
|
||||
)
|
||||
)
|
||||
|
||||
requireNotNull(evidence)
|
||||
assertFalse(evidence.groups.first().complete)
|
||||
assertTrue(evidence.groups.last().complete)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `missing group and transaction controls are rejected`() {
|
||||
assertNull(
|
||||
PinduoduoSpecificationParser.parse(
|
||||
listOf(
|
||||
element("颜色分类", top = 100),
|
||||
element("立即购买", top = 140, clickable = true)
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `safe entry allowlist excludes purchase controls`() {
|
||||
assertTrue(
|
||||
PinduoduoSpecificationParser.isSafeEntryText("请选择规格")
|
||||
)
|
||||
assertTrue(
|
||||
PinduoduoSpecificationParser.isSafeEntryText("已选:黑色,L")
|
||||
)
|
||||
assertTrue(
|
||||
PinduoduoSpecificationParser.isSafeEntryText("颜色\n款式")
|
||||
)
|
||||
assertFalse(
|
||||
PinduoduoSpecificationParser.isSafeEntryText("免拼购买")
|
||||
)
|
||||
}
|
||||
|
||||
private fun element(
|
||||
text: String,
|
||||
top: Int,
|
||||
clickable: Boolean = false,
|
||||
enabled: Boolean = true,
|
||||
selected: Boolean = false,
|
||||
scrollable: Boolean = false
|
||||
) = PinduoduoUiElement(
|
||||
text = text,
|
||||
contentDescription = null,
|
||||
className = "android.widget.TextView",
|
||||
resourceId = null,
|
||||
clickable = clickable,
|
||||
editable = false,
|
||||
enabled = enabled,
|
||||
visibleToUser = true,
|
||||
selected = selected,
|
||||
scrollable = scrollable,
|
||||
boundsTop = top
|
||||
)
|
||||
}
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
package com.roubao.autopilot.procurement
|
||||
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertThrows
|
||||
import org.junit.Test
|
||||
|
||||
class ExecutionEvidenceAssociationPolicyTest {
|
||||
@Test
|
||||
fun `binds detail and specification assets to each ranked candidate`() {
|
||||
val grouped = ExecutionEvidenceAssociationPolicy.group(
|
||||
candidateOrdinals = listOf(1, 2),
|
||||
evidence = listOf(
|
||||
draft(2, ExecutionEvidenceKind.SPECIFICATION),
|
||||
draft(1, ExecutionEvidenceKind.DETAIL),
|
||||
draft(2, ExecutionEvidenceKind.DETAIL),
|
||||
draft(1, ExecutionEvidenceKind.SPECIFICATION)
|
||||
)
|
||||
)
|
||||
|
||||
assertEquals(listOf(1, 2), grouped.keys.toList())
|
||||
assertEquals(
|
||||
ExecutionEvidenceKind.entries,
|
||||
grouped.getValue(1).map { it.kind }
|
||||
)
|
||||
assertEquals(
|
||||
ExecutionEvidenceKind.entries,
|
||||
grouped.getValue(2).map { it.kind }
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `rejects missing duplicate and wrong candidate assets`() {
|
||||
listOf(
|
||||
listOf(
|
||||
draft(1, ExecutionEvidenceKind.DETAIL)
|
||||
),
|
||||
listOf(
|
||||
draft(1, ExecutionEvidenceKind.DETAIL),
|
||||
draft(1, ExecutionEvidenceKind.DETAIL)
|
||||
),
|
||||
listOf(
|
||||
draft(1, ExecutionEvidenceKind.DETAIL),
|
||||
draft(2, ExecutionEvidenceKind.SPECIFICATION)
|
||||
)
|
||||
).forEach { evidence ->
|
||||
assertThrows(IllegalArgumentException::class.java) {
|
||||
ExecutionEvidenceAssociationPolicy.group(
|
||||
candidateOrdinals = listOf(1),
|
||||
evidence = evidence
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun draft(
|
||||
ordinal: Int,
|
||||
kind: ExecutionEvidenceKind
|
||||
) = ExecutionEvidenceDraft(
|
||||
ordinal = ordinal,
|
||||
pngBytes = byteArrayOf(ordinal.toByte(), kind.ordinal.toByte()),
|
||||
sha256 = "a".repeat(64),
|
||||
kind = kind
|
||||
)
|
||||
}
|
||||
@@ -159,7 +159,12 @@ class CandidateEvaluatorTest {
|
||||
)
|
||||
}
|
||||
|
||||
val result = evaluator.evaluate(input(candidateCount = 2))
|
||||
val result = evaluator.evaluate(
|
||||
input(
|
||||
candidateCount = 2,
|
||||
hardStatus = HardConstraintMatchStatus.MISMATCH
|
||||
)
|
||||
)
|
||||
as CandidateEvaluationResult.Completed
|
||||
|
||||
assertEquals(CandidateBatchConclusion.NO_MATCH, result.batch.conclusion)
|
||||
@@ -305,13 +310,40 @@ class CandidateEvaluatorTest {
|
||||
hardStatus = HardConstraintMatchStatus.UNKNOWN
|
||||
)
|
||||
)
|
||||
}.evaluate(input(candidateCount = 1))
|
||||
}.evaluate(
|
||||
input(
|
||||
candidateCount = 1,
|
||||
hardStatus = HardConstraintMatchStatus.UNKNOWN
|
||||
)
|
||||
)
|
||||
as CandidateEvaluationResult.Completed
|
||||
|
||||
assertTrue(CandidateTopFivePolicy.select(result.batch).isEmpty())
|
||||
assertNull(result.batch.recommendedCandidateOrdinal)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `model cannot override local specification status`() = runTest {
|
||||
val response = validResponse(
|
||||
ordinal = 1,
|
||||
decision = CandidateDecision.REJECT,
|
||||
score = 0.1,
|
||||
matched = emptyList(),
|
||||
rejectionReasons = listOf("模型声称颜色不匹配"),
|
||||
hardStatus = HardConstraintMatchStatus.MISMATCH
|
||||
)
|
||||
|
||||
val result = evaluator { Result.success(response) }
|
||||
.evaluate(input(candidateCount = 1))
|
||||
as CandidateEvaluationResult.Completed
|
||||
|
||||
assertEquals(
|
||||
CandidateDecision.MANUAL_REQUIRED,
|
||||
result.batch.assessments.single().decision
|
||||
)
|
||||
assertNull(result.batch.recommendedCandidateOrdinal)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `encoded review batch fixes order submitted to false`() = runTest {
|
||||
val result = evaluator { Result.success(validResponse(1)) }
|
||||
@@ -323,6 +355,12 @@ class CandidateEvaluatorTest {
|
||||
assertFalse(json.getBoolean("order_submitted"))
|
||||
assertTrue(json.getBoolean("manual_review_required"))
|
||||
assertEquals(1, json.getJSONArray("candidates").length())
|
||||
assertEquals(
|
||||
"e".repeat(64),
|
||||
json.getJSONArray("candidates")
|
||||
.getJSONObject(0)
|
||||
.getString("specification_evidence_sha256")
|
||||
)
|
||||
}
|
||||
|
||||
private fun evaluator(
|
||||
@@ -336,18 +374,44 @@ class CandidateEvaluatorTest {
|
||||
model = "fake-model"
|
||||
)
|
||||
|
||||
private fun input(candidateCount: Int): CandidateEvaluationInput =
|
||||
private fun input(
|
||||
candidateCount: Int,
|
||||
hardStatus: HardConstraintMatchStatus =
|
||||
HardConstraintMatchStatus.MATCH
|
||||
): CandidateEvaluationInput =
|
||||
CandidateEvaluationInput(
|
||||
requirement = requirement(),
|
||||
candidates = (1..candidateCount).map(::candidate)
|
||||
candidates = (1..candidateCount).map {
|
||||
candidate(it, hardStatus)
|
||||
}
|
||||
)
|
||||
|
||||
private fun candidate(ordinal: Int): CandidateEvaluationImage =
|
||||
private fun candidate(
|
||||
ordinal: Int,
|
||||
hardStatus: HardConstraintMatchStatus =
|
||||
HardConstraintMatchStatus.MATCH
|
||||
): CandidateEvaluationImage =
|
||||
CandidateEvaluationImage(
|
||||
ordinal = ordinal,
|
||||
mediaType = "image/png",
|
||||
bytes = byteArrayOf(ordinal.toByte()),
|
||||
sha256 = ordinal.toString(16).padStart(64, '0')
|
||||
sha256 = ordinal.toString(16).padStart(64, '0'),
|
||||
specificationBytes = byteArrayOf((ordinal + 10).toByte()),
|
||||
specificationSha256 = "e".repeat(64),
|
||||
localHardConstraintResults = listOf(
|
||||
CandidateHardConstraintResult(
|
||||
kind = SkuConstraintKind.COLOR,
|
||||
expected = "BLACK",
|
||||
status = hardStatus,
|
||||
evidence = "本地规格颜色证据"
|
||||
),
|
||||
CandidateHardConstraintResult(
|
||||
kind = SkuConstraintKind.SIZE,
|
||||
expected = "L",
|
||||
status = hardStatus,
|
||||
evidence = "本地规格尺码证据"
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
private fun requirement(): RequirementExtraction =
|
||||
|
||||
+148
@@ -0,0 +1,148 @@
|
||||
package com.roubao.autopilot.vlm
|
||||
|
||||
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationEvidence
|
||||
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationGroup
|
||||
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationGroupKind
|
||||
import com.roubao.autopilot.pinduoduo.PinduoduoSpecificationOption
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
import java.nio.charset.StandardCharsets
|
||||
|
||||
class CandidateSpecificationHardConstraintPolicyTest {
|
||||
@Test
|
||||
fun `both explicit enabled targets match`() {
|
||||
assertEquals(
|
||||
listOf(
|
||||
HardConstraintMatchStatus.MATCH,
|
||||
HardConstraintMatchStatus.MATCH
|
||||
),
|
||||
evaluate(
|
||||
colorOptions = listOf(option("经典黑色")),
|
||||
sizeOptions = listOf(option("XXL"))
|
||||
).map { it.status }
|
||||
)
|
||||
assertTrue(
|
||||
evaluate(
|
||||
colorOptions = listOf(option("经典黑色")),
|
||||
sizeOptions = listOf(option("XXL"))
|
||||
).all {
|
||||
it.evidence.toByteArray(StandardCharsets.UTF_8).size <= 160
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `complete missing target is mismatch`() {
|
||||
assertEquals(
|
||||
HardConstraintMatchStatus.MISMATCH,
|
||||
evaluate(
|
||||
colorOptions = listOf(option("白色")),
|
||||
sizeOptions = listOf(option("2XL"))
|
||||
).first().status
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `disabled target is mismatch`() {
|
||||
assertEquals(
|
||||
HardConstraintMatchStatus.MISMATCH,
|
||||
evaluate(
|
||||
colorOptions = listOf(option("黑色", enabled = false)),
|
||||
sizeOptions = listOf(option("2XL"))
|
||||
).first().status
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `truncated group and duplicated matching option are unknown`() {
|
||||
assertEquals(
|
||||
HardConstraintMatchStatus.UNKNOWN,
|
||||
evaluate(
|
||||
colorOptions = listOf(option("白色")),
|
||||
sizeOptions = listOf(option("2XL")),
|
||||
colorComplete = false
|
||||
).first().status
|
||||
)
|
||||
assertEquals(
|
||||
HardConstraintMatchStatus.UNKNOWN,
|
||||
evaluate(
|
||||
colorOptions = listOf(option("黑色"), option("纯黑")),
|
||||
sizeOptions = listOf(option("2XL"))
|
||||
).first().status
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `missing group is unknown`() {
|
||||
val evidence = specification(
|
||||
colorOptions = listOf(option("黑色")),
|
||||
sizeOptions = listOf(option("2XL"))
|
||||
).copy(groups = specification(
|
||||
colorOptions = listOf(option("黑色")),
|
||||
sizeOptions = listOf(option("2XL"))
|
||||
).groups.filter { it.kind == PinduoduoSpecificationGroupKind.COLOR })
|
||||
|
||||
val results = CandidateSpecificationHardConstraintPolicy.evaluate(
|
||||
constraints = constraints(),
|
||||
evidence = evidence,
|
||||
evidenceSha256 = "b".repeat(64)
|
||||
)
|
||||
|
||||
assertEquals(
|
||||
HardConstraintMatchStatus.UNKNOWN,
|
||||
results.last().status
|
||||
)
|
||||
}
|
||||
|
||||
private fun evaluate(
|
||||
colorOptions: List<PinduoduoSpecificationOption>,
|
||||
sizeOptions: List<PinduoduoSpecificationOption>,
|
||||
colorComplete: Boolean = true
|
||||
) = CandidateSpecificationHardConstraintPolicy.evaluate(
|
||||
constraints = constraints(),
|
||||
evidence = specification(
|
||||
colorOptions,
|
||||
sizeOptions,
|
||||
colorComplete
|
||||
),
|
||||
evidenceSha256 = "b".repeat(64)
|
||||
)
|
||||
|
||||
private fun constraints() = listOf(
|
||||
SkuHardConstraint(SkuConstraintKind.COLOR, "BLACK"),
|
||||
SkuHardConstraint(SkuConstraintKind.SIZE, "2XL")
|
||||
)
|
||||
|
||||
private fun specification(
|
||||
colorOptions: List<PinduoduoSpecificationOption>,
|
||||
sizeOptions: List<PinduoduoSpecificationOption>,
|
||||
colorComplete: Boolean = true
|
||||
) = PinduoduoSpecificationEvidence(
|
||||
signature = "specification",
|
||||
semanticTextCount = 8,
|
||||
groups = listOf(
|
||||
PinduoduoSpecificationGroup(
|
||||
PinduoduoSpecificationGroupKind.COLOR,
|
||||
"颜色分类",
|
||||
colorOptions,
|
||||
colorComplete
|
||||
),
|
||||
PinduoduoSpecificationGroup(
|
||||
PinduoduoSpecificationGroupKind.SIZE,
|
||||
"尺码",
|
||||
sizeOptions,
|
||||
true
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
private fun option(
|
||||
text: String,
|
||||
enabled: Boolean = true
|
||||
) = PinduoduoSpecificationOption(
|
||||
text = text,
|
||||
selected = false,
|
||||
enabled = enabled
|
||||
)
|
||||
}
|
||||
+33
-11
@@ -257,22 +257,44 @@ T-102/T-104 在搜索结果后追加一个有界候选步骤:
|
||||
-> 最多 2 次滚动预算
|
||||
-> 最多 5 个去重商品卡
|
||||
-> 验证详情页
|
||||
-> 无障碍截图
|
||||
-> App cache/candidate-NN.png + manifest.json
|
||||
-> 详情页无障碍截图
|
||||
-> 唯一、安全、非交易规格入口
|
||||
-> 规格弹层只读语义和截图
|
||||
-> App cache/candidate-NN-{detail,specification}.png + manifest.json
|
||||
-> 关闭规格弹层并复核详情页
|
||||
-> 一次全局返回并复核固定词结果页
|
||||
```
|
||||
|
||||
候选卡只保留语义指纹和计数,详情截图保存在 App 内部 cache。manifest 记录匿名文件名、
|
||||
截图 SHA-256、字节数和尺寸,不保存商品标题或页面原文。T-104 使用这些证据时必须通过
|
||||
受控 evidence 边界读取,不能让 VLM adapter 自行遍历 cache。Android 10/API 29 及
|
||||
以下不能运行当前截图探针,应在预检时明确不支持,不使用媒体投影或 shell 绕过。
|
||||
候选卡只保留语义指纹和计数,详情/规格截图保存在 App 内部 cache。manifest v2 记录
|
||||
匿名文件名、截图 SHA-256、字节数、尺寸和有界规格语义,不保存商品标题或完整页面
|
||||
原文。T-104/T-213 使用这些证据时必须通过受控 evidence 边界读取,不能让 VLM adapter
|
||||
自行遍历 cache。Android 10/API 29 及以下不能运行当前截图探针,应在预检时明确
|
||||
不支持,不使用媒体投影或 shell 绕过。
|
||||
|
||||
`CandidateEvidenceSource` 只接受当前 workflow 内存中的连续 ordinal 元数据,文件名
|
||||
固定为 `candidate-01.png` 至 `candidate-05.png`;每张 PNG 最多 8 MiB、全批最多
|
||||
32 MiB、声明尺寸最长边不超过 10000 px,并复核规范路径、PNG/IHDR、精确字节数、
|
||||
尺寸和 SHA-256。Android gateway 再解码并把最长边缩至 2048 px。当前需求快照与
|
||||
搜索词必须同时匹配候选 session,否则拒绝评估;另一个关键词的结果页只允许重新进入
|
||||
搜索框,不能采集候选。
|
||||
固定为 `candidate-01-detail.png`/`candidate-01-specification.png` 至第五组;每张
|
||||
PNG 最多 8 MiB、全批最多 64 MiB、声明尺寸最长边不超过 10000 px,并复核规范路径、
|
||||
PNG/IHDR、精确字节数、尺寸和 SHA-256。每个候选回传时必须恰好绑定
|
||||
`DETAIL`/`SPECIFICATION` 两份证据,生成独立 asset ID;先验证全批 ordinal、类型和
|
||||
哈希再写入加密 outbox,候选的 asset ID 数组固定按 `DETAIL`、`SPECIFICATION`
|
||||
排序,重试沿用各自 idempotency key。Android gateway 再解码详情图并把最长边缩至
|
||||
2048 px。当前需求快照与搜索词必须同时匹配候选 session,否则拒绝评估;另一个
|
||||
关键词的结果页只允许重新进入搜索框,不能采集候选。
|
||||
|
||||
T-213 规格核验只允许精确语义为“选择规格/请选择规格/颜色分类/颜色款式/选择颜色/
|
||||
选择尺码”或“已选/请选择”前缀的唯一节点及其可点击祖先。弹层内不点击任何规格选项,
|
||||
只读取最多 160 个语义节点、每组最多 30 个选项及 selected/enabled/scrollable 语义。
|
||||
颜色和尺码各自必须是唯一分组:目标值明确且可用为 `MATCH`,完整分组明确缺失或目标
|
||||
禁用为 `MISMATCH`,重复、缺组、滚动截断或无障碍语义不足为 `UNKNOWN`。
|
||||
|
||||
候选评估 schema/prompt v3 把本地规格结果作为权威输入;模型返回的颜色/尺码状态必须
|
||||
逐项与本地结果相同,否则整批降级人工检查。详情和规格证据 SHA-256 同时绑定
|
||||
assessment、重排身份和 outbox 资产,VLM 不能通过主图推断覆盖本地硬约束。
|
||||
|
||||
PKG110 上的拼多多 8.17.0 服装详情只暴露不可点击的“颜色/款式”预览;完整颜色/尺码
|
||||
弹层只能通过“单独购买/免拼购买”控件进入。自动化不得点击这两个交易语义控件,也
|
||||
不得用坐标降级,因此当前版本在没有独立可点击规格入口时返回安全阻塞。是否把购买
|
||||
语义控件重新定义为只读弹层入口,必须单独做产品/安全决策后变更本边界。
|
||||
|
||||
评估结束后 automation 不再产生动作,UI 进入 `AWAITING_CONFIRMATION`、
|
||||
`MANUAL_REVIEW` 或 `NO_MATCH`。人员可以把本地建议项标记为可用,或拒绝本次候选;
|
||||
|
||||
+20
-15
@@ -5,9 +5,9 @@
|
||||
## 当前快照
|
||||
|
||||
- 日期:2026-07-27
|
||||
- 阶段:T-212 候选重排身份映射完成;下一步 T-213 只读核验拼多多规格弹窗
|
||||
- 阶段:T-212 已完成;T-213 只读规格核验已实现但被拼多多 8.17 入口语义阻塞
|
||||
- Git:当前分支为 `main`;T-001 至 T-004、T-101 至 T-104、T-201 至 T-207、T-209、
|
||||
T-210、T-211、T-212 均已纳入 Git 历史
|
||||
T-210、T-211、T-212 均已纳入 Git 历史;T-213 等待安全策略决策
|
||||
- 生产代码:`android-buyer/` 已接入 Roubao Android 源码
|
||||
- Android:固定 `main@c8a6d7f03422eb01744b01f3ee77bf7757741f7e`;MIT 许可证已保留
|
||||
|
||||
@@ -17,10 +17,10 @@
|
||||
- 本机 Android 工具:JDK 17.0.13、Command-line Tools 22.0、SDK 34、
|
||||
Build Tools 34.0.0、Platform Tools/ADB 37.0.0;用户级 SDK 环境变量已设置
|
||||
- Android Studio:未安装;`winget` 静默安装卡住后已终止,不阻塞命令行构建
|
||||
- 测试:T-212 运行 `:app:testDebugUnitTest` 和 `:app:assembleDebug` 通过,21 个 suite
|
||||
共 113 个测试、0 失败;T-211 Debug APK 已安装到 PKG110
|
||||
- 后端测试:T-211 运行 `GOTOOLCHAIN=local go test -count=1 ./...` 和
|
||||
`go vet ./...` 通过;T-207 的全包 race 与 migration 验证继续有效
|
||||
- 测试:T-213 运行 `:app:testDebugUnitTest` 和 `:app:assembleDebug` 通过,24 个 suite
|
||||
共 128 个测试、0 失败;Debug APK `1.4.3 (8)` 已安装到 PKG110
|
||||
- 后端测试:T-213 运行 `go test ./...` 通过;T-207 的全包 race 与 migration
|
||||
验证继续有效
|
||||
- 原型:4 个管理 Web 页面和 7 个 Android 页面均可离线独立打开;Playwright
|
||||
以 1440×900、390×844、360×800 验证 36 个页面/视口组合,无页面横向溢出、
|
||||
脚本错误或外部请求,Android 可见交互控件均不小于 44px
|
||||
@@ -47,9 +47,13 @@
|
||||
顺序回传,授权到期补报单独审计,所有终态固定 `order_submitted=false`。管理任务
|
||||
详情展示模型/候选/人工理由/事件/证据摘要,不保存 VLM Key、完整 endpoint 或原始响应。
|
||||
- T-211 图片检索:后台任务固定使用经 SHA-256 校验的参考 JPEG,经一次性 MediaStore
|
||||
图片进入拼多多拍照搜索;App 本地从 SKU 唯一提取颜色和尺码,schema v2 逐项返回
|
||||
图片进入拼多多拍照搜索;App 本地从 SKU 唯一提取颜色和尺码,schema v3 逐项返回
|
||||
`MATCH/MISMATCH/UNKNOWN`。只有两项均匹配且分数/置信度不低于 `0.75` 的候选按
|
||||
分数、置信度和曝光顺序回传 `0..5` 项,弱匹配和未知项不凑数。
|
||||
- T-213 规格核验:只允许唯一、非交易规格入口;弹层内不点击选项,本地根据有界的
|
||||
颜色/尺码分组、可用/禁用和完整性语义产生硬约束结果。每个候选绑定详情/规格两张
|
||||
PNG 及两个 SHA-256/asset ID;模型不能覆盖本地状态。拼多多 8.17 服装详情的完整
|
||||
规格弹层只挂在“单独购买/免拼购买”后,当前安全边界拒绝该入口,故真机流程阻塞。
|
||||
- Workflow:纯 Kotlin runner 已支持步骤 timeout、最多 3 次 retry、安全阻塞、
|
||||
用户停止和单 runner 并发拒绝;T-102 已接入搜索加有界候选采集五步
|
||||
- TaskSource:严格 CLI 已生成并验证真实私有 ProbeTask;默认 APK 不含私有 fixture
|
||||
@@ -65,7 +69,7 @@
|
||||
- 离线执行:默认 30 分钟有限授权和 30 秒 best-effort heartbeat 已实现;PKG110
|
||||
真机断开临时后端 95 秒后保持同一 execution,重连后滑动续期;到期持久安全停止,
|
||||
RUNNING 不自动重新分配
|
||||
- 测试设备:OnePlus PKG110,Android 16/API 36;肉包 `1.4.2 (7)`;拼多多
|
||||
- 测试设备:OnePlus PKG110,Android 16/API 36;肉包 `1.4.3 (8)`;拼多多
|
||||
`8.17.0 (81700)`
|
||||
- 设备就绪:拼多多首页、文字/图片搜索、双列候选卡、详情截图和返回均已通过 8.17.0
|
||||
真机验证;最终 APK 重装/force-stop 后 ColorOS 已关闭肉包采购无障碍,当前需采购员
|
||||
@@ -80,8 +84,10 @@
|
||||
已用 CLI 真实导入并逐字段/图片哈希验证,生成物位于被忽略的 `.local/`
|
||||
- 标准启动路径:`$env:RUN_START_COMMAND="1"; .\init.ps1`
|
||||
- 标准验证路径:`.\init.ps1`
|
||||
- 当前 blocker:真实 VLM 服务地址、模型、设备级测试凭证、成本上限和数据留存尚未确认;
|
||||
当前只支持单 SKU/JPEG;候选探针截图要求 Android 11/API 30+
|
||||
- 当前 blocker:T-213 需要确认是否允许点击“单独购买/免拼购买”仅用于打开规格弹层,
|
||||
或提供独立规格入口/受支持数据源;ColorOS 重装后需采购员手动重新启用肉包无障碍。
|
||||
真实 VLM 服务地址、模型、设备级测试凭证、成本上限和数据留存也尚未确认;当前只
|
||||
支持单 SKU/JPEG;候选探针截图要求 Android 11/API 30+
|
||||
|
||||
## 当前目录
|
||||
|
||||
@@ -108,7 +114,7 @@
|
||||
| `docs/tasks/T-210.md` | DONE | 兼容拼多多 8.17 搜索结果页与长词省略显示 |
|
||||
| `docs/tasks/T-211.md` | DONE | 参考图召回、SKU 颜色尺码硬匹配与 0..5 候选回传 |
|
||||
| `docs/tasks/T-212.md` | DONE | 修复重排候选、推荐、证据与人工接受的身份映射 |
|
||||
| `docs/tasks/T-213.md` | TODO | 只读采集拼多多规格弹窗中的颜色和尺码证据 |
|
||||
| `docs/tasks/T-213.md` | BLOCKED | 实现只读规格核验;等待拼多多购买语义入口安全决策 |
|
||||
| `docs/design/` | 已确认 | T-202 原型索引、4 个管理页和 7 个 Android 页面 |
|
||||
| `deepseek总结.txt` | 已有 | 历史讨论摘要,不是正式需求权威 |
|
||||
| `android-buyer/` | 已有 | Roubao `main` 固定 commit 的 Android 基线 |
|
||||
@@ -121,10 +127,9 @@
|
||||
|
||||
- 已完成:T-001 至 T-004、T-101 至 T-104、T-201 至 T-207、T-209、T-210、T-211、
|
||||
T-212。
|
||||
- 正在进行:T-213 拼多多规格弹窗只读核验。
|
||||
- 下一个可领取任务:无;T-208 依赖 T-213 完成。
|
||||
- 后置任务:T-213 拼多多规格弹窗只读核验;完成后再做 T-208 候选决策数据与人工
|
||||
理由闭环。
|
||||
- 阻塞中:T-213 拼多多规格弹窗只读核验,等待购买语义入口的安全策略决策。
|
||||
- 下一个可领取任务:无;T-208 依赖 T-213 真机验收完成。
|
||||
- 后置任务:解除 T-213 阻塞后再做 T-208 候选决策数据与人工理由闭环。
|
||||
|
||||
## 当前可运行内容
|
||||
|
||||
|
||||
+24
-7
@@ -4,11 +4,12 @@ title: 拼多多规格弹窗只读核验
|
||||
phase: 2
|
||||
deps:
|
||||
- T-212
|
||||
status: DOING
|
||||
status: BLOCKED
|
||||
created: 2026-07-27
|
||||
context_ref: edfee2c
|
||||
work_branch: null
|
||||
write_paths:
|
||||
- android-buyer/app/build.gradle.kts
|
||||
- docs/tasks/T-213.md
|
||||
- docs/04-architecture.md
|
||||
- docs/current-state.md
|
||||
@@ -53,12 +54,12 @@ Top 5 为空。仅凭商品主图推断颜色或尺码不满足硬约束要求
|
||||
|
||||
## 验收要点
|
||||
|
||||
- [ ] 页面分类测试覆盖规格弹窗、普通详情、登录、验证码、风控、购物车、订单和支付。
|
||||
- [ ] 自动化只点击唯一规格入口,不点击颜色/尺码选项、购买、拼单、购物车或支付控件。
|
||||
- [ ] 颜色、尺码分组及选项解析有数量/文本长度边界,截断或歧义不会判为 `MATCH`。
|
||||
- [ ] 单元测试覆盖两个目标均匹配、明确不匹配、禁用、缺失和未知。
|
||||
- [ ] 每个候选的详情/规格证据哈希和后端 asset ID 对应正确,离线重试不重复或串位。
|
||||
- [ ] Android 单元测试和 Debug 构建通过。
|
||||
- [x] 页面分类测试覆盖规格弹窗、普通详情、登录、验证码、风控、购物车、订单和支付。
|
||||
- [x] 自动化只点击唯一规格入口,不点击颜色/尺码选项、购买、拼单、购物车或支付控件。
|
||||
- [x] 颜色、尺码分组及选项解析有数量/文本长度边界,截断或歧义不会判为 `MATCH`。
|
||||
- [x] 单元测试覆盖两个目标均匹配、明确不匹配、禁用、缺失和未知。
|
||||
- [x] 每个候选的详情/规格证据哈希和后端 asset ID 对应正确,离线重试不重复或串位。
|
||||
- [x] Android 单元测试和 Debug 构建通过。
|
||||
- [ ] PKG110、Android 16、拼多多 8.17.0 真机 smoke 完成规格弹窗打开、只读采集、
|
||||
关闭和返回;记录 App/拼多多版本并确认未进入订单或支付。
|
||||
|
||||
@@ -75,3 +76,19 @@ Top 5 为空。仅凭商品主图推断颜色或尺码不满足硬约束要求
|
||||
关联到错误的重排候选。
|
||||
- 2026-07-27:T-212 完成提交 `4b795a4` 后领取。先冻结拼多多 8.17 规格弹窗的只读
|
||||
页面/动作边界和候选多证据模型,再进行真机验证。
|
||||
- 2026-07-27:实现规格弹层分类、安全入口 allowlist、颜色/尺码有界解析、本地确定性
|
||||
硬匹配、candidate evaluation schema/prompt v3,以及每个候选详情/规格两份证据的
|
||||
cache 校验、重排身份绑定和加密 outbox 关联。模型返回状态与本地状态不一致时强制
|
||||
降级人工检查。
|
||||
- 2026-07-27:Android `testDebugUnitTest` 24 个 suite、128 个测试全部通过,
|
||||
`assembleDebug` 通过;后端 `go test ./...` 通过。Debug APK `1.4.3 (8)` 已安装到
|
||||
PKG110,拼多多版本为 `8.17.0 (81700)`。
|
||||
- 2026-07-27:真机检查图片结果中的定制商品和文字搜索 `black tshirt L` 的普通服装
|
||||
商品。服装详情只暴露不可点击的“颜色/款式”预览,完整颜色/尺码弹层只能通过
|
||||
“单独购买/免拼购买”控件进入;前两个定制商品甚至没有规格预览。全程未点击购买、
|
||||
拼单、购物车、订单或支付控件。
|
||||
- 2026-07-27:任务标记 `BLOCKED`。代码在缺少独立可点击规格入口时按
|
||||
`UNKNOWN_PAGE` 安全停止,符合边界但无法完成“打开/采集/关闭/返回”的真机验收。
|
||||
外部解除条件是:确认允许把购买语义控件仅用于打开规格弹层,或提供不经过购买控件
|
||||
的稳定规格入口/受支持数据源。重新安装后 ColorOS 也关闭了肉包无障碍服务,继续
|
||||
smoke 前需采购员在系统设置手动重新启用。
|
||||
|
||||
Reference in New Issue
Block a user