fix(collect): handle login target close race
Tests / Python 3.11 / Windows (push) Has been cancelled
Tests / Python 3.11 / Windows (push) Has been cancelled
This commit is contained in:
@@ -327,6 +327,10 @@ def login_status_text(status) -> str:
|
||||
return "未登录"
|
||||
if reason == "NO_SESSION_COOKIE":
|
||||
return "未登录"
|
||||
if reason == "LOGIN_CHECK_TARGET_UNAVAILABLE" or str(reason).startswith(
|
||||
"LOGIN_CHECK_FAILED"
|
||||
):
|
||||
return "登录状态暂不可确认"
|
||||
if reason:
|
||||
return f"未登录({reason})"
|
||||
return "未登录"
|
||||
|
||||
+41
-2
@@ -22,12 +22,12 @@ def _base(host=None):
|
||||
return f"http://{host or CDP_HOST}"
|
||||
|
||||
|
||||
def http_get(path, host=None):
|
||||
def http_get(path, host=None, timeout=10):
|
||||
import requests
|
||||
|
||||
s = requests.Session()
|
||||
s.trust_env = False # 忽略环境代理
|
||||
return s.get(f"{_base(host)}{path}", timeout=10).json()
|
||||
return s.get(f"{_base(host)}{path}", timeout=timeout).json()
|
||||
|
||||
|
||||
def close_tab(target_id, host=None):
|
||||
@@ -43,6 +43,45 @@ def close_tab(target_id, host=None):
|
||||
return response.ok
|
||||
|
||||
|
||||
def wait_target_closed(target_id, host=None, timeout=2.0, poll_interval=0.1):
|
||||
"""Wait briefly for a closed target to disappear from Chrome's target list."""
|
||||
|
||||
if not target_id:
|
||||
return True
|
||||
timeout = max(0.0, float(timeout))
|
||||
poll_interval = max(0.01, float(poll_interval))
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
remaining = max(0.0, deadline - time.monotonic())
|
||||
request_timeout = max(0.05, min(1.0, remaining))
|
||||
try:
|
||||
targets = http_get("/json", host=host, timeout=request_timeout)
|
||||
except Exception:
|
||||
targets = None
|
||||
if targets is not None and not any(
|
||||
str(target.get("id") or "") == str(target_id)
|
||||
for target in targets
|
||||
):
|
||||
return True
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
return False
|
||||
time.sleep(min(poll_interval, remaining))
|
||||
|
||||
|
||||
def close_tab_and_wait(target_id, host=None, timeout=2.0, poll_interval=0.1):
|
||||
"""Request target closure and confirm disappearance within a bounded budget."""
|
||||
|
||||
if not close_tab(target_id, host=host):
|
||||
return False
|
||||
return wait_target_closed(
|
||||
target_id,
|
||||
host=host,
|
||||
timeout=timeout,
|
||||
poll_interval=poll_interval,
|
||||
)
|
||||
|
||||
|
||||
def activate_tab(target_id, host=None):
|
||||
"""Bring an existing Chrome target/page to the foreground."""
|
||||
|
||||
|
||||
+215
-60
@@ -6,7 +6,15 @@ import time
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from . import appconfig, image_paths
|
||||
from .cdp import CDP, close_tab, create_tab, create_tab_info, find_product_tab, http_get
|
||||
from .cdp import (
|
||||
CDP,
|
||||
close_tab,
|
||||
close_tab_and_wait,
|
||||
create_tab,
|
||||
create_tab_info,
|
||||
find_product_tab,
|
||||
http_get,
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_REGION_HOST = "seller.shopee.tw"
|
||||
@@ -669,71 +677,206 @@ def _current_url(cdp):
|
||||
return ""
|
||||
|
||||
|
||||
def _wait_for_login_probe(cdp, timeout=8):
|
||||
end = time.time() + timeout
|
||||
last_url = ""
|
||||
def _wait_for_login_probe(cdp, timeout=8, initial_url=""):
|
||||
deadline = time.monotonic() + max(0.0, float(timeout))
|
||||
last_url = str(initial_url or "")
|
||||
cookie_names = set()
|
||||
while time.time() < end:
|
||||
last_url = _current_url(cdp) or last_url
|
||||
cookie_read_succeeded = False
|
||||
probe_error = None
|
||||
first_probe = True
|
||||
while first_probe or time.monotonic() < deadline:
|
||||
first_probe = False
|
||||
try:
|
||||
last_url = cdp.val("location.href") or last_url
|
||||
except Exception as exc:
|
||||
return {
|
||||
"url": last_url,
|
||||
"cookie_names": cookie_names,
|
||||
"cookie_read_succeeded": cookie_read_succeeded,
|
||||
"target_available": False,
|
||||
"probe_error": exc.__class__.__name__,
|
||||
}
|
||||
if _is_login_url(last_url):
|
||||
return {
|
||||
"url": last_url,
|
||||
"cookie_names": cookie_names,
|
||||
"cookie_read_succeeded": cookie_read_succeeded,
|
||||
"target_available": True,
|
||||
"probe_error": probe_error,
|
||||
}
|
||||
try:
|
||||
cookies = cdp.send("Network.getAllCookies").get("cookies", [])
|
||||
cookie_names = {
|
||||
c.get("name")
|
||||
for c in cookies
|
||||
if "shopee" in (c.get("domain") or "")
|
||||
except Exception as exc:
|
||||
return {
|
||||
"url": last_url,
|
||||
"cookie_names": cookie_names,
|
||||
"cookie_read_succeeded": cookie_read_succeeded,
|
||||
"target_available": False,
|
||||
"probe_error": exc.__class__.__name__,
|
||||
}
|
||||
except Exception:
|
||||
cookie_names = set()
|
||||
if _is_login_url(last_url) or "SPC_ST" in cookie_names or "SPC_U" in cookie_names:
|
||||
cookie_read_succeeded = True
|
||||
cookie_names = {
|
||||
cookie.get("name")
|
||||
for cookie in cookies
|
||||
if "shopee" in (cookie.get("domain") or "")
|
||||
}
|
||||
if "SPC_ST" in cookie_names or "SPC_U" in cookie_names:
|
||||
break
|
||||
time.sleep(0.5)
|
||||
return last_url, cookie_names
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
break
|
||||
time.sleep(min(0.5, remaining))
|
||||
return {
|
||||
"url": last_url,
|
||||
"cookie_names": cookie_names,
|
||||
"cookie_read_succeeded": cookie_read_succeeded,
|
||||
"target_available": cookie_read_succeeded,
|
||||
"probe_error": probe_error,
|
||||
}
|
||||
|
||||
|
||||
def _login_target_priority(target):
|
||||
url = str((target or {}).get("url") or "").lower()
|
||||
if _is_login_url(url):
|
||||
return 3
|
||||
if "/portal/product/" in url:
|
||||
return 2
|
||||
if "/portal/" in url:
|
||||
return 0
|
||||
return 1
|
||||
|
||||
|
||||
def _login_target_key(target):
|
||||
target = target or {}
|
||||
return str(target.get("webSocketDebuggerUrl") or target.get("id") or target.get("url") or "")
|
||||
|
||||
|
||||
def _shopee_page_targets(host):
|
||||
pages = [
|
||||
target
|
||||
for target in http_get("/json", host=host)
|
||||
if target.get("type") == "page"
|
||||
and "shopee" in str(target.get("url") or "").lower()
|
||||
and target.get("webSocketDebuggerUrl")
|
||||
]
|
||||
return sorted(pages, key=_login_target_priority)
|
||||
|
||||
|
||||
def login_status(account, timeout=8) -> dict:
|
||||
"""Return detailed Shopee login status for an account's CDP session."""
|
||||
|
||||
host = _cdp_host(account)
|
||||
pages = [t for t in http_get("/json", host=host) if t.get("type") == "page"]
|
||||
shopee_page = next((p for p in pages if "shopee" in (p.get("url") or "")), None)
|
||||
if not shopee_page:
|
||||
ws = create_tab(_seller_home_url(account), host=host)
|
||||
initial_url = _seller_home_url(account)
|
||||
cdp = CDP(ws)
|
||||
else:
|
||||
initial_url = shopee_page.get("url") or ""
|
||||
if _is_login_url(initial_url):
|
||||
deadline = time.monotonic() + max(0.0, float(timeout))
|
||||
attempted_targets = set()
|
||||
opened_home = False
|
||||
probe_attempts = 0
|
||||
last_probe = {
|
||||
"url": "",
|
||||
"cookie_names": set(),
|
||||
"cookie_read_succeeded": False,
|
||||
"target_available": False,
|
||||
"probe_error": None,
|
||||
}
|
||||
while True:
|
||||
candidates = [
|
||||
target
|
||||
for target in _shopee_page_targets(host)
|
||||
if _login_target_key(target) not in attempted_targets
|
||||
]
|
||||
if not candidates and not opened_home:
|
||||
home_url = _seller_home_url(account)
|
||||
ws = create_tab(home_url, host=host)
|
||||
candidates = [
|
||||
{
|
||||
"type": "page",
|
||||
"url": home_url,
|
||||
"webSocketDebuggerUrl": ws,
|
||||
}
|
||||
]
|
||||
opened_home = True
|
||||
if not candidates:
|
||||
break
|
||||
for target in candidates:
|
||||
key = _login_target_key(target)
|
||||
if key:
|
||||
attempted_targets.add(key)
|
||||
probe_attempts += 1
|
||||
initial_url = str(target.get("url") or "")
|
||||
if _is_login_url(initial_url):
|
||||
return {
|
||||
"logged_in": False,
|
||||
"reason": "LOGIN_PAGE",
|
||||
"url": initial_url,
|
||||
"host": host,
|
||||
"cookie_names": [],
|
||||
"cookie_read_succeeded": False,
|
||||
"probe_error": None,
|
||||
"probe_attempts": probe_attempts,
|
||||
}
|
||||
cdp = None
|
||||
try:
|
||||
cdp = CDP(target["webSocketDebuggerUrl"])
|
||||
_ensure_page_domains(cdp)
|
||||
remaining = max(0.0, deadline - time.monotonic())
|
||||
probe = _wait_for_login_probe(
|
||||
cdp,
|
||||
timeout=remaining,
|
||||
initial_url=initial_url,
|
||||
)
|
||||
except Exception as exc:
|
||||
probe = {
|
||||
"url": initial_url,
|
||||
"cookie_names": set(),
|
||||
"cookie_read_succeeded": False,
|
||||
"target_available": False,
|
||||
"probe_error": exc.__class__.__name__,
|
||||
}
|
||||
finally:
|
||||
if cdp is not None:
|
||||
cdp.close()
|
||||
last_probe = probe
|
||||
url = str(probe.get("url") or initial_url)
|
||||
names = set(probe.get("cookie_names") or [])
|
||||
if _is_login_url(url):
|
||||
return {
|
||||
"logged_in": False,
|
||||
"reason": "LOGIN_PAGE",
|
||||
"url": url,
|
||||
"host": host,
|
||||
"cookie_names": sorted(name for name in names if name),
|
||||
"cookie_read_succeeded": bool(probe.get("cookie_read_succeeded")),
|
||||
"probe_error": probe.get("probe_error"),
|
||||
"probe_attempts": probe_attempts,
|
||||
}
|
||||
if not probe.get("cookie_read_succeeded") or not probe.get(
|
||||
"target_available"
|
||||
):
|
||||
continue
|
||||
logged_in = "SPC_ST" in names or "SPC_U" in names
|
||||
return {
|
||||
"logged_in": False,
|
||||
"reason": "LOGIN_PAGE",
|
||||
"url": initial_url,
|
||||
"logged_in": logged_in,
|
||||
"reason": None if logged_in else "NO_SESSION_COOKIE",
|
||||
"url": url,
|
||||
"host": host,
|
||||
"cookie_names": [],
|
||||
"cookie_names": sorted(name for name in names if name),
|
||||
"cookie_read_succeeded": True,
|
||||
"probe_error": probe.get("probe_error"),
|
||||
"probe_attempts": probe_attempts,
|
||||
}
|
||||
cdp = CDP(shopee_page["webSocketDebuggerUrl"])
|
||||
|
||||
try:
|
||||
_ensure_page_domains(cdp)
|
||||
url, names = _wait_for_login_probe(cdp, timeout=timeout)
|
||||
url = url or initial_url
|
||||
if _is_login_url(url):
|
||||
reason = "LOGIN_PAGE"
|
||||
logged_in = False
|
||||
elif "SPC_ST" in names or "SPC_U" in names:
|
||||
reason = None
|
||||
logged_in = True
|
||||
else:
|
||||
reason = "NO_SESSION_COOKIE"
|
||||
logged_in = False
|
||||
return {
|
||||
"logged_in": logged_in,
|
||||
"reason": reason,
|
||||
"url": url,
|
||||
"host": host,
|
||||
"cookie_names": sorted(name for name in names if name),
|
||||
}
|
||||
finally:
|
||||
cdp.close()
|
||||
if time.monotonic() >= deadline:
|
||||
break
|
||||
return {
|
||||
"logged_in": False,
|
||||
"reason": "LOGIN_CHECK_TARGET_UNAVAILABLE",
|
||||
"url": str(last_probe.get("url") or ""),
|
||||
"host": host,
|
||||
"cookie_names": sorted(
|
||||
name for name in (last_probe.get("cookie_names") or []) if name
|
||||
),
|
||||
"cookie_read_succeeded": False,
|
||||
"probe_error": last_probe.get("probe_error"),
|
||||
"probe_attempts": probe_attempts,
|
||||
}
|
||||
|
||||
|
||||
def is_logged_in(account) -> bool:
|
||||
@@ -742,7 +885,7 @@ def is_logged_in(account) -> bool:
|
||||
return bool(login_status(account).get("logged_in"))
|
||||
|
||||
|
||||
def _close_open_product_failure(cdp):
|
||||
def _close_open_product_failure(cdp, confirm_target_closed=False):
|
||||
target_id = getattr(cdp, "target_id", None)
|
||||
created_by_app = bool(getattr(cdp, "created_by_app", False))
|
||||
host = getattr(cdp, "cdp_host", None)
|
||||
@@ -751,7 +894,10 @@ def _close_open_product_failure(cdp):
|
||||
finally:
|
||||
if created_by_app and target_id:
|
||||
try:
|
||||
close_tab(target_id, host=host)
|
||||
if confirm_target_closed:
|
||||
close_tab_and_wait(target_id, host=host, timeout=2.0)
|
||||
else:
|
||||
close_tab(target_id, host=host)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -786,7 +932,7 @@ def open_product(account, item_id, on_step=None, bring_to_front=True) -> CDP:
|
||||
_wait_ready(cdp)
|
||||
return cdp
|
||||
except Exception:
|
||||
_close_open_product_failure(cdp)
|
||||
_close_open_product_failure(cdp, confirm_target_closed=not bring_to_front)
|
||||
raise
|
||||
|
||||
|
||||
@@ -847,6 +993,7 @@ def collect(account, task, on_step=None) -> dict:
|
||||
|
||||
item_id = _item_id(task)
|
||||
cdp = open_product(account, item_id, on_step=on_step, bring_to_front=False)
|
||||
result = None
|
||||
try:
|
||||
_notify_collect_step(on_step, "read_title")
|
||||
old_title = read_title(cdp)
|
||||
@@ -862,13 +1009,15 @@ def collect(account, task, on_step=None) -> dict:
|
||||
)
|
||||
_notify_collect_step(on_step, "download_cover")
|
||||
old_cover_path = download_cover(old_cover_src, out_path)
|
||||
return {
|
||||
result = {
|
||||
"old_title": old_title,
|
||||
"old_cover_src": old_cover_src,
|
||||
"old_cover_path": old_cover_path,
|
||||
}
|
||||
finally:
|
||||
_close_collected_product(cdp)
|
||||
close_target_confirmed = _close_collected_product(cdp)
|
||||
result["close_target_confirmed"] = close_target_confirmed
|
||||
return result
|
||||
|
||||
|
||||
|
||||
@@ -897,20 +1046,26 @@ def _close_collected_product(cdp):
|
||||
target_id = getattr(cdp, "target_id", None)
|
||||
created_by_app = bool(getattr(cdp, "created_by_app", False))
|
||||
host = getattr(cdp, "cdp_host", None)
|
||||
close_target_confirmed = None
|
||||
try:
|
||||
cdp.close()
|
||||
finally:
|
||||
if created_by_app and target_id:
|
||||
try:
|
||||
close_tab(target_id, host=host)
|
||||
close_target_confirmed = close_tab_and_wait(
|
||||
target_id,
|
||||
host=host,
|
||||
timeout=2.0,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
close_target_confirmed = False
|
||||
return close_target_confirmed
|
||||
|
||||
|
||||
def close_readonly_product(cdp):
|
||||
"""Close a read-only product CDP session using the collection tab cleanup rules."""
|
||||
|
||||
_close_collected_product(cdp)
|
||||
return _close_collected_product(cdp)
|
||||
|
||||
|
||||
def change_title(cdp, new_title) -> dict:
|
||||
|
||||
+58
-4
@@ -1997,6 +1997,25 @@ class CollectWorker(BaseWorker):
|
||||
},
|
||||
on_step=on_step,
|
||||
)
|
||||
if result.get("close_target_confirmed") is False:
|
||||
self._log_run_event(
|
||||
"step=close_product result=uncertain detail=任务 {task_id} 商品 {item_id} 商品页已请求关闭,但未在短时间内确认关闭;采集结果已保留,继续处理后续任务".format(
|
||||
task_id=task.id,
|
||||
item_id=task.item_id,
|
||||
),
|
||||
task=task,
|
||||
level="warning",
|
||||
)
|
||||
self._write_diagnostic_log(
|
||||
"采集商品页关闭确认超时",
|
||||
level="WARNING",
|
||||
step="close_product",
|
||||
task=task,
|
||||
payload={
|
||||
"alias": getattr(account, "alias", None),
|
||||
"close_target_confirmed": False,
|
||||
},
|
||||
)
|
||||
current_step = "db_write"
|
||||
self._emit_activity(
|
||||
"task_step",
|
||||
@@ -2250,17 +2269,31 @@ class CollectWorker(BaseWorker):
|
||||
}
|
||||
|
||||
def _confirmed_login_status(self, account, context, task=None):
|
||||
started = time.monotonic()
|
||||
subject = self._login_check_subject(account, task)
|
||||
last_status = {}
|
||||
for attempt in range(1, self.LOGIN_CHECK_ATTEMPTS + 1):
|
||||
status = dict(self._login_status(account) or {})
|
||||
status["login_check_attempts"] = attempt
|
||||
last_status = status
|
||||
if status.get("logged_in") or self._is_definitive_logged_out(status):
|
||||
if status.get("logged_in"):
|
||||
if attempt > 1:
|
||||
self._log_run_event(
|
||||
"step=login_check result=recovered detail={subject} 登录检测已恢复,第{attempt}/{total}次确认已登录 elapsed_ms={elapsed_ms}".format(
|
||||
subject=subject,
|
||||
attempt=attempt,
|
||||
total=self.LOGIN_CHECK_ATTEMPTS,
|
||||
elapsed_ms=self._elapsed_ms(started),
|
||||
),
|
||||
task=task,
|
||||
)
|
||||
return status
|
||||
if self._is_definitive_logged_out(status):
|
||||
return status
|
||||
if attempt < self.LOGIN_CHECK_ATTEMPTS:
|
||||
self._log_run_event(
|
||||
"step=login_check result=retry detail=账号 {alias} 第{attempt}/{total}次登录检测暂不确定,{delay:g}秒后重试: {detail}".format(
|
||||
alias=getattr(account, "alias", ""),
|
||||
"step=login_check result=retry detail={subject} 登录状态暂时无法读取,第{attempt}/{total}次检测后将在{delay:g}秒后重试:{detail}".format(
|
||||
subject=subject,
|
||||
attempt=attempt,
|
||||
total=self.LOGIN_CHECK_ATTEMPTS,
|
||||
delay=self.LOGIN_CHECK_RETRY_DELAY_SECONDS,
|
||||
@@ -2282,11 +2315,22 @@ class CollectWorker(BaseWorker):
|
||||
"reason": last_status.get("reason"),
|
||||
"url": last_status.get("url"),
|
||||
"cookie_names": list(last_status.get("cookie_names") or []),
|
||||
"cookie_read_succeeded": bool(
|
||||
last_status.get("cookie_read_succeeded")
|
||||
),
|
||||
"probe_error": last_status.get("probe_error"),
|
||||
"probe_attempts": last_status.get("probe_attempts"),
|
||||
"attempts": last_status.get("login_check_attempts"),
|
||||
},
|
||||
)
|
||||
return last_status
|
||||
|
||||
def _login_check_subject(self, account, task=None):
|
||||
alias = str(getattr(account, "alias", "") or "未知账号")
|
||||
if task is None:
|
||||
return f"账号 {alias}"
|
||||
return f"任务 {task.id} 商品 {task.item_id} 账号 {alias}"
|
||||
|
||||
def _is_definitive_logged_out(self, status):
|
||||
reason = str((status or {}).get("reason") or "").strip()
|
||||
url = str((status or {}).get("url") or "").lower()
|
||||
@@ -2296,7 +2340,17 @@ class CollectWorker(BaseWorker):
|
||||
|
||||
def _login_status_detail(self, status):
|
||||
status = status or {}
|
||||
reason = status.get("reason") or "未知原因"
|
||||
raw_reason = str(status.get("reason") or "").strip()
|
||||
if raw_reason == "LOGIN_CHECK_TARGET_UNAVAILABLE":
|
||||
reason = "CDP页面暂时不可用"
|
||||
elif raw_reason.startswith("LOGIN_CHECK_FAILED"):
|
||||
reason = "登录检测调用失败"
|
||||
elif raw_reason == "NO_SESSION_COOKIE":
|
||||
reason = "暂未读取到登录会话"
|
||||
elif raw_reason == "LOGIN_PAGE":
|
||||
reason = "检测到登录页面"
|
||||
else:
|
||||
reason = raw_reason or "未知原因"
|
||||
url = status.get("url") or "未知URL"
|
||||
cookie_names = [str(name) for name in (status.get("cookie_names") or []) if name]
|
||||
cookie_text = ",".join(sorted(cookie_names)) if cookie_names else "未读到登录Cookie"
|
||||
|
||||
Reference in New Issue
Block a user