butubb
44cbe8e2aa
顺序反了：原先先开页面、读二维码，读完才发现「已经登录、没有二维码」。而读二维码内部 会 wait_for_selector 等满 30 秒才放弃 —— 用户点一下按钮要干等半分钟，还白开一个标签页。 实测日志里就是 `Page.wait_for_selector: Timeout 30000ms exceeded`。 改成先问登录状态（那是一次接口调用，很快），已登录就直接返回，根本不碰页面。 新增测试守住这个顺序：已登录时 context.new_page 不得被调用。
426 lines
16 KiB
Python
426 lines
16 KiB
Python
# -*- coding: utf-8 -*-
|
||
# Copyright (c) 2025 [email protected]
|
||
#
|
||
# This file is part of MediaCrawler project.
|
||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/monitor/qrlogin.py
|
||
# GitHub: https://github.com/NanmiCoder
|
||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||
#
|
||
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
||
# 1. 不得用于任何商业用途。
|
||
# 2. 使用时应遵守对应平台的使用条款和robots.txt规则。
|
||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
||
# 5. 不得用于任何非法或不当的用途。
|
||
#
|
||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||
|
||
"""扫码登录，以及"现在到底登没登录"的查询。
|
||
|
||
**为什么需要这个模块**：服务器上 Chrome 跑在 Xvfb 里没有显示器，爬虫原本用
|
||
`show_qrcode`（PIL 的 `Image.show()`）弹窗展示二维码，那需要桌面看图程序，服务器上
|
||
没有。所以改成经 CDP 把二维码从页面里读出来交给 WebUI。
|
||
|
||
**为什么登录状态要能独立查询**：扫码会话是内存里的临时状态，进程一重启就没了
|
||
（部署、崩溃都算）。把"是否已登录"绑在它上面，就会出现"扫完了但界面没反应、
|
||
也不知道到底成没成"。所以状态查询是独立的、随时可调用的，二维码只是达成它的手段之一。
|
||
|
||
三个容易搞错的地方：
|
||
|
||
* **必须复用浏览器默认 context**。`browser.new_context()` 会造出一个无痕式的 profile，
|
||
扫了也白扫——爬虫读不到那份 cookie。真正的 profile 在 `browser.contexts[0]`。
|
||
* **绝不能调 `browser.close()`**。对 CDP 连接而言那会关掉操作者自己的 Chrome，
|
||
连带所有无关标签页。只能关本模块自己开的那一个。
|
||
* **不能靠 `web_session` 判断登录**。实测：一个全新的空 profile 首次访问小红书就会
|
||
被发一个 `web_session`，所以"有这个 cookie"什么都证明不了。可信信号是页面自己的
|
||
`__INITIAL_STATE__.user.loggedIn`。
|
||
"""
|
||
|
||
import asyncio
|
||
import os
|
||
import time
|
||
from typing import Any, Dict, Optional
|
||
|
||
import config
|
||
from playwright.async_api import async_playwright
|
||
from tools import utils
|
||
|
||
from ..creator.client import CreatorApiError, CreatorClient
|
||
from .platforms import PLATFORM_XHS
|
||
|
||
|
||
def _cookie_string(cookies) -> str:
|
||
"""把 CDP 拿到的 cookie 列表拼成请求头用的字符串。"""
|
||
return "; ".join(f"{c['name']}={c['value']}" for c in cookies)
|
||
|
||
# 二维码有效期。平台自己会更早轮换；这个上限只是为了让一次被放弃的尝试不会
|
||
# 永久占着一个标签页。
|
||
QR_TTL_SECONDS = 300
|
||
|
||
# 登录状态查询的缓存时长。轮询时不必每次都去问浏览器。
|
||
STATE_CACHE_SECONDS = 5
|
||
|
||
STATUS_IDLE = "idle"
|
||
STATUS_WAITING = "waiting"
|
||
STATUS_SUCCESS = "success"
|
||
STATUS_EXPIRED = "expired"
|
||
STATUS_ERROR = "error"
|
||
|
||
# 只有小红书接了监控流程，所以扫码也只对它开放。给别的平台显示一个按不动的按钮
|
||
# 是在假装功能存在。
|
||
LOGIN_URL: Dict[str, str] = {PLATFORM_XHS: "https://www.xiaohongshu.com"}
|
||
EXPLORE_URL: Dict[str, str] = {PLATFORM_XHS: "https://www.xiaohongshu.com/explore"}
|
||
QR_SELECTOR: Dict[str, str] = {PLATFORM_XHS: "xpath=//img[@class='qrcode-img']"}
|
||
LOGIN_BUTTON_SELECTOR: Dict[str, str] = {
|
||
PLATFORM_XHS: "xpath=//*[@id='app']/div[1]/div[2]/div[1]/ul/div[1]/button"
|
||
}
|
||
|
||
# 这里本来有一个读 window.__INITIAL_STATE__ 的 JS 探针，**已删除，不要加回来**。
|
||
#
|
||
# 它是页面加载那一刻的快照：浏览器本来就登录着时它是对的，但扫码是加载**之后**才
|
||
# 登录的，快照不会翻转，检测于是永远等不到 —— 表现为"扫了码却一直停在二维码上"。
|
||
# 运营模块踩过同一个坑。现在的判据是拿 cookie 问后台接口，见 check_login_state。
|
||
|
||
_lock = asyncio.Lock()
|
||
_current: Optional["QrLoginSession"] = None
|
||
|
||
# 常驻的 Playwright 客户端和本模块自己的标签页。长期持有是有意的：状态查询要能
|
||
# 随时回答，而每次都新建一个标签页会在操作者的浏览器里堆垃圾。
|
||
_playwright: Any = None
|
||
_page: Any = None
|
||
|
||
# (时间戳, 结果)，避免轮询时反复问浏览器。
|
||
_state_cache: Optional[tuple[float, Dict[str, Any]]] = None
|
||
|
||
|
||
def _cdp_url() -> str:
|
||
"""浏览器 DevTools 端点。``MC_CDP_URL`` 优先，便于换主机而不用改代码。"""
|
||
return os.getenv("MC_CDP_URL") or f"http://127.0.0.1:{config.CDP_DEBUG_PORT}"
|
||
|
||
|
||
def _login_url(platform: str) -> str:
|
||
if platform == PLATFORM_XHS and getattr(config, "XHS_INTERNATIONAL", False):
|
||
return "https://www.rednote.com"
|
||
return LOGIN_URL[platform]
|
||
|
||
|
||
async def _ensure_context() -> Any:
|
||
"""连上浏览器并返回它的默认 context。"""
|
||
global _playwright
|
||
|
||
if _playwright is None:
|
||
_playwright = await async_playwright().start()
|
||
try:
|
||
browser = await _playwright.chromium.connect_over_cdp(_cdp_url(), timeout=15000)
|
||
except Exception as exc:
|
||
await _reset_playwright()
|
||
raise RuntimeError(
|
||
f"连接浏览器失败（{_cdp_url()}）。请确认服务器上的 Chrome 以 "
|
||
f"--remote-debugging-port 启动。原始错误：{exc}"
|
||
) from exc
|
||
|
||
if not browser.contexts:
|
||
raise RuntimeError(
|
||
"浏览器没有可用上下文。CDP 已连上，但读不到 profile —— "
|
||
"请确认 Chrome 不是以无痕模式启动的。"
|
||
)
|
||
# contexts[0] 就是真实 profile，用它，不要 new_context()。
|
||
return browser.contexts[0]
|
||
|
||
|
||
async def _ensure_page(platform: str = PLATFORM_XHS, reload: bool = False) -> Any:
|
||
"""本模块在操作者浏览器里的那一个标签页，复用而不是反复新建。
|
||
|
||
若已有一个停在目标站点的标签页就认领它——进程重启后页柄会丢，但标签页还在，
|
||
认领可以避免在浏览器里留下一堆没人关的孤儿页。
|
||
"""
|
||
global _page
|
||
|
||
context = await _ensure_context()
|
||
|
||
if _page is not None:
|
||
try:
|
||
if _page.is_closed():
|
||
_page = None
|
||
except Exception:
|
||
_page = None
|
||
|
||
if _page is None:
|
||
for candidate in context.pages:
|
||
try:
|
||
if "xiaohongshu.com" in candidate.url or "rednote.com" in candidate.url:
|
||
_page = candidate
|
||
break
|
||
except Exception:
|
||
continue
|
||
|
||
if _page is None:
|
||
_page = await context.new_page()
|
||
|
||
try:
|
||
url = _page.url
|
||
except Exception:
|
||
url = ""
|
||
|
||
if reload or "xiaohongshu.com" not in url and "rednote.com" not in url:
|
||
await _page.goto(
|
||
EXPLORE_URL.get(platform, EXPLORE_URL[PLATFORM_XHS]),
|
||
wait_until="domcontentloaded",
|
||
timeout=45000,
|
||
)
|
||
|
||
return _page
|
||
|
||
|
||
async def check_login_state(force: bool = False) -> Dict[str, Any]:
|
||
"""问浏览器：现在登录了吗？
|
||
|
||
``force`` 会先重新加载页面。SPA 的状态会随登录实时更新，所以轮询时不必重载；
|
||
但若登录态是在别处失效的，页面上的副本可能是陈旧的，重新检测就该重载。
|
||
"""
|
||
global _state_cache
|
||
|
||
now = time.time()
|
||
if not force and _state_cache is not None:
|
||
cached_at, cached = _state_cache
|
||
if now - cached_at < STATE_CACHE_SECONDS:
|
||
return cached
|
||
|
||
try:
|
||
context = await _ensure_context()
|
||
cookies = await context.cookies()
|
||
except Exception as exc:
|
||
result = {
|
||
"known": False,
|
||
"logged_in": False,
|
||
"nickname": None,
|
||
"error": f"{exc.__class__.__name__}: {exc}",
|
||
}
|
||
_state_cache = (now, result)
|
||
return result
|
||
|
||
cookie = _cookie_string(cookies)
|
||
|
||
# 判据不再是页面里的 window.__INITIAL_STATE__ —— 那是**页面加载那一刻的快照**：
|
||
# 浏览器已登录时它是对的，但扫码是加载**之后**才登录的，快照不会翻转，检测就永远
|
||
# 等不到（运营模块踩过同一个坑）。改成拿 cookie 问后台「我是谁」，那是权威的：
|
||
# 实测游客也会被发一个 web_session，所以「有这个 cookie」什么都证明不了，
|
||
# 后台认了才算。
|
||
try:
|
||
info = await CreatorClient(cookie).fetch_user_info()
|
||
except CreatorApiError:
|
||
result = {"known": True, "logged_in": False, "nickname": None}
|
||
else:
|
||
result = {
|
||
"known": True,
|
||
"logged_in": bool(info.get("user_id")),
|
||
"nickname": info.get("nickname"),
|
||
}
|
||
|
||
_state_cache = (now, result)
|
||
return result
|
||
|
||
|
||
async def _current_cookie() -> str:
|
||
"""默认 profile 当前的小红书 cookie 串。
|
||
|
||
扫码面板要的不只是「登录了吗」，而是**把登录态拿出来存一份** —— 存进库之后，
|
||
即使 CDP 关掉、任务改用 --cookies_file 注入，也照样能跑。
|
||
"""
|
||
context = await _ensure_context()
|
||
return _cookie_string(await context.cookies())
|
||
|
||
|
||
async def _reset_playwright() -> None:
|
||
global _playwright, _page
|
||
_page = None
|
||
if _playwright is not None:
|
||
try:
|
||
await _playwright.stop()
|
||
except Exception:
|
||
pass
|
||
_playwright = None
|
||
|
||
|
||
class QrLoginSession:
|
||
"""一次进行中的扫码尝试。"""
|
||
|
||
def __init__(self, platform: str, page: Any) -> None:
|
||
self.platform = platform
|
||
self.status = STATUS_WAITING
|
||
self.message = "请用手机扫描二维码"
|
||
self.image = ""
|
||
self.started_at = time.time()
|
||
self.logged_in = False
|
||
self.nickname: Optional[str] = None
|
||
# 登录成功后从默认 profile 取出来的 cookie，供调用方存库。
|
||
self.cookie: str = ""
|
||
self.cookie_taken = False
|
||
self._page = page
|
||
|
||
@property
|
||
def elapsed(self) -> float:
|
||
return time.time() - self.started_at
|
||
|
||
async def refresh(self) -> None:
|
||
"""轮询一次，看扫码是否完成。"""
|
||
if self.status != STATUS_WAITING:
|
||
return
|
||
if self.elapsed > QR_TTL_SECONDS:
|
||
self.status = STATUS_EXPIRED
|
||
self.message = "二维码已超时，请重新获取"
|
||
return
|
||
|
||
state = await check_login_state()
|
||
if state.get("logged_in"):
|
||
self.cookie = await _current_cookie()
|
||
self.logged_in = True
|
||
self.nickname = state.get("nickname")
|
||
self.status = STATUS_SUCCESS
|
||
who = f"（{self.nickname}）" if self.nickname else ""
|
||
self.message = f"登录成功{who}，登录态已写入浏览器 profile"
|
||
return
|
||
|
||
try:
|
||
if self._page.is_closed():
|
||
self.status = STATUS_ERROR
|
||
self.message = "二维码所在页面已被关闭，请重新获取"
|
||
except Exception:
|
||
pass
|
||
|
||
def snapshot(self) -> Dict[str, Any]:
|
||
return {
|
||
"status": self.status,
|
||
"platform": self.platform,
|
||
"image": self.image,
|
||
"message": self.message,
|
||
"elapsed": int(self.elapsed),
|
||
"expires_in": max(0, int(QR_TTL_SECONDS - self.elapsed)),
|
||
"logged_in": self.logged_in,
|
||
"nickname": self.nickname,
|
||
}
|
||
|
||
|
||
async def _read_qr(page: Any, platform: str) -> str:
|
||
"""把二维码从页面里取出来，必要时先点开登录框。"""
|
||
image = await utils.find_login_qrcode(page, selector=QR_SELECTOR[platform])
|
||
if image:
|
||
return image
|
||
# 登录框不一定自己弹出来。这是爬虫自身扫码流程里同款兜底。
|
||
await asyncio.sleep(0.5)
|
||
try:
|
||
await page.locator(LOGIN_BUTTON_SELECTOR[platform]).click(timeout=5000)
|
||
except Exception:
|
||
return ""
|
||
return await utils.find_login_qrcode(page, selector=QR_SELECTOR[platform])
|
||
|
||
|
||
async def _discard_current_locked() -> None:
|
||
global _current
|
||
_current = None
|
||
|
||
|
||
async def start(platform: str = PLATFORM_XHS) -> Dict[str, Any]:
|
||
"""在 CDP 浏览器里打开登录页，取回二维码。"""
|
||
global _current
|
||
|
||
if platform not in LOGIN_URL:
|
||
raise ValueError(f"平台 {platform} 尚未接入扫码登录（目前仅支持小红书）")
|
||
|
||
async with _lock:
|
||
await _discard_current_locked()
|
||
|
||
# **先问状态，再决定要不要开页面。** 顺序反过来是有代价的：读二维码内部会
|
||
# wait_for_selector 等满 30 秒才放弃，而已经登录时页面上根本没有二维码 ——
|
||
# 用户点一下按钮要干等半分钟，还白开一个标签页。
|
||
state = await check_login_state(force=True)
|
||
if state.get("logged_in"):
|
||
# 已经是登录状态时站点不显示二维码 —— 这本身就是成功，不是失败。
|
||
# 顺带把 cookie 取出来，让调用方可以存进库。
|
||
session = QrLoginSession(platform, None)
|
||
session.cookie = await _current_cookie()
|
||
session.status = STATUS_SUCCESS
|
||
session.logged_in = True
|
||
session.nickname = state.get("nickname")
|
||
who = f"（{session.nickname}）" if session.nickname else ""
|
||
session.message = f"浏览器已经是登录状态{who}，无需扫码"
|
||
_current = session
|
||
return session.snapshot()
|
||
|
||
page = await _ensure_page(platform)
|
||
try:
|
||
await page.goto(
|
||
_login_url(platform), wait_until="domcontentloaded", timeout=45000
|
||
)
|
||
image = await _read_qr(page, platform)
|
||
except Exception as exc:
|
||
raise RuntimeError(f"打开登录页失败：{exc}") from exc
|
||
|
||
session = QrLoginSession(platform, page)
|
||
session.image = image
|
||
if not image:
|
||
session.status = STATUS_ERROR
|
||
session.message = "页面上没找到二维码，请确认站点结构没有变化"
|
||
|
||
_current = session
|
||
return session.snapshot()
|
||
|
||
|
||
async def status() -> Dict[str, Any]:
|
||
async with _lock:
|
||
if _current is None:
|
||
state = await check_login_state()
|
||
return {
|
||
"status": STATUS_IDLE,
|
||
"platform": None,
|
||
"image": "",
|
||
"message": "",
|
||
"elapsed": 0,
|
||
"expires_in": 0,
|
||
"logged_in": bool(state.get("logged_in")),
|
||
"nickname": state.get("nickname"),
|
||
}
|
||
await _current.refresh()
|
||
return _current.snapshot()
|
||
|
||
|
||
async def take_cookie() -> Optional[str]:
|
||
"""取走已登录会话的 cookie，且只给一次。
|
||
|
||
由路由层在落库时调用。**cookie 不进响应体** —— 它是凭证，前端没有理由看到它。
|
||
|
||
这里**刻意不结束会话**（与运营模块不同）：那里取完即拆，因为临时上下文用完就该丢；
|
||
这里的浏览器 profile 是长期存在的，面板还该继续显示「已登录」。所以只标记已取过，
|
||
让重复轮询拿不到第二份、也就不会反复写库。
|
||
"""
|
||
async with _lock:
|
||
if _current is None or _current.status != STATUS_SUCCESS or _current.cookie_taken:
|
||
return None
|
||
_current.cookie_taken = True
|
||
return _current.cookie
|
||
|
||
|
||
async def cancel() -> Dict[str, Any]:
|
||
async with _lock:
|
||
await _discard_current_locked()
|
||
state = await check_login_state()
|
||
return {
|
||
"status": STATUS_IDLE,
|
||
"platform": None,
|
||
"image": "",
|
||
"message": "已取消",
|
||
"elapsed": 0,
|
||
"expires_in": 0,
|
||
"logged_in": bool(state.get("logged_in")),
|
||
"nickname": state.get("nickname"),
|
||
}
|
||
|
||
|
||
async def shutdown() -> None:
|
||
"""进程退出时断开连接。刻意不关那个标签页——它是操作者浏览器的一部分。"""
|
||
global _current
|
||
async with _lock:
|
||
_current = None
|
||
await _reset_playwright()
|