From 37ca1b1cd6334992ef0f2ad80a27c44e7207baa6 Mon Sep 17 00:00:00 2001 From: butubb <1422726308@qq.com> Date: Wed, 7 Oct 2026 10:41:11 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20CDP=20=E6=8E=A5=E7=AE=A1=E5=BC=80?= =?UTF-8?q?=E5=85=B3=20+=20=E6=89=AB=E7=A0=81=E7=99=BB=E5=BD=95=E9=9D=A2?= =?UTF-8?q?=E6=9D=BF=20+=20Docker=20=E9=83=A8=E7=BD=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - runner: enable_cdp_mode 从硬编码 False 改为系统设置 cdp_enabled。服务器部署下爬虫接管已开启远程调试的 Chrome（默认 9222），复用其 profile 登录态；本机桌面默认仍为关，行为不变 - qrlogin: 新增 CDP 扫码登录。Chrome 在服务器上跑于 Xvfb，show_qrcode 依赖的 PIL 桌面看图程序不存在，二维码无处可显示；改为经 CDP 从页面取出二维码交给 WebUI 渲染。刻意复用 browser.contexts[0]（新建 context 是无痕 profile，扫了也白扫），且绝不调用 browser.close()（会连带关掉操作者自己的 Chrome） - webui: 设置页新增扫码面板，替换原本跳到采集页看终端二维码的入口 - Dockerfile / .dockerignore / docker-compose.yml: 服务器部署。host 网络是必需而非图省事——容器里 127.0.0.1:9222 必须落到宿主机回环 - UPSTREAM.md: 补充 gitcode 镜像，用于 GitHub 大包传输必断时补历史 --- .dockerignore | 29 ++ Dockerfile | 43 +++ UPSTREAM.md | 25 +- api/main.py | 4 + api/monitor/app_settings.py | 12 + api/monitor/qrlogin.py | 268 ++++++++++++++++++ api/monitor/runner.py | 10 +- api/routers/monitor.py | 39 ++- docker-compose.yml | 24 ++ tests/test_qrlogin.py | 190 +++++++++++++ webui/src/components/monitor/QrLoginPanel.tsx | 154 ++++++++++ .../src/components/settings/SettingsView.tsx | 19 +- webui/src/hooks/useMonitor.ts | 40 +++ webui/src/lib/api.ts | 8 + webui/src/types/monitor.ts | 20 ++ 15 files changed, 872 insertions(+), 13 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 api/monitor/qrlogin.py create mode 100644 docker-compose.yml create mode 100644 tests/test_qrlogin.py create mode 100644 webui/src/components/monitor/QrLoginPanel.tsx diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..090bfcc --- /dev/null +++ b/.dockerignore @@ -0,0 +1,29 @@ +# Keep the build context to what the server actually runs. +.git +.github +.venv +venv + +# The WebUI sources are not needed -- only the bundle they produce, which lands +# in api/webui and is therefore NOT excluded. +webui/node_modules +webui/src +webui/dist + +# Runtime state: per-run crawler output and the login browser profile. Mounted +# as a volume instead, so it survives image rebuilds. +data +browser_data + +# Secrets come from the environment via compose, never baked into a layer. +.env + +tests +docs +__pycache__ +**/__pycache__ +*.pyc +*.pyo +.pytest_cache +*.db +*.log diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7fcbb49 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,43 @@ +# Server deployment image. +# +# No browser is bundled on purpose. On this deployment the crawler attaches over +# CDP to the Chrome already running on the host (see the 接管已有 Chrome setting), +# so shipping a second copy of Chromium would only add hundreds of megabytes and +# a login state that nothing uses. The Playwright Python package is still needed +# -- that is what speaks CDP -- hence PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD. +FROM python:3.11-slim + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 \ + TZ=Asia/Shanghai + +# asyncmy compiles a Cython extension, so a toolchain has to exist at build time. +# It is left installed: purging it risks taking libmysqlclient with it, and a +# slightly larger image is cheaper than a runtime that fails months later. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential pkg-config default-libmysqlclient-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Requirements first: this layer only rebuilds when the pins actually change. +ARG PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple +COPY requirements.txt ./ +RUN pip install --no-cache-dir -i "$PIP_INDEX" -r requirements.txt + +COPY . . + +# The WebUI bundle lives at api/webui, which is gitignored -- it is built on the +# workstation (`cd webui && npm run build`) and shipped inside the build context. +# Without this check a missing bundle is invisible until someone opens the page +# and gets the bare JSON stub from serve_frontend(). +RUN test -f api/webui/index.html \ + || (echo "ERROR: api/webui/index.html is missing. Run 'cd webui && npm run build' before building the image." >&2; exit 1) + +EXPOSE 18051 + +# api.main reads MC_HOST / MC_PORT from the environment; compose supplies both. +CMD ["python", "-m", "api.main"] diff --git a/UPSTREAM.md b/UPSTREAM.md index b6b74ae..fa57e4e 100644 --- a/UPSTREAM.md +++ b/UPSTREAM.md @@ -25,9 +25,14 @@ webui/src/components/{monitor,settings,auth}/ 新视图 webui/src/components/layout/{PlatformSwitcher,UnwiredPlatformNotice}.tsx webui/src/{hooks/useMonitor.ts,hooks/usePlatform.ts,store/platformStore.ts,lib/monitorFormat.ts,types/monitor.ts} docs/监控功能使用说明.md -tests/test_{auth,settings,platforms,monitor_*}.py +tests/test_{auth,settings,platforms,qrlogin,monitor_*}.py +Dockerfile / .dockerignore / docker-compose.yml 服务器部署用 ``` +其中 `api/monitor/qrlogin.py` + `webui/.../QrLoginPanel.tsx` 是**服务器专用**的扫码登录： +那台机器上 Chrome 跑在 Xvfb 里，`show_qrcode` 调的 PIL `Image.show()` 需要桌面看图程序， +服务器没有，二维码会无处可去。所以改成用 CDP 把二维码从页面里读出来交给前端 `` 显示。 + ### 2. 加法改动（低冲突） 只在既有文件里**新增**内容，不改动原有行： @@ -90,9 +95,25 @@ tests/test_{auth,settings,platforms,monitor_*}.py git stash # 或先 commit 到自己的分支（推荐） git fetch origin main git rebase origin/main # 冲突只会出现在上表第 3、4 类文件里 -./.venv/Scripts/python.exe -m pytest tests/ -q # 486 个测试就是回归网 +./.venv/Scripts/python.exe -m pytest tests/ -q # 502 个测试就是回归网 ``` +### 直连 GitHub 不通时（本机常见） + +本机到 `github.com` 时通时不通，**大包传输必断**（`Recv failure: Connection was reset` +或 `unexpected disconnect while reading sideband packet`），所以 `git clone` / `--unshallow` +这类一次性拉全量的操作基本必失败。可用的替代源： + +```bash +# gitcode 的 GitHub 镜像，国内直连，比 GitHub 本身还新一天以内 +git remote add gitcode https://gitcode.com/gh_mirrors/me/MediaCrawler.git +git fetch --no-tags --unshallow gitcode # 本仓库当初就是这样补全历史的，约 2 秒 +``` + +注意 `git fetch` 只写 `refs/remotes/*`，**不会动本地 `main`**； +但拉镜像会把 `upstream/main` 指到镜像的 tip（可能比 GitHub 晚一天）， +等 GitHub 通了再 `git fetch upstream` 正回来即可。 + ### 强烈建议：先把改动提交掉 当前状态是**未提交**的（25 个上游文件被改 + 31 个新文件）。在 `main` 分支上裸着工作区， diff --git a/api/main.py b/api/main.py index 87c4111..5be7df6 100644 --- a/api/main.py +++ b/api/main.py @@ -65,6 +65,7 @@ async def lifespan(_app: FastAPI): happen whether or not anyone has the UI open. """ from .monitor.db import dispose_engine, init_db + from .monitor.qrlogin import shutdown as shutdown_qrlogin from .monitor.scheduler import monitor_scheduler await init_db() @@ -89,6 +90,9 @@ async def lifespan(_app: FastAPI): yield finally: await monitor_scheduler.stop() + # Drops the tab a QR login may have opened and stops the Playwright + # client; leaving them would strand a driver process on every restart. + await shutdown_qrlogin() await dispose_engine() diff --git a/api/monitor/app_settings.py b/api/monitor/app_settings.py index 54ab30e..b98fabc 100644 --- a/api/monitor/app_settings.py +++ b/api/monitor/app_settings.py @@ -204,6 +204,18 @@ SETTING_SPECS: List[SettingSpec] = [ maximum=23, affects_new_runs=False, ), + SettingSpec( + name="cdp_enabled", + scope=SCOPE_SYSTEM, + type=TYPE_BOOL, + label="接管已有 Chrome（CDP）", + help=( + "开启后爬虫不再自己启动浏览器，而是接管本机已开放远程调试端口的 Chrome" + "（默认 127.0.0.1:9222），复用它的登录态与扩展。" + "服务器部署请开启；本机桌面使用请保持关闭。" + ), + default=False, + ), ] SPECS_BY_NAME = {spec.name: spec for spec in SETTING_SPECS} diff --git a/api/monitor/qrlogin.py b/api/monitor/qrlogin.py new file mode 100644 index 0000000..cef70d1 --- /dev/null +++ b/api/monitor/qrlogin.py @@ -0,0 +1,268 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/monitor/qrlogin.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + +"""Show a login QR code to the operator through the WebUI. + +Why this module exists: on a server deployment Chrome runs under Xvfb, so there +is no display to look at, and the helper the crawler normally uses to present the +QR (`show_qrcode` in `tools/crawler_util.py`) calls PIL's ``Image.show()`` -- it +needs a desktop image viewer that such a machine does not have, so the code would +go nowhere and the operator would be stuck. Instead the QR is read straight out +of the page over CDP and handed to the WebUI, which renders it as an ````. + +Three details that are easy to get wrong: + +* **Reuse the browser's default context.** ``browser.new_context()`` would give + an incognito-like profile, so the scan would land in a cookie jar the crawler + never reads and every run would still look logged out. The real profile -- the + one the crawler attaches to -- is ``browser.contexts[0]``. +* **Never call ``browser.close()``.** On a CDP connection that tears down the + operator's own Chrome and takes every unrelated tab with it. Only the page this + module opened is closed, and the Playwright client is stopped to drop the + socket. +* **A scan is not a login until the cookie changes.** The page can be showing a + QR for an account that is in fact already signed in, so success is decided by + ``web_session`` appearing or changing against the value captured at start, + never by anything the page displays. +""" + +import asyncio +import os +import time +from typing import Any, Dict, Optional + +import config +from playwright.async_api import async_playwright +from tools import utils + +from .platforms import PLATFORM_XHS + +# How long a QR stays valid before the session is written off. The platform +# rotates the code well before this; the limit exists so an abandoned attempt +# cannot pin a browser tab open indefinitely. +QR_TTL_SECONDS = 180 + +STATUS_IDLE = "idle" +STATUS_WAITING = "waiting" +STATUS_SUCCESS = "success" +STATUS_EXPIRED = "expired" +STATUS_ERROR = "error" + +# Only xhs is wired: it is the only platform whose monitor pipeline works, and +# pretending otherwise would offer the operator a button that cannot succeed. +LOGIN_URL: Dict[str, str] = {PLATFORM_XHS: "https://www.xiaohongshu.com"} +QR_SELECTOR: Dict[str, str] = {PLATFORM_XHS: "xpath=//img[@class='qrcode-img']"} +LOGIN_BUTTON_SELECTOR: Dict[str, str] = { + PLATFORM_XHS: "xpath=//*[@id='app']/div[1]/div[2]/div[1]/ul/div[1]/button" +} +SESSION_COOKIE: Dict[str, str] = {PLATFORM_XHS: "web_session"} + +_IDLE_SNAPSHOT: Dict[str, Any] = { + "status": STATUS_IDLE, + "platform": None, + "image": "", + "message": "", + "elapsed": 0, + "expires_in": 0, +} + +_lock = asyncio.Lock() +_current: Optional["QrLoginSession"] = None + + +def _cdp_url() -> str: + """Where to reach the browser's DevTools endpoint. + + ``MC_CDP_URL`` wins so a deployment can point at another host without a code + change; otherwise the port comes from the same config the crawler itself + reads, so the two can never drift apart. + """ + return os.getenv("MC_CDP_URL") or f"http://127.0.0.1:{config.CDP_DEBUG_PORT}" + + +def _login_url(platform: str) -> str: + if platform == PLATFORM_XHS and getattr(config, "XHS_INTERNATIONAL", False): + return "https://www.rednote.com" + return LOGIN_URL[platform] + + +class QrLoginSession: + """One live QR-login attempt against the CDP browser.""" + + def __init__(self, platform: str, playwright: Any, page: Any, baseline: str) -> None: + self.platform = platform + self.status = STATUS_WAITING + self.message = "请用小红书 App 扫描二维码" + self.image = "" + self.started_at = time.time() + self._playwright = playwright + self._page = page + # The `web_session` value present *before* the scan. An account already + # signed in has a non-empty baseline, which is why success is "changed", + # not merely "present". + self._baseline = baseline + + @property + def elapsed(self) -> float: + return time.time() - self.started_at + + async def refresh(self) -> None: + """Poll the browser once for a completed scan.""" + if self.status != STATUS_WAITING: + return + if self.elapsed > QR_TTL_SECONDS: + self.status = STATUS_EXPIRED + self.message = "二维码已超时，请重新获取" + return + try: + cookies = await self._page.context.cookies() + except Exception: + # The operator may have closed the tab we opened. + self.status = STATUS_ERROR + self.message = "二维码所在页面已被关闭，请重新获取" + return + token = {c["name"]: c["value"] for c in cookies}.get( + SESSION_COOKIE[self.platform], "" + ) + if token and token != self._baseline: + self.status = STATUS_SUCCESS + self.message = "登录成功，登录态已写入浏览器 profile" + + def snapshot(self) -> Dict[str, Any]: + return { + "status": self.status, + "platform": self.platform, + "image": self.image, + "message": self.message, + "elapsed": int(self.elapsed), + "expires_in": max(0, int(QR_TTL_SECONDS - self.elapsed)), + } + + async def close(self) -> None: + """Drop our page and the Playwright client, leaving Chrome untouched.""" + try: + await self._page.close() + except Exception: + pass + try: + await self._playwright.stop() + except Exception: + pass + + +async def _read_qr(page: Any, platform: str) -> str: + """Pull the QR image out of the page, opening the login dialog if needed.""" + image = await utils.find_login_qrcode(page, selector=QR_SELECTOR[platform]) + if image: + return image + # The dialog does not always open on its own. This is the same fallback the + # crawler's own QR flow performs before giving up. + await asyncio.sleep(0.5) + try: + await page.locator(LOGIN_BUTTON_SELECTOR[platform]).click(timeout=5000) + except Exception: + return "" + return await utils.find_login_qrcode(page, selector=QR_SELECTOR[platform]) + + +async def _reset_locked() -> None: + global _current + if _current is not None: + await _current.close() + _current = None + + +async def start(platform: str = PLATFORM_XHS) -> Dict[str, Any]: + """Open a login page in the CDP browser and return its QR code.""" + if platform not in LOGIN_URL: + raise ValueError(f"平台 {platform} 尚未接入扫码登录（目前仅支持小红书）") + + async with _lock: + await _reset_locked() + + playwright = await async_playwright().start() + try: + browser = await playwright.chromium.connect_over_cdp(_cdp_url(), timeout=15000) + except Exception as exc: + await playwright.stop() + raise RuntimeError( + f"连接浏览器失败（{_cdp_url()}）。请确认服务器上的 Chrome 以 " + f"--remote-debugging-port 启动。原始错误：{exc}" + ) from exc + + if not browser.contexts: + await playwright.stop() + raise RuntimeError( + "浏览器没有可用上下文。CDP 已连上，但读不到 profile —— " + "请确认 Chrome 不是以无痕模式启动的。" + ) + + # contexts[0] is the real profile. See the module docstring. + context = browser.contexts[0] + page = await context.new_page() + try: + await page.goto(_login_url(platform), wait_until="domcontentloaded", timeout=30000) + image = await _read_qr(page, platform) + cookies = await context.cookies() + except Exception as exc: + try: + await page.close() + except Exception: + pass + await playwright.stop() + raise RuntimeError(f"打开登录页失败：{exc}") from exc + + baseline = {c["name"]: c["value"] for c in cookies}.get( + SESSION_COOKIE[platform], "" + ) + + session = QrLoginSession(platform, playwright, page, baseline) + session.image = image + if not image: + if baseline: + session.status = STATUS_SUCCESS + session.message = "浏览器已经是登录状态，无需扫码" + else: + session.status = STATUS_ERROR + session.message = "页面上没找到二维码，请确认站点结构没有变化" + + global _current + _current = session + return session.snapshot() + + +async def status() -> Dict[str, Any]: + async with _lock: + if _current is None: + return dict(_IDLE_SNAPSHOT) + await _current.refresh() + return _current.snapshot() + + +async def cancel() -> Dict[str, Any]: + async with _lock: + await _reset_locked() + snapshot = dict(_IDLE_SNAPSHOT) + snapshot["message"] = "已取消" + return snapshot + + +async def shutdown() -> None: + """Release the browser tab at application shutdown.""" + async with _lock: + await _reset_locked() diff --git a/api/monitor/runner.py b/api/monitor/runner.py index d0895aa..9ed077e 100644 --- a/api/monitor/runner.py +++ b/api/monitor/runner.py @@ -162,6 +162,11 @@ async def execute_task(task_id: int, trigger: str = "manual") -> IngestResult: urls = build_target_urls(task.mode, targets) cookie = await get_cookie(session, platform) strategy = await _strategy_settings(session, platform) + # System-wide switch. On a headless server the crawler must attach to the + # Chrome already listening on the debug port -- that browser is where the + # operator scanned the login QR, so its profile is the login. Default False + # keeps desktop runs launching a private browser exactly as before. + cdp_enabled = await app_settings.get_value(session, "cdp_enabled", fallback=False) run = MonitorRun( task_id=task.id, @@ -209,8 +214,9 @@ async def execute_task(task_id: int, trigger: str = "manual") -> IngestResult: # Isolate this run's output: the crawler names files by date only, so # otherwise same-day runs would append into one shared file. save_data_path=str(out_dir), - # Unattended runs must not try to attach to the user's desktop Chrome. - enable_cdp_mode=False, + # Attach to the browser already running on CDP_DEBUG_PORT when the + # operator enabled it; otherwise launch a private, throwaway browser. + enable_cdp_mode=cdp_enabled, # Only injecting web_session is not enough to sign requests from a cold # browser profile. inject_all_cookies=True, diff --git a/api/routers/monitor.py b/api/routers/monitor.py index 1957b7f..3b283ac 100644 --- a/api/routers/monitor.py +++ b/api/routers/monitor.py @@ -23,7 +23,7 @@ from typing import Any, Dict, List, Optional from fastapi import APIRouter, HTTPException, Query, Response -from ..monitor import notify, report, service +from ..monitor import notify, qrlogin, report, service from ..monitor.db import get_session from ..monitor.platforms import PLATFORM_XHS from ..monitor.settings import ( @@ -243,6 +243,43 @@ async def clear_cookie_endpoint(platform: str = Query(default=PLATFORM_XHS)): return {"message": "Cookie cleared"} +# --------------------------------------------------------------------------- +# QR login +# --------------------------------------------------------------------------- + +# These drive the browser already listening on the CDP debug port, which is the +# same browser -- and therefore the same profile -- that monitor runs attach to. +# Scanning once is what makes later unattended runs logged in. + + +@router.post("/login/qr") +async def start_qr_login(platform: str = Query(default=PLATFORM_XHS)): + """Open the login page in the CDP browser and return its QR code. + + A server deployment has no display (Chrome sits under Xvfb), so the code is + surfaced here for the operator to scan instead of in a desktop window that + does not exist. + """ + try: + return await qrlogin.start(platform) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) + except RuntimeError as exc: + raise HTTPException(status_code=502, detail=str(exc)) + + +@router.get("/login/qr") +async def get_qr_login(): + """Poll the live session: waiting -> success / expired / error.""" + return await qrlogin.status() + + +@router.delete("/login/qr") +async def cancel_qr_login(): + """Drop our tab and stop polling.""" + return await qrlogin.cancel() + + # --------------------------------------------------------------------------- # Report # --------------------------------------------------------------------------- diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..e768cff --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,24 @@ +services: + mediacrawler: + build: . + image: mediacrawler:latest + container_name: mediacrawler + restart: unless-stopped + + # host networking is a requirement, not a convenience: the crawler attaches + # to the operator's Chrome at 127.0.0.1:9222, and inside a bridge network + # that loopback is the container's own, where no browser is listening. + # It also puts the app port directly on the host, so `ports:` is not used. + network_mode: host + + env_file: + - .env + environment: + MC_HOST: 0.0.0.0 + MC_PORT: "18051" + TZ: Asia/Shanghai + + volumes: + # Per-run crawler output, read back by the monitor layer. Without this the + # data would live inside the container and vanish on every rebuild. + - ./data:/app/data diff --git a/tests/test_qrlogin.py b/tests/test_qrlogin.py new file mode 100644 index 0000000..7f287a9 --- /dev/null +++ b/tests/test_qrlogin.py @@ -0,0 +1,190 @@ +# -*- coding: utf-8 -*- +"""Tests for CDP-driven QR login. + +The behaviour worth pinning down is not "does it call Playwright" but the two +decisions that silently produce a logged-out crawler if they regress: + +* the QR must be read from the browser's **default** context, because a new + context is an incognito-like profile whose cookies the crawler never sees; +* success must be decided by ``web_session`` *changing*, not merely existing -- + an account already signed in has a value before the operator ever scans. +""" + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from api.monitor import qrlogin + + +@pytest.fixture(autouse=True) +def _reset_module_state(): + """Each test starts from "nothing on screen" and leaves it that way.""" + qrlogin._current = None + yield + qrlogin._current = None + + +def _fake_stack(qr_image: str = "data:image/png;base64,AAAA", cookies=None): + """A Playwright/Chrome stand-in wired the way the real one behaves.""" + context = MagicMock() + context.cookies = AsyncMock(return_value=list(cookies or [])) + page = MagicMock() + page.goto = AsyncMock() + page.close = AsyncMock() + context.new_page = AsyncMock(return_value=page) + + browser = MagicMock() + browser.contexts = [context] + # Reaching for a fresh context is the bug this guards against, so make it + # blow up loudly rather than quietly returning a throwaway profile. + browser.new_context = AsyncMock( + side_effect=AssertionError("must reuse browser.contexts[0], not a new context") + ) + + playwright = MagicMock() + playwright.chromium.connect_over_cdp = AsyncMock(return_value=browser) + playwright.stop = AsyncMock() + + manager = MagicMock() + manager.start = AsyncMock(return_value=playwright) + + return manager, playwright, browser, context, page + + +def _patch(monkeypatch, manager, qr_image="data:image/png;base64,AAAA"): + monkeypatch.setattr(qrlogin, "async_playwright", lambda: manager) + monkeypatch.setattr( + qrlogin.utils, "find_login_qrcode", AsyncMock(return_value=qr_image) + ) + + +@pytest.mark.asyncio +async def test_idle_before_any_session(): + snapshot = await qrlogin.status() + + assert snapshot["status"] == qrlogin.STATUS_IDLE + assert snapshot["image"] == "" + + +@pytest.mark.asyncio +async def test_unwired_platform_is_rejected(): + """Only xhs is wired; anything else must fail loudly, not show a dead button.""" + with pytest.raises(ValueError): + await qrlogin.start("dy") + + +@pytest.mark.asyncio +async def test_start_returns_the_qr_and_reuses_the_default_context(monkeypatch): + manager, playwright, browser, context, _page = _fake_stack() + _patch(monkeypatch, manager) + + snapshot = await qrlogin.start(qrlogin.PLATFORM_XHS) + + assert snapshot["status"] == qrlogin.STATUS_WAITING + assert snapshot["image"] == "data:image/png;base64,AAAA" + context.new_page.assert_awaited_once() + browser.new_context.assert_not_called() + playwright.chromium.connect_over_cdp.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_success_requires_the_cookie_to_change(): + session = qrlogin.QrLoginSession( + qrlogin.PLATFORM_XHS, + MagicMock(), + _page_with_cookies([{"name": "web_session", "value": "before"}]), + baseline="before", + ) + + await session.refresh() + assert session.status == qrlogin.STATUS_WAITING + + session._page = _page_with_cookies([{"name": "web_session", "value": "after"}]) + await session.refresh() + assert session.status == qrlogin.STATUS_SUCCESS + + +@pytest.mark.asyncio +async def test_an_already_signed_in_profile_needs_no_scan(monkeypatch): + """No QR on the page plus a session cookie means someone is already logged in.""" + manager, _playwright, _browser, _context, _page = _fake_stack( + qr_image="", cookies=[{"name": "web_session", "value": "existing"}] + ) + _patch(monkeypatch, manager, qr_image="") + + snapshot = await qrlogin.start(qrlogin.PLATFORM_XHS) + + assert snapshot["status"] == qrlogin.STATUS_SUCCESS + assert "登录" in snapshot["message"] + + +@pytest.mark.asyncio +async def test_no_qr_and_no_session_is_an_error(monkeypatch): + manager, _playwright, _browser, _context, _page = _fake_stack(qr_image="", cookies=[]) + _patch(monkeypatch, manager, qr_image="") + + snapshot = await qrlogin.start(qrlogin.PLATFORM_XHS) + + assert snapshot["status"] == qrlogin.STATUS_ERROR + + +@pytest.mark.asyncio +async def test_session_expires(monkeypatch): + session = qrlogin.QrLoginSession( + qrlogin.PLATFORM_XHS, MagicMock(), _page_with_cookies([]), baseline="" + ) + session.started_at -= qrlogin.QR_TTL_SECONDS + 1 + + await session.refresh() + + assert session.status == qrlogin.STATUS_EXPIRED + + +@pytest.mark.asyncio +async def test_closed_tab_is_reported_rather_than_crashing(): + page = MagicMock() + page.context.cookies = AsyncMock(side_effect=RuntimeError("Target closed")) + session = qrlogin.QrLoginSession(qrlogin.PLATFORM_XHS, MagicMock(), page, baseline="") + + await session.refresh() + + assert session.status == qrlogin.STATUS_ERROR + + +@pytest.mark.asyncio +async def test_unreachable_browser_is_reported(monkeypatch): + """A server whose Chrome is not listening must say so, not 500 anonymously.""" + playwright = MagicMock() + playwright.chromium.connect_over_cdp = AsyncMock( + side_effect=OSError("Connection refused") + ) + playwright.stop = AsyncMock() + manager = MagicMock() + manager.start = AsyncMock(return_value=playwright) + monkeypatch.setattr(qrlogin, "async_playwright", lambda: manager) + + with pytest.raises(RuntimeError) as excinfo: + await qrlogin.start(qrlogin.PLATFORM_XHS) + + assert "连接浏览器失败" in str(excinfo.value) + playwright.stop.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_cancel_closes_the_tab_and_resets(monkeypatch): + manager, _playwright, _browser, _context, page = _fake_stack() + _patch(monkeypatch, manager) + await qrlogin.start(qrlogin.PLATFORM_XHS) + + snapshot = await qrlogin.cancel() + + assert snapshot["status"] == qrlogin.STATUS_IDLE + page.close.assert_awaited_once() + assert (await qrlogin.status())["status"] == qrlogin.STATUS_IDLE + + +def _page_with_cookies(cookies): + page = MagicMock() + page.context.cookies = AsyncMock(return_value=list(cookies)) + return page diff --git a/webui/src/components/monitor/QrLoginPanel.tsx b/webui/src/components/monitor/QrLoginPanel.tsx new file mode 100644 index 0000000..9bc7df6 --- /dev/null +++ b/webui/src/components/monitor/QrLoginPanel.tsx @@ -0,0 +1,154 @@ +import { useEffect, useState } from 'react' +import { useQueryClient } from '@tanstack/react-query' +import { AlertTriangle, CheckCircle2, Loader2, QrCode, RefreshCw, X } from 'lucide-react' + +import { Badge } from '@/components/ui/badge' +import { Button } from '@/components/ui/button' +import { useCancelQrLogin, useQrLoginStatus, useStartQrLogin } from '@/hooks/useMonitor' +import { useCurrentPlatform } from '@/hooks/usePlatform' + +/** + * Scan-to-login for a host with no display. + * + * The crawler's own QR flow prints the code into the terminal via PIL's + * `Image.show()`, which needs a desktop image viewer. On a server Chrome runs + * under Xvfb and there is no such viewer, so the backend reads the code out of + * that same browser over CDP and hands it here instead. + * + * That browser is the one monitor runs attach to, which is the point: scanning + * once leaves the login in the profile that every later unattended run reuses. + */ +export function QrLoginPanel() { + const { capability, platform } = useCurrentPlatform() + const queryClient = useQueryClient() + const [polling, setPolling] = useState(false) + + const { data: state } = useQrLoginStatus(polling) + const start = useStartQrLogin() + const cancel = useCancelQrLogin() + + const label = capability?.label ?? platform + const status = state?.status ?? 'idle' + + // Stop polling the moment the outcome is known, and refresh the cookie panel: + // a completed scan is what makes it start reporting a healthy login. + useEffect(() => { + if (status === 'waiting' || status === 'idle') return + setPolling(false) + if (status === 'success') { + queryClient.invalidateQueries({ queryKey: ['monitorCookie'] }) + } + }, [status, queryClient]) + + const begin = () => { + setPolling(true) + start.mutate() + } + + const busy = start.isPending || cancel.isPending + + return ( +
+
+
+ + + {label}扫码登录 + + {status === 'waiting' && ( + + 等待扫码 + + )} + {status === 'success' && ( + + 已登录 + + )} +
+ + {status === 'waiting' && ( + + )} +
+ + {status === 'waiting' && state?.image && ( +
+ {/* The code is a data: URL straight from the page, so nothing is + fetched from a third party and no file is written server-side. */} + 登录二维码 +
+

+ 用{label} App扫码。 + 扫码成功后登录态会写入服务器上那台 Chrome 的 profile， + 之后定时监控无需再登录。 +

+

+ 剩余 {state.expires_in} 秒 +

+
+
+ )} + + {status === 'waiting' && !state?.image && ( +

+ + 正在从浏览器取二维码… +

+ )} + + {status === 'success' && ( +

+ + {state?.message || '登录成功'} +

+ )} + + {(status === 'error' || status === 'expired') && ( +
+

+ + {state?.message || '获取二维码失败'} +

+ +
+ )} + + {status === 'idle' && ( +
+

+ 走 CDP 接管服务器上已开启远程调试的 Chrome，把二维码取回来显示在这里。 + 需要先在「系统设置」里打开 + 接管已有 Chrome（CDP）， + 并确保那台 Chrome 正以 9222 端口运行。 +

+ +
+ )} +
+ ) +} diff --git a/webui/src/components/settings/SettingsView.tsx b/webui/src/components/settings/SettingsView.tsx index 4817bd1..548af47 100644 --- a/webui/src/components/settings/SettingsView.tsx +++ b/webui/src/components/settings/SettingsView.tsx @@ -3,6 +3,7 @@ import { KeyRound, QrCode, Save } from 'lucide-react' import { Button } from '@/components/ui/button' import { CookiePanel } from '@/components/monitor/CookiePanel' +import { QrLoginPanel } from '@/components/monitor/QrLoginPanel' import { Section, SettingField } from '@/components/settings/SettingFields' import { useSettings, useUpdateSettings } from '@/hooks/useMonitor' import { useCurrentPlatform } from '@/hooks/usePlatform' @@ -66,17 +67,19 @@ export function SettingsView({ onNavigate }: { onNavigate?: (view: AppView) => v

-
+
+ {/* QR first: it is the one that keeps working unattended, since the + scan lands in the very browser profile the monitor runs reuse. */} + - {/* Not a duplicate login flow: the crawler has no login-only mode, so a - QR login is a side effect of a real crawl -- which is exactly what - the 采集 page already does. This preselects it rather than - reimplementing it. */}

- Cookie 不好使时，可以走一次**扫码登录**：二维码会显示在「采集」页的终端里。 - 扫码成功后浏览器 profile 会被更新，Cookie 的可靠性也会显著提升。 + 在本机桌面运行时，也可以走「采集」页的扫码：二维码会打印在终端里。 + 服务器没有显示器，那条路走不通，用上面的面板。

diff --git a/webui/src/hooks/useMonitor.ts b/webui/src/hooks/useMonitor.ts index d19de35..73e6b25 100644 --- a/webui/src/hooks/useMonitor.ts +++ b/webui/src/hooks/useMonitor.ts @@ -204,6 +204,46 @@ export function useClearCookie() { }) } +// --- QR login (over CDP) --------------------------------------------------- + +/** Polls only while a code is on screen; an idle session has nothing to watch. */ +export function useQrLoginStatus(polling: boolean) { + return useQuery({ + queryKey: ['monitorQrLogin'], + queryFn: async () => (await monitorApi.getQrLogin()).data, + refetchInterval: polling ? 2000 : false, + }) +} + +export function useStartQrLogin() { + const queryClient = useQueryClient() + const platform = usePlatformParam() + return useMutation({ + mutationFn: () => monitorApi.startQrLogin(platform), + onSuccess: (response) => { + queryClient.setQueryData(['monitorQrLogin'], response.data) + }, + onError: (error: Error) => { + // The backend's reason -- "Chrome is not reachable on ...", "no QR on the + // page" -- is the entire value here, and axios would flatten it to + // "Request failed with status code 502". + const detail = (error as { response?: { data?: { detail?: string } } })?.response + ?.data?.detail + toast.error(`获取二维码失败：${detail ?? error.message}`) + }, + }) +} + +export function useCancelQrLogin() { + const queryClient = useQueryClient() + return useMutation({ + mutationFn: () => monitorApi.cancelQrLogin(), + onSuccess: (response) => { + queryClient.setQueryData(['monitorQrLogin'], response.data) + }, + }) +} + // --- Settings ------------------------------------------------------------- export function useSettings() { diff --git a/webui/src/lib/api.ts b/webui/src/lib/api.ts index 1d8c5d1..4bc2283 100644 --- a/webui/src/lib/api.ts +++ b/webui/src/lib/api.ts @@ -11,6 +11,7 @@ import type { MonitorRun, MonitorTask, PlatformCapability, + QrLoginState, ReportResult, SettingsResponse, TaskCreatePayload, @@ -256,6 +257,13 @@ export const monitorApi = { return api.get(`/monitor/report?${params.toString()}`) }, + // QR login over CDP. Reads the code out of the browser the crawler attaches + // to, so the operator can scan even when the host has no display. + startQrLogin: (platform?: string) => + api.post('/monitor/login/qr', null, { params: { platform } }), + getQrLogin: () => api.get('/monitor/login/qr'), + cancelQrLogin: () => api.delete('/monitor/login/qr'), + getWebhook: () => api.get('/monitor/webhook'), setWebhook: (url: string) => api.post('/monitor/webhook', { url }), clearWebhook: () => api.delete('/monitor/webhook'), diff --git a/webui/src/types/monitor.ts b/webui/src/types/monitor.ts index 9827ba4..2983bca 100644 --- a/webui/src/types/monitor.ts +++ b/webui/src/types/monitor.ts @@ -161,6 +161,26 @@ export interface CookieStatus { last_ok_at: number | null } +export type QrLoginStatus = 'idle' | 'waiting' | 'success' | 'expired' | 'error' + +/** + * A QR login driven over CDP, for servers with no display to show one on. + * + * `success` also covers "the browser was already signed in" — the scan and the + * existing session both just mean the profile the crawler attaches to is usable. + */ +export interface QrLoginState { + status: QrLoginStatus + platform: string | null + /** `data:image/...;base64,...`. Empty unless status is `waiting`. */ + image: string + message: string + /** Seconds since the code was fetched. */ + elapsed: number + /** Seconds left before the code is written off. */ + expires_in: number +} + export interface MonitorOverview { tasks: number enabled_tasks: number