feat: CDP 接管开关 + 扫码登录面板 + Docker 部署
- runner: enable_cdp_mode 从硬编码 False 改为系统设置 cdp_enabled。服务器部署下爬虫接管已开启远程调试的 Chrome(默认 9222),复用其 profile 登录态;本机桌面默认仍为关,行为不变 - qrlogin: 新增 CDP 扫码登录。Chrome 在服务器上跑于 Xvfb,show_qrcode 依赖的 PIL 桌面看图程序不存在,二维码无处可显示;改为经 CDP 从页面取出二维码交给 WebUI 渲染。刻意复用 browser.contexts[0](新建 context 是无痕 profile,扫了也白扫),且绝不调用 browser.close()(会连带关掉操作者自己的 Chrome) - webui: 设置页新增扫码面板,替换原本跳到采集页看终端二维码的入口 - Dockerfile / .dockerignore / docker-compose.yml: 服务器部署。host 网络是必需而非图省事——容器里 127.0.0.1:9222 必须落到宿主机回环 - UPSTREAM.md: 补充 gitcode 镜像,用于 GitHub 大包传输必断时补历史
This commit is contained in:
@@ -0,0 +1,29 @@
|
|||||||
|
# Keep the build context to what the server actually runs.
|
||||||
|
.git
|
||||||
|
.github
|
||||||
|
.venv
|
||||||
|
venv
|
||||||
|
|
||||||
|
# The WebUI sources are not needed -- only the bundle they produce, which lands
|
||||||
|
# in api/webui and is therefore NOT excluded.
|
||||||
|
webui/node_modules
|
||||||
|
webui/src
|
||||||
|
webui/dist
|
||||||
|
|
||||||
|
# Runtime state: per-run crawler output and the login browser profile. Mounted
|
||||||
|
# as a volume instead, so it survives image rebuilds.
|
||||||
|
data
|
||||||
|
browser_data
|
||||||
|
|
||||||
|
# Secrets come from the environment via compose, never baked into a layer.
|
||||||
|
.env
|
||||||
|
|
||||||
|
tests
|
||||||
|
docs
|
||||||
|
__pycache__
|
||||||
|
**/__pycache__
|
||||||
|
*.pyc
|
||||||
|
*.pyo
|
||||||
|
.pytest_cache
|
||||||
|
*.db
|
||||||
|
*.log
|
||||||
+43
@@ -0,0 +1,43 @@
|
|||||||
|
# Server deployment image.
|
||||||
|
#
|
||||||
|
# No browser is bundled on purpose. On this deployment the crawler attaches over
|
||||||
|
# CDP to the Chrome already running on the host (see the 接管已有 Chrome setting),
|
||||||
|
# so shipping a second copy of Chromium would only add hundreds of megabytes and
|
||||||
|
# a login state that nothing uses. The Playwright Python package is still needed
|
||||||
|
# -- that is what speaks CDP -- hence PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD.
|
||||||
|
FROM python:3.11-slim
|
||||||
|
|
||||||
|
ENV PYTHONUNBUFFERED=1 \
|
||||||
|
PYTHONDONTWRITEBYTECODE=1 \
|
||||||
|
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||||
|
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 \
|
||||||
|
TZ=Asia/Shanghai
|
||||||
|
|
||||||
|
# asyncmy compiles a Cython extension, so a toolchain has to exist at build time.
|
||||||
|
# It is left installed: purging it risks taking libmysqlclient with it, and a
|
||||||
|
# slightly larger image is cheaper than a runtime that fails months later.
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends \
|
||||||
|
build-essential pkg-config default-libmysqlclient-dev \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Requirements first: this layer only rebuilds when the pins actually change.
|
||||||
|
ARG PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple
|
||||||
|
COPY requirements.txt ./
|
||||||
|
RUN pip install --no-cache-dir -i "$PIP_INDEX" -r requirements.txt
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
# The WebUI bundle lives at api/webui, which is gitignored -- it is built on the
|
||||||
|
# workstation (`cd webui && npm run build`) and shipped inside the build context.
|
||||||
|
# Without this check a missing bundle is invisible until someone opens the page
|
||||||
|
# and gets the bare JSON stub from serve_frontend().
|
||||||
|
RUN test -f api/webui/index.html \
|
||||||
|
|| (echo "ERROR: api/webui/index.html is missing. Run 'cd webui && npm run build' before building the image." >&2; exit 1)
|
||||||
|
|
||||||
|
EXPOSE 18051
|
||||||
|
|
||||||
|
# api.main reads MC_HOST / MC_PORT from the environment; compose supplies both.
|
||||||
|
CMD ["python", "-m", "api.main"]
|
||||||
+23
-2
@@ -25,9 +25,14 @@ webui/src/components/{monitor,settings,auth}/ 新视图
|
|||||||
webui/src/components/layout/{PlatformSwitcher,UnwiredPlatformNotice}.tsx
|
webui/src/components/layout/{PlatformSwitcher,UnwiredPlatformNotice}.tsx
|
||||||
webui/src/{hooks/useMonitor.ts,hooks/usePlatform.ts,store/platformStore.ts,lib/monitorFormat.ts,types/monitor.ts}
|
webui/src/{hooks/useMonitor.ts,hooks/usePlatform.ts,store/platformStore.ts,lib/monitorFormat.ts,types/monitor.ts}
|
||||||
docs/监控功能使用说明.md
|
docs/监控功能使用说明.md
|
||||||
tests/test_{auth,settings,platforms,monitor_*}.py
|
tests/test_{auth,settings,platforms,qrlogin,monitor_*}.py
|
||||||
|
Dockerfile / .dockerignore / docker-compose.yml 服务器部署用
|
||||||
```
|
```
|
||||||
|
|
||||||
|
其中 `api/monitor/qrlogin.py` + `webui/.../QrLoginPanel.tsx` 是**服务器专用**的扫码登录:
|
||||||
|
那台机器上 Chrome 跑在 Xvfb 里,`show_qrcode` 调的 PIL `Image.show()` 需要桌面看图程序,
|
||||||
|
服务器没有,二维码会无处可去。所以改成用 CDP 把二维码从页面里读出来交给前端 `<img>` 显示。
|
||||||
|
|
||||||
### 2. 加法改动(低冲突)
|
### 2. 加法改动(低冲突)
|
||||||
|
|
||||||
只在既有文件里**新增**内容,不改动原有行:
|
只在既有文件里**新增**内容,不改动原有行:
|
||||||
@@ -90,9 +95,25 @@ tests/test_{auth,settings,platforms,monitor_*}.py
|
|||||||
git stash # 或先 commit 到自己的分支(推荐)
|
git stash # 或先 commit 到自己的分支(推荐)
|
||||||
git fetch origin main
|
git fetch origin main
|
||||||
git rebase origin/main # 冲突只会出现在上表第 3、4 类文件里
|
git rebase origin/main # 冲突只会出现在上表第 3、4 类文件里
|
||||||
./.venv/Scripts/python.exe -m pytest tests/ -q # 486 个测试就是回归网
|
./.venv/Scripts/python.exe -m pytest tests/ -q # 502 个测试就是回归网
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### 直连 GitHub 不通时(本机常见)
|
||||||
|
|
||||||
|
本机到 `github.com` 时通时不通,**大包传输必断**(`Recv failure: Connection was reset`
|
||||||
|
或 `unexpected disconnect while reading sideband packet`),所以 `git clone` / `--unshallow`
|
||||||
|
这类一次性拉全量的操作基本必失败。可用的替代源:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# gitcode 的 GitHub 镜像,国内直连,比 GitHub 本身还新一天以内
|
||||||
|
git remote add gitcode https://gitcode.com/gh_mirrors/me/MediaCrawler.git
|
||||||
|
git fetch --no-tags --unshallow gitcode # 本仓库当初就是这样补全历史的,约 2 秒
|
||||||
|
```
|
||||||
|
|
||||||
|
注意 `git fetch` 只写 `refs/remotes/*`,**不会动本地 `main`**;
|
||||||
|
但拉镜像会把 `upstream/main` 指到镜像的 tip(可能比 GitHub 晚一天),
|
||||||
|
等 GitHub 通了再 `git fetch upstream` 正回来即可。
|
||||||
|
|
||||||
### 强烈建议:先把改动提交掉
|
### 强烈建议:先把改动提交掉
|
||||||
|
|
||||||
当前状态是**未提交**的(25 个上游文件被改 + 31 个新文件)。在 `main` 分支上裸着工作区,
|
当前状态是**未提交**的(25 个上游文件被改 + 31 个新文件)。在 `main` 分支上裸着工作区,
|
||||||
|
|||||||
@@ -65,6 +65,7 @@ async def lifespan(_app: FastAPI):
|
|||||||
happen whether or not anyone has the UI open.
|
happen whether or not anyone has the UI open.
|
||||||
"""
|
"""
|
||||||
from .monitor.db import dispose_engine, init_db
|
from .monitor.db import dispose_engine, init_db
|
||||||
|
from .monitor.qrlogin import shutdown as shutdown_qrlogin
|
||||||
from .monitor.scheduler import monitor_scheduler
|
from .monitor.scheduler import monitor_scheduler
|
||||||
|
|
||||||
await init_db()
|
await init_db()
|
||||||
@@ -89,6 +90,9 @@ async def lifespan(_app: FastAPI):
|
|||||||
yield
|
yield
|
||||||
finally:
|
finally:
|
||||||
await monitor_scheduler.stop()
|
await monitor_scheduler.stop()
|
||||||
|
# Drops the tab a QR login may have opened and stops the Playwright
|
||||||
|
# client; leaving them would strand a driver process on every restart.
|
||||||
|
await shutdown_qrlogin()
|
||||||
await dispose_engine()
|
await dispose_engine()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -204,6 +204,18 @@ SETTING_SPECS: List[SettingSpec] = [
|
|||||||
maximum=23,
|
maximum=23,
|
||||||
affects_new_runs=False,
|
affects_new_runs=False,
|
||||||
),
|
),
|
||||||
|
SettingSpec(
|
||||||
|
name="cdp_enabled",
|
||||||
|
scope=SCOPE_SYSTEM,
|
||||||
|
type=TYPE_BOOL,
|
||||||
|
label="接管已有 Chrome(CDP)",
|
||||||
|
help=(
|
||||||
|
"开启后爬虫不再自己启动浏览器,而是接管本机已开放远程调试端口的 Chrome"
|
||||||
|
"(默认 127.0.0.1:9222),复用它的登录态与扩展。"
|
||||||
|
"服务器部署请开启;本机桌面使用请保持关闭。"
|
||||||
|
),
|
||||||
|
default=False,
|
||||||
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
SPECS_BY_NAME = {spec.name: spec for spec in SETTING_SPECS}
|
SPECS_BY_NAME = {spec.name: spec for spec in SETTING_SPECS}
|
||||||
|
|||||||
@@ -0,0 +1,268 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Copyright (c) 2025 [email protected]
|
||||||
|
#
|
||||||
|
# This file is part of MediaCrawler project.
|
||||||
|
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/monitor/qrlogin.py
|
||||||
|
# GitHub: https://github.com/NanmiCoder
|
||||||
|
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||||
|
#
|
||||||
|
# 声明:本代码仅供学习和研究目的使用。使用者应遵守以下原则:
|
||||||
|
# 1. 不得用于任何商业用途。
|
||||||
|
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||||
|
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||||
|
# 4. 应合理控制请求频率,避免给目标平台带来不必要的负担。
|
||||||
|
# 5. 不得用于任何非法或不当的用途。
|
||||||
|
#
|
||||||
|
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||||
|
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||||
|
|
||||||
|
"""Show a login QR code to the operator through the WebUI.
|
||||||
|
|
||||||
|
Why this module exists: on a server deployment Chrome runs under Xvfb, so there
|
||||||
|
is no display to look at, and the helper the crawler normally uses to present the
|
||||||
|
QR (`show_qrcode` in `tools/crawler_util.py`) calls PIL's ``Image.show()`` -- it
|
||||||
|
needs a desktop image viewer that such a machine does not have, so the code would
|
||||||
|
go nowhere and the operator would be stuck. Instead the QR is read straight out
|
||||||
|
of the page over CDP and handed to the WebUI, which renders it as an ``<img>``.
|
||||||
|
|
||||||
|
Three details that are easy to get wrong:
|
||||||
|
|
||||||
|
* **Reuse the browser's default context.** ``browser.new_context()`` would give
|
||||||
|
an incognito-like profile, so the scan would land in a cookie jar the crawler
|
||||||
|
never reads and every run would still look logged out. The real profile -- the
|
||||||
|
one the crawler attaches to -- is ``browser.contexts[0]``.
|
||||||
|
* **Never call ``browser.close()``.** On a CDP connection that tears down the
|
||||||
|
operator's own Chrome and takes every unrelated tab with it. Only the page this
|
||||||
|
module opened is closed, and the Playwright client is stopped to drop the
|
||||||
|
socket.
|
||||||
|
* **A scan is not a login until the cookie changes.** The page can be showing a
|
||||||
|
QR for an account that is in fact already signed in, so success is decided by
|
||||||
|
``web_session`` appearing or changing against the value captured at start,
|
||||||
|
never by anything the page displays.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
from typing import Any, Dict, Optional
|
||||||
|
|
||||||
|
import config
|
||||||
|
from playwright.async_api import async_playwright
|
||||||
|
from tools import utils
|
||||||
|
|
||||||
|
from .platforms import PLATFORM_XHS
|
||||||
|
|
||||||
|
# How long a QR stays valid before the session is written off. The platform
|
||||||
|
# rotates the code well before this; the limit exists so an abandoned attempt
|
||||||
|
# cannot pin a browser tab open indefinitely.
|
||||||
|
QR_TTL_SECONDS = 180
|
||||||
|
|
||||||
|
STATUS_IDLE = "idle"
|
||||||
|
STATUS_WAITING = "waiting"
|
||||||
|
STATUS_SUCCESS = "success"
|
||||||
|
STATUS_EXPIRED = "expired"
|
||||||
|
STATUS_ERROR = "error"
|
||||||
|
|
||||||
|
# Only xhs is wired: it is the only platform whose monitor pipeline works, and
|
||||||
|
# pretending otherwise would offer the operator a button that cannot succeed.
|
||||||
|
LOGIN_URL: Dict[str, str] = {PLATFORM_XHS: "https://www.xiaohongshu.com"}
|
||||||
|
QR_SELECTOR: Dict[str, str] = {PLATFORM_XHS: "xpath=//img[@class='qrcode-img']"}
|
||||||
|
LOGIN_BUTTON_SELECTOR: Dict[str, str] = {
|
||||||
|
PLATFORM_XHS: "xpath=//*[@id='app']/div[1]/div[2]/div[1]/ul/div[1]/button"
|
||||||
|
}
|
||||||
|
SESSION_COOKIE: Dict[str, str] = {PLATFORM_XHS: "web_session"}
|
||||||
|
|
||||||
|
_IDLE_SNAPSHOT: Dict[str, Any] = {
|
||||||
|
"status": STATUS_IDLE,
|
||||||
|
"platform": None,
|
||||||
|
"image": "",
|
||||||
|
"message": "",
|
||||||
|
"elapsed": 0,
|
||||||
|
"expires_in": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
_lock = asyncio.Lock()
|
||||||
|
_current: Optional["QrLoginSession"] = None
|
||||||
|
|
||||||
|
|
||||||
|
def _cdp_url() -> str:
|
||||||
|
"""Where to reach the browser's DevTools endpoint.
|
||||||
|
|
||||||
|
``MC_CDP_URL`` wins so a deployment can point at another host without a code
|
||||||
|
change; otherwise the port comes from the same config the crawler itself
|
||||||
|
reads, so the two can never drift apart.
|
||||||
|
"""
|
||||||
|
return os.getenv("MC_CDP_URL") or f"http://127.0.0.1:{config.CDP_DEBUG_PORT}"
|
||||||
|
|
||||||
|
|
||||||
|
def _login_url(platform: str) -> str:
|
||||||
|
if platform == PLATFORM_XHS and getattr(config, "XHS_INTERNATIONAL", False):
|
||||||
|
return "https://www.rednote.com"
|
||||||
|
return LOGIN_URL[platform]
|
||||||
|
|
||||||
|
|
||||||
|
class QrLoginSession:
|
||||||
|
"""One live QR-login attempt against the CDP browser."""
|
||||||
|
|
||||||
|
def __init__(self, platform: str, playwright: Any, page: Any, baseline: str) -> None:
|
||||||
|
self.platform = platform
|
||||||
|
self.status = STATUS_WAITING
|
||||||
|
self.message = "请用小红书 App 扫描二维码"
|
||||||
|
self.image = ""
|
||||||
|
self.started_at = time.time()
|
||||||
|
self._playwright = playwright
|
||||||
|
self._page = page
|
||||||
|
# The `web_session` value present *before* the scan. An account already
|
||||||
|
# signed in has a non-empty baseline, which is why success is "changed",
|
||||||
|
# not merely "present".
|
||||||
|
self._baseline = baseline
|
||||||
|
|
||||||
|
@property
|
||||||
|
def elapsed(self) -> float:
|
||||||
|
return time.time() - self.started_at
|
||||||
|
|
||||||
|
async def refresh(self) -> None:
|
||||||
|
"""Poll the browser once for a completed scan."""
|
||||||
|
if self.status != STATUS_WAITING:
|
||||||
|
return
|
||||||
|
if self.elapsed > QR_TTL_SECONDS:
|
||||||
|
self.status = STATUS_EXPIRED
|
||||||
|
self.message = "二维码已超时,请重新获取"
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
cookies = await self._page.context.cookies()
|
||||||
|
except Exception:
|
||||||
|
# The operator may have closed the tab we opened.
|
||||||
|
self.status = STATUS_ERROR
|
||||||
|
self.message = "二维码所在页面已被关闭,请重新获取"
|
||||||
|
return
|
||||||
|
token = {c["name"]: c["value"] for c in cookies}.get(
|
||||||
|
SESSION_COOKIE[self.platform], ""
|
||||||
|
)
|
||||||
|
if token and token != self._baseline:
|
||||||
|
self.status = STATUS_SUCCESS
|
||||||
|
self.message = "登录成功,登录态已写入浏览器 profile"
|
||||||
|
|
||||||
|
def snapshot(self) -> Dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"status": self.status,
|
||||||
|
"platform": self.platform,
|
||||||
|
"image": self.image,
|
||||||
|
"message": self.message,
|
||||||
|
"elapsed": int(self.elapsed),
|
||||||
|
"expires_in": max(0, int(QR_TTL_SECONDS - self.elapsed)),
|
||||||
|
}
|
||||||
|
|
||||||
|
async def close(self) -> None:
|
||||||
|
"""Drop our page and the Playwright client, leaving Chrome untouched."""
|
||||||
|
try:
|
||||||
|
await self._page.close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
await self._playwright.stop()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
async def _read_qr(page: Any, platform: str) -> str:
|
||||||
|
"""Pull the QR image out of the page, opening the login dialog if needed."""
|
||||||
|
image = await utils.find_login_qrcode(page, selector=QR_SELECTOR[platform])
|
||||||
|
if image:
|
||||||
|
return image
|
||||||
|
# The dialog does not always open on its own. This is the same fallback the
|
||||||
|
# crawler's own QR flow performs before giving up.
|
||||||
|
await asyncio.sleep(0.5)
|
||||||
|
try:
|
||||||
|
await page.locator(LOGIN_BUTTON_SELECTOR[platform]).click(timeout=5000)
|
||||||
|
except Exception:
|
||||||
|
return ""
|
||||||
|
return await utils.find_login_qrcode(page, selector=QR_SELECTOR[platform])
|
||||||
|
|
||||||
|
|
||||||
|
async def _reset_locked() -> None:
|
||||||
|
global _current
|
||||||
|
if _current is not None:
|
||||||
|
await _current.close()
|
||||||
|
_current = None
|
||||||
|
|
||||||
|
|
||||||
|
async def start(platform: str = PLATFORM_XHS) -> Dict[str, Any]:
|
||||||
|
"""Open a login page in the CDP browser and return its QR code."""
|
||||||
|
if platform not in LOGIN_URL:
|
||||||
|
raise ValueError(f"平台 {platform} 尚未接入扫码登录(目前仅支持小红书)")
|
||||||
|
|
||||||
|
async with _lock:
|
||||||
|
await _reset_locked()
|
||||||
|
|
||||||
|
playwright = await async_playwright().start()
|
||||||
|
try:
|
||||||
|
browser = await playwright.chromium.connect_over_cdp(_cdp_url(), timeout=15000)
|
||||||
|
except Exception as exc:
|
||||||
|
await playwright.stop()
|
||||||
|
raise RuntimeError(
|
||||||
|
f"连接浏览器失败({_cdp_url()})。请确认服务器上的 Chrome 以 "
|
||||||
|
f"--remote-debugging-port 启动。原始错误:{exc}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
if not browser.contexts:
|
||||||
|
await playwright.stop()
|
||||||
|
raise RuntimeError(
|
||||||
|
"浏览器没有可用上下文。CDP 已连上,但读不到 profile —— "
|
||||||
|
"请确认 Chrome 不是以无痕模式启动的。"
|
||||||
|
)
|
||||||
|
|
||||||
|
# contexts[0] is the real profile. See the module docstring.
|
||||||
|
context = browser.contexts[0]
|
||||||
|
page = await context.new_page()
|
||||||
|
try:
|
||||||
|
await page.goto(_login_url(platform), wait_until="domcontentloaded", timeout=30000)
|
||||||
|
image = await _read_qr(page, platform)
|
||||||
|
cookies = await context.cookies()
|
||||||
|
except Exception as exc:
|
||||||
|
try:
|
||||||
|
await page.close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
await playwright.stop()
|
||||||
|
raise RuntimeError(f"打开登录页失败:{exc}") from exc
|
||||||
|
|
||||||
|
baseline = {c["name"]: c["value"] for c in cookies}.get(
|
||||||
|
SESSION_COOKIE[platform], ""
|
||||||
|
)
|
||||||
|
|
||||||
|
session = QrLoginSession(platform, playwright, page, baseline)
|
||||||
|
session.image = image
|
||||||
|
if not image:
|
||||||
|
if baseline:
|
||||||
|
session.status = STATUS_SUCCESS
|
||||||
|
session.message = "浏览器已经是登录状态,无需扫码"
|
||||||
|
else:
|
||||||
|
session.status = STATUS_ERROR
|
||||||
|
session.message = "页面上没找到二维码,请确认站点结构没有变化"
|
||||||
|
|
||||||
|
global _current
|
||||||
|
_current = session
|
||||||
|
return session.snapshot()
|
||||||
|
|
||||||
|
|
||||||
|
async def status() -> Dict[str, Any]:
|
||||||
|
async with _lock:
|
||||||
|
if _current is None:
|
||||||
|
return dict(_IDLE_SNAPSHOT)
|
||||||
|
await _current.refresh()
|
||||||
|
return _current.snapshot()
|
||||||
|
|
||||||
|
|
||||||
|
async def cancel() -> Dict[str, Any]:
|
||||||
|
async with _lock:
|
||||||
|
await _reset_locked()
|
||||||
|
snapshot = dict(_IDLE_SNAPSHOT)
|
||||||
|
snapshot["message"] = "已取消"
|
||||||
|
return snapshot
|
||||||
|
|
||||||
|
|
||||||
|
async def shutdown() -> None:
|
||||||
|
"""Release the browser tab at application shutdown."""
|
||||||
|
async with _lock:
|
||||||
|
await _reset_locked()
|
||||||
@@ -162,6 +162,11 @@ async def execute_task(task_id: int, trigger: str = "manual") -> IngestResult:
|
|||||||
urls = build_target_urls(task.mode, targets)
|
urls = build_target_urls(task.mode, targets)
|
||||||
cookie = await get_cookie(session, platform)
|
cookie = await get_cookie(session, platform)
|
||||||
strategy = await _strategy_settings(session, platform)
|
strategy = await _strategy_settings(session, platform)
|
||||||
|
# System-wide switch. On a headless server the crawler must attach to the
|
||||||
|
# Chrome already listening on the debug port -- that browser is where the
|
||||||
|
# operator scanned the login QR, so its profile is the login. Default False
|
||||||
|
# keeps desktop runs launching a private browser exactly as before.
|
||||||
|
cdp_enabled = await app_settings.get_value(session, "cdp_enabled", fallback=False)
|
||||||
|
|
||||||
run = MonitorRun(
|
run = MonitorRun(
|
||||||
task_id=task.id,
|
task_id=task.id,
|
||||||
@@ -209,8 +214,9 @@ async def execute_task(task_id: int, trigger: str = "manual") -> IngestResult:
|
|||||||
# Isolate this run's output: the crawler names files by date only, so
|
# Isolate this run's output: the crawler names files by date only, so
|
||||||
# otherwise same-day runs would append into one shared file.
|
# otherwise same-day runs would append into one shared file.
|
||||||
save_data_path=str(out_dir),
|
save_data_path=str(out_dir),
|
||||||
# Unattended runs must not try to attach to the user's desktop Chrome.
|
# Attach to the browser already running on CDP_DEBUG_PORT when the
|
||||||
enable_cdp_mode=False,
|
# operator enabled it; otherwise launch a private, throwaway browser.
|
||||||
|
enable_cdp_mode=cdp_enabled,
|
||||||
# Only injecting web_session is not enough to sign requests from a cold
|
# Only injecting web_session is not enough to sign requests from a cold
|
||||||
# browser profile.
|
# browser profile.
|
||||||
inject_all_cookies=True,
|
inject_all_cookies=True,
|
||||||
|
|||||||
+38
-1
@@ -23,7 +23,7 @@ from typing import Any, Dict, List, Optional
|
|||||||
|
|
||||||
from fastapi import APIRouter, HTTPException, Query, Response
|
from fastapi import APIRouter, HTTPException, Query, Response
|
||||||
|
|
||||||
from ..monitor import notify, report, service
|
from ..monitor import notify, qrlogin, report, service
|
||||||
from ..monitor.db import get_session
|
from ..monitor.db import get_session
|
||||||
from ..monitor.platforms import PLATFORM_XHS
|
from ..monitor.platforms import PLATFORM_XHS
|
||||||
from ..monitor.settings import (
|
from ..monitor.settings import (
|
||||||
@@ -243,6 +243,43 @@ async def clear_cookie_endpoint(platform: str = Query(default=PLATFORM_XHS)):
|
|||||||
return {"message": "Cookie cleared"}
|
return {"message": "Cookie cleared"}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# QR login
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
# These drive the browser already listening on the CDP debug port, which is the
|
||||||
|
# same browser -- and therefore the same profile -- that monitor runs attach to.
|
||||||
|
# Scanning once is what makes later unattended runs logged in.
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/login/qr")
|
||||||
|
async def start_qr_login(platform: str = Query(default=PLATFORM_XHS)):
|
||||||
|
"""Open the login page in the CDP browser and return its QR code.
|
||||||
|
|
||||||
|
A server deployment has no display (Chrome sits under Xvfb), so the code is
|
||||||
|
surfaced here for the operator to scan instead of in a desktop window that
|
||||||
|
does not exist.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return await qrlogin.start(platform)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise HTTPException(status_code=400, detail=str(exc))
|
||||||
|
except RuntimeError as exc:
|
||||||
|
raise HTTPException(status_code=502, detail=str(exc))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/login/qr")
|
||||||
|
async def get_qr_login():
|
||||||
|
"""Poll the live session: waiting -> success / expired / error."""
|
||||||
|
return await qrlogin.status()
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/login/qr")
|
||||||
|
async def cancel_qr_login():
|
||||||
|
"""Drop our tab and stop polling."""
|
||||||
|
return await qrlogin.cancel()
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Report
|
# Report
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
services:
|
||||||
|
mediacrawler:
|
||||||
|
build: .
|
||||||
|
image: mediacrawler:latest
|
||||||
|
container_name: mediacrawler
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
# host networking is a requirement, not a convenience: the crawler attaches
|
||||||
|
# to the operator's Chrome at 127.0.0.1:9222, and inside a bridge network
|
||||||
|
# that loopback is the container's own, where no browser is listening.
|
||||||
|
# It also puts the app port directly on the host, so `ports:` is not used.
|
||||||
|
network_mode: host
|
||||||
|
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
environment:
|
||||||
|
MC_HOST: 0.0.0.0
|
||||||
|
MC_PORT: "18051"
|
||||||
|
TZ: Asia/Shanghai
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
# Per-run crawler output, read back by the monitor layer. Without this the
|
||||||
|
# data would live inside the container and vanish on every rebuild.
|
||||||
|
- ./data:/app/data
|
||||||
@@ -0,0 +1,190 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""Tests for CDP-driven QR login.
|
||||||
|
|
||||||
|
The behaviour worth pinning down is not "does it call Playwright" but the two
|
||||||
|
decisions that silently produce a logged-out crawler if they regress:
|
||||||
|
|
||||||
|
* the QR must be read from the browser's **default** context, because a new
|
||||||
|
context is an incognito-like profile whose cookies the crawler never sees;
|
||||||
|
* success must be decided by ``web_session`` *changing*, not merely existing --
|
||||||
|
an account already signed in has a value before the operator ever scans.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from unittest.mock import AsyncMock, MagicMock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from api.monitor import qrlogin
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _reset_module_state():
|
||||||
|
"""Each test starts from "nothing on screen" and leaves it that way."""
|
||||||
|
qrlogin._current = None
|
||||||
|
yield
|
||||||
|
qrlogin._current = None
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_stack(qr_image: str = "data:image/png;base64,AAAA", cookies=None):
|
||||||
|
"""A Playwright/Chrome stand-in wired the way the real one behaves."""
|
||||||
|
context = MagicMock()
|
||||||
|
context.cookies = AsyncMock(return_value=list(cookies or []))
|
||||||
|
page = MagicMock()
|
||||||
|
page.goto = AsyncMock()
|
||||||
|
page.close = AsyncMock()
|
||||||
|
context.new_page = AsyncMock(return_value=page)
|
||||||
|
|
||||||
|
browser = MagicMock()
|
||||||
|
browser.contexts = [context]
|
||||||
|
# Reaching for a fresh context is the bug this guards against, so make it
|
||||||
|
# blow up loudly rather than quietly returning a throwaway profile.
|
||||||
|
browser.new_context = AsyncMock(
|
||||||
|
side_effect=AssertionError("must reuse browser.contexts[0], not a new context")
|
||||||
|
)
|
||||||
|
|
||||||
|
playwright = MagicMock()
|
||||||
|
playwright.chromium.connect_over_cdp = AsyncMock(return_value=browser)
|
||||||
|
playwright.stop = AsyncMock()
|
||||||
|
|
||||||
|
manager = MagicMock()
|
||||||
|
manager.start = AsyncMock(return_value=playwright)
|
||||||
|
|
||||||
|
return manager, playwright, browser, context, page
|
||||||
|
|
||||||
|
|
||||||
|
def _patch(monkeypatch, manager, qr_image="data:image/png;base64,AAAA"):
|
||||||
|
monkeypatch.setattr(qrlogin, "async_playwright", lambda: manager)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
qrlogin.utils, "find_login_qrcode", AsyncMock(return_value=qr_image)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_idle_before_any_session():
|
||||||
|
snapshot = await qrlogin.status()
|
||||||
|
|
||||||
|
assert snapshot["status"] == qrlogin.STATUS_IDLE
|
||||||
|
assert snapshot["image"] == ""
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_unwired_platform_is_rejected():
|
||||||
|
"""Only xhs is wired; anything else must fail loudly, not show a dead button."""
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
await qrlogin.start("dy")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_start_returns_the_qr_and_reuses_the_default_context(monkeypatch):
|
||||||
|
manager, playwright, browser, context, _page = _fake_stack()
|
||||||
|
_patch(monkeypatch, manager)
|
||||||
|
|
||||||
|
snapshot = await qrlogin.start(qrlogin.PLATFORM_XHS)
|
||||||
|
|
||||||
|
assert snapshot["status"] == qrlogin.STATUS_WAITING
|
||||||
|
assert snapshot["image"] == "data:image/png;base64,AAAA"
|
||||||
|
context.new_page.assert_awaited_once()
|
||||||
|
browser.new_context.assert_not_called()
|
||||||
|
playwright.chromium.connect_over_cdp.assert_awaited_once()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_success_requires_the_cookie_to_change():
|
||||||
|
session = qrlogin.QrLoginSession(
|
||||||
|
qrlogin.PLATFORM_XHS,
|
||||||
|
MagicMock(),
|
||||||
|
_page_with_cookies([{"name": "web_session", "value": "before"}]),
|
||||||
|
baseline="before",
|
||||||
|
)
|
||||||
|
|
||||||
|
await session.refresh()
|
||||||
|
assert session.status == qrlogin.STATUS_WAITING
|
||||||
|
|
||||||
|
session._page = _page_with_cookies([{"name": "web_session", "value": "after"}])
|
||||||
|
await session.refresh()
|
||||||
|
assert session.status == qrlogin.STATUS_SUCCESS
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_an_already_signed_in_profile_needs_no_scan(monkeypatch):
|
||||||
|
"""No QR on the page plus a session cookie means someone is already logged in."""
|
||||||
|
manager, _playwright, _browser, _context, _page = _fake_stack(
|
||||||
|
qr_image="", cookies=[{"name": "web_session", "value": "existing"}]
|
||||||
|
)
|
||||||
|
_patch(monkeypatch, manager, qr_image="")
|
||||||
|
|
||||||
|
snapshot = await qrlogin.start(qrlogin.PLATFORM_XHS)
|
||||||
|
|
||||||
|
assert snapshot["status"] == qrlogin.STATUS_SUCCESS
|
||||||
|
assert "登录" in snapshot["message"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_no_qr_and_no_session_is_an_error(monkeypatch):
|
||||||
|
manager, _playwright, _browser, _context, _page = _fake_stack(qr_image="", cookies=[])
|
||||||
|
_patch(monkeypatch, manager, qr_image="")
|
||||||
|
|
||||||
|
snapshot = await qrlogin.start(qrlogin.PLATFORM_XHS)
|
||||||
|
|
||||||
|
assert snapshot["status"] == qrlogin.STATUS_ERROR
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_session_expires(monkeypatch):
|
||||||
|
session = qrlogin.QrLoginSession(
|
||||||
|
qrlogin.PLATFORM_XHS, MagicMock(), _page_with_cookies([]), baseline=""
|
||||||
|
)
|
||||||
|
session.started_at -= qrlogin.QR_TTL_SECONDS + 1
|
||||||
|
|
||||||
|
await session.refresh()
|
||||||
|
|
||||||
|
assert session.status == qrlogin.STATUS_EXPIRED
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_closed_tab_is_reported_rather_than_crashing():
|
||||||
|
page = MagicMock()
|
||||||
|
page.context.cookies = AsyncMock(side_effect=RuntimeError("Target closed"))
|
||||||
|
session = qrlogin.QrLoginSession(qrlogin.PLATFORM_XHS, MagicMock(), page, baseline="")
|
||||||
|
|
||||||
|
await session.refresh()
|
||||||
|
|
||||||
|
assert session.status == qrlogin.STATUS_ERROR
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_unreachable_browser_is_reported(monkeypatch):
|
||||||
|
"""A server whose Chrome is not listening must say so, not 500 anonymously."""
|
||||||
|
playwright = MagicMock()
|
||||||
|
playwright.chromium.connect_over_cdp = AsyncMock(
|
||||||
|
side_effect=OSError("Connection refused")
|
||||||
|
)
|
||||||
|
playwright.stop = AsyncMock()
|
||||||
|
manager = MagicMock()
|
||||||
|
manager.start = AsyncMock(return_value=playwright)
|
||||||
|
monkeypatch.setattr(qrlogin, "async_playwright", lambda: manager)
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError) as excinfo:
|
||||||
|
await qrlogin.start(qrlogin.PLATFORM_XHS)
|
||||||
|
|
||||||
|
assert "连接浏览器失败" in str(excinfo.value)
|
||||||
|
playwright.stop.assert_awaited_once()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cancel_closes_the_tab_and_resets(monkeypatch):
|
||||||
|
manager, _playwright, _browser, _context, page = _fake_stack()
|
||||||
|
_patch(monkeypatch, manager)
|
||||||
|
await qrlogin.start(qrlogin.PLATFORM_XHS)
|
||||||
|
|
||||||
|
snapshot = await qrlogin.cancel()
|
||||||
|
|
||||||
|
assert snapshot["status"] == qrlogin.STATUS_IDLE
|
||||||
|
page.close.assert_awaited_once()
|
||||||
|
assert (await qrlogin.status())["status"] == qrlogin.STATUS_IDLE
|
||||||
|
|
||||||
|
|
||||||
|
def _page_with_cookies(cookies):
|
||||||
|
page = MagicMock()
|
||||||
|
page.context.cookies = AsyncMock(return_value=list(cookies))
|
||||||
|
return page
|
||||||
@@ -0,0 +1,154 @@
|
|||||||
|
import { useEffect, useState } from 'react'
|
||||||
|
import { useQueryClient } from '@tanstack/react-query'
|
||||||
|
import { AlertTriangle, CheckCircle2, Loader2, QrCode, RefreshCw, X } from 'lucide-react'
|
||||||
|
|
||||||
|
import { Badge } from '@/components/ui/badge'
|
||||||
|
import { Button } from '@/components/ui/button'
|
||||||
|
import { useCancelQrLogin, useQrLoginStatus, useStartQrLogin } from '@/hooks/useMonitor'
|
||||||
|
import { useCurrentPlatform } from '@/hooks/usePlatform'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Scan-to-login for a host with no display.
|
||||||
|
*
|
||||||
|
* The crawler's own QR flow prints the code into the terminal via PIL's
|
||||||
|
* `Image.show()`, which needs a desktop image viewer. On a server Chrome runs
|
||||||
|
* under Xvfb and there is no such viewer, so the backend reads the code out of
|
||||||
|
* that same browser over CDP and hands it here instead.
|
||||||
|
*
|
||||||
|
* That browser is the one monitor runs attach to, which is the point: scanning
|
||||||
|
* once leaves the login in the profile that every later unattended run reuses.
|
||||||
|
*/
|
||||||
|
export function QrLoginPanel() {
|
||||||
|
const { capability, platform } = useCurrentPlatform()
|
||||||
|
const queryClient = useQueryClient()
|
||||||
|
const [polling, setPolling] = useState(false)
|
||||||
|
|
||||||
|
const { data: state } = useQrLoginStatus(polling)
|
||||||
|
const start = useStartQrLogin()
|
||||||
|
const cancel = useCancelQrLogin()
|
||||||
|
|
||||||
|
const label = capability?.label ?? platform
|
||||||
|
const status = state?.status ?? 'idle'
|
||||||
|
|
||||||
|
// Stop polling the moment the outcome is known, and refresh the cookie panel:
|
||||||
|
// a completed scan is what makes it start reporting a healthy login.
|
||||||
|
useEffect(() => {
|
||||||
|
if (status === 'waiting' || status === 'idle') return
|
||||||
|
setPolling(false)
|
||||||
|
if (status === 'success') {
|
||||||
|
queryClient.invalidateQueries({ queryKey: ['monitorCookie'] })
|
||||||
|
}
|
||||||
|
}, [status, queryClient])
|
||||||
|
|
||||||
|
const begin = () => {
|
||||||
|
setPolling(true)
|
||||||
|
start.mutate()
|
||||||
|
}
|
||||||
|
|
||||||
|
const busy = start.isPending || cancel.isPending
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="rounded-lg glass-panel float-panel p-4 space-y-3">
|
||||||
|
<div className="flex items-center justify-between gap-3">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<QrCode className="w-4 h-4 text-cyber-neon-cyan" />
|
||||||
|
<span className="font-mono text-xs tracking-wider text-cyber-text-primary">
|
||||||
|
{label}扫码登录
|
||||||
|
</span>
|
||||||
|
{status === 'waiting' && (
|
||||||
|
<Badge variant="warning" className="text-[10px]">
|
||||||
|
等待扫码
|
||||||
|
</Badge>
|
||||||
|
)}
|
||||||
|
{status === 'success' && (
|
||||||
|
<Badge variant="success" className="text-[10px]">
|
||||||
|
已登录
|
||||||
|
</Badge>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{status === 'waiting' && (
|
||||||
|
<Button
|
||||||
|
variant="ghost"
|
||||||
|
size="sm"
|
||||||
|
disabled={busy}
|
||||||
|
onClick={() => {
|
||||||
|
setPolling(false)
|
||||||
|
cancel.mutate()
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<X className="w-3 h-3" />
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{status === 'waiting' && state?.image && (
|
||||||
|
<div className="flex items-start gap-4">
|
||||||
|
{/* The code is a data: URL straight from the page, so nothing is
|
||||||
|
fetched from a third party and no file is written server-side. */}
|
||||||
|
<img
|
||||||
|
src={state.image}
|
||||||
|
alt="登录二维码"
|
||||||
|
className="w-44 h-44 rounded-md border border-cyber-border-DEFAULT bg-white p-1"
|
||||||
|
/>
|
||||||
|
<div className="space-y-2 pt-1">
|
||||||
|
<p className="text-[11px] font-mono text-cyber-text-secondary leading-relaxed">
|
||||||
|
用<span className="text-cyber-neon-cyan">{label} App</span>扫码。
|
||||||
|
扫码成功后登录态会写入服务器上那台 Chrome 的 profile,
|
||||||
|
之后定时监控无需再登录。
|
||||||
|
</p>
|
||||||
|
<p className="text-[11px] font-mono text-cyber-text-muted">
|
||||||
|
剩余 <span className="text-cyber-neon-cyan">{state.expires_in}</span> 秒
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{status === 'waiting' && !state?.image && (
|
||||||
|
<p className="flex items-center gap-2 text-[11px] font-mono text-cyber-text-muted">
|
||||||
|
<Loader2 className="w-3 h-3 animate-spin" />
|
||||||
|
正在从浏览器取二维码…
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{status === 'success' && (
|
||||||
|
<p className="flex items-start gap-2 text-[11px] font-mono text-cyber-neon-green leading-relaxed">
|
||||||
|
<CheckCircle2 className="w-3 h-3 mt-0.5 shrink-0" />
|
||||||
|
{state?.message || '登录成功'}
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{(status === 'error' || status === 'expired') && (
|
||||||
|
<div className="space-y-2">
|
||||||
|
<p className="flex items-start gap-2 text-[11px] font-mono text-cyber-neon-orange leading-relaxed">
|
||||||
|
<AlertTriangle className="w-3 h-3 mt-0.5 shrink-0" />
|
||||||
|
{state?.message || '获取二维码失败'}
|
||||||
|
</p>
|
||||||
|
<Button variant="outline" size="sm" disabled={busy} onClick={begin}>
|
||||||
|
<RefreshCw className="w-3 h-3 mr-1" />
|
||||||
|
重新获取
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{status === 'idle' && (
|
||||||
|
<div className="space-y-2">
|
||||||
|
<p className="text-[11px] font-mono text-cyber-text-muted leading-relaxed">
|
||||||
|
走 CDP 接管服务器上已开启远程调试的 Chrome,把二维码取回来显示在这里。
|
||||||
|
需要先在「系统设置」里打开
|
||||||
|
<span className="text-cyber-neon-cyan">接管已有 Chrome(CDP)</span>,
|
||||||
|
并确保那台 Chrome 正以 <span className="text-cyber-neon-cyan">9222</span> 端口运行。
|
||||||
|
</p>
|
||||||
|
<Button size="sm" disabled={busy} onClick={begin}>
|
||||||
|
{start.isPending ? (
|
||||||
|
<Loader2 className="w-3 h-3 mr-1 animate-spin" />
|
||||||
|
) : (
|
||||||
|
<QrCode className="w-3 h-3 mr-1" />
|
||||||
|
)}
|
||||||
|
获取二维码
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -3,6 +3,7 @@ import { KeyRound, QrCode, Save } from 'lucide-react'
|
|||||||
|
|
||||||
import { Button } from '@/components/ui/button'
|
import { Button } from '@/components/ui/button'
|
||||||
import { CookiePanel } from '@/components/monitor/CookiePanel'
|
import { CookiePanel } from '@/components/monitor/CookiePanel'
|
||||||
|
import { QrLoginPanel } from '@/components/monitor/QrLoginPanel'
|
||||||
import { Section, SettingField } from '@/components/settings/SettingFields'
|
import { Section, SettingField } from '@/components/settings/SettingFields'
|
||||||
import { useSettings, useUpdateSettings } from '@/hooks/useMonitor'
|
import { useSettings, useUpdateSettings } from '@/hooks/useMonitor'
|
||||||
import { useCurrentPlatform } from '@/hooks/usePlatform'
|
import { useCurrentPlatform } from '@/hooks/usePlatform'
|
||||||
@@ -66,17 +67,19 @@ export function SettingsView({ onNavigate }: { onNavigate?: (view: AppView) => v
|
|||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<Section title="登录态" description={`${label}的登录 Cookie,定时监控必须持久化登录态。`}>
|
<Section
|
||||||
|
title="登录态"
|
||||||
|
description={`${label}的登录态。定时监控必须持久化,扫码一次最省事。`}
|
||||||
|
>
|
||||||
|
{/* QR first: it is the one that keeps working unattended, since the
|
||||||
|
scan lands in the very browser profile the monitor runs reuse. */}
|
||||||
|
<QrLoginPanel />
|
||||||
<CookiePanel />
|
<CookiePanel />
|
||||||
|
|
||||||
{/* Not a duplicate login flow: the crawler has no login-only mode, so a
|
|
||||||
QR login is a side effect of a real crawl -- which is exactly what
|
|
||||||
the 采集 page already does. This preselects it rather than
|
|
||||||
reimplementing it. */}
|
|
||||||
<div className="pt-3 mt-1 border-t border-cyber-border-subtle space-y-2">
|
<div className="pt-3 mt-1 border-t border-cyber-border-subtle space-y-2">
|
||||||
<p className="text-[10px] font-mono text-cyber-text-muted leading-relaxed">
|
<p className="text-[10px] font-mono text-cyber-text-muted leading-relaxed">
|
||||||
Cookie 不好使时,可以走一次**扫码登录**:二维码会显示在「采集」页的终端里。
|
在本机桌面运行时,也可以走「采集」页的扫码:二维码会打印在终端里。
|
||||||
扫码成功后浏览器 profile 会被更新,Cookie 的可靠性也会显著提升。
|
服务器没有显示器,那条路走不通,用上面的面板。
|
||||||
</p>
|
</p>
|
||||||
<Button
|
<Button
|
||||||
variant="outline"
|
variant="outline"
|
||||||
@@ -89,7 +92,7 @@ export function SettingsView({ onNavigate }: { onNavigate?: (view: AppView) => v
|
|||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
<QrCode className="w-3 h-3 mr-1" />
|
<QrCode className="w-3 h-3 mr-1" />
|
||||||
去扫码登录
|
去采集页扫码
|
||||||
</Button>
|
</Button>
|
||||||
</div>
|
</div>
|
||||||
</Section>
|
</Section>
|
||||||
|
|||||||
@@ -204,6 +204,46 @@ export function useClearCookie() {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- QR login (over CDP) ---------------------------------------------------
|
||||||
|
|
||||||
|
/** Polls only while a code is on screen; an idle session has nothing to watch. */
|
||||||
|
export function useQrLoginStatus(polling: boolean) {
|
||||||
|
return useQuery({
|
||||||
|
queryKey: ['monitorQrLogin'],
|
||||||
|
queryFn: async () => (await monitorApi.getQrLogin()).data,
|
||||||
|
refetchInterval: polling ? 2000 : false,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
export function useStartQrLogin() {
|
||||||
|
const queryClient = useQueryClient()
|
||||||
|
const platform = usePlatformParam()
|
||||||
|
return useMutation({
|
||||||
|
mutationFn: () => monitorApi.startQrLogin(platform),
|
||||||
|
onSuccess: (response) => {
|
||||||
|
queryClient.setQueryData(['monitorQrLogin'], response.data)
|
||||||
|
},
|
||||||
|
onError: (error: Error) => {
|
||||||
|
// The backend's reason -- "Chrome is not reachable on ...", "no QR on the
|
||||||
|
// page" -- is the entire value here, and axios would flatten it to
|
||||||
|
// "Request failed with status code 502".
|
||||||
|
const detail = (error as { response?: { data?: { detail?: string } } })?.response
|
||||||
|
?.data?.detail
|
||||||
|
toast.error(`获取二维码失败:${detail ?? error.message}`)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
export function useCancelQrLogin() {
|
||||||
|
const queryClient = useQueryClient()
|
||||||
|
return useMutation({
|
||||||
|
mutationFn: () => monitorApi.cancelQrLogin(),
|
||||||
|
onSuccess: (response) => {
|
||||||
|
queryClient.setQueryData(['monitorQrLogin'], response.data)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// --- Settings -------------------------------------------------------------
|
// --- Settings -------------------------------------------------------------
|
||||||
|
|
||||||
export function useSettings() {
|
export function useSettings() {
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import type {
|
|||||||
MonitorRun,
|
MonitorRun,
|
||||||
MonitorTask,
|
MonitorTask,
|
||||||
PlatformCapability,
|
PlatformCapability,
|
||||||
|
QrLoginState,
|
||||||
ReportResult,
|
ReportResult,
|
||||||
SettingsResponse,
|
SettingsResponse,
|
||||||
TaskCreatePayload,
|
TaskCreatePayload,
|
||||||
@@ -256,6 +257,13 @@ export const monitorApi = {
|
|||||||
return api.get<ReportResult>(`/monitor/report?${params.toString()}`)
|
return api.get<ReportResult>(`/monitor/report?${params.toString()}`)
|
||||||
},
|
},
|
||||||
|
|
||||||
|
// QR login over CDP. Reads the code out of the browser the crawler attaches
|
||||||
|
// to, so the operator can scan even when the host has no display.
|
||||||
|
startQrLogin: (platform?: string) =>
|
||||||
|
api.post<QrLoginState>('/monitor/login/qr', null, { params: { platform } }),
|
||||||
|
getQrLogin: () => api.get<QrLoginState>('/monitor/login/qr'),
|
||||||
|
cancelQrLogin: () => api.delete<QrLoginState>('/monitor/login/qr'),
|
||||||
|
|
||||||
getWebhook: () => api.get<WebhookStatus>('/monitor/webhook'),
|
getWebhook: () => api.get<WebhookStatus>('/monitor/webhook'),
|
||||||
setWebhook: (url: string) => api.post('/monitor/webhook', { url }),
|
setWebhook: (url: string) => api.post('/monitor/webhook', { url }),
|
||||||
clearWebhook: () => api.delete('/monitor/webhook'),
|
clearWebhook: () => api.delete('/monitor/webhook'),
|
||||||
|
|||||||
@@ -161,6 +161,26 @@ export interface CookieStatus {
|
|||||||
last_ok_at: number | null
|
last_ok_at: number | null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type QrLoginStatus = 'idle' | 'waiting' | 'success' | 'expired' | 'error'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A QR login driven over CDP, for servers with no display to show one on.
|
||||||
|
*
|
||||||
|
* `success` also covers "the browser was already signed in" — the scan and the
|
||||||
|
* existing session both just mean the profile the crawler attaches to is usable.
|
||||||
|
*/
|
||||||
|
export interface QrLoginState {
|
||||||
|
status: QrLoginStatus
|
||||||
|
platform: string | null
|
||||||
|
/** `data:image/...;base64,...`. Empty unless status is `waiting`. */
|
||||||
|
image: string
|
||||||
|
message: string
|
||||||
|
/** Seconds since the code was fetched. */
|
||||||
|
elapsed: number
|
||||||
|
/** Seconds left before the code is written off. */
|
||||||
|
expires_in: number
|
||||||
|
}
|
||||||
|
|
||||||
export interface MonitorOverview {
|
export interface MonitorOverview {
|
||||||
tasks: number
|
tasks: number
|
||||||
enabled_tasks: number
|
enabled_tasks: number
|
||||||
|
|||||||
Reference in New Issue
Block a user