Files
MediaCrawler/api/routers/monitor.py
T
butubb de9ff58371
Deploy VitePress site to Pages / build (push) Canceled after 0s
Deploy VitePress site to Pages / Deploy (push) Canceled after 0s
feat(monitor): 扫码同时存一份 Cookie,并把登录判定换成权威判据
用户问「现在是不是扫码就能自动获取 cookie」—— 不是,而且这正是那个面板显得没用的根源:
它只干了半件事,扫码**只写浏览器 profile,完全不提取 cookie**(qrlogin.py 里连一行
取 cookie 的代码都没有)。于是:
- CDP 开着时任务能用(复用 profile),但 Cookie 面板始终显示「未配置」
- CDP 一关,任务立刻断,因为库里那份 cookie 从来没被填过

现在扫码把两件事一起做了:写 profile(CDP 用)+ 存一份到库(Cookie 注入用)。
两种机制同时填上,开关怎么切都不断。cookie 只在内存里从 qrlogin 传到路由,不进响应体。

同时修掉一个同类 bug:监控侧的登录判定还在用页面里的 window.__INITIAL_STATE__,
而那是**页面加载那一刻的快照** —— 浏览器本来就登录着时它是对的,但扫码是加载之后
才登录的,快照不会翻转,表现为「扫了码却一直停在二维码上」。运营模块踩过同一个坑,
当时只修了那一处。现在两边统一为:拿 cookie 问后台接口「我是谁」。顺带不再需要页面导航,
检测变轻了。

前端:已登录时按钮原先被我藏起来了,面板于是变成一块只能看、不能操作的区域 ——
用户的原话是「没用」。现在两种状态都给按钮,含义不同:未登录=取二维码,
已登录=把当前登录态同步成 Cookie。

测试:tests/test_qrlogin.py 重写(stub 从页面探针换成后台接口),新增「成功会话必须
交出 cookie」「只能取一次」两例。
2026-10-09 13:46:48 +08:00

599 lines
22 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# -*- coding: utf-8 -*-
# Copyright (c) 2025 [email protected]
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/routers/monitor.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
# 声明:本代码仅供学习和研究目的使用。使用者应遵守以下原则:
# 1. 不得用于任何商业用途。
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
# 3. 不得进行大规模爬取或对平台造成运营干扰。
# 4. 应合理控制请求频率,避免给目标平台带来不必要的负担。
# 5. 不得用于任何非法或不当的用途。
#
# 详细许可条款请参阅项目根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
"""HTTP API for scheduled monitoring tasks."""
from datetime import date, timedelta
from typing import Any, Dict, List, Optional
from fastapi import APIRouter, HTTPException, Query, Response
from fastapi.responses import FileResponse
from ..monitor import covers, notify, qrlogin, report, service
from ..monitor.db import get_session
from ..monitor.platforms import PLATFORM_XHS
from ..monitor.settings import (
cookie_key,
delete_setting,
get_cookie_status,
get_setting,
set_cookie,
set_setting,
)
from ..monitor.models import SETTING_WECOM_WEBHOOK, MonitorTask
from ..schemas.monitor import (
CookiePayload,
MonitorTaskCreate,
MonitorTaskUpdate,
WebhookPayload,
WebhookTestPayload,
)
router = APIRouter(prefix="/monitor", tags=["monitor"])
@router.get("/overview")
async def get_overview(platform: Optional[str] = None):
"""Headline numbers for the dashboard tiles, scoped to one platform."""
async with get_session() as session:
return await service.overview(session, platform)
# ---------------------------------------------------------------------------
# Tasks
# ---------------------------------------------------------------------------
@router.get("/tasks")
async def list_tasks(platform: Optional[str] = None):
async with get_session() as session:
return {"tasks": await service.list_tasks(session, platform)}
@router.post("/tasks", status_code=201)
async def create_task(payload: MonitorTaskCreate):
async with get_session() as session:
try:
task = await service.create_task(session, payload.model_dump())
except service.TargetParseError as exc:
raise HTTPException(status_code=400, detail=str(exc))
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc))
return {"id": task.id, "message": "Monitoring task created"}
@router.patch("/tasks/{task_id}")
async def update_task(task_id: int, payload: MonitorTaskUpdate):
async with get_session() as session:
try:
await service.update_task(session, task_id, payload.model_dump(exclude_unset=True))
except service.TargetParseError as exc:
raise HTTPException(status_code=400, detail=str(exc))
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc))
return {"message": "Monitoring task updated"}
@router.delete("/tasks/{task_id}")
async def delete_task(task_id: int):
async with get_session() as session:
try:
await service.delete_task(session, task_id)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc))
return {"message": "Monitoring task deleted"}
@router.post("/tasks/{task_id}/run")
async def run_task_now(task_id: int):
"""Queue a run immediately and return; the crawl itself takes minutes."""
async with get_session() as session:
task = await session.get(MonitorTask, task_id)
if task is None:
raise HTTPException(status_code=404, detail=f"Task {task_id} not found")
if not any(target.enabled for target in task.targets):
raise HTTPException(status_code=400, detail="Task has no enabled targets")
service.trigger_manual_run(task_id)
return {"message": "Run queued"}
@router.get("/tasks/{task_id}/runs")
async def list_runs(task_id: int, limit: int = Query(default=50, ge=1, le=500)):
async with get_session() as session:
return {"runs": await service.list_runs(session, task_id, limit=limit)}
# ---------------------------------------------------------------------------
# Collected data
# ---------------------------------------------------------------------------
@router.get("/notes")
async def list_notes(
task_id: Optional[int] = None,
only_new: bool = False,
limit: int = Query(default=200, ge=1, le=2000),
platform: Optional[str] = None,
):
async with get_session() as session:
return {
"notes": await service.list_notes(session, task_id, only_new, limit, platform)
}
@router.get("/notes/{note_id}/series")
async def note_series(note_id: str, task_id: Optional[int] = None):
"""Metric time series for a single note."""
async with get_session() as session:
return {"series": await service.note_series(session, note_id, task_id)}
@router.get("/comments")
async def list_comments(
task_id: Optional[int] = None,
note_id: Optional[str] = None,
group_by: Optional[str] = Query(
default=None, description="传 note 则按作品分组返回,便于阅读"
),
limit: int = Query(default=200, ge=1, le=2000),
platform: Optional[str] = None,
):
"""Comments, each carrying the work it belongs to.
``note_id`` filters to one work; ``group_by=note`` returns them bucketed per
work instead of as a flat stream.
"""
async with get_session() as session:
comments = await service.list_comments(session, task_id, note_id, limit, platform)
if group_by != "note":
return {"comments": comments, "total": len(comments)}
buckets: Dict[str, Dict[str, Any]] = {}
for comment in comments:
bucket = buckets.setdefault(
comment["note_id"],
{
"note_id": comment["note_id"],
"note_title": comment["note_title"],
"note_cover": comment["note_cover"],
"note_url": comment["note_url"],
"comments": [],
},
)
bucket["comments"].append(comment)
ordered = sorted(
buckets.values(),
key=lambda group: group["comments"][0]["first_seen_at"],
reverse=True,
)
return {"groups": ordered, "total": len(comments)}
@router.get("/comment-notes")
async def list_comment_notes(task_id: Optional[int] = None, platform: Optional[str] = None):
"""Works that have comments, newest first, with counts.
Feeds the comment filter dropdown so the operator can pick by title.
"""
async with get_session() as session:
return {"notes": await service.comment_note_groups(session, task_id, platform)}
@router.get("/events")
async def list_events(
task_id: Optional[int] = None,
type: Optional[str] = None,
since_id: Optional[int] = None,
limit: int = Query(default=200, ge=1, le=2000),
platform: Optional[str] = None,
):
async with get_session() as session:
events = await service.list_events(session, task_id, type, since_id, limit, platform)
return {"events": events, "latest_id": events[0]["id"] if events else since_id}
@router.post("/events/read")
async def mark_events_read(task_id: Optional[int] = None):
async with get_session() as session:
count = await service.mark_events_read(session, task_id)
return {"marked": count}
# ---------------------------------------------------------------------------
# Cookie / login health
# ---------------------------------------------------------------------------
@router.get("/cookie")
async def get_cookie_endpoint(platform: str = Query(default=PLATFORM_XHS)):
"""Cookie health only -- deliberately never returns the cookie value.
``platform`` defaults to Xiaohongshu so existing callers keep working; the
key it reads is the namespaced one.
"""
async with get_session() as session:
return await get_cookie_status(session, platform)
@router.post("/cookie")
async def set_cookie_endpoint(payload: CookiePayload, platform: str = Query(default=PLATFORM_XHS)):
async with get_session() as session:
await set_cookie(session, payload.cookie.strip(), platform)
return {"message": "Cookie saved"}
@router.delete("/cookie")
async def clear_cookie_endpoint(platform: str = Query(default=PLATFORM_XHS)):
async with get_session() as session:
await delete_setting(session, cookie_key(platform))
return {"message": "Cookie cleared"}
# ---------------------------------------------------------------------------
# QR login
# ---------------------------------------------------------------------------
# These drive the browser already listening on the CDP debug port, which is the
# same browser -- and therefore the same profile -- that monitor runs attach to.
# Scanning once is what makes later unattended runs logged in.
@router.get("/covers/{note_id}")
async def get_cover(note_id: str):
"""作品封面,从本地缓存读。
**为什么不让前端直连图床**:图床地址是带签名、会过期的 —— 实测隔天即 403,
而且带不带 Referer 都一样,所以那是过期而不是防盗链。本地那份与签名无关。
这个路由是带鉴权的(整条 monitor 路由都挂了 require_auth),所以封面不会被
匿名读走;前端用同源的 <img> 请求会自动带上会话 cookie。
"""
path = covers.find_cached(note_id)
if path is None:
raise HTTPException(status_code=404, detail="封面未缓存")
media_types = {
".jpg": "image/jpeg",
".png": "image/png",
".webp": "image/webp",
".gif": "image/gif",
".heic": "image/heic",
}
return FileResponse(
path,
media_type=media_types.get(path.suffix.lower(), "application/octet-stream"),
# 本地文件不会变(note_id 唯一),让浏览器自己缓存,省掉重复请求。
headers={"Cache-Control": "private, max-age=86400"},
)
@router.post("/login/qr")
async def start_qr_login(platform: str = Query(default=PLATFORM_XHS)):
"""Open the login page in the CDP browser and return its QR code.
A server deployment has no display (Chrome sits under Xvfb), so the code is
surfaced here for the operator to scan instead of in a desktop window that
does not exist.
"""
try:
return await qrlogin.start(platform)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc))
except RuntimeError as exc:
raise HTTPException(status_code=502, detail=str(exc))
@router.get("/login/qr")
async def get_qr_login():
"""Poll the live session: waiting -> success / expired / error.
扫码成功时**把 cookie 一并存进库**。扫码本来只写浏览器 profile,那只够 CDP 模式用;
存一份之后,CDP 关掉、任务改用 --cookies_file 注入也照样能跑 —— 两种机制同时填上,
开关怎么切都不会断。
"""
snapshot = await qrlogin.status()
if snapshot["status"] == qrlogin.STATUS_SUCCESS:
cookie = await qrlogin.take_cookie()
if cookie:
async with get_session() as session:
await set_cookie(session, cookie)
snapshot["cookie_saved"] = True
snapshot["message"] = f"{snapshot['message']};登录态已同时存入 Cookie"
return snapshot
@router.delete("/login/qr")
async def cancel_qr_login():
"""Drop our tab and stop polling."""
return await qrlogin.cancel()
@router.get("/login/state")
async def get_login_state(force: bool = Query(default=False)):
"""Ask the browser itself whether it is signed in.
Deliberately separate from the QR session above. That session is in-memory and
dies with the process -- a redeploy is enough -- so "am I logged in?" must not
hinge on it, or a successful scan looks like nothing happened.
``force`` reloads the page first, for when the login may have lapsed somewhere
else and the page's copy of the state is stale.
"""
return await qrlogin.check_login_state(force=force)
# ---------------------------------------------------------------------------
# Report
# ---------------------------------------------------------------------------
async def _resolve_scope(
session, task_ids: Optional[List[int]], platform: Optional[str]
) -> Optional[List[int]]:
"""Combine an explicit task selection with an optional platform filter.
``None`` means "no restriction"; an explicit list is intersected with the
platform's tasks so a stale selection cannot leak another platform's data
into a scoped report.
"""
if platform is None:
return task_ids
platform_ids = set(await service.platform_task_ids(session, platform))
if task_ids is None:
return list(platform_ids)
return [task for task in task_ids if task in platform_ids]
@router.get("/export")
async def export_data(
kind: str = Query(..., description="notes | comments | report"),
task_id: Optional[List[int]] = Query(default=None),
note_id: Optional[str] = None,
start_date: Optional[str] = Query(default=None, description="YYYY-MM-DD,report 用"),
end_date: Optional[str] = Query(default=None, description="YYYY-MM-DD,report 用"),
days: int = Query(default=7, ge=1, le=365),
file_format: str = Query(default="csv", alias="format", description="csv | xlsx"),
platform: Optional[str] = None,
):
"""Download collected data as CSV or Excel.
Reached by the browser as a navigation (``window.open``), which cannot carry
an Authorization header -- this is one of the reasons the session lives in a
cookie.
"""
if kind not in ("notes", "comments", "report"):
raise HTTPException(status_code=400, detail="kind 必须是 notes / comments / report")
if file_format not in ("csv", "xlsx"):
raise HTTPException(status_code=400, detail="format 必须是 csv 或 xlsx")
async with get_session() as session:
scoped = await _resolve_scope(session, task_id, platform)
if kind == "notes":
single_task = scoped[0] if scoped and len(scoped) == 1 else None
rows = await service.list_notes(session, single_task, False, 5000, platform)
elif kind == "comments":
single_task = scoped[0] if scoped and len(scoped) == 1 else None
rows = await service.list_comments(session, single_task, note_id, 5000, platform)
else:
try:
end_day = date.fromisoformat(end_date) if end_date else date.today()
start_day = (
date.fromisoformat(start_date)
if start_date
else end_day - timedelta(days=days - 1)
)
except ValueError:
raise HTTPException(status_code=400, detail="日期格式应为 YYYY-MM-DD")
# Not `report = ...`: that would make `report` a local name for the
# whole function and shadow the module import on this very line.
report_data = await report.build_report(session, scoped, start_day, end_day)
rows = report_data["rows"]
if not rows:
raise HTTPException(status_code=404, detail="该范围内没有数据可导出")
columns = _export_columns(kind)
stamp = date.today().isoformat()
# ASCII on purpose: a non-ASCII filename needs RFC 5987 encoding in
# Content-Disposition, and the plain `filename="..."` form used below would
# mangle it.
filename = f"export_{kind}_{stamp}.{file_format}"
if file_format == "xlsx":
payload = _to_xlsx(rows, columns, kind)
media_type = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
else:
payload = _to_csv(rows, columns)
media_type = "text/csv; charset=utf-8"
return Response(
content=payload,
media_type=media_type,
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
def _export_columns(kind: str) -> List[tuple[str, str]]:
"""(key, header) pairs per export kind."""
if kind == "notes":
return [
("note_id", "作品ID"),
("title", "标题"),
("note_url", "链接"),
("liked_count", "点赞"),
("comment_count", "评论"),
("collected_count", "收藏"),
("share_count", "分享"),
("liked_count_delta", "点赞增量"),
("comment_count_delta", "评论增量"),
("first_seen_at", "首次发现"),
("last_seen_at", "最近采集"),
]
if kind == "comments":
return [
("note_title", "所属作品"),
("note_id", "作品ID"),
("comment_id", "评论ID"),
("content", "内容"),
("nickname", "昵称"),
("like_count", "点赞"),
("sub_comment_count", "子评论数"),
("create_time", "发布时间"),
("first_seen_at", "首次发现"),
]
return [
("date", "日期"),
("new_notes", "新增作品"),
("new_comments", "新增评论"),
("liked_count_delta", "点赞增量"),
("comment_count_delta", "评论增量"),
("collected_count_delta", "收藏增量"),
("share_count_delta", "分享增量"),
]
def _cell(value: Any) -> Any:
if value is None:
return ""
if isinstance(value, (list, dict)):
return ", ".join(str(v) for v in value) if isinstance(value, list) else str(value)
return value
def _to_csv(rows: List[Dict[str, Any]], columns: List[tuple[str, str]]) -> bytes:
import csv
import io
buffer = io.StringIO()
writer = csv.writer(buffer)
writer.writerow([header for _, header in columns])
for row in rows:
writer.writerow([_cell(row.get(key)) for key, _ in columns])
# utf-8-sig: without the BOM Excel opens Chinese CSV as mojibake, which is
# the single most common complaint about CSV exports here.
return buffer.getvalue().encode("utf-8-sig")
def _to_xlsx(rows: List[Dict[str, Any]], columns: List[tuple[str, str]], sheet: str) -> bytes:
import io
from openpyxl import Workbook
workbook = Workbook()
worksheet = workbook.active
worksheet.title = {"notes": "作品", "comments": "评论"}.get(sheet, "报表")
worksheet.append([header for _, header in columns])
for row in rows:
worksheet.append([_cell(row.get(key)) for key, _ in columns])
output = io.BytesIO()
workbook.save(output)
return output.getvalue()
@router.get("/report")
async def get_report(
task_id: Optional[List[int]] = Query(
default=None, description="Repeat to include several tasks; omit for all"
),
start_date: Optional[str] = Query(default=None, description="YYYY-MM-DD"),
end_date: Optional[str] = Query(default=None, description="YYYY-MM-DD"),
days: int = Query(default=7, ge=1, le=365, description="Window used when dates are omitted"),
platform: Optional[str] = None,
):
"""Daily new-content counts and interaction deltas for the selected tasks."""
try:
end_day = date.fromisoformat(end_date) if end_date else date.today()
start_day = date.fromisoformat(start_date) if start_date else end_day - timedelta(days=days - 1)
except ValueError:
raise HTTPException(status_code=400, detail="日期格式应为 YYYY-MM-DD")
if start_day > end_day:
raise HTTPException(status_code=400, detail="开始日期不能晚于结束日期")
async with get_session() as session:
scoped = await _resolve_scope(session, task_id, platform)
return await report.build_report(session, scoped, start_day, end_day)
# ---------------------------------------------------------------------------
# WeCom webhook
# ---------------------------------------------------------------------------
def _mask_webhook(url: str) -> str:
"""Show enough of the URL to recognise it, without exposing the robot key."""
if not url:
return ""
key_marker = "key="
index = url.find(key_marker)
if index == -1:
return url[:12] + "..." if len(url) > 12 else url
prefix = url[: index + len(key_marker)]
key = url[index + len(key_marker) :]
if len(key) <= 8:
return prefix + "*" * len(key)
return f"{prefix}{key[:4]}...{key[-4:]}"
@router.get("/webhook")
async def get_webhook():
async with get_session() as session:
url = (await get_setting(session, SETTING_WECOM_WEBHOOK)) or ""
return {"configured": bool(url), "masked": _mask_webhook(url)}
@router.post("/webhook")
async def set_webhook(payload: WebhookPayload):
url = payload.url.strip()
if url and "qyapi.weixin.qq.com" not in url:
# Catches the common mistake of pasting a group-chat invite or the app
# URL instead of the robot webhook.
raise HTTPException(
status_code=400,
detail="这不像企业微信机器人 Webhook 地址(应包含 qyapi.weixin.qq.com)",
)
async with get_session() as session:
await set_setting(session, SETTING_WECOM_WEBHOOK, url)
return {"message": "Webhook 已保存" if url else "Webhook 已清空", "configured": bool(url)}
@router.delete("/webhook")
async def clear_webhook():
async with get_session() as session:
await delete_setting(session, SETTING_WECOM_WEBHOOK)
return {"message": "Webhook 已删除"}
@router.post("/webhook/test")
async def test_webhook(payload: WebhookTestPayload):
"""Send a test message so the user can verify the robot works before relying on it."""
async with get_session() as session:
url = payload.url.strip() if payload.url else await notify.get_webhook_url(session)
ok, detail = await notify.send_wecom(
url, "**综合采集平台 通知测试**\n> 如果你看到这条消息,说明 Webhook 配置成功。"
)
if not ok:
raise HTTPException(status_code=400, detail=detail)
return {"message": detail}