feat: 监控面板 / 登录鉴权 / 多平台切换 / MySQL
Deploy VitePress site to Pages / build (push) Canceled after 0s
Deploy VitePress site to Pages / Deploy (push) Canceled after 0s

在上游 MediaCrawler 之上新增一层:

- 监控层 api/monitor/ —— 多博主/多笔记的定时采集、指标快照差分、报表、
  企业微信通知。每轮采集写入独立目录,差分才成立。
- WebUI 登录鉴权 api/auth.py —— PBKDF2 口令 + 服务端会话,/api 全接口防护。
  WebSocket 单独加依赖:BaseHTTPMiddleware 对 ws 作用域直接放行,覆盖不到。
- 全局平台切换 + 能力矩阵 —— 如实区分「爬虫模块支持」与「监控层已接线」,
  未接通的平台直接拒绝建任务,而不是静默跑空。
- 监控库改用 MySQL 5.7(可回退 SQLite 供测试):逐表强制 utf8mb4
  (服务端与库默认都是 latin1),启动校验所连 schema 以防写错库,
  连接池 recycle + pre_ping 应对 MySQL 的 8 小时空闲断连。

修复上游缺陷:

- xhs/core.py: 主页抓取失败会跳掉整个博主,导致一条作品都抓不到,
  而那份资料只喂给一个空函数。改为尽力而为,失败不中断。
- xhs/login.py: cookie 登录只注入 web_session,冷启动签名会失败。
  新增 INJECT_ALL_COOKIES 开关(默认关闭,原有行为不变)。
- requirements.txt: 补上 websockets。它在上游 pyproject.toml 里有声明、
  这里漏了,导致 uvicorn 没有 WebSocket 能力,实时日志流从未工作。

改动过的上游文件清单及合并方式见 UPSTREAM.md。

测试:492 passed(另有 1 个既有的 Windows/gbk 上游测试失败,与本改动无关)
This commit is contained in:
2026-10-07 09:58:40 +08:00
parent 5d547f4586
commit 4e60524f37
88 changed files with 13224 additions and 436 deletions
+11 -1
View File
@@ -16,8 +16,18 @@
# 详细许可条款请参阅项目根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
from .auth import router as auth_router
from .crawler import router as crawler_router
from .data import router as data_router
from .monitor import router as monitor_router
from .settings import router as settings_router
from .websocket import router as websocket_router
__all__ = ["crawler_router", "data_router", "websocket_router"]
__all__ = [
"auth_router",
"crawler_router",
"data_router",
"monitor_router",
"settings_router",
"websocket_router",
]
+173
View File
@@ -0,0 +1,173 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2025 [email protected]
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/routers/auth.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
# 声明:本代码仅供学习和研究目的使用。使用者应遵守以下原则:
# 1. 不得用于任何商业用途。
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
# 3. 不得进行大规模爬取或对平台造成运营干扰。
# 4. 应合理控制请求频率,避免给目标平台带来不必要的负担。
# 5. 不得用于任何非法或不当的用途。
#
# 详细许可条款请参阅项目根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
"""Login / logout endpoints.
Deliberately exempt from ``require_auth``:
* ``/login`` -- it is the way in.
* ``/logout`` -- exempt so an already-expired session still gets a clean 200
and a cleared cookie instead of a confusing 401, which
would leave the browser holding a stale cookie.
"""
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
from ..auth import (
INVALID_CREDENTIALS,
SESSION_COOKIE_NAME,
check_password,
require_auth,
clear_failures,
client_key,
cookie_secure,
create_session,
purge_expired_sessions,
record_failure,
resolve_session,
retry_after_seconds,
revoke_all_sessions,
revoke_session,
set_password,
token_from_request,
)
from ..monitor.db import get_session
from ..schemas.auth import ChangePasswordPayload, LoginPayload
from tools.time_util import get_current_timestamp
router = APIRouter(prefix="/auth", tags=["auth"])
def _apply_session_cookie(response: Response, token: str, expires_at: int) -> None:
"""Attach the session cookie.
``secure`` is off by default because the panel is served over plain HTTP on
a LAN; setting it there means the browser silently discards the cookie and
the login page just loops with no error. ``SameSite=lax`` is also what
blocks cross-site POSTs, i.e. the CSRF defence for the write endpoints.
"""
max_age = max((expires_at - get_current_timestamp()) // 1000, 60)
response.set_cookie(
key=SESSION_COOKIE_NAME,
value=token,
max_age=max_age,
httponly=True,
secure=cookie_secure(),
samesite="lax",
path="/",
)
@router.post("/login")
async def login(payload: LoginPayload, request: Request, response: Response):
key = client_key(request)
wait = await retry_after_seconds(key)
if wait:
raise HTTPException(
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
detail=f"尝试过于频繁,请 {wait} 秒后再试",
headers={"Retry-After": str(wait)},
)
async with get_session() as session:
valid = await check_password(session, payload.password)
token = ""
expires_at = 0
if valid:
await purge_expired_sessions(session)
token, expires_at = await create_session(session)
if not valid:
await record_failure(key)
# One generic message regardless of whether the password was wrong,
# empty, or simply not set yet -- no oracle.
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED, detail=INVALID_CREDENTIALS
)
await clear_failures(key)
_apply_session_cookie(response, token, expires_at)
return {"expires_at": expires_at}
@router.post("/logout")
async def logout(request: Request, response: Response):
token = token_from_request(request)
if token:
async with get_session() as session:
await revoke_session(session, token)
response.delete_cookie(SESSION_COOKIE_NAME, path="/")
return {"message": "已退出登录"}
@router.get("/me")
async def me(request: Request):
"""Identity probe. The SPA treats a 401 here as "show the login page".
Does its own resolution rather than using ``require_auth`` so it can also
report the expiry.
"""
token = token_from_request(request)
if not token:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED, detail=INVALID_CREDENTIALS
)
async with get_session() as session:
row = await resolve_session(session, token)
if row is None:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED, detail=INVALID_CREDENTIALS
)
return {"authenticated": True, "expires_at": row.expires_at}
# Auth as a route dependency, not only inside the handler: FastAPI validates the
# request body before the endpoint body runs, so an unauthenticated caller would
# otherwise get a 422 that confirms the endpoint and its schema exist.
@router.post("/password", dependencies=[Depends(require_auth)])
async def change_password(
payload: ChangePasswordPayload, request: Request, response: Response
):
"""Change the password and log every device out.
Revoking all sessions is the point: a password change is usually a response
to suspicion, and leaving other sessions alive would defeat it.
"""
token = token_from_request(request)
async with get_session() as session:
current = await resolve_session(session, token)
if current is None:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED, detail=INVALID_CREDENTIALS
)
if not await check_password(session, payload.current):
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED, detail="当前密码不正确"
)
await set_password(session, payload.new)
await revoke_all_sessions(session)
# Issue a fresh session so the caller is not bounced mid-use.
new_token, expires_at = await create_session(session)
_apply_session_cookie(response, new_token, expires_at)
return {"message": "密码已更新,其他设备的登录已全部失效", "expires_at": expires_at}
+25 -2
View File
@@ -18,7 +18,9 @@
from fastapi import APIRouter, HTTPException
from ..schemas import CrawlerStartRequest, CrawlerStatusResponse
from ..monitor.db import get_session
from ..monitor.settings import get_cookie
from ..schemas import CrawlerStartRequest, CrawlerStatusResponse, LoginTypeEnum
from ..services import crawler_manager
router = APIRouter(prefix="/crawler", tags=["crawler"])
@@ -26,7 +28,28 @@ router = APIRouter(prefix="/crawler", tags=["crawler"])
@router.post("/start")
async def start_crawler(request: CrawlerStartRequest):
"""Start crawler task"""
"""Start crawler task.
A cookie login with no cookie supplied falls back to the one stored for the
selected platform. The manual crawl and the monitor therefore share a single
credential; keeping a second paste field on the crawl page meant it went
stale and could silently disagree with the monitor's.
"""
if (
request.login_type == LoginTypeEnum.COOKIE
and not request.cookies
and not request.cookies_file
):
async with get_session() as session:
stored = await get_cookie(session, request.platform.value)
if not stored:
raise HTTPException(
status_code=400,
detail="该平台尚未保存 Cookie,请到「设置 → 登录态」配置,或改用扫码登录",
)
request.cookies = stored
success = await crawler_manager.start(request)
if not success:
# Handle concurrent/duplicate requests: if process is already running, return 400 instead of 500
+502
View File
@@ -0,0 +1,502 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2025 [email protected]
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/routers/monitor.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
# 声明:本代码仅供学习和研究目的使用。使用者应遵守以下原则:
# 1. 不得用于任何商业用途。
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
# 3. 不得进行大规模爬取或对平台造成运营干扰。
# 4. 应合理控制请求频率,避免给目标平台带来不必要的负担。
# 5. 不得用于任何非法或不当的用途。
#
# 详细许可条款请参阅项目根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
"""HTTP API for scheduled monitoring tasks."""
from datetime import date, timedelta
from typing import Any, Dict, List, Optional
from fastapi import APIRouter, HTTPException, Query, Response
from ..monitor import notify, report, service
from ..monitor.db import get_session
from ..monitor.platforms import PLATFORM_XHS
from ..monitor.settings import (
cookie_key,
delete_setting,
get_cookie_status,
get_setting,
set_cookie,
set_setting,
)
from ..monitor.models import SETTING_WECOM_WEBHOOK, MonitorTask
from ..schemas.monitor import (
CookiePayload,
MonitorTaskCreate,
MonitorTaskUpdate,
WebhookPayload,
WebhookTestPayload,
)
router = APIRouter(prefix="/monitor", tags=["monitor"])
@router.get("/overview")
async def get_overview(platform: Optional[str] = None):
"""Headline numbers for the dashboard tiles, scoped to one platform."""
async with get_session() as session:
return await service.overview(session, platform)
# ---------------------------------------------------------------------------
# Tasks
# ---------------------------------------------------------------------------
@router.get("/tasks")
async def list_tasks(platform: Optional[str] = None):
async with get_session() as session:
return {"tasks": await service.list_tasks(session, platform)}
@router.post("/tasks", status_code=201)
async def create_task(payload: MonitorTaskCreate):
async with get_session() as session:
try:
task = await service.create_task(session, payload.model_dump())
except service.TargetParseError as exc:
raise HTTPException(status_code=400, detail=str(exc))
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc))
return {"id": task.id, "message": "Monitoring task created"}
@router.patch("/tasks/{task_id}")
async def update_task(task_id: int, payload: MonitorTaskUpdate):
async with get_session() as session:
try:
await service.update_task(session, task_id, payload.model_dump(exclude_unset=True))
except service.TargetParseError as exc:
raise HTTPException(status_code=400, detail=str(exc))
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc))
return {"message": "Monitoring task updated"}
@router.delete("/tasks/{task_id}")
async def delete_task(task_id: int):
async with get_session() as session:
try:
await service.delete_task(session, task_id)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc))
return {"message": "Monitoring task deleted"}
@router.post("/tasks/{task_id}/run")
async def run_task_now(task_id: int):
"""Queue a run immediately and return; the crawl itself takes minutes."""
async with get_session() as session:
task = await session.get(MonitorTask, task_id)
if task is None:
raise HTTPException(status_code=404, detail=f"Task {task_id} not found")
if not any(target.enabled for target in task.targets):
raise HTTPException(status_code=400, detail="Task has no enabled targets")
service.trigger_manual_run(task_id)
return {"message": "Run queued"}
@router.get("/tasks/{task_id}/runs")
async def list_runs(task_id: int, limit: int = Query(default=50, ge=1, le=500)):
async with get_session() as session:
return {"runs": await service.list_runs(session, task_id, limit=limit)}
# ---------------------------------------------------------------------------
# Collected data
# ---------------------------------------------------------------------------
@router.get("/notes")
async def list_notes(
task_id: Optional[int] = None,
only_new: bool = False,
limit: int = Query(default=200, ge=1, le=2000),
platform: Optional[str] = None,
):
async with get_session() as session:
return {
"notes": await service.list_notes(session, task_id, only_new, limit, platform)
}
@router.get("/notes/{note_id}/series")
async def note_series(note_id: str, task_id: Optional[int] = None):
"""Metric time series for a single note."""
async with get_session() as session:
return {"series": await service.note_series(session, note_id, task_id)}
@router.get("/comments")
async def list_comments(
task_id: Optional[int] = None,
note_id: Optional[str] = None,
group_by: Optional[str] = Query(
default=None, description="传 note 则按作品分组返回,便于阅读"
),
limit: int = Query(default=200, ge=1, le=2000),
platform: Optional[str] = None,
):
"""Comments, each carrying the work it belongs to.
``note_id`` filters to one work; ``group_by=note`` returns them bucketed per
work instead of as a flat stream.
"""
async with get_session() as session:
comments = await service.list_comments(session, task_id, note_id, limit, platform)
if group_by != "note":
return {"comments": comments, "total": len(comments)}
buckets: Dict[str, Dict[str, Any]] = {}
for comment in comments:
bucket = buckets.setdefault(
comment["note_id"],
{
"note_id": comment["note_id"],
"note_title": comment["note_title"],
"note_cover": comment["note_cover"],
"note_url": comment["note_url"],
"comments": [],
},
)
bucket["comments"].append(comment)
ordered = sorted(
buckets.values(),
key=lambda group: group["comments"][0]["first_seen_at"],
reverse=True,
)
return {"groups": ordered, "total": len(comments)}
@router.get("/comment-notes")
async def list_comment_notes(task_id: Optional[int] = None, platform: Optional[str] = None):
"""Works that have comments, newest first, with counts.
Feeds the comment filter dropdown so the operator can pick by title.
"""
async with get_session() as session:
return {"notes": await service.comment_note_groups(session, task_id, platform)}
@router.get("/events")
async def list_events(
task_id: Optional[int] = None,
type: Optional[str] = None,
since_id: Optional[int] = None,
limit: int = Query(default=200, ge=1, le=2000),
platform: Optional[str] = None,
):
async with get_session() as session:
events = await service.list_events(session, task_id, type, since_id, limit, platform)
return {"events": events, "latest_id": events[0]["id"] if events else since_id}
@router.post("/events/read")
async def mark_events_read(task_id: Optional[int] = None):
async with get_session() as session:
count = await service.mark_events_read(session, task_id)
return {"marked": count}
# ---------------------------------------------------------------------------
# Cookie / login health
# ---------------------------------------------------------------------------
@router.get("/cookie")
async def get_cookie_endpoint(platform: str = Query(default=PLATFORM_XHS)):
"""Cookie health only -- deliberately never returns the cookie value.
``platform`` defaults to Xiaohongshu so existing callers keep working; the
key it reads is the namespaced one.
"""
async with get_session() as session:
return await get_cookie_status(session, platform)
@router.post("/cookie")
async def set_cookie_endpoint(payload: CookiePayload, platform: str = Query(default=PLATFORM_XHS)):
async with get_session() as session:
await set_cookie(session, payload.cookie.strip(), platform)
return {"message": "Cookie saved"}
@router.delete("/cookie")
async def clear_cookie_endpoint(platform: str = Query(default=PLATFORM_XHS)):
async with get_session() as session:
await delete_setting(session, cookie_key(platform))
return {"message": "Cookie cleared"}
# ---------------------------------------------------------------------------
# Report
# ---------------------------------------------------------------------------
async def _resolve_scope(
session, task_ids: Optional[List[int]], platform: Optional[str]
) -> Optional[List[int]]:
"""Combine an explicit task selection with an optional platform filter.
``None`` means "no restriction"; an explicit list is intersected with the
platform's tasks so a stale selection cannot leak another platform's data
into a scoped report.
"""
if platform is None:
return task_ids
platform_ids = set(await service.platform_task_ids(session, platform))
if task_ids is None:
return list(platform_ids)
return [task for task in task_ids if task in platform_ids]
@router.get("/export")
async def export_data(
kind: str = Query(..., description="notes | comments | report"),
task_id: Optional[List[int]] = Query(default=None),
note_id: Optional[str] = None,
start_date: Optional[str] = Query(default=None, description="YYYY-MM-DD,report 用"),
end_date: Optional[str] = Query(default=None, description="YYYY-MM-DD,report 用"),
days: int = Query(default=7, ge=1, le=365),
file_format: str = Query(default="csv", alias="format", description="csv | xlsx"),
platform: Optional[str] = None,
):
"""Download collected data as CSV or Excel.
Reached by the browser as a navigation (``window.open``), which cannot carry
an Authorization header -- this is one of the reasons the session lives in a
cookie.
"""
if kind not in ("notes", "comments", "report"):
raise HTTPException(status_code=400, detail="kind 必须是 notes / comments / report")
if file_format not in ("csv", "xlsx"):
raise HTTPException(status_code=400, detail="format 必须是 csv 或 xlsx")
async with get_session() as session:
scoped = await _resolve_scope(session, task_id, platform)
if kind == "notes":
single_task = scoped[0] if scoped and len(scoped) == 1 else None
rows = await service.list_notes(session, single_task, False, 5000, platform)
elif kind == "comments":
single_task = scoped[0] if scoped and len(scoped) == 1 else None
rows = await service.list_comments(session, single_task, note_id, 5000, platform)
else:
try:
end_day = date.fromisoformat(end_date) if end_date else date.today()
start_day = (
date.fromisoformat(start_date)
if start_date
else end_day - timedelta(days=days - 1)
)
except ValueError:
raise HTTPException(status_code=400, detail="日期格式应为 YYYY-MM-DD")
# Not `report = ...`: that would make `report` a local name for the
# whole function and shadow the module import on this very line.
report_data = await report.build_report(session, scoped, start_day, end_day)
rows = report_data["rows"]
if not rows:
raise HTTPException(status_code=404, detail="该范围内没有数据可导出")
columns = _export_columns(kind)
stamp = date.today().isoformat()
# ASCII on purpose: a non-ASCII filename needs RFC 5987 encoding in
# Content-Disposition, and the plain `filename="..."` form used below would
# mangle it.
filename = f"export_{kind}_{stamp}.{file_format}"
if file_format == "xlsx":
payload = _to_xlsx(rows, columns, kind)
media_type = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
else:
payload = _to_csv(rows, columns)
media_type = "text/csv; charset=utf-8"
return Response(
content=payload,
media_type=media_type,
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
def _export_columns(kind: str) -> List[tuple[str, str]]:
"""(key, header) pairs per export kind."""
if kind == "notes":
return [
("note_id", "作品ID"),
("title", "标题"),
("note_url", "链接"),
("liked_count", "点赞"),
("comment_count", "评论"),
("collected_count", "收藏"),
("share_count", "分享"),
("liked_count_delta", "点赞增量"),
("comment_count_delta", "评论增量"),
("first_seen_at", "首次发现"),
("last_seen_at", "最近采集"),
]
if kind == "comments":
return [
("note_title", "所属作品"),
("note_id", "作品ID"),
("comment_id", "评论ID"),
("content", "内容"),
("nickname", "昵称"),
("like_count", "点赞"),
("sub_comment_count", "子评论数"),
("create_time", "发布时间"),
("first_seen_at", "首次发现"),
]
return [
("date", "日期"),
("new_notes", "新增作品"),
("new_comments", "新增评论"),
("liked_count_delta", "点赞增量"),
("comment_count_delta", "评论增量"),
("collected_count_delta", "收藏增量"),
("share_count_delta", "分享增量"),
]
def _cell(value: Any) -> Any:
if value is None:
return ""
if isinstance(value, (list, dict)):
return ", ".join(str(v) for v in value) if isinstance(value, list) else str(value)
return value
def _to_csv(rows: List[Dict[str, Any]], columns: List[tuple[str, str]]) -> bytes:
import csv
import io
buffer = io.StringIO()
writer = csv.writer(buffer)
writer.writerow([header for _, header in columns])
for row in rows:
writer.writerow([_cell(row.get(key)) for key, _ in columns])
# utf-8-sig: without the BOM Excel opens Chinese CSV as mojibake, which is
# the single most common complaint about CSV exports here.
return buffer.getvalue().encode("utf-8-sig")
def _to_xlsx(rows: List[Dict[str, Any]], columns: List[tuple[str, str]], sheet: str) -> bytes:
import io
from openpyxl import Workbook
workbook = Workbook()
worksheet = workbook.active
worksheet.title = {"notes": "作品", "comments": "评论"}.get(sheet, "报表")
worksheet.append([header for _, header in columns])
for row in rows:
worksheet.append([_cell(row.get(key)) for key, _ in columns])
output = io.BytesIO()
workbook.save(output)
return output.getvalue()
@router.get("/report")
async def get_report(
task_id: Optional[List[int]] = Query(
default=None, description="Repeat to include several tasks; omit for all"
),
start_date: Optional[str] = Query(default=None, description="YYYY-MM-DD"),
end_date: Optional[str] = Query(default=None, description="YYYY-MM-DD"),
days: int = Query(default=7, ge=1, le=365, description="Window used when dates are omitted"),
platform: Optional[str] = None,
):
"""Daily new-content counts and interaction deltas for the selected tasks."""
try:
end_day = date.fromisoformat(end_date) if end_date else date.today()
start_day = date.fromisoformat(start_date) if start_date else end_day - timedelta(days=days - 1)
except ValueError:
raise HTTPException(status_code=400, detail="日期格式应为 YYYY-MM-DD")
if start_day > end_day:
raise HTTPException(status_code=400, detail="开始日期不能晚于结束日期")
async with get_session() as session:
scoped = await _resolve_scope(session, task_id, platform)
return await report.build_report(session, scoped, start_day, end_day)
# ---------------------------------------------------------------------------
# WeCom webhook
# ---------------------------------------------------------------------------
def _mask_webhook(url: str) -> str:
"""Show enough of the URL to recognise it, without exposing the robot key."""
if not url:
return ""
key_marker = "key="
index = url.find(key_marker)
if index == -1:
return url[:12] + "..." if len(url) > 12 else url
prefix = url[: index + len(key_marker)]
key = url[index + len(key_marker) :]
if len(key) <= 8:
return prefix + "*" * len(key)
return f"{prefix}{key[:4]}...{key[-4:]}"
@router.get("/webhook")
async def get_webhook():
async with get_session() as session:
url = (await get_setting(session, SETTING_WECOM_WEBHOOK)) or ""
return {"configured": bool(url), "masked": _mask_webhook(url)}
@router.post("/webhook")
async def set_webhook(payload: WebhookPayload):
url = payload.url.strip()
if url and "qyapi.weixin.qq.com" not in url:
# Catches the common mistake of pasting a group-chat invite or the app
# URL instead of the robot webhook.
raise HTTPException(
status_code=400,
detail="这不像企业微信机器人 Webhook 地址(应包含 qyapi.weixin.qq.com)",
)
async with get_session() as session:
await set_setting(session, SETTING_WECOM_WEBHOOK, url)
return {"message": "Webhook 已保存" if url else "Webhook 已清空", "configured": bool(url)}
@router.delete("/webhook")
async def clear_webhook():
async with get_session() as session:
await delete_setting(session, SETTING_WECOM_WEBHOOK)
return {"message": "Webhook 已删除"}
@router.post("/webhook/test")
async def test_webhook(payload: WebhookTestPayload):
"""Send a test message so the user can verify the robot works before relying on it."""
async with get_session() as session:
url = payload.url.strip() if payload.url else await notify.get_webhook_url(session)
ok, detail = await notify.send_wecom(
url, "**综合采集平台 通知测试**\n> 如果你看到这条消息,说明 Webhook 配置成功。"
)
if not ok:
raise HTTPException(status_code=400, detail=detail)
return {"message": detail}
+70
View File
@@ -0,0 +1,70 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2025 [email protected]
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/routers/settings.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
# 声明:本代码仅供学习和研究目的使用。使用者应遵守以下原则:
# 1. 不得用于任何商业用途。
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
# 3. 不得进行大规模爬取或对平台造成运营干扰。
# 4. 应合理控制请求频率,避免给目标平台带来不必要的负担。
# 5. 不得用于任何非法或不当的用途。
#
# 详细许可条款请参阅项目根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
"""Unified settings endpoint.
Consolidates what used to be scattered across the monitor router. The older
``/api/monitor/cookie`` and ``/api/monitor/webhook`` endpoints are deliberately
left in place -- they still work and removing them would be a breaking change for
no gain.
"""
from typing import Any, Dict
from fastapi import APIRouter, HTTPException, Query
from ..monitor import app_settings
from ..monitor.db import get_session
from ..monitor.platforms import PLATFORM_XHS
from ..schemas.settings import SettingsUpdatePayload
router = APIRouter(prefix="/settings", tags=["settings"])
@router.get("")
async def read_settings(platform: str = Query(default=PLATFORM_XHS)):
"""Settings for one platform, plus the system-wide ones.
Platform-scoped values are returned for the requested platform; system-scoped
values are the same regardless. Every spec carries its resolved ``key`` so the
UI can PUT changes straight back.
Sensitive values are returned as ``{present, length}`` only.
"""
async with get_session() as session:
return await app_settings.get_all(session, platform)
@router.put("")
async def write_settings(
payload: SettingsUpdatePayload, platform: str = Query(default=PLATFORM_XHS)
):
"""Partial update: only the keys present in the body are written.
A key belonging to a different platform is rejected rather than written
somewhere unexpected.
"""
values: Dict[str, Any] = payload.values()
async with get_session() as session:
try:
changed = await app_settings.update(session, values, platform)
except app_settings.SettingValidationError as exc:
raise HTTPException(status_code=400, detail=str(exc))
return {"message": f"已保存 {len(changed)} 项设置", "changed": changed}
+7 -3
View File
@@ -19,8 +19,9 @@
import asyncio
from typing import Set, Optional
from fastapi import APIRouter, WebSocket, WebSocketDisconnect
from fastapi import APIRouter, Depends, WebSocket, WebSocketDisconnect
from ..auth import require_ws_auth
from ..services import crawler_manager
router = APIRouter(tags=["websocket"])
@@ -86,7 +87,10 @@ def start_broadcaster():
_broadcaster_task = asyncio.create_task(log_broadcaster())
@router.websocket("/ws/logs")
# Websocket routes need their own auth dependency: BaseHTTPMiddleware returns
# early for any non-http scope, and HTTP router-level dependencies do not reach
# websocket routes. Without this the live crawl log stream would be wide open.
@router.websocket("/ws/logs", dependencies=[Depends(require_ws_auth)])
async def websocket_logs(websocket: WebSocket):
"""WebSocket log stream"""
print("[WS] New connection attempt")
@@ -134,7 +138,7 @@ async def websocket_logs(websocket: WebSocket):
print(f"[WS] Cleanup done, active connections: {len(manager.active_connections)}")
@router.websocket("/ws/status")
@router.websocket("/ws/status", dependencies=[Depends(require_ws_auth)])
async def websocket_status(websocket: WebSocket):
"""WebSocket status stream"""
await websocket.accept()