feat: 监控面板 / 登录鉴权 / 多平台切换 / MySQL
在上游 MediaCrawler 之上新增一层: - 监控层 api/monitor/ —— 多博主/多笔记的定时采集、指标快照差分、报表、 企业微信通知。每轮采集写入独立目录,差分才成立。 - WebUI 登录鉴权 api/auth.py —— PBKDF2 口令 + 服务端会话,/api 全接口防护。 WebSocket 单独加依赖:BaseHTTPMiddleware 对 ws 作用域直接放行,覆盖不到。 - 全局平台切换 + 能力矩阵 —— 如实区分「爬虫模块支持」与「监控层已接线」, 未接通的平台直接拒绝建任务,而不是静默跑空。 - 监控库改用 MySQL 5.7(可回退 SQLite 供测试):逐表强制 utf8mb4 (服务端与库默认都是 latin1),启动校验所连 schema 以防写错库, 连接池 recycle + pre_ping 应对 MySQL 的 8 小时空闲断连。 修复上游缺陷: - xhs/core.py: 主页抓取失败会跳掉整个博主,导致一条作品都抓不到, 而那份资料只喂给一个空函数。改为尽力而为,失败不中断。 - xhs/login.py: cookie 登录只注入 web_session,冷启动签名会失败。 新增 INJECT_ALL_COOKIES 开关(默认关闭,原有行为不变)。 - requirements.txt: 补上 websockets。它在上游 pyproject.toml 里有声明、 这里漏了,导致 uvicorn 没有 WebSocket 能力,实时日志流从未工作。 改动过的上游文件清单及合并方式见 UPSTREAM.md。 测试:492 passed(另有 1 个既有的 Windows/gbk 上游测试失败,与本改动无关)
This commit is contained in:
+11
-1
@@ -16,8 +16,18 @@
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
from .auth import router as auth_router
|
||||
from .crawler import router as crawler_router
|
||||
from .data import router as data_router
|
||||
from .monitor import router as monitor_router
|
||||
from .settings import router as settings_router
|
||||
from .websocket import router as websocket_router
|
||||
|
||||
__all__ = ["crawler_router", "data_router", "websocket_router"]
|
||||
__all__ = [
|
||||
"auth_router",
|
||||
"crawler_router",
|
||||
"data_router",
|
||||
"monitor_router",
|
||||
"settings_router",
|
||||
"websocket_router",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2025 [email protected]
|
||||
#
|
||||
# This file is part of MediaCrawler project.
|
||||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/routers/auth.py
|
||||
# GitHub: https://github.com/NanmiCoder
|
||||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||
#
|
||||
# 声明:本代码仅供学习和研究目的使用。使用者应遵守以下原则:
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||
# 4. 应合理控制请求频率,避免给目标平台带来不必要的负担。
|
||||
# 5. 不得用于任何非法或不当的用途。
|
||||
#
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
"""Login / logout endpoints.
|
||||
|
||||
Deliberately exempt from ``require_auth``:
|
||||
* ``/login`` -- it is the way in.
|
||||
* ``/logout`` -- exempt so an already-expired session still gets a clean 200
|
||||
and a cleared cookie instead of a confusing 401, which
|
||||
would leave the browser holding a stale cookie.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
|
||||
|
||||
from ..auth import (
|
||||
INVALID_CREDENTIALS,
|
||||
SESSION_COOKIE_NAME,
|
||||
check_password,
|
||||
require_auth,
|
||||
clear_failures,
|
||||
client_key,
|
||||
cookie_secure,
|
||||
create_session,
|
||||
purge_expired_sessions,
|
||||
record_failure,
|
||||
resolve_session,
|
||||
retry_after_seconds,
|
||||
revoke_all_sessions,
|
||||
revoke_session,
|
||||
set_password,
|
||||
token_from_request,
|
||||
)
|
||||
from ..monitor.db import get_session
|
||||
from ..schemas.auth import ChangePasswordPayload, LoginPayload
|
||||
from tools.time_util import get_current_timestamp
|
||||
|
||||
router = APIRouter(prefix="/auth", tags=["auth"])
|
||||
|
||||
|
||||
def _apply_session_cookie(response: Response, token: str, expires_at: int) -> None:
|
||||
"""Attach the session cookie.
|
||||
|
||||
``secure`` is off by default because the panel is served over plain HTTP on
|
||||
a LAN; setting it there means the browser silently discards the cookie and
|
||||
the login page just loops with no error. ``SameSite=lax`` is also what
|
||||
blocks cross-site POSTs, i.e. the CSRF defence for the write endpoints.
|
||||
"""
|
||||
max_age = max((expires_at - get_current_timestamp()) // 1000, 60)
|
||||
response.set_cookie(
|
||||
key=SESSION_COOKIE_NAME,
|
||||
value=token,
|
||||
max_age=max_age,
|
||||
httponly=True,
|
||||
secure=cookie_secure(),
|
||||
samesite="lax",
|
||||
path="/",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/login")
|
||||
async def login(payload: LoginPayload, request: Request, response: Response):
|
||||
key = client_key(request)
|
||||
|
||||
wait = await retry_after_seconds(key)
|
||||
if wait:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
|
||||
detail=f"尝试过于频繁,请 {wait} 秒后再试",
|
||||
headers={"Retry-After": str(wait)},
|
||||
)
|
||||
|
||||
async with get_session() as session:
|
||||
valid = await check_password(session, payload.password)
|
||||
token = ""
|
||||
expires_at = 0
|
||||
if valid:
|
||||
await purge_expired_sessions(session)
|
||||
token, expires_at = await create_session(session)
|
||||
|
||||
if not valid:
|
||||
await record_failure(key)
|
||||
# One generic message regardless of whether the password was wrong,
|
||||
# empty, or simply not set yet -- no oracle.
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED, detail=INVALID_CREDENTIALS
|
||||
)
|
||||
|
||||
await clear_failures(key)
|
||||
_apply_session_cookie(response, token, expires_at)
|
||||
return {"expires_at": expires_at}
|
||||
|
||||
|
||||
@router.post("/logout")
|
||||
async def logout(request: Request, response: Response):
|
||||
token = token_from_request(request)
|
||||
if token:
|
||||
async with get_session() as session:
|
||||
await revoke_session(session, token)
|
||||
|
||||
response.delete_cookie(SESSION_COOKIE_NAME, path="/")
|
||||
return {"message": "已退出登录"}
|
||||
|
||||
|
||||
@router.get("/me")
|
||||
async def me(request: Request):
|
||||
"""Identity probe. The SPA treats a 401 here as "show the login page".
|
||||
|
||||
Does its own resolution rather than using ``require_auth`` so it can also
|
||||
report the expiry.
|
||||
"""
|
||||
token = token_from_request(request)
|
||||
if not token:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED, detail=INVALID_CREDENTIALS
|
||||
)
|
||||
|
||||
async with get_session() as session:
|
||||
row = await resolve_session(session, token)
|
||||
if row is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED, detail=INVALID_CREDENTIALS
|
||||
)
|
||||
return {"authenticated": True, "expires_at": row.expires_at}
|
||||
|
||||
|
||||
# Auth as a route dependency, not only inside the handler: FastAPI validates the
|
||||
# request body before the endpoint body runs, so an unauthenticated caller would
|
||||
# otherwise get a 422 that confirms the endpoint and its schema exist.
|
||||
@router.post("/password", dependencies=[Depends(require_auth)])
|
||||
async def change_password(
|
||||
payload: ChangePasswordPayload, request: Request, response: Response
|
||||
):
|
||||
"""Change the password and log every device out.
|
||||
|
||||
Revoking all sessions is the point: a password change is usually a response
|
||||
to suspicion, and leaving other sessions alive would defeat it.
|
||||
"""
|
||||
token = token_from_request(request)
|
||||
|
||||
async with get_session() as session:
|
||||
current = await resolve_session(session, token)
|
||||
if current is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED, detail=INVALID_CREDENTIALS
|
||||
)
|
||||
|
||||
if not await check_password(session, payload.current):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED, detail="当前密码不正确"
|
||||
)
|
||||
|
||||
await set_password(session, payload.new)
|
||||
await revoke_all_sessions(session)
|
||||
# Issue a fresh session so the caller is not bounced mid-use.
|
||||
new_token, expires_at = await create_session(session)
|
||||
|
||||
_apply_session_cookie(response, new_token, expires_at)
|
||||
return {"message": "密码已更新,其他设备的登录已全部失效", "expires_at": expires_at}
|
||||
+25
-2
@@ -18,7 +18,9 @@
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
|
||||
from ..schemas import CrawlerStartRequest, CrawlerStatusResponse
|
||||
from ..monitor.db import get_session
|
||||
from ..monitor.settings import get_cookie
|
||||
from ..schemas import CrawlerStartRequest, CrawlerStatusResponse, LoginTypeEnum
|
||||
from ..services import crawler_manager
|
||||
|
||||
router = APIRouter(prefix="/crawler", tags=["crawler"])
|
||||
@@ -26,7 +28,28 @@ router = APIRouter(prefix="/crawler", tags=["crawler"])
|
||||
|
||||
@router.post("/start")
|
||||
async def start_crawler(request: CrawlerStartRequest):
|
||||
"""Start crawler task"""
|
||||
"""Start crawler task.
|
||||
|
||||
A cookie login with no cookie supplied falls back to the one stored for the
|
||||
selected platform. The manual crawl and the monitor therefore share a single
|
||||
credential; keeping a second paste field on the crawl page meant it went
|
||||
stale and could silently disagree with the monitor's.
|
||||
"""
|
||||
if (
|
||||
request.login_type == LoginTypeEnum.COOKIE
|
||||
and not request.cookies
|
||||
and not request.cookies_file
|
||||
):
|
||||
async with get_session() as session:
|
||||
stored = await get_cookie(session, request.platform.value)
|
||||
|
||||
if not stored:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="该平台尚未保存 Cookie,请到「设置 → 登录态」配置,或改用扫码登录",
|
||||
)
|
||||
request.cookies = stored
|
||||
|
||||
success = await crawler_manager.start(request)
|
||||
if not success:
|
||||
# Handle concurrent/duplicate requests: if process is already running, return 400 instead of 500
|
||||
|
||||
@@ -0,0 +1,502 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2025 [email protected]
|
||||
#
|
||||
# This file is part of MediaCrawler project.
|
||||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/routers/monitor.py
|
||||
# GitHub: https://github.com/NanmiCoder
|
||||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||
#
|
||||
# 声明:本代码仅供学习和研究目的使用。使用者应遵守以下原则:
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||
# 4. 应合理控制请求频率,避免给目标平台带来不必要的负担。
|
||||
# 5. 不得用于任何非法或不当的用途。
|
||||
#
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
"""HTTP API for scheduled monitoring tasks."""
|
||||
|
||||
from datetime import date, timedelta
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Response
|
||||
|
||||
from ..monitor import notify, report, service
|
||||
from ..monitor.db import get_session
|
||||
from ..monitor.platforms import PLATFORM_XHS
|
||||
from ..monitor.settings import (
|
||||
cookie_key,
|
||||
delete_setting,
|
||||
get_cookie_status,
|
||||
get_setting,
|
||||
set_cookie,
|
||||
set_setting,
|
||||
)
|
||||
from ..monitor.models import SETTING_WECOM_WEBHOOK, MonitorTask
|
||||
from ..schemas.monitor import (
|
||||
CookiePayload,
|
||||
MonitorTaskCreate,
|
||||
MonitorTaskUpdate,
|
||||
WebhookPayload,
|
||||
WebhookTestPayload,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/monitor", tags=["monitor"])
|
||||
|
||||
|
||||
@router.get("/overview")
|
||||
async def get_overview(platform: Optional[str] = None):
|
||||
"""Headline numbers for the dashboard tiles, scoped to one platform."""
|
||||
async with get_session() as session:
|
||||
return await service.overview(session, platform)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tasks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/tasks")
|
||||
async def list_tasks(platform: Optional[str] = None):
|
||||
async with get_session() as session:
|
||||
return {"tasks": await service.list_tasks(session, platform)}
|
||||
|
||||
|
||||
@router.post("/tasks", status_code=201)
|
||||
async def create_task(payload: MonitorTaskCreate):
|
||||
async with get_session() as session:
|
||||
try:
|
||||
task = await service.create_task(session, payload.model_dump())
|
||||
except service.TargetParseError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
return {"id": task.id, "message": "Monitoring task created"}
|
||||
|
||||
|
||||
@router.patch("/tasks/{task_id}")
|
||||
async def update_task(task_id: int, payload: MonitorTaskUpdate):
|
||||
async with get_session() as session:
|
||||
try:
|
||||
await service.update_task(session, task_id, payload.model_dump(exclude_unset=True))
|
||||
except service.TargetParseError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc))
|
||||
return {"message": "Monitoring task updated"}
|
||||
|
||||
|
||||
@router.delete("/tasks/{task_id}")
|
||||
async def delete_task(task_id: int):
|
||||
async with get_session() as session:
|
||||
try:
|
||||
await service.delete_task(session, task_id)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc))
|
||||
return {"message": "Monitoring task deleted"}
|
||||
|
||||
|
||||
@router.post("/tasks/{task_id}/run")
|
||||
async def run_task_now(task_id: int):
|
||||
"""Queue a run immediately and return; the crawl itself takes minutes."""
|
||||
async with get_session() as session:
|
||||
task = await session.get(MonitorTask, task_id)
|
||||
if task is None:
|
||||
raise HTTPException(status_code=404, detail=f"Task {task_id} not found")
|
||||
if not any(target.enabled for target in task.targets):
|
||||
raise HTTPException(status_code=400, detail="Task has no enabled targets")
|
||||
|
||||
service.trigger_manual_run(task_id)
|
||||
return {"message": "Run queued"}
|
||||
|
||||
|
||||
@router.get("/tasks/{task_id}/runs")
|
||||
async def list_runs(task_id: int, limit: int = Query(default=50, ge=1, le=500)):
|
||||
async with get_session() as session:
|
||||
return {"runs": await service.list_runs(session, task_id, limit=limit)}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Collected data
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/notes")
|
||||
async def list_notes(
|
||||
task_id: Optional[int] = None,
|
||||
only_new: bool = False,
|
||||
limit: int = Query(default=200, ge=1, le=2000),
|
||||
platform: Optional[str] = None,
|
||||
):
|
||||
async with get_session() as session:
|
||||
return {
|
||||
"notes": await service.list_notes(session, task_id, only_new, limit, platform)
|
||||
}
|
||||
|
||||
|
||||
@router.get("/notes/{note_id}/series")
|
||||
async def note_series(note_id: str, task_id: Optional[int] = None):
|
||||
"""Metric time series for a single note."""
|
||||
async with get_session() as session:
|
||||
return {"series": await service.note_series(session, note_id, task_id)}
|
||||
|
||||
|
||||
@router.get("/comments")
|
||||
async def list_comments(
|
||||
task_id: Optional[int] = None,
|
||||
note_id: Optional[str] = None,
|
||||
group_by: Optional[str] = Query(
|
||||
default=None, description="传 note 则按作品分组返回,便于阅读"
|
||||
),
|
||||
limit: int = Query(default=200, ge=1, le=2000),
|
||||
platform: Optional[str] = None,
|
||||
):
|
||||
"""Comments, each carrying the work it belongs to.
|
||||
|
||||
``note_id`` filters to one work; ``group_by=note`` returns them bucketed per
|
||||
work instead of as a flat stream.
|
||||
"""
|
||||
async with get_session() as session:
|
||||
comments = await service.list_comments(session, task_id, note_id, limit, platform)
|
||||
|
||||
if group_by != "note":
|
||||
return {"comments": comments, "total": len(comments)}
|
||||
|
||||
buckets: Dict[str, Dict[str, Any]] = {}
|
||||
for comment in comments:
|
||||
bucket = buckets.setdefault(
|
||||
comment["note_id"],
|
||||
{
|
||||
"note_id": comment["note_id"],
|
||||
"note_title": comment["note_title"],
|
||||
"note_cover": comment["note_cover"],
|
||||
"note_url": comment["note_url"],
|
||||
"comments": [],
|
||||
},
|
||||
)
|
||||
bucket["comments"].append(comment)
|
||||
|
||||
ordered = sorted(
|
||||
buckets.values(),
|
||||
key=lambda group: group["comments"][0]["first_seen_at"],
|
||||
reverse=True,
|
||||
)
|
||||
return {"groups": ordered, "total": len(comments)}
|
||||
|
||||
|
||||
@router.get("/comment-notes")
|
||||
async def list_comment_notes(task_id: Optional[int] = None, platform: Optional[str] = None):
|
||||
"""Works that have comments, newest first, with counts.
|
||||
|
||||
Feeds the comment filter dropdown so the operator can pick by title.
|
||||
"""
|
||||
async with get_session() as session:
|
||||
return {"notes": await service.comment_note_groups(session, task_id, platform)}
|
||||
|
||||
|
||||
@router.get("/events")
|
||||
async def list_events(
|
||||
task_id: Optional[int] = None,
|
||||
type: Optional[str] = None,
|
||||
since_id: Optional[int] = None,
|
||||
limit: int = Query(default=200, ge=1, le=2000),
|
||||
platform: Optional[str] = None,
|
||||
):
|
||||
async with get_session() as session:
|
||||
events = await service.list_events(session, task_id, type, since_id, limit, platform)
|
||||
return {"events": events, "latest_id": events[0]["id"] if events else since_id}
|
||||
|
||||
|
||||
@router.post("/events/read")
|
||||
async def mark_events_read(task_id: Optional[int] = None):
|
||||
async with get_session() as session:
|
||||
count = await service.mark_events_read(session, task_id)
|
||||
return {"marked": count}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cookie / login health
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/cookie")
|
||||
async def get_cookie_endpoint(platform: str = Query(default=PLATFORM_XHS)):
|
||||
"""Cookie health only -- deliberately never returns the cookie value.
|
||||
|
||||
``platform`` defaults to Xiaohongshu so existing callers keep working; the
|
||||
key it reads is the namespaced one.
|
||||
"""
|
||||
async with get_session() as session:
|
||||
return await get_cookie_status(session, platform)
|
||||
|
||||
|
||||
@router.post("/cookie")
|
||||
async def set_cookie_endpoint(payload: CookiePayload, platform: str = Query(default=PLATFORM_XHS)):
|
||||
async with get_session() as session:
|
||||
await set_cookie(session, payload.cookie.strip(), platform)
|
||||
return {"message": "Cookie saved"}
|
||||
|
||||
|
||||
@router.delete("/cookie")
|
||||
async def clear_cookie_endpoint(platform: str = Query(default=PLATFORM_XHS)):
|
||||
async with get_session() as session:
|
||||
await delete_setting(session, cookie_key(platform))
|
||||
return {"message": "Cookie cleared"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Report
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def _resolve_scope(
|
||||
session, task_ids: Optional[List[int]], platform: Optional[str]
|
||||
) -> Optional[List[int]]:
|
||||
"""Combine an explicit task selection with an optional platform filter.
|
||||
|
||||
``None`` means "no restriction"; an explicit list is intersected with the
|
||||
platform's tasks so a stale selection cannot leak another platform's data
|
||||
into a scoped report.
|
||||
"""
|
||||
if platform is None:
|
||||
return task_ids
|
||||
|
||||
platform_ids = set(await service.platform_task_ids(session, platform))
|
||||
if task_ids is None:
|
||||
return list(platform_ids)
|
||||
return [task for task in task_ids if task in platform_ids]
|
||||
|
||||
|
||||
@router.get("/export")
|
||||
async def export_data(
|
||||
kind: str = Query(..., description="notes | comments | report"),
|
||||
task_id: Optional[List[int]] = Query(default=None),
|
||||
note_id: Optional[str] = None,
|
||||
start_date: Optional[str] = Query(default=None, description="YYYY-MM-DD,report 用"),
|
||||
end_date: Optional[str] = Query(default=None, description="YYYY-MM-DD,report 用"),
|
||||
days: int = Query(default=7, ge=1, le=365),
|
||||
file_format: str = Query(default="csv", alias="format", description="csv | xlsx"),
|
||||
platform: Optional[str] = None,
|
||||
):
|
||||
"""Download collected data as CSV or Excel.
|
||||
|
||||
Reached by the browser as a navigation (``window.open``), which cannot carry
|
||||
an Authorization header -- this is one of the reasons the session lives in a
|
||||
cookie.
|
||||
"""
|
||||
if kind not in ("notes", "comments", "report"):
|
||||
raise HTTPException(status_code=400, detail="kind 必须是 notes / comments / report")
|
||||
if file_format not in ("csv", "xlsx"):
|
||||
raise HTTPException(status_code=400, detail="format 必须是 csv 或 xlsx")
|
||||
|
||||
async with get_session() as session:
|
||||
scoped = await _resolve_scope(session, task_id, platform)
|
||||
|
||||
if kind == "notes":
|
||||
single_task = scoped[0] if scoped and len(scoped) == 1 else None
|
||||
rows = await service.list_notes(session, single_task, False, 5000, platform)
|
||||
elif kind == "comments":
|
||||
single_task = scoped[0] if scoped and len(scoped) == 1 else None
|
||||
rows = await service.list_comments(session, single_task, note_id, 5000, platform)
|
||||
else:
|
||||
try:
|
||||
end_day = date.fromisoformat(end_date) if end_date else date.today()
|
||||
start_day = (
|
||||
date.fromisoformat(start_date)
|
||||
if start_date
|
||||
else end_day - timedelta(days=days - 1)
|
||||
)
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail="日期格式应为 YYYY-MM-DD")
|
||||
# Not `report = ...`: that would make `report` a local name for the
|
||||
# whole function and shadow the module import on this very line.
|
||||
report_data = await report.build_report(session, scoped, start_day, end_day)
|
||||
rows = report_data["rows"]
|
||||
|
||||
if not rows:
|
||||
raise HTTPException(status_code=404, detail="该范围内没有数据可导出")
|
||||
|
||||
columns = _export_columns(kind)
|
||||
stamp = date.today().isoformat()
|
||||
# ASCII on purpose: a non-ASCII filename needs RFC 5987 encoding in
|
||||
# Content-Disposition, and the plain `filename="..."` form used below would
|
||||
# mangle it.
|
||||
filename = f"export_{kind}_{stamp}.{file_format}"
|
||||
|
||||
if file_format == "xlsx":
|
||||
payload = _to_xlsx(rows, columns, kind)
|
||||
media_type = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||
else:
|
||||
payload = _to_csv(rows, columns)
|
||||
media_type = "text/csv; charset=utf-8"
|
||||
|
||||
return Response(
|
||||
content=payload,
|
||||
media_type=media_type,
|
||||
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
||||
)
|
||||
|
||||
|
||||
def _export_columns(kind: str) -> List[tuple[str, str]]:
|
||||
"""(key, header) pairs per export kind."""
|
||||
if kind == "notes":
|
||||
return [
|
||||
("note_id", "作品ID"),
|
||||
("title", "标题"),
|
||||
("note_url", "链接"),
|
||||
("liked_count", "点赞"),
|
||||
("comment_count", "评论"),
|
||||
("collected_count", "收藏"),
|
||||
("share_count", "分享"),
|
||||
("liked_count_delta", "点赞增量"),
|
||||
("comment_count_delta", "评论增量"),
|
||||
("first_seen_at", "首次发现"),
|
||||
("last_seen_at", "最近采集"),
|
||||
]
|
||||
if kind == "comments":
|
||||
return [
|
||||
("note_title", "所属作品"),
|
||||
("note_id", "作品ID"),
|
||||
("comment_id", "评论ID"),
|
||||
("content", "内容"),
|
||||
("nickname", "昵称"),
|
||||
("like_count", "点赞"),
|
||||
("sub_comment_count", "子评论数"),
|
||||
("create_time", "发布时间"),
|
||||
("first_seen_at", "首次发现"),
|
||||
]
|
||||
return [
|
||||
("date", "日期"),
|
||||
("new_notes", "新增作品"),
|
||||
("new_comments", "新增评论"),
|
||||
("liked_count_delta", "点赞增量"),
|
||||
("comment_count_delta", "评论增量"),
|
||||
("collected_count_delta", "收藏增量"),
|
||||
("share_count_delta", "分享增量"),
|
||||
]
|
||||
|
||||
|
||||
def _cell(value: Any) -> Any:
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, (list, dict)):
|
||||
return ", ".join(str(v) for v in value) if isinstance(value, list) else str(value)
|
||||
return value
|
||||
|
||||
|
||||
def _to_csv(rows: List[Dict[str, Any]], columns: List[tuple[str, str]]) -> bytes:
|
||||
import csv
|
||||
import io
|
||||
|
||||
buffer = io.StringIO()
|
||||
writer = csv.writer(buffer)
|
||||
writer.writerow([header for _, header in columns])
|
||||
for row in rows:
|
||||
writer.writerow([_cell(row.get(key)) for key, _ in columns])
|
||||
|
||||
# utf-8-sig: without the BOM Excel opens Chinese CSV as mojibake, which is
|
||||
# the single most common complaint about CSV exports here.
|
||||
return buffer.getvalue().encode("utf-8-sig")
|
||||
|
||||
|
||||
def _to_xlsx(rows: List[Dict[str, Any]], columns: List[tuple[str, str]], sheet: str) -> bytes:
|
||||
import io
|
||||
|
||||
from openpyxl import Workbook
|
||||
|
||||
workbook = Workbook()
|
||||
worksheet = workbook.active
|
||||
worksheet.title = {"notes": "作品", "comments": "评论"}.get(sheet, "报表")
|
||||
worksheet.append([header for _, header in columns])
|
||||
for row in rows:
|
||||
worksheet.append([_cell(row.get(key)) for key, _ in columns])
|
||||
|
||||
output = io.BytesIO()
|
||||
workbook.save(output)
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
@router.get("/report")
|
||||
async def get_report(
|
||||
task_id: Optional[List[int]] = Query(
|
||||
default=None, description="Repeat to include several tasks; omit for all"
|
||||
),
|
||||
start_date: Optional[str] = Query(default=None, description="YYYY-MM-DD"),
|
||||
end_date: Optional[str] = Query(default=None, description="YYYY-MM-DD"),
|
||||
days: int = Query(default=7, ge=1, le=365, description="Window used when dates are omitted"),
|
||||
platform: Optional[str] = None,
|
||||
):
|
||||
"""Daily new-content counts and interaction deltas for the selected tasks."""
|
||||
try:
|
||||
end_day = date.fromisoformat(end_date) if end_date else date.today()
|
||||
start_day = date.fromisoformat(start_date) if start_date else end_day - timedelta(days=days - 1)
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail="日期格式应为 YYYY-MM-DD")
|
||||
|
||||
if start_day > end_day:
|
||||
raise HTTPException(status_code=400, detail="开始日期不能晚于结束日期")
|
||||
|
||||
async with get_session() as session:
|
||||
scoped = await _resolve_scope(session, task_id, platform)
|
||||
return await report.build_report(session, scoped, start_day, end_day)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# WeCom webhook
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _mask_webhook(url: str) -> str:
|
||||
"""Show enough of the URL to recognise it, without exposing the robot key."""
|
||||
if not url:
|
||||
return ""
|
||||
key_marker = "key="
|
||||
index = url.find(key_marker)
|
||||
if index == -1:
|
||||
return url[:12] + "..." if len(url) > 12 else url
|
||||
prefix = url[: index + len(key_marker)]
|
||||
key = url[index + len(key_marker) :]
|
||||
if len(key) <= 8:
|
||||
return prefix + "*" * len(key)
|
||||
return f"{prefix}{key[:4]}...{key[-4:]}"
|
||||
|
||||
|
||||
@router.get("/webhook")
|
||||
async def get_webhook():
|
||||
async with get_session() as session:
|
||||
url = (await get_setting(session, SETTING_WECOM_WEBHOOK)) or ""
|
||||
return {"configured": bool(url), "masked": _mask_webhook(url)}
|
||||
|
||||
|
||||
@router.post("/webhook")
|
||||
async def set_webhook(payload: WebhookPayload):
|
||||
url = payload.url.strip()
|
||||
if url and "qyapi.weixin.qq.com" not in url:
|
||||
# Catches the common mistake of pasting a group-chat invite or the app
|
||||
# URL instead of the robot webhook.
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="这不像企业微信机器人 Webhook 地址(应包含 qyapi.weixin.qq.com)",
|
||||
)
|
||||
|
||||
async with get_session() as session:
|
||||
await set_setting(session, SETTING_WECOM_WEBHOOK, url)
|
||||
return {"message": "Webhook 已保存" if url else "Webhook 已清空", "configured": bool(url)}
|
||||
|
||||
|
||||
@router.delete("/webhook")
|
||||
async def clear_webhook():
|
||||
async with get_session() as session:
|
||||
await delete_setting(session, SETTING_WECOM_WEBHOOK)
|
||||
return {"message": "Webhook 已删除"}
|
||||
|
||||
|
||||
@router.post("/webhook/test")
|
||||
async def test_webhook(payload: WebhookTestPayload):
|
||||
"""Send a test message so the user can verify the robot works before relying on it."""
|
||||
async with get_session() as session:
|
||||
url = payload.url.strip() if payload.url else await notify.get_webhook_url(session)
|
||||
|
||||
ok, detail = await notify.send_wecom(
|
||||
url, "**综合采集平台 通知测试**\n> 如果你看到这条消息,说明 Webhook 配置成功。"
|
||||
)
|
||||
if not ok:
|
||||
raise HTTPException(status_code=400, detail=detail)
|
||||
return {"message": detail}
|
||||
@@ -0,0 +1,70 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2025 [email protected]
|
||||
#
|
||||
# This file is part of MediaCrawler project.
|
||||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/routers/settings.py
|
||||
# GitHub: https://github.com/NanmiCoder
|
||||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||
#
|
||||
# 声明:本代码仅供学习和研究目的使用。使用者应遵守以下原则:
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||
# 4. 应合理控制请求频率,避免给目标平台带来不必要的负担。
|
||||
# 5. 不得用于任何非法或不当的用途。
|
||||
#
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
"""Unified settings endpoint.
|
||||
|
||||
Consolidates what used to be scattered across the monitor router. The older
|
||||
``/api/monitor/cookie`` and ``/api/monitor/webhook`` endpoints are deliberately
|
||||
left in place -- they still work and removing them would be a breaking change for
|
||||
no gain.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
|
||||
from ..monitor import app_settings
|
||||
from ..monitor.db import get_session
|
||||
from ..monitor.platforms import PLATFORM_XHS
|
||||
from ..schemas.settings import SettingsUpdatePayload
|
||||
|
||||
router = APIRouter(prefix="/settings", tags=["settings"])
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def read_settings(platform: str = Query(default=PLATFORM_XHS)):
|
||||
"""Settings for one platform, plus the system-wide ones.
|
||||
|
||||
Platform-scoped values are returned for the requested platform; system-scoped
|
||||
values are the same regardless. Every spec carries its resolved ``key`` so the
|
||||
UI can PUT changes straight back.
|
||||
|
||||
Sensitive values are returned as ``{present, length}`` only.
|
||||
"""
|
||||
async with get_session() as session:
|
||||
return await app_settings.get_all(session, platform)
|
||||
|
||||
|
||||
@router.put("")
|
||||
async def write_settings(
|
||||
payload: SettingsUpdatePayload, platform: str = Query(default=PLATFORM_XHS)
|
||||
):
|
||||
"""Partial update: only the keys present in the body are written.
|
||||
|
||||
A key belonging to a different platform is rejected rather than written
|
||||
somewhere unexpected.
|
||||
"""
|
||||
values: Dict[str, Any] = payload.values()
|
||||
|
||||
async with get_session() as session:
|
||||
try:
|
||||
changed = await app_settings.update(session, values, platform)
|
||||
except app_settings.SettingValidationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
|
||||
return {"message": f"已保存 {len(changed)} 项设置", "changed": changed}
|
||||
@@ -19,8 +19,9 @@
|
||||
import asyncio
|
||||
from typing import Set, Optional
|
||||
|
||||
from fastapi import APIRouter, WebSocket, WebSocketDisconnect
|
||||
from fastapi import APIRouter, Depends, WebSocket, WebSocketDisconnect
|
||||
|
||||
from ..auth import require_ws_auth
|
||||
from ..services import crawler_manager
|
||||
|
||||
router = APIRouter(tags=["websocket"])
|
||||
@@ -86,7 +87,10 @@ def start_broadcaster():
|
||||
_broadcaster_task = asyncio.create_task(log_broadcaster())
|
||||
|
||||
|
||||
@router.websocket("/ws/logs")
|
||||
# Websocket routes need their own auth dependency: BaseHTTPMiddleware returns
|
||||
# early for any non-http scope, and HTTP router-level dependencies do not reach
|
||||
# websocket routes. Without this the live crawl log stream would be wide open.
|
||||
@router.websocket("/ws/logs", dependencies=[Depends(require_ws_auth)])
|
||||
async def websocket_logs(websocket: WebSocket):
|
||||
"""WebSocket log stream"""
|
||||
print("[WS] New connection attempt")
|
||||
@@ -134,7 +138,7 @@ async def websocket_logs(websocket: WebSocket):
|
||||
print(f"[WS] Cleanup done, active connections: {len(manager.active_connections)}")
|
||||
|
||||
|
||||
@router.websocket("/ws/status")
|
||||
@router.websocket("/ws/status", dependencies=[Depends(require_ws_auth)])
|
||||
async def websocket_status(websocket: WebSocket):
|
||||
"""WebSocket status stream"""
|
||||
await websocket.accept()
|
||||
|
||||
Reference in New Issue
Block a user