feat: 监控面板 / 登录鉴权 / 多平台切换 / MySQL
在上游 MediaCrawler 之上新增一层： - 监控层 api/monitor/ —— 多博主/多笔记的定时采集、指标快照差分、报表、 企业微信通知。每轮采集写入独立目录，差分才成立。 - WebUI 登录鉴权 api/auth.py —— PBKDF2 口令 + 服务端会话，/api 全接口防护。 WebSocket 单独加依赖：BaseHTTPMiddleware 对 ws 作用域直接放行，覆盖不到。 - 全局平台切换 + 能力矩阵 —— 如实区分「爬虫模块支持」与「监控层已接线」， 未接通的平台直接拒绝建任务，而不是静默跑空。 - 监控库改用 MySQL 5.7（可回退 SQLite 供测试）：逐表强制 utf8mb4 （服务端与库默认都是 latin1），启动校验所连 schema 以防写错库， 连接池 recycle + pre_ping 应对 MySQL 的 8 小时空闲断连。 修复上游缺陷： - xhs/core.py: 主页抓取失败会跳掉整个博主，导致一条作品都抓不到， 而那份资料只喂给一个空函数。改为尽力而为，失败不中断。 - xhs/login.py: cookie 登录只注入 web_session，冷启动签名会失败。 新增 INJECT_ALL_COOKIES 开关（默认关闭，原有行为不变）。 - requirements.txt: 补上 websockets。它在上游 pyproject.toml 里有声明、 这里漏了，导致 uvicorn 没有 WebSocket 能力，实时日志流从未工作。 改动过的上游文件清单及合并方式见 UPSTREAM.md。 测试：492 passed（另有 1 个既有的 Windows/gbk 上游测试失败，与本改动无关）
This commit is contained in:
+139
-43
@@ -17,7 +17,7 @@
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
"""
|
||||
MediaCrawler WebUI API Server
|
||||
综合采集平台 API Server
|
||||
Start command: uvicorn api.main:app --port 8080 --reload
|
||||
Or: python -m api.main
|
||||
"""
|
||||
@@ -25,28 +25,95 @@ import asyncio
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
import uvicorn
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from .routers import crawler_router, data_router, websocket_router
|
||||
|
||||
# Project root directory (used for running subprocesses like uv run main.py)
|
||||
PROJECT_ROOT = Path(__file__).parent.parent
|
||||
|
||||
# Load .env before importing anything that reads os.getenv at module import time
|
||||
# (config/db_config.py does). python-dotenv was already a declared dependency but
|
||||
# nothing ever called it, so the shipped .env.example had no effect.
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv(PROJECT_ROOT / ".env")
|
||||
|
||||
import uvicorn
|
||||
from fastapi import Depends, FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from .auth import ensure_initial_credential, require_auth
|
||||
from .routers import (
|
||||
auth_router,
|
||||
crawler_router,
|
||||
data_router,
|
||||
monitor_router,
|
||||
settings_router,
|
||||
websocket_router,
|
||||
)
|
||||
from .services.interpreter import describe_interpreter, resolve_python_cmd
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app: FastAPI):
|
||||
"""Start the monitor scheduler with the server, and shut it down cleanly.
|
||||
|
||||
The scheduler is a background asyncio task, so it must not be tied to a
|
||||
browser session the way the log broadcaster is -- a scheduled run has to
|
||||
happen whether or not anyone has the UI open.
|
||||
"""
|
||||
from .monitor.db import dispose_engine, init_db
|
||||
from .monitor.scheduler import monitor_scheduler
|
||||
|
||||
await init_db()
|
||||
|
||||
generated = await ensure_initial_credential()
|
||||
if generated:
|
||||
# Printed once, on the run that creates it. There is no unauthenticated
|
||||
# "set your password" endpoint on purpose: on a LAN bind that would be a
|
||||
# claim-the-instance race.
|
||||
rule = "=" * 68
|
||||
print(
|
||||
f"\n{rule}\n"
|
||||
" WebUI 首次启动，已生成登录密码：\n"
|
||||
f"\n {generated}\n"
|
||||
"\n 请立即登录并修改。忘记密码时可设置环境变量 MC_PASSWORD 后重启。\n"
|
||||
f"{rule}\n",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
await monitor_scheduler.start()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
await monitor_scheduler.stop()
|
||||
await dispose_engine()
|
||||
|
||||
|
||||
# Docs are disabled deliberately: /docs, /redoc and /openapi.json are
|
||||
# unauthenticated by default, which would hand out a complete map of the API
|
||||
# (and a "Try it out" console that 401s anyway).
|
||||
app = FastAPI(
|
||||
title="MediaCrawler WebUI API",
|
||||
description="API for controlling MediaCrawler from WebUI",
|
||||
version="1.0.0"
|
||||
title="综合采集平台 API",
|
||||
description="API for controlling 综合采集平台 from WebUI",
|
||||
version="1.0.0",
|
||||
lifespan=lifespan,
|
||||
docs_url=None,
|
||||
redoc_url=None,
|
||||
openapi_url=None,
|
||||
)
|
||||
|
||||
# Get webui static files directory
|
||||
WEBUI_DIR = os.path.join(os.path.dirname(__file__), "webui")
|
||||
|
||||
# CORS configuration - allow frontend dev server access
|
||||
# CORS only matters for a split-origin setup. In production this app serves the
|
||||
# SPA itself, and in development Vite proxies /api here (see webui/vite.config.ts),
|
||||
# so the browser always sees a single origin and CORS never actually triggers.
|
||||
# Kept as an explicit allowlist -- never "*", which is invalid next to
|
||||
# allow_credentials -- and extensible via env for a dev server reached over LAN.
|
||||
_extra_origins = [o.strip() for o in os.getenv("MC_CORS_ORIGINS", "").split(",") if o.strip()]
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=[
|
||||
@@ -54,15 +121,24 @@ app.add_middleware(
|
||||
"http://localhost:3000", # Backup port
|
||||
"http://127.0.0.1:5173",
|
||||
"http://127.0.0.1:3000",
|
||||
*_extra_origins,
|
||||
],
|
||||
allow_origin_regex=os.getenv("MC_CORS_ORIGIN_REGEX") or None,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# Register routers
|
||||
app.include_router(crawler_router, prefix="/api")
|
||||
app.include_router(data_router, prefix="/api")
|
||||
# Register routers.
|
||||
# The auth router stays open -- it is the way in. Everything else under /api
|
||||
# requires a session. Enforcement is a Depends applied per router rather than
|
||||
# app-wide middleware, because middleware needs a hand-rolled path allowlist and,
|
||||
# more importantly, never sees WebSocket scopes at all.
|
||||
app.include_router(auth_router, prefix="/api")
|
||||
app.include_router(crawler_router, prefix="/api", dependencies=[Depends(require_auth)])
|
||||
app.include_router(data_router, prefix="/api", dependencies=[Depends(require_auth)])
|
||||
app.include_router(monitor_router, prefix="/api", dependencies=[Depends(require_auth)])
|
||||
app.include_router(settings_router, prefix="/api", dependencies=[Depends(require_auth)])
|
||||
app.include_router(websocket_router, prefix="/api")
|
||||
|
||||
|
||||
@@ -73,9 +149,8 @@ async def serve_frontend():
|
||||
if os.path.exists(index_path):
|
||||
return FileResponse(index_path)
|
||||
return {
|
||||
"message": "MediaCrawler WebUI API",
|
||||
"message": "综合采集平台 API",
|
||||
"version": "1.0.0",
|
||||
"docs": "/docs",
|
||||
"note": "WebUI not found, please build it first: cd webui && npm run build"
|
||||
}
|
||||
|
||||
@@ -85,18 +160,21 @@ async def health_check():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/api/env/check")
|
||||
@app.get("/api/env/check", dependencies=[Depends(require_auth)])
|
||||
async def check_environment():
|
||||
"""Check if MediaCrawler environment is configured correctly"""
|
||||
"""Check whether the crawler environment is configured correctly"""
|
||||
try:
|
||||
# Run uv run main.py --help command to check environment
|
||||
# Use PROJECT_ROOT so it works regardless of where uvicorn was started
|
||||
# Run `main.py --help` to check the environment.
|
||||
# Resolve the interpreter the same way the crawler manager does, so this
|
||||
# check can never disagree with how main.py is actually executed.
|
||||
# Use PROJECT_ROOT so it works regardless of where uvicorn was started.
|
||||
python_cmd = resolve_python_cmd()
|
||||
if sys.platform == "win32":
|
||||
loop = asyncio.get_running_loop()
|
||||
process = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: subprocess.run(
|
||||
["uv", "run", "main.py", "--help"],
|
||||
[*python_cmd, "main.py", "--help"],
|
||||
capture_output=True,
|
||||
timeout=30.0,
|
||||
cwd=str(PROJECT_ROOT)
|
||||
@@ -105,7 +183,7 @@ async def check_environment():
|
||||
stdout, stderr = process.stdout, process.stderr # bytes
|
||||
else:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
"uv", "run", "main.py", "--help",
|
||||
*python_cmd, "main.py", "--help",
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
cwd=str(PROJECT_ROOT) # Project root directory
|
||||
@@ -117,7 +195,8 @@ async def check_environment():
|
||||
if process.returncode == 0:
|
||||
return {
|
||||
"success": True,
|
||||
"message": "MediaCrawler environment configured correctly",
|
||||
"message": "环境配置正确",
|
||||
"interpreter": describe_interpreter(),
|
||||
"output": stdout.decode("utf-8", errors="ignore")[:500] # Truncate to first 500 characters
|
||||
}
|
||||
else:
|
||||
@@ -136,8 +215,11 @@ async def check_environment():
|
||||
except FileNotFoundError:
|
||||
return {
|
||||
"success": False,
|
||||
"message": "uv command not found",
|
||||
"error": "Please ensure uv is installed and configured in system PATH"
|
||||
"message": "Python interpreter not found",
|
||||
"error": (
|
||||
"Neither uv nor a usable interpreter was found. Install uv, or create a "
|
||||
"project virtualenv (.venv) with the requirements installed."
|
||||
)
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
@@ -147,29 +229,29 @@ async def check_environment():
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/config/platforms")
|
||||
@app.get("/api/config/platforms", dependencies=[Depends(require_auth)])
|
||||
async def get_platforms():
|
||||
"""Get list of supported platforms"""
|
||||
return {
|
||||
"platforms": [
|
||||
{"value": "xhs", "label": "Xiaohongshu", "icon": "book-open"},
|
||||
{"value": "dy", "label": "Douyin", "icon": "music"},
|
||||
{"value": "ks", "label": "Kuaishou", "icon": "video"},
|
||||
{"value": "bili", "label": "Bilibili", "icon": "tv"},
|
||||
{"value": "wb", "label": "Weibo", "icon": "message-circle"},
|
||||
{"value": "tieba", "label": "Baidu Tieba", "icon": "messages-square"},
|
||||
{"value": "zhihu", "label": "Zhihu", "icon": "help-circle"},
|
||||
]
|
||||
}
|
||||
"""Platform capability matrix.
|
||||
|
||||
Returns what each platform's crawler supports (modes, metrics, comment
|
||||
levels, media) *and* whether the monitoring layer has been wired up for it.
|
||||
The UI renders its platform switcher and metric columns from this, so the
|
||||
two are never allowed to drift apart.
|
||||
"""
|
||||
from .monitor.platforms import describe_all
|
||||
|
||||
return {"platforms": describe_all()}
|
||||
|
||||
|
||||
@app.get("/api/config/options")
|
||||
@app.get("/api/config/options", dependencies=[Depends(require_auth)])
|
||||
async def get_config_options():
|
||||
"""Get all configuration options"""
|
||||
return {
|
||||
"login_types": [
|
||||
{"value": "qrcode", "label": "QR Code Login"},
|
||||
{"value": "cookie", "label": "Cookie Login"},
|
||||
{"value": "qrcode", "label": "扫码登录"},
|
||||
# Named for what it now does: the value itself is no longer typed
|
||||
# here, it is reused from Settings.
|
||||
{"value": "cookie", "label": "复用已保存的 Cookie"},
|
||||
],
|
||||
"crawler_types": [
|
||||
{"value": "search", "label": "Search Mode"},
|
||||
@@ -202,4 +284,18 @@ if os.path.exists(WEBUI_DIR):
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
uvicorn.run(app, host="0.0.0.0", port=8080)
|
||||
# Loopback by default: the safe choice for anyone who has not thought about
|
||||
# exposure. Set MC_HOST=0.0.0.0 (e.g. in .env) for LAN access. Before this,
|
||||
# `python -m api.main` bound 0.0.0.0 while the documented `uvicorn api.main:app`
|
||||
# bound loopback -- two launch paths with different exposure.
|
||||
host = os.getenv("MC_HOST", "127.0.0.1")
|
||||
port = int(os.getenv("MC_PORT", "8080"))
|
||||
|
||||
if host not in ("127.0.0.1", "localhost", "::1"):
|
||||
print(
|
||||
f"[综合采集平台] 监听 {host}:{port}，局域网内其他机器可访问。\n"
|
||||
f"[综合采集平台] 已启用密码鉴权；如需暴露到可信网络之外，请走 HTTPS 反向代理。",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
uvicorn.run(app, host=host, port=port)
|
||||
|
||||
Reference in New Issue
Block a user