feat: 监控面板 / 登录鉴权 / 多平台切换 / MySQL
在上游 MediaCrawler 之上新增一层： - 监控层 api/monitor/ —— 多博主/多笔记的定时采集、指标快照差分、报表、 企业微信通知。每轮采集写入独立目录，差分才成立。 - WebUI 登录鉴权 api/auth.py —— PBKDF2 口令 + 服务端会话，/api 全接口防护。 WebSocket 单独加依赖：BaseHTTPMiddleware 对 ws 作用域直接放行，覆盖不到。 - 全局平台切换 + 能力矩阵 —— 如实区分「爬虫模块支持」与「监控层已接线」， 未接通的平台直接拒绝建任务，而不是静默跑空。 - 监控库改用 MySQL 5.7（可回退 SQLite 供测试）：逐表强制 utf8mb4 （服务端与库默认都是 latin1），启动校验所连 schema 以防写错库， 连接池 recycle + pre_ping 应对 MySQL 的 8 小时空闲断连。 修复上游缺陷： - xhs/core.py: 主页抓取失败会跳掉整个博主，导致一条作品都抓不到， 而那份资料只喂给一个空函数。改为尽力而为，失败不中断。 - xhs/login.py: cookie 登录只注入 web_session，冷启动签名会失败。 新增 INJECT_ALL_COOKIES 开关（默认关闭，原有行为不变）。 - requirements.txt: 补上 websockets。它在上游 pyproject.toml 里有声明、 这里漏了，导致 uvicorn 没有 WebSocket 能力，实时日志流从未工作。 改动过的上游文件清单及合并方式见 UPSTREAM.md。 测试：492 passed（另有 1 个既有的 Windows/gbk 上游测试失败，与本改动无关）
This commit is contained in:
@@ -0,0 +1,197 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2025 [email protected]
|
||||
#
|
||||
# This file is part of MediaCrawler project.
|
||||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/monitor/platforms.py
|
||||
# GitHub: https://github.com/NanmiCoder
|
||||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||
#
|
||||
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
||||
# 5. 不得用于任何非法或不当的用途。
|
||||
#
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
"""Platform capability matrix.
|
||||
|
||||
The single source of truth for what each platform can do. The UI renders its
|
||||
platform switcher and metric columns from this, and the API validates against
|
||||
it.
|
||||
|
||||
Two distinct things are recorded here, and conflating them would be misleading:
|
||||
|
||||
* ``crawler_modes`` / ``metrics`` / ``comment_levels`` / ``media`` describe what
|
||||
the upstream crawler module actually supports. These were read out of the
|
||||
platform modules, not assumed -- all seven implement search/detail/creator;
|
||||
the real differences are in which interaction metrics they capture.
|
||||
* ``monitor_wired`` says whether the *monitoring layer* has been hooked up. It
|
||||
currently covers only Xiaohongshu: ``runner.py`` pins the platform,
|
||||
``ingest.py`` reads a fixed ``xhs/jsonl`` directory, and ``service.py`` only
|
||||
parses Xiaohongshu target URLs.
|
||||
|
||||
A platform can therefore be fully crawlable by upstream and still not usable for
|
||||
monitoring, which is exactly the state of the other six today.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
PLATFORM_XHS = "xhs"
|
||||
|
||||
PLATFORM_LABELS = {
|
||||
"xhs": "小红书",
|
||||
"dy": "抖音",
|
||||
"ks": "快手",
|
||||
"bili": "B站",
|
||||
"wb": "微博",
|
||||
"tieba": "贴吧",
|
||||
"zhihu": "知乎",
|
||||
}
|
||||
|
||||
# Interaction metrics each platform's store actually persists. Xiaohongshu has no
|
||||
# play count or danmaku; Bilibili has both and the widest set; Kuaishou carries
|
||||
# no comment/share/collect at all; Tieba stores only reply counts.
|
||||
PLATFORM_CAPABILITIES: Dict[str, Dict[str, Any]] = {
|
||||
"xhs": {
|
||||
"crawler_modes": ["search", "detail", "creator"],
|
||||
"metrics": ["liked_count", "comment_count", "collected_count", "share_count"],
|
||||
"comment_levels": 2,
|
||||
"media": True,
|
||||
"monitor_wired": True,
|
||||
},
|
||||
"dy": {
|
||||
"crawler_modes": ["search", "detail", "creator"],
|
||||
"metrics": ["liked_count", "comment_count", "collected_count", "share_count"],
|
||||
"comment_levels": 2,
|
||||
"media": True,
|
||||
"monitor_wired": False,
|
||||
},
|
||||
"ks": {
|
||||
"crawler_modes": ["search", "detail", "creator"],
|
||||
# No comment/share/collect in the Kuaishou store; sub-comments are stored
|
||||
# flat with no parent link and carry no like count.
|
||||
"metrics": ["liked_count", "view_count"],
|
||||
"comment_levels": 1,
|
||||
"media": True,
|
||||
"monitor_wired": False,
|
||||
},
|
||||
"bili": {
|
||||
"crawler_modes": ["search", "detail", "creator"],
|
||||
"metrics": [
|
||||
"liked_count",
|
||||
"video_play_count",
|
||||
"video_danmaku",
|
||||
"comment_count",
|
||||
"video_favorite_count",
|
||||
"video_coin_count",
|
||||
"video_share_count",
|
||||
],
|
||||
"comment_levels": 2,
|
||||
"media": True,
|
||||
"monitor_wired": False,
|
||||
},
|
||||
"wb": {
|
||||
"crawler_modes": ["search", "detail", "creator"],
|
||||
# Weibo has no collect count, and its comment count field is named
|
||||
# differently in the model.
|
||||
"metrics": ["liked_count", "comments_count", "shared_count"],
|
||||
"comment_levels": 2,
|
||||
"media": True,
|
||||
"monitor_wired": False,
|
||||
},
|
||||
"tieba": {
|
||||
"crawler_modes": ["search", "detail", "creator"],
|
||||
"metrics": ["total_replay_num", "total_replay_page"],
|
||||
"comment_levels": 2,
|
||||
"media": False,
|
||||
"monitor_wired": False,
|
||||
},
|
||||
"zhihu": {
|
||||
"crawler_modes": ["search", "detail", "creator"],
|
||||
"metrics": ["voteup_count", "comment_count"],
|
||||
"comment_levels": 2,
|
||||
"media": False,
|
||||
"monitor_wired": False,
|
||||
},
|
||||
}
|
||||
|
||||
METRIC_LABELS = {
|
||||
"liked_count": "点赞",
|
||||
"comment_count": "评论",
|
||||
"collected_count": "收藏",
|
||||
"share_count": "分享",
|
||||
"view_count": "播放",
|
||||
"video_play_count": "播放",
|
||||
"video_danmaku": "弹幕",
|
||||
"video_favorite_count": "收藏",
|
||||
"video_coin_count": "投币",
|
||||
"video_share_count": "分享",
|
||||
"comments_count": "评论",
|
||||
"shared_count": "转发",
|
||||
"total_replay_num": "回复数",
|
||||
"total_replay_page": "回复页数",
|
||||
"voteup_count": "赞同",
|
||||
}
|
||||
|
||||
# Monitoring modes, mapped to the CLI crawler types upstream understands.
|
||||
MONITOR_MODE_CREATOR = "creator"
|
||||
MONITOR_MODE_NOTE = "note"
|
||||
CLI_TYPE_FOR_MODE = {
|
||||
MONITOR_MODE_CREATOR: "creator",
|
||||
MONITOR_MODE_NOTE: "detail",
|
||||
}
|
||||
|
||||
|
||||
class UnsupportedPlatformError(ValueError):
|
||||
"""Raised for an unknown platform, or one the monitor layer cannot run."""
|
||||
|
||||
|
||||
def all_platforms() -> List[str]:
|
||||
return list(PLATFORM_CAPABILITIES)
|
||||
|
||||
|
||||
def is_known(platform: str) -> bool:
|
||||
return platform in PLATFORM_CAPABILITIES
|
||||
|
||||
|
||||
def is_monitor_wired(platform: str) -> bool:
|
||||
return bool(PLATFORM_CAPABILITIES.get(platform, {}).get("monitor_wired"))
|
||||
|
||||
|
||||
def describe(platform: str) -> Optional[Dict[str, Any]]:
|
||||
capability = PLATFORM_CAPABILITIES.get(platform)
|
||||
if capability is None:
|
||||
return None
|
||||
return {
|
||||
"value": platform,
|
||||
"label": PLATFORM_LABELS.get(platform, platform),
|
||||
**capability,
|
||||
"metric_labels": {
|
||||
metric: METRIC_LABELS.get(metric, metric) for metric in capability["metrics"]
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def describe_all() -> List[Dict[str, Any]]:
|
||||
return [describe(platform) for platform in all_platforms()]
|
||||
|
||||
|
||||
def ensure_runnable(platform: str) -> None:
|
||||
"""Validate a platform for a monitoring task.
|
||||
|
||||
An unwired platform is rejected outright rather than accepted and left to
|
||||
silently produce nothing -- the same silent-failure shape that made a valid
|
||||
creator look like an expired login earlier.
|
||||
"""
|
||||
if not is_known(platform):
|
||||
raise UnsupportedPlatformError(
|
||||
f"未知平台：{platform}（支持：{', '.join(all_platforms())}）"
|
||||
)
|
||||
if not is_monitor_wired(platform):
|
||||
label = PLATFORM_LABELS.get(platform, platform)
|
||||
raise UnsupportedPlatformError(
|
||||
f"{label}的爬虫已支持，但监控层尚未接通，暂时无法创建监控任务。"
|
||||
)
|
||||
Reference in New Issue
Block a user