feat: 监控面板 / 登录鉴权 / 多平台切换 / MySQL
在上游 MediaCrawler 之上新增一层： - 监控层 api/monitor/ —— 多博主/多笔记的定时采集、指标快照差分、报表、 企业微信通知。每轮采集写入独立目录，差分才成立。 - WebUI 登录鉴权 api/auth.py —— PBKDF2 口令 + 服务端会话，/api 全接口防护。 WebSocket 单独加依赖：BaseHTTPMiddleware 对 ws 作用域直接放行，覆盖不到。 - 全局平台切换 + 能力矩阵 —— 如实区分「爬虫模块支持」与「监控层已接线」， 未接通的平台直接拒绝建任务，而不是静默跑空。 - 监控库改用 MySQL 5.7（可回退 SQLite 供测试）：逐表强制 utf8mb4 （服务端与库默认都是 latin1），启动校验所连 schema 以防写错库， 连接池 recycle + pre_ping 应对 MySQL 的 8 小时空闲断连。 修复上游缺陷： - xhs/core.py: 主页抓取失败会跳掉整个博主，导致一条作品都抓不到， 而那份资料只喂给一个空函数。改为尽力而为，失败不中断。 - xhs/login.py: cookie 登录只注入 web_session，冷启动签名会失败。 新增 INJECT_ALL_COOKIES 开关（默认关闭，原有行为不变）。 - requirements.txt: 补上 websockets。它在上游 pyproject.toml 里有声明、 这里漏了，导致 uvicorn 没有 WebSocket 能力，实时日志流从未工作。 改动过的上游文件清单及合并方式见 UPSTREAM.md。 测试：492 passed（另有 1 个既有的 Windows/gbk 上游测试失败，与本改动无关）
This commit is contained in:
@@ -0,0 +1,200 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2025 [email protected]
|
||||
#
|
||||
# This file is part of MediaCrawler project.
|
||||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/monitor/notify.py
|
||||
# GitHub: https://github.com/NanmiCoder
|
||||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||
#
|
||||
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
||||
# 5. 不得用于任何非法或不当的用途。
|
||||
#
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
"""Push notifications via a WeCom (企业微信) group robot webhook.
|
||||
|
||||
Two rules shape this module:
|
||||
|
||||
* **One message per run, not per event.** A run that finds twenty new notes must
|
||||
produce one summary, not twenty pushes.
|
||||
* **A failed push never fails the crawl.** Notification is best-effort: the run's
|
||||
data is already committed by the time we get here, so every error is logged
|
||||
and swallowed.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from tools.time_util import get_current_timestamp
|
||||
|
||||
from .models import (
|
||||
EVENT_AUTH_FAILURE,
|
||||
EVENT_NEW_NOTE,
|
||||
EVENT_NO_DATA,
|
||||
EVENT_RUN_FAILED,
|
||||
MonitorEvent,
|
||||
MonitorRun,
|
||||
MonitorTask,
|
||||
)
|
||||
from .settings import get_setting
|
||||
|
||||
# Short on purpose: the scheduler awaits the run, so a hanging webhook would
|
||||
# stall every other task behind it.
|
||||
WEBHOOK_TIMEOUT_SECONDS = 10.0
|
||||
|
||||
# Only these event types are worth interrupting someone for. NO_DATA is included
|
||||
# because a run that fetched nothing at all is always anomalous -- a creator
|
||||
# always has *some* notes -- even when the login is not the culprit.
|
||||
NOTIFIABLE_EVENT_TYPES = (
|
||||
EVENT_AUTH_FAILURE,
|
||||
EVENT_RUN_FAILED,
|
||||
EVENT_NO_DATA,
|
||||
EVENT_NEW_NOTE,
|
||||
)
|
||||
|
||||
# WeCom markdown is a limited subset; coloured text is the one bit of flair it
|
||||
# supports and it makes failures stand out in a busy group chat.
|
||||
_COLOR_WARNING = "warning"
|
||||
_COLOR_INFO = "info"
|
||||
|
||||
|
||||
async def send_wecom(webhook_url: str, content: str) -> tuple[bool, str]:
|
||||
"""Post a markdown message to a WeCom group robot.
|
||||
|
||||
Returns (ok, detail) rather than raising, so callers can surface the reason
|
||||
in the UI when the user clicks "send test".
|
||||
"""
|
||||
if not webhook_url:
|
||||
return False, "Webhook 未配置"
|
||||
|
||||
payload = {"msgtype": "markdown", "markdown": {"content": content}}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=WEBHOOK_TIMEOUT_SECONDS) as client:
|
||||
response = await client.post(webhook_url, json=payload)
|
||||
response.raise_for_status()
|
||||
body = response.json()
|
||||
except httpx.HTTPError as exc:
|
||||
return False, f"请求失败：{exc}"
|
||||
except json.JSONDecodeError:
|
||||
return False, "返回内容不是合法 JSON，请检查 Webhook 地址"
|
||||
|
||||
# WeCom answers 200 with a non-zero errcode on failure.
|
||||
errcode = body.get("errcode")
|
||||
if errcode != 0:
|
||||
return False, f"企业微信返回 errcode={errcode} {body.get('errmsg', '')}"
|
||||
|
||||
return True, "发送成功"
|
||||
|
||||
|
||||
async def get_webhook_url(session: AsyncSession) -> str:
|
||||
from .models import SETTING_WECOM_WEBHOOK
|
||||
|
||||
return (await get_setting(session, SETTING_WECOM_WEBHOOK)) or ""
|
||||
|
||||
|
||||
async def build_run_message(
|
||||
session: AsyncSession,
|
||||
task: MonitorTask,
|
||||
run: MonitorRun,
|
||||
) -> Optional[str]:
|
||||
"""Compose one markdown summary for a finished run, or None if nothing to say."""
|
||||
events = list(
|
||||
(
|
||||
await session.scalars(
|
||||
select(MonitorEvent)
|
||||
.where(
|
||||
MonitorEvent.run_id == run.id,
|
||||
MonitorEvent.type.in_(NOTIFIABLE_EVENT_TYPES),
|
||||
)
|
||||
.order_by(MonitorEvent.id)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
if not events:
|
||||
return None
|
||||
|
||||
failures = [
|
||||
e for e in events if e.type in (EVENT_AUTH_FAILURE, EVENT_RUN_FAILED, EVENT_NO_DATA)
|
||||
]
|
||||
new_notes = [e for e in events if e.type == EVENT_NEW_NOTE]
|
||||
|
||||
lines: list[str] = []
|
||||
|
||||
if failures:
|
||||
# Word the header from what actually happened, not from whether new notes
|
||||
# accompanied it: a login outage usually brings no new notes either.
|
||||
unavailable = any(e.type == EVENT_NO_DATA for e in failures) and not any(
|
||||
e.type in (EVENT_AUTH_FAILURE, EVENT_RUN_FAILED) for e in failures
|
||||
)
|
||||
header = "监控任务未抓到数据" if unavailable else "监控任务异常"
|
||||
lines.append(f"**⚠️ {header}：{task.name}**")
|
||||
for event in failures:
|
||||
lines.append(f'> <font color="{_COLOR_WARNING}">{event.title}</font>')
|
||||
else:
|
||||
lines.append(f"**📢 监控任务有新作品：{task.name}**")
|
||||
|
||||
if new_notes:
|
||||
lines.append(f"> 新增作品 **{len(new_notes)}** 篇")
|
||||
# Cap the listing: a first-ever run or a long gap can produce a lot, and
|
||||
# a wall of text is worse than a count.
|
||||
for event in new_notes[:10]:
|
||||
payload = _load_payload(event.payload_json)
|
||||
title = payload.get("title") or event.target_id
|
||||
note_id = payload.get("note_id") or event.target_id
|
||||
url = f"https://www.xiaohongshu.com/explore/{note_id}"
|
||||
lines.append(f"> [{title}]({url})")
|
||||
if len(new_notes) > 10:
|
||||
lines.append(f"> …等共 {len(new_notes)} 篇")
|
||||
|
||||
if run.is_baseline:
|
||||
lines.append("> （首轮基线，未计入新增统计）")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _load_payload(raw: str) -> dict:
|
||||
try:
|
||||
payload = json.loads(raw or "{}")
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
async def notify_run(session: AsyncSession, task: MonitorTask, run: MonitorRun) -> Optional[str]:
|
||||
"""Push a summary for a finished run if the task opted in.
|
||||
|
||||
Returns the message that was sent, or None. Never raises.
|
||||
"""
|
||||
try:
|
||||
if not task.notify_enabled:
|
||||
return None
|
||||
|
||||
webhook_url = await get_webhook_url(session)
|
||||
if not webhook_url:
|
||||
return None
|
||||
|
||||
message = await build_run_message(session, task, run)
|
||||
if not message:
|
||||
return None
|
||||
|
||||
ok, detail = await send_wecom(webhook_url, message)
|
||||
if not ok:
|
||||
print(f"[monitor.notify] task {task.id} push failed: {detail}")
|
||||
return None
|
||||
|
||||
task.last_notified_at = get_current_timestamp()
|
||||
return message
|
||||
|
||||
except Exception as exc: # pragma: no cover - notification must never break a run
|
||||
print(f"[monitor.notify] unexpected error: {exc}")
|
||||
return None
|
||||
Reference in New Issue
Block a user