butubb
4e60524f37
在上游 MediaCrawler 之上新增一层： - 监控层 api/monitor/ —— 多博主/多笔记的定时采集、指标快照差分、报表、 企业微信通知。每轮采集写入独立目录，差分才成立。 - WebUI 登录鉴权 api/auth.py —— PBKDF2 口令 + 服务端会话，/api 全接口防护。 WebSocket 单独加依赖：BaseHTTPMiddleware 对 ws 作用域直接放行，覆盖不到。 - 全局平台切换 + 能力矩阵 —— 如实区分「爬虫模块支持」与「监控层已接线」， 未接通的平台直接拒绝建任务，而不是静默跑空。 - 监控库改用 MySQL 5.7（可回退 SQLite 供测试）：逐表强制 utf8mb4 （服务端与库默认都是 latin1），启动校验所连 schema 以防写错库， 连接池 recycle + pre_ping 应对 MySQL 的 8 小时空闲断连。 修复上游缺陷： - xhs/core.py: 主页抓取失败会跳掉整个博主，导致一条作品都抓不到， 而那份资料只喂给一个空函数。改为尽力而为，失败不中断。 - xhs/login.py: cookie 登录只注入 web_session，冷启动签名会失败。 新增 INJECT_ALL_COOKIES 开关（默认关闭，原有行为不变）。 - requirements.txt: 补上 websockets。它在上游 pyproject.toml 里有声明、 这里漏了，导致 uvicorn 没有 WebSocket 能力，实时日志流从未工作。 改动过的上游文件清单及合并方式见 UPSTREAM.md。 测试：492 passed（另有 1 个既有的 Windows/gbk 上游测试失败，与本改动无关）
109 lines
3.9 KiB
Python
109 lines
3.9 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright (c) 2025 [email protected]
|
|
#
|
|
# This file is part of MediaCrawler project.
|
|
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/monitor/settings.py
|
|
# GitHub: https://github.com/NanmiCoder
|
|
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
|
#
|
|
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
|
# 1. 不得用于任何商业用途。
|
|
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
|
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
|
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
|
# 5. 不得用于任何非法或不当的用途。
|
|
#
|
|
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
|
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
|
|
|
"""Key/value settings for the monitoring layer, plus cookie health helpers.
|
|
|
|
The XHS cookie is what makes scheduled runs unattended. It expires every few
|
|
weeks, so alongside the value we track when it was last seen working -- that is
|
|
what lets the UI warn before a task silently stops collecting.
|
|
"""
|
|
|
|
from typing import Optional
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from tools.time_util import get_current_timestamp
|
|
|
|
from .models import MonitorSetting
|
|
from .platforms import PLATFORM_XHS
|
|
|
|
|
|
def platform_key(platform: str, name: str) -> str:
|
|
"""Key for a setting that each platform keeps its own copy of."""
|
|
return f"platform.{platform}.{name}"
|
|
|
|
|
|
def system_key(name: str) -> str:
|
|
"""Key for a setting shared across every platform."""
|
|
return f"system.{name}"
|
|
|
|
|
|
def cookie_key(platform: str) -> str:
|
|
return platform_key(platform, "cookie")
|
|
|
|
|
|
def cookie_updated_key(platform: str) -> str:
|
|
return platform_key(platform, "cookie_updated_at")
|
|
|
|
|
|
def cookie_last_ok_key(platform: str) -> str:
|
|
return platform_key(platform, "cookie_last_ok_at")
|
|
|
|
|
|
async def get_setting(session: AsyncSession, key: str) -> Optional[str]:
|
|
return await session.scalar(select(MonitorSetting.value).where(MonitorSetting.key == key))
|
|
|
|
|
|
async def set_setting(session: AsyncSession, key: str, value: str) -> None:
|
|
row = await session.get(MonitorSetting, key)
|
|
now = get_current_timestamp()
|
|
if row is None:
|
|
session.add(MonitorSetting(key=key, value=value, updated_at=now))
|
|
else:
|
|
row.value = value
|
|
row.updated_at = now
|
|
|
|
|
|
async def delete_setting(session: AsyncSession, key: str) -> None:
|
|
row = await session.get(MonitorSetting, key)
|
|
if row is not None:
|
|
await session.delete(row)
|
|
|
|
|
|
async def get_cookie(session: AsyncSession, platform: str = PLATFORM_XHS) -> str:
|
|
return (await get_setting(session, cookie_key(platform))) or ""
|
|
|
|
|
|
async def set_cookie(
|
|
session: AsyncSession, cookie: str, platform: str = PLATFORM_XHS
|
|
) -> None:
|
|
await set_setting(session, cookie_key(platform), cookie)
|
|
await set_setting(session, cookie_updated_key(platform), str(get_current_timestamp()))
|
|
|
|
|
|
async def mark_cookie_ok(session: AsyncSession, platform: str = PLATFORM_XHS) -> None:
|
|
"""Record that a run authenticated successfully."""
|
|
await set_setting(session, cookie_last_ok_key(platform), str(get_current_timestamp()))
|
|
|
|
|
|
async def get_cookie_status(session: AsyncSession, platform: str = PLATFORM_XHS) -> dict:
|
|
"""Cookie health for the UI. Never returns the cookie value itself."""
|
|
cookie = await get_cookie(session, platform)
|
|
updated_at = await get_setting(session, cookie_updated_key(platform))
|
|
last_ok_at = await get_setting(session, cookie_last_ok_key(platform))
|
|
|
|
return {
|
|
"platform": platform,
|
|
"present": bool(cookie),
|
|
# Enough to eyeball whether the pasted value looks right, not enough to leak it.
|
|
"length": len(cookie),
|
|
"updated_at": int(updated_at) if updated_at else None,
|
|
"last_ok_at": int(last_ok_at) if last_ok_at else None,
|
|
}
|