butubb
e348de48d3
本仓库在上游 MediaCrawler 之上加了一整层（见 UPSTREAM.md），可合并流程默认 「有人知道上游动了」。而部署是 git pull --ff-only，只从自己的 Gitea 拉——上游的 提交不主动 fetch 就永远看不见。拖着不合并的代价是复利的：越久越难合。 于是把「上游动了没有」变成一条会自己跑、会推企业微信的通知： * api/monitor/upstream.py：git fetch <url> <branch> 到 FETCH_HEAD，用 rev-list --count HEAD..FETCH_HEAD 算落后数、FETCH_HEAD..HEAD 算领先数。 用 git 而非托管商 API，因为只有 git 知道共同祖先在哪——本仓库含有上游没有的 提交，直接比 tip 会得出错误结论。增量 fetch 只传几个新提交，不会遇到 UPSTREAM.md 里说的「大包必断」。 * 只 fetch 到 FETCH_HEAD：不配 remote、不写 refs/remotes、不碰索引与工作区， 所以不打断正在跑的采集，也不和 deploy.sh 的 git pull 抢锁。 * 挂在调度器 tick 上（不是采集，所以不看 is_busy、不受活跃时段限制——定时检查 放在半夜反而最合适），按 checked_at + 间隔 到期才跑；失败也写 checked_at， 于是 GitHub 不通时是每间隔重试一次，而不是每个 tick 撞一次墙。 * 同一个 tip 只推一次（记 tip 而不是「推过没」），上游真又动了会再推。 * 两个接口：GET /monitor/upstream 只读缓存；POST /monitor/upstream/check 手动 查一次且刻意不推通知——点按钮的人正看着结果。 * 默认关闭，间隔默认一天。 Dockerfile 显式装 git（python:slim 不带，而这是唯一的依赖）；deploy.sh 顺带补上 一个真 bug 的提示：Dockerfile/requirements.txt 变了只 up -d 用的还是旧镜像。
464 lines
15 KiB
Python
464 lines
15 KiB
Python
# -*- coding: utf-8 -*-
|
||
# Copyright (c) 2025 [email protected]
|
||
#
|
||
# This file is part of MediaCrawler project.
|
||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/monitor/app_settings.py
|
||
# GitHub: https://github.com/NanmiCoder
|
||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||
#
|
||
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
||
# 1. 不得用于任何商业用途。
|
||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
||
# 5. 不得用于任何非法或不当的用途。
|
||
#
|
||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||
|
||
"""Application settings, declared once and rendered from that declaration.
|
||
|
||
Every setting carries a **scope**, which is the whole reason this is not a flat
|
||
list:
|
||
|
||
* ``platform`` -- each platform keeps its own copy. A cookie obviously differs,
|
||
but so do crawl pacing and proxies: what is safe on one platform is a rate
|
||
limit on another. Stored as ``platform.<p>.<name>``.
|
||
* ``system`` -- one value for the whole instance. The notification webhook is
|
||
a single group chat, and the scheduler has a single active-hours window, so
|
||
scoping those per platform would be a fiction.
|
||
|
||
The registry is the single source of truth: the API returns it and the Settings
|
||
page builds its form from it, so adding a setting does not mean editing a
|
||
matching list on the frontend.
|
||
|
||
Two rules carry over from how the cookie and webhook were already handled:
|
||
|
||
* **Secrets are never returned.** A sensitive key comes back as
|
||
``{present, length, updated_at}``, never as a value.
|
||
* **Update is partial.** Only keys present in the request are written, so a form
|
||
that does not resubmit a secret cannot silently wipe it.
|
||
"""
|
||
|
||
from dataclasses import dataclass
|
||
from typing import Any, Dict, List, Optional
|
||
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from .platforms import PLATFORM_XHS
|
||
from .settings import (
|
||
delete_setting,
|
||
get_setting,
|
||
platform_key,
|
||
set_setting,
|
||
system_key,
|
||
)
|
||
from .upstream import DEFAULT_BRANCH, DEFAULT_REMOTE_URL
|
||
|
||
SCOPE_PLATFORM = "platform"
|
||
SCOPE_SYSTEM = "system"
|
||
|
||
TYPE_BOOL = "bool"
|
||
TYPE_INT = "int"
|
||
TYPE_STR = "str"
|
||
TYPE_SECRET = "secret"
|
||
|
||
# Mirrors config/base_config.py. Nothing is written until the operator changes
|
||
# something; an unset value simply means "pass no CLI flag, so the config file's
|
||
# value applies".
|
||
_DEFAULT_SLEEP_SEC = 2
|
||
|
||
|
||
@dataclass
|
||
class SettingSpec:
|
||
name: str
|
||
scope: str
|
||
type: str
|
||
label: str
|
||
help: str = ""
|
||
default: Any = None
|
||
minimum: Optional[int] = None
|
||
maximum: Optional[int] = None
|
||
choices: Optional[List[str]] = None
|
||
affects_new_runs: bool = True
|
||
|
||
def key(self, platform: str = PLATFORM_XHS) -> str:
|
||
if self.scope == SCOPE_SYSTEM:
|
||
return system_key(self.name)
|
||
return platform_key(platform, self.name)
|
||
|
||
|
||
SETTING_SPECS: List[SettingSpec] = [
|
||
# --- 平台设置 -----------------------------------------------------------
|
||
SettingSpec(
|
||
name="cookie",
|
||
scope=SCOPE_PLATFORM,
|
||
type=TYPE_SECRET,
|
||
label="登录 Cookie",
|
||
help="定时监控必须持久化登录态。建议先手动登录一次再粘贴 Cookie。",
|
||
),
|
||
SettingSpec(
|
||
name="default_interval_minutes",
|
||
scope=SCOPE_PLATFORM,
|
||
type=TYPE_INT,
|
||
label="新任务默认采集间隔（分钟）",
|
||
help="仅影响新建任务时的默认值，不会改动已有任务。",
|
||
default=360,
|
||
minimum=30,
|
||
maximum=10080,
|
||
),
|
||
SettingSpec(
|
||
name="default_max_notes",
|
||
scope=SCOPE_PLATFORM,
|
||
type=TYPE_INT,
|
||
label="默认单轮作品上限",
|
||
default=20,
|
||
minimum=1,
|
||
maximum=500,
|
||
),
|
||
SettingSpec(
|
||
name="default_max_comments",
|
||
scope=SCOPE_PLATFORM,
|
||
type=TYPE_INT,
|
||
label="默认每篇评论抓取条数",
|
||
help="接口无时间排序，只取平台默认排序的前 N 条；N 越大越容易发现新评论。",
|
||
default=50,
|
||
minimum=1,
|
||
maximum=500,
|
||
),
|
||
SettingSpec(
|
||
name="enable_sub_comments",
|
||
scope=SCOPE_PLATFORM,
|
||
type=TYPE_BOOL,
|
||
label="抓取二级评论",
|
||
help="请求量显著增加，风控风险更高。",
|
||
default=False,
|
||
),
|
||
SettingSpec(
|
||
name="crawl_sleep_sec",
|
||
scope=SCOPE_PLATFORM,
|
||
type=TYPE_INT,
|
||
label="请求间隔（秒）",
|
||
help="调大更慢但更不容易触发平台限流。各平台风控容忍度不同，故分开配置。",
|
||
default=_DEFAULT_SLEEP_SEC,
|
||
minimum=0,
|
||
maximum=600,
|
||
),
|
||
SettingSpec(
|
||
name="enable_ip_proxy",
|
||
scope=SCOPE_PLATFORM,
|
||
type=TYPE_BOOL,
|
||
label="启用 IP 代理",
|
||
default=False,
|
||
),
|
||
SettingSpec(
|
||
name="proxy_provider",
|
||
scope=SCOPE_PLATFORM,
|
||
type=TYPE_STR,
|
||
label="代理提供方",
|
||
default="kuaidaili",
|
||
choices=["kuaidaili", "wandouhttp", "static"],
|
||
),
|
||
SettingSpec(
|
||
name="proxy_pool_count",
|
||
scope=SCOPE_PLATFORM,
|
||
type=TYPE_INT,
|
||
label="代理 IP 池大小",
|
||
default=2,
|
||
minimum=1,
|
||
maximum=100,
|
||
),
|
||
SettingSpec(
|
||
name="static_proxy_url",
|
||
scope=SCOPE_PLATFORM,
|
||
type=TYPE_STR,
|
||
label="静态代理地址",
|
||
help="仅当提供方选择 static 时使用，格式 http://host:port",
|
||
default="",
|
||
),
|
||
# --- 系统设置 -----------------------------------------------------------
|
||
SettingSpec(
|
||
name="wecom_webhook",
|
||
scope=SCOPE_SYSTEM,
|
||
type=TYPE_SECRET,
|
||
label="企业微信 Webhook",
|
||
help="企业微信群机器人地址。所有平台共用同一个群，只有开了推送开关的任务才会发消息。",
|
||
),
|
||
SettingSpec(
|
||
name="active_hours_start",
|
||
scope=SCOPE_SYSTEM,
|
||
type=TYPE_INT,
|
||
label="活跃时段开始（小时）",
|
||
help="只在此时段内触发定时采集。默认 0–23 即全天；支持跨午夜，如 22–6。",
|
||
default=0,
|
||
minimum=0,
|
||
maximum=23,
|
||
affects_new_runs=False,
|
||
),
|
||
SettingSpec(
|
||
name="active_hours_end",
|
||
scope=SCOPE_SYSTEM,
|
||
type=TYPE_INT,
|
||
label="活跃时段结束（小时）",
|
||
default=23,
|
||
minimum=0,
|
||
maximum=23,
|
||
affects_new_runs=False,
|
||
),
|
||
SettingSpec(
|
||
name="cdp_enabled",
|
||
scope=SCOPE_SYSTEM,
|
||
type=TYPE_BOOL,
|
||
label="接管已有 Chrome（CDP）",
|
||
help=(
|
||
"开启后爬虫不再自己启动浏览器，而是接管本机已开放远程调试端口的 Chrome"
|
||
"（默认 127.0.0.1:9222），复用它的登录态与扩展。"
|
||
"服务器部署请开启；本机桌面使用请保持关闭。"
|
||
),
|
||
default=False,
|
||
),
|
||
# --- 上游更新检查 -------------------------------------------------------
|
||
# 这几项不作用于采集，所以都标 affects_new_runs=False：改动它们不需要等下一轮，
|
||
# 也不影响采集命令的拼装。
|
||
SettingSpec(
|
||
name="upstream_check_enabled",
|
||
scope=SCOPE_SYSTEM,
|
||
type=TYPE_BOOL,
|
||
label="检查上游仓库更新",
|
||
help=(
|
||
"定期 fetch 上游仓库，看看它有没有新提交，并在有更新时推送通知。"
|
||
"本仓库在上游之上加了一整层（见 UPSTREAM.md），不定期看一眼就会越拖越难合并。"
|
||
),
|
||
default=False,
|
||
affects_new_runs=False,
|
||
),
|
||
SettingSpec(
|
||
name="upstream_check_interval_minutes",
|
||
scope=SCOPE_SYSTEM,
|
||
type=TYPE_INT,
|
||
label="上游检查间隔（分钟）",
|
||
help="默认 1440 分钟（每天一次）。检查只是 fetch，不需要太频繁。",
|
||
default=1440,
|
||
minimum=30,
|
||
maximum=10080,
|
||
affects_new_runs=False,
|
||
),
|
||
SettingSpec(
|
||
name="upstream_remote_url",
|
||
scope=SCOPE_SYSTEM,
|
||
type=TYPE_STR,
|
||
label="上游仓库地址",
|
||
help=(
|
||
"默认是 GitHub 上的上游。国内直连 GitHub 不稳时改成 gitcode 镜像"
|
||
"（见 UPSTREAM.md），或任意能访问到上游的地址。"
|
||
),
|
||
default=DEFAULT_REMOTE_URL,
|
||
affects_new_runs=False,
|
||
),
|
||
SettingSpec(
|
||
name="upstream_branch",
|
||
scope=SCOPE_SYSTEM,
|
||
type=TYPE_STR,
|
||
label="上游分支",
|
||
default=DEFAULT_BRANCH,
|
||
affects_new_runs=False,
|
||
),
|
||
SettingSpec(
|
||
name="upstream_notify",
|
||
scope=SCOPE_SYSTEM,
|
||
type=TYPE_BOOL,
|
||
label="上游有更新时推送通知",
|
||
help="只在出现此前没推过的上游提交时发一条，同一个更新不会反复推。",
|
||
default=True,
|
||
affects_new_runs=False,
|
||
),
|
||
]
|
||
|
||
SPECS_BY_NAME = {spec.name: spec for spec in SETTING_SPECS}
|
||
|
||
# Managed by their own endpoints; never writable through the settings API.
|
||
# Suffix-matched rather than enumerated, because the cookie bookkeeping keys
|
||
# exist once per platform.
|
||
_HIDDEN_KEY_SUFFIXES = (".cookie_updated_at", ".cookie_last_ok_at")
|
||
_HIDDEN_KEYS = {"auth_password_hash", "auth_password_updated_at"}
|
||
|
||
|
||
def _is_hidden(key: str) -> bool:
|
||
return key in _HIDDEN_KEYS or key.endswith(_HIDDEN_KEY_SUFFIXES)
|
||
|
||
|
||
class SettingValidationError(ValueError):
|
||
"""Raised for a value the registry will not accept."""
|
||
|
||
|
||
def _coerce(spec: SettingSpec, raw: Any) -> Any:
|
||
if spec.type == TYPE_SECRET:
|
||
return str(raw) if raw is not None else ""
|
||
|
||
if spec.type == TYPE_BOOL:
|
||
if isinstance(raw, bool):
|
||
return raw
|
||
text = str(raw).strip().lower()
|
||
if text in ("1", "true", "yes", "y", "on"):
|
||
return True
|
||
if text in ("0", "false", "no", "n", "off", ""):
|
||
return False
|
||
raise SettingValidationError(f"{spec.label}: 需要是/否")
|
||
|
||
if spec.type == TYPE_INT:
|
||
try:
|
||
value = int(raw)
|
||
except (TypeError, ValueError):
|
||
raise SettingValidationError(f"{spec.label}: 需要整数")
|
||
if spec.minimum is not None and value < spec.minimum:
|
||
raise SettingValidationError(f"{spec.label}: 不能小于 {spec.minimum}")
|
||
if spec.maximum is not None and value > spec.maximum:
|
||
raise SettingValidationError(f"{spec.label}: 不能大于 {spec.maximum}")
|
||
return value
|
||
|
||
value = str(raw) if raw is not None else ""
|
||
if spec.choices and value not in spec.choices:
|
||
raise SettingValidationError(f"{spec.label}: 只能是 {'/'.join(spec.choices)}")
|
||
return value
|
||
|
||
|
||
def _decode(spec: SettingSpec, raw: Optional[str]) -> Any:
|
||
if raw is None:
|
||
return spec.default
|
||
if spec.type == TYPE_BOOL:
|
||
return raw.strip().lower() in ("1", "true", "yes", "y", "on")
|
||
if spec.type == TYPE_INT:
|
||
try:
|
||
return int(raw)
|
||
except ValueError:
|
||
return spec.default
|
||
return raw
|
||
|
||
|
||
def _encode(spec: SettingSpec, value: Any) -> str:
|
||
if spec.type == TYPE_BOOL:
|
||
return "true" if value else "false"
|
||
return str(value)
|
||
|
||
|
||
def _describe(spec: SettingSpec, platform: str) -> Dict[str, Any]:
|
||
return {
|
||
"key": spec.key(platform),
|
||
"name": spec.name,
|
||
"scope": spec.scope,
|
||
"type": spec.type,
|
||
"label": spec.label,
|
||
"help": spec.help,
|
||
"default": spec.default,
|
||
"minimum": spec.minimum,
|
||
"maximum": spec.maximum,
|
||
"choices": spec.choices,
|
||
"affects_new_runs": spec.affects_new_runs,
|
||
}
|
||
|
||
|
||
async def get_all(session: AsyncSession, platform: str = PLATFORM_XHS) -> Dict[str, Any]:
|
||
"""Every editable setting for one platform, plus the system-wide ones.
|
||
|
||
Secrets come back masked, never in the clear.
|
||
"""
|
||
values: Dict[str, Any] = {}
|
||
secrets: Dict[str, Any] = {}
|
||
|
||
for spec in SETTING_SPECS:
|
||
key = spec.key(platform)
|
||
raw = await get_setting(session, key)
|
||
|
||
if spec.type == TYPE_SECRET:
|
||
secrets[key] = {"present": bool(raw), "length": len(raw or "")}
|
||
else:
|
||
values[key] = _decode(spec, raw)
|
||
|
||
return {
|
||
"platform": platform,
|
||
"values": values,
|
||
"secrets": secrets,
|
||
"specs": [_describe(spec, platform) for spec in SETTING_SPECS],
|
||
}
|
||
|
||
|
||
def _spec_for_key(key: str, platform: str) -> Optional[SettingSpec]:
|
||
"""Resolve a full key back to its spec, rejecting keys for another platform."""
|
||
for spec in SETTING_SPECS:
|
||
if spec.key(platform) == key:
|
||
return spec
|
||
return None
|
||
|
||
|
||
async def update(
|
||
session: AsyncSession, payload: Dict[str, Any], platform: str = PLATFORM_XHS
|
||
) -> List[str]:
|
||
"""Apply a partial update. Returns the keys that changed.
|
||
|
||
Only keys present in ``payload`` are touched: a form that omits a secret must
|
||
not blank it. Keys belonging to a different platform are rejected rather than
|
||
silently written somewhere unexpected.
|
||
"""
|
||
changed: List[str] = []
|
||
|
||
for key, raw in payload.items():
|
||
if _is_hidden(key):
|
||
continue
|
||
|
||
spec = _spec_for_key(key, platform)
|
||
if spec is None:
|
||
raise SettingValidationError(f"未知的设置项：{key}")
|
||
|
||
# An explicit empty string clears a secret -- that is how the UI removes
|
||
# one. For everything else it is just a value.
|
||
if spec.type == TYPE_SECRET and raw == "":
|
||
await delete_setting(session, key)
|
||
changed.append(key)
|
||
continue
|
||
|
||
value = _coerce(spec, raw)
|
||
await set_setting(session, key, _encode(spec, value))
|
||
changed.append(key)
|
||
|
||
return changed
|
||
|
||
|
||
async def get_value(
|
||
session: AsyncSession,
|
||
name: str,
|
||
platform: str = PLATFORM_XHS,
|
||
fallback: Any = None,
|
||
) -> Any:
|
||
"""Read one typed setting for internal callers (the runner, the scheduler)."""
|
||
spec = SPECS_BY_NAME.get(name)
|
||
if spec is None:
|
||
return fallback
|
||
raw = await get_setting(session, spec.key(platform))
|
||
if raw is None:
|
||
return spec.default if fallback is None else fallback
|
||
return _decode(spec, raw)
|
||
|
||
|
||
async def defaults(session: AsyncSession, platform: str = PLATFORM_XHS) -> Dict[str, Any]:
|
||
"""Defaults applied when creating a task on this platform.
|
||
|
||
This is what makes the Settings page govern new tasks: the create endpoint
|
||
falls back to these for anything the caller omits.
|
||
"""
|
||
return {
|
||
"interval_minutes": int(
|
||
await get_value(session, "default_interval_minutes", platform, 360)
|
||
),
|
||
"max_notes_count": int(await get_value(session, "default_max_notes", platform, 20)),
|
||
"max_comments_count": int(
|
||
await get_value(session, "default_max_comments", platform, 50)
|
||
),
|
||
}
|
||
|
||
|
||
async def active_hours(session: AsyncSession) -> tuple[int, int]:
|
||
"""The (start, end) hour window for scheduled runs. System-wide."""
|
||
start = await get_value(session, "active_hours_start", fallback=0)
|
||
end = await get_value(session, "active_hours_end", fallback=23)
|
||
return int(start), int(end)
|