butubb
37ca1b1cd6
- runner: enable_cdp_mode 从硬编码 False 改为系统设置 cdp_enabled。服务器部署下爬虫接管已开启远程调试的 Chrome（默认 9222），复用其 profile 登录态；本机桌面默认仍为关，行为不变 - qrlogin: 新增 CDP 扫码登录。Chrome 在服务器上跑于 Xvfb，show_qrcode 依赖的 PIL 桌面看图程序不存在，二维码无处可显示；改为经 CDP 从页面取出二维码交给 WebUI 渲染。刻意复用 browser.contexts[0]（新建 context 是无痕 profile，扫了也白扫），且绝不调用 browser.close()（会连带关掉操作者自己的 Chrome） - webui: 设置页新增扫码面板，替换原本跳到采集页看终端二维码的入口 - Dockerfile / .dockerignore / docker-compose.yml: 服务器部署。host 网络是必需而非图省事——容器里 127.0.0.1:9222 必须落到宿主机回环 - UPSTREAM.md: 补充 gitcode 镜像，用于 GitHub 大包传输必断时补历史
408 lines
13 KiB
Python
408 lines
13 KiB
Python
# -*- coding: utf-8 -*-
|
||
# Copyright (c) 2025 [email protected]
|
||
#
|
||
# This file is part of MediaCrawler project.
|
||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/monitor/app_settings.py
|
||
# GitHub: https://github.com/NanmiCoder
|
||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||
#
|
||
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
||
# 1. 不得用于任何商业用途。
|
||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
||
# 5. 不得用于任何非法或不当的用途。
|
||
#
|
||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||
|
||
"""Application settings, declared once and rendered from that declaration.
|
||
|
||
Every setting carries a **scope**, which is the whole reason this is not a flat
|
||
list:
|
||
|
||
* ``platform`` -- each platform keeps its own copy. A cookie obviously differs,
|
||
but so do crawl pacing and proxies: what is safe on one platform is a rate
|
||
limit on another. Stored as ``platform.<p>.<name>``.
|
||
* ``system`` -- one value for the whole instance. The notification webhook is
|
||
a single group chat, and the scheduler has a single active-hours window, so
|
||
scoping those per platform would be a fiction.
|
||
|
||
The registry is the single source of truth: the API returns it and the Settings
|
||
page builds its form from it, so adding a setting does not mean editing a
|
||
matching list on the frontend.
|
||
|
||
Two rules carry over from how the cookie and webhook were already handled:
|
||
|
||
* **Secrets are never returned.** A sensitive key comes back as
|
||
``{present, length, updated_at}``, never as a value.
|
||
* **Update is partial.** Only keys present in the request are written, so a form
|
||
that does not resubmit a secret cannot silently wipe it.
|
||
"""
|
||
|
||
from dataclasses import dataclass
|
||
from typing import Any, Dict, List, Optional
|
||
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from .platforms import PLATFORM_XHS
|
||
from .settings import (
|
||
delete_setting,
|
||
get_setting,
|
||
platform_key,
|
||
set_setting,
|
||
system_key,
|
||
)
|
||
|
||
SCOPE_PLATFORM = "platform"
|
||
SCOPE_SYSTEM = "system"
|
||
|
||
TYPE_BOOL = "bool"
|
||
TYPE_INT = "int"
|
||
TYPE_STR = "str"
|
||
TYPE_SECRET = "secret"
|
||
|
||
# Mirrors config/base_config.py. Nothing is written until the operator changes
|
||
# something; an unset value simply means "pass no CLI flag, so the config file's
|
||
# value applies".
|
||
_DEFAULT_SLEEP_SEC = 2
|
||
|
||
|
||
@dataclass
|
||
class SettingSpec:
|
||
name: str
|
||
scope: str
|
||
type: str
|
||
label: str
|
||
help: str = ""
|
||
default: Any = None
|
||
minimum: Optional[int] = None
|
||
maximum: Optional[int] = None
|
||
choices: Optional[List[str]] = None
|
||
affects_new_runs: bool = True
|
||
|
||
def key(self, platform: str = PLATFORM_XHS) -> str:
|
||
if self.scope == SCOPE_SYSTEM:
|
||
return system_key(self.name)
|
||
return platform_key(platform, self.name)
|
||
|
||
|
||
SETTING_SPECS: List[SettingSpec] = [
|
||
# --- 平台设置 -----------------------------------------------------------
|
||
SettingSpec(
|
||
name="cookie",
|
||
scope=SCOPE_PLATFORM,
|
||
type=TYPE_SECRET,
|
||
label="登录 Cookie",
|
||
help="定时监控必须持久化登录态。建议先手动登录一次再粘贴 Cookie。",
|
||
),
|
||
SettingSpec(
|
||
name="default_interval_minutes",
|
||
scope=SCOPE_PLATFORM,
|
||
type=TYPE_INT,
|
||
label="新任务默认采集间隔（分钟）",
|
||
help="仅影响新建任务时的默认值，不会改动已有任务。",
|
||
default=360,
|
||
minimum=30,
|
||
maximum=10080,
|
||
),
|
||
SettingSpec(
|
||
name="default_max_notes",
|
||
scope=SCOPE_PLATFORM,
|
||
type=TYPE_INT,
|
||
label="默认单轮作品上限",
|
||
default=20,
|
||
minimum=1,
|
||
maximum=500,
|
||
),
|
||
SettingSpec(
|
||
name="default_max_comments",
|
||
scope=SCOPE_PLATFORM,
|
||
type=TYPE_INT,
|
||
label="默认每篇评论抓取条数",
|
||
help="接口无时间排序，只取平台默认排序的前 N 条；N 越大越容易发现新评论。",
|
||
default=50,
|
||
minimum=1,
|
||
maximum=500,
|
||
),
|
||
SettingSpec(
|
||
name="enable_sub_comments",
|
||
scope=SCOPE_PLATFORM,
|
||
type=TYPE_BOOL,
|
||
label="抓取二级评论",
|
||
help="请求量显著增加，风控风险更高。",
|
||
default=False,
|
||
),
|
||
SettingSpec(
|
||
name="crawl_sleep_sec",
|
||
scope=SCOPE_PLATFORM,
|
||
type=TYPE_INT,
|
||
label="请求间隔（秒）",
|
||
help="调大更慢但更不容易触发平台限流。各平台风控容忍度不同，故分开配置。",
|
||
default=_DEFAULT_SLEEP_SEC,
|
||
minimum=0,
|
||
maximum=600,
|
||
),
|
||
SettingSpec(
|
||
name="enable_ip_proxy",
|
||
scope=SCOPE_PLATFORM,
|
||
type=TYPE_BOOL,
|
||
label="启用 IP 代理",
|
||
default=False,
|
||
),
|
||
SettingSpec(
|
||
name="proxy_provider",
|
||
scope=SCOPE_PLATFORM,
|
||
type=TYPE_STR,
|
||
label="代理提供方",
|
||
default="kuaidaili",
|
||
choices=["kuaidaili", "wandouhttp", "static"],
|
||
),
|
||
SettingSpec(
|
||
name="proxy_pool_count",
|
||
scope=SCOPE_PLATFORM,
|
||
type=TYPE_INT,
|
||
label="代理 IP 池大小",
|
||
default=2,
|
||
minimum=1,
|
||
maximum=100,
|
||
),
|
||
SettingSpec(
|
||
name="static_proxy_url",
|
||
scope=SCOPE_PLATFORM,
|
||
type=TYPE_STR,
|
||
label="静态代理地址",
|
||
help="仅当提供方选择 static 时使用，格式 http://host:port",
|
||
default="",
|
||
),
|
||
# --- 系统设置 -----------------------------------------------------------
|
||
SettingSpec(
|
||
name="wecom_webhook",
|
||
scope=SCOPE_SYSTEM,
|
||
type=TYPE_SECRET,
|
||
label="企业微信 Webhook",
|
||
help="企业微信群机器人地址。所有平台共用同一个群，只有开了推送开关的任务才会发消息。",
|
||
),
|
||
SettingSpec(
|
||
name="active_hours_start",
|
||
scope=SCOPE_SYSTEM,
|
||
type=TYPE_INT,
|
||
label="活跃时段开始（小时）",
|
||
help="只在此时段内触发定时采集。默认 0–23 即全天；支持跨午夜，如 22–6。",
|
||
default=0,
|
||
minimum=0,
|
||
maximum=23,
|
||
affects_new_runs=False,
|
||
),
|
||
SettingSpec(
|
||
name="active_hours_end",
|
||
scope=SCOPE_SYSTEM,
|
||
type=TYPE_INT,
|
||
label="活跃时段结束（小时）",
|
||
default=23,
|
||
minimum=0,
|
||
maximum=23,
|
||
affects_new_runs=False,
|
||
),
|
||
SettingSpec(
|
||
name="cdp_enabled",
|
||
scope=SCOPE_SYSTEM,
|
||
type=TYPE_BOOL,
|
||
label="接管已有 Chrome（CDP）",
|
||
help=(
|
||
"开启后爬虫不再自己启动浏览器，而是接管本机已开放远程调试端口的 Chrome"
|
||
"（默认 127.0.0.1:9222），复用它的登录态与扩展。"
|
||
"服务器部署请开启；本机桌面使用请保持关闭。"
|
||
),
|
||
default=False,
|
||
),
|
||
]
|
||
|
||
SPECS_BY_NAME = {spec.name: spec for spec in SETTING_SPECS}
|
||
|
||
# Managed by their own endpoints; never writable through the settings API.
|
||
# Suffix-matched rather than enumerated, because the cookie bookkeeping keys
|
||
# exist once per platform.
|
||
_HIDDEN_KEY_SUFFIXES = (".cookie_updated_at", ".cookie_last_ok_at")
|
||
_HIDDEN_KEYS = {"auth_password_hash", "auth_password_updated_at"}
|
||
|
||
|
||
def _is_hidden(key: str) -> bool:
|
||
return key in _HIDDEN_KEYS or key.endswith(_HIDDEN_KEY_SUFFIXES)
|
||
|
||
|
||
class SettingValidationError(ValueError):
|
||
"""Raised for a value the registry will not accept."""
|
||
|
||
|
||
def _coerce(spec: SettingSpec, raw: Any) -> Any:
|
||
if spec.type == TYPE_SECRET:
|
||
return str(raw) if raw is not None else ""
|
||
|
||
if spec.type == TYPE_BOOL:
|
||
if isinstance(raw, bool):
|
||
return raw
|
||
text = str(raw).strip().lower()
|
||
if text in ("1", "true", "yes", "y", "on"):
|
||
return True
|
||
if text in ("0", "false", "no", "n", "off", ""):
|
||
return False
|
||
raise SettingValidationError(f"{spec.label}: 需要是/否")
|
||
|
||
if spec.type == TYPE_INT:
|
||
try:
|
||
value = int(raw)
|
||
except (TypeError, ValueError):
|
||
raise SettingValidationError(f"{spec.label}: 需要整数")
|
||
if spec.minimum is not None and value < spec.minimum:
|
||
raise SettingValidationError(f"{spec.label}: 不能小于 {spec.minimum}")
|
||
if spec.maximum is not None and value > spec.maximum:
|
||
raise SettingValidationError(f"{spec.label}: 不能大于 {spec.maximum}")
|
||
return value
|
||
|
||
value = str(raw) if raw is not None else ""
|
||
if spec.choices and value not in spec.choices:
|
||
raise SettingValidationError(f"{spec.label}: 只能是 {'/'.join(spec.choices)}")
|
||
return value
|
||
|
||
|
||
def _decode(spec: SettingSpec, raw: Optional[str]) -> Any:
|
||
if raw is None:
|
||
return spec.default
|
||
if spec.type == TYPE_BOOL:
|
||
return raw.strip().lower() in ("1", "true", "yes", "y", "on")
|
||
if spec.type == TYPE_INT:
|
||
try:
|
||
return int(raw)
|
||
except ValueError:
|
||
return spec.default
|
||
return raw
|
||
|
||
|
||
def _encode(spec: SettingSpec, value: Any) -> str:
|
||
if spec.type == TYPE_BOOL:
|
||
return "true" if value else "false"
|
||
return str(value)
|
||
|
||
|
||
def _describe(spec: SettingSpec, platform: str) -> Dict[str, Any]:
|
||
return {
|
||
"key": spec.key(platform),
|
||
"name": spec.name,
|
||
"scope": spec.scope,
|
||
"type": spec.type,
|
||
"label": spec.label,
|
||
"help": spec.help,
|
||
"default": spec.default,
|
||
"minimum": spec.minimum,
|
||
"maximum": spec.maximum,
|
||
"choices": spec.choices,
|
||
"affects_new_runs": spec.affects_new_runs,
|
||
}
|
||
|
||
|
||
async def get_all(session: AsyncSession, platform: str = PLATFORM_XHS) -> Dict[str, Any]:
|
||
"""Every editable setting for one platform, plus the system-wide ones.
|
||
|
||
Secrets come back masked, never in the clear.
|
||
"""
|
||
values: Dict[str, Any] = {}
|
||
secrets: Dict[str, Any] = {}
|
||
|
||
for spec in SETTING_SPECS:
|
||
key = spec.key(platform)
|
||
raw = await get_setting(session, key)
|
||
|
||
if spec.type == TYPE_SECRET:
|
||
secrets[key] = {"present": bool(raw), "length": len(raw or "")}
|
||
else:
|
||
values[key] = _decode(spec, raw)
|
||
|
||
return {
|
||
"platform": platform,
|
||
"values": values,
|
||
"secrets": secrets,
|
||
"specs": [_describe(spec, platform) for spec in SETTING_SPECS],
|
||
}
|
||
|
||
|
||
def _spec_for_key(key: str, platform: str) -> Optional[SettingSpec]:
|
||
"""Resolve a full key back to its spec, rejecting keys for another platform."""
|
||
for spec in SETTING_SPECS:
|
||
if spec.key(platform) == key:
|
||
return spec
|
||
return None
|
||
|
||
|
||
async def update(
|
||
session: AsyncSession, payload: Dict[str, Any], platform: str = PLATFORM_XHS
|
||
) -> List[str]:
|
||
"""Apply a partial update. Returns the keys that changed.
|
||
|
||
Only keys present in ``payload`` are touched: a form that omits a secret must
|
||
not blank it. Keys belonging to a different platform are rejected rather than
|
||
silently written somewhere unexpected.
|
||
"""
|
||
changed: List[str] = []
|
||
|
||
for key, raw in payload.items():
|
||
if _is_hidden(key):
|
||
continue
|
||
|
||
spec = _spec_for_key(key, platform)
|
||
if spec is None:
|
||
raise SettingValidationError(f"未知的设置项：{key}")
|
||
|
||
# An explicit empty string clears a secret -- that is how the UI removes
|
||
# one. For everything else it is just a value.
|
||
if spec.type == TYPE_SECRET and raw == "":
|
||
await delete_setting(session, key)
|
||
changed.append(key)
|
||
continue
|
||
|
||
value = _coerce(spec, raw)
|
||
await set_setting(session, key, _encode(spec, value))
|
||
changed.append(key)
|
||
|
||
return changed
|
||
|
||
|
||
async def get_value(
|
||
session: AsyncSession,
|
||
name: str,
|
||
platform: str = PLATFORM_XHS,
|
||
fallback: Any = None,
|
||
) -> Any:
|
||
"""Read one typed setting for internal callers (the runner, the scheduler)."""
|
||
spec = SPECS_BY_NAME.get(name)
|
||
if spec is None:
|
||
return fallback
|
||
raw = await get_setting(session, spec.key(platform))
|
||
if raw is None:
|
||
return spec.default if fallback is None else fallback
|
||
return _decode(spec, raw)
|
||
|
||
|
||
async def defaults(session: AsyncSession, platform: str = PLATFORM_XHS) -> Dict[str, Any]:
|
||
"""Defaults applied when creating a task on this platform.
|
||
|
||
This is what makes the Settings page govern new tasks: the create endpoint
|
||
falls back to these for anything the caller omits.
|
||
"""
|
||
return {
|
||
"interval_minutes": int(
|
||
await get_value(session, "default_interval_minutes", platform, 360)
|
||
),
|
||
"max_notes_count": int(await get_value(session, "default_max_notes", platform, 20)),
|
||
"max_comments_count": int(
|
||
await get_value(session, "default_max_comments", platform, 50)
|
||
),
|
||
}
|
||
|
||
|
||
async def active_hours(session: AsyncSession) -> tuple[int, int]:
|
||
"""The (start, end) hour window for scheduled runs. System-wide."""
|
||
start = await get_value(session, "active_hours_start", fallback=0)
|
||
end = await get_value(session, "active_hours_end", fallback=23)
|
||
return int(start), int(end)
|