# -*- coding: utf-8 -*- # Copyright (c) 2025 relakkes@gmail.com # # This file is part of MediaCrawler project. # Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/monitor/app_settings.py # GitHub: https://github.com/NanmiCoder # Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 # # 声明:本代码仅供学习和研究目的使用。使用者应遵守以下原则: # 1. 不得用于任何商业用途。 # 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 # 3. 不得进行大规模爬取或对平台造成运营干扰。 # 4. 应合理控制请求频率,避免给目标平台带来不必要的负担。 # 5. 不得用于任何非法或不当的用途。 # # 详细许可条款请参阅项目根目录下的LICENSE文件。 # 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 """Application settings, declared once and rendered from that declaration. Every setting carries a **scope**, which is the whole reason this is not a flat list: * ``platform`` -- each platform keeps its own copy. A cookie obviously differs, but so do crawl pacing and proxies: what is safe on one platform is a rate limit on another. Stored as ``platform.

.``. * ``system`` -- one value for the whole instance. The notification webhook is a single group chat, and the scheduler has a single active-hours window, so scoping those per platform would be a fiction. The registry is the single source of truth: the API returns it and the Settings page builds its form from it, so adding a setting does not mean editing a matching list on the frontend. Two rules carry over from how the cookie and webhook were already handled: * **Secrets are never returned.** A sensitive key comes back as ``{present, length, updated_at}``, never as a value. * **Update is partial.** Only keys present in the request are written, so a form that does not resubmit a secret cannot silently wipe it. """ from dataclasses import dataclass from typing import Any, Dict, List, Optional from sqlalchemy.ext.asyncio import AsyncSession from .platforms import PLATFORM_XHS from .settings import ( delete_setting, get_setting, platform_key, set_setting, system_key, ) SCOPE_PLATFORM = "platform" SCOPE_SYSTEM = "system" TYPE_BOOL = "bool" TYPE_INT = "int" TYPE_STR = "str" TYPE_SECRET = "secret" # Mirrors config/base_config.py. Nothing is written until the operator changes # something; an unset value simply means "pass no CLI flag, so the config file's # value applies". _DEFAULT_SLEEP_SEC = 2 @dataclass class SettingSpec: name: str scope: str type: str label: str help: str = "" default: Any = None minimum: Optional[int] = None maximum: Optional[int] = None choices: Optional[List[str]] = None affects_new_runs: bool = True def key(self, platform: str = PLATFORM_XHS) -> str: if self.scope == SCOPE_SYSTEM: return system_key(self.name) return platform_key(platform, self.name) SETTING_SPECS: List[SettingSpec] = [ # --- 平台设置 ----------------------------------------------------------- SettingSpec( name="cookie", scope=SCOPE_PLATFORM, type=TYPE_SECRET, label="登录 Cookie", help="定时监控必须持久化登录态。建议先手动登录一次再粘贴 Cookie。", ), SettingSpec( name="default_interval_minutes", scope=SCOPE_PLATFORM, type=TYPE_INT, label="新任务默认采集间隔(分钟)", help="仅影响新建任务时的默认值,不会改动已有任务。", default=360, minimum=30, maximum=10080, ), SettingSpec( name="default_max_notes", scope=SCOPE_PLATFORM, type=TYPE_INT, label="默认单轮作品上限", default=20, minimum=1, maximum=500, ), SettingSpec( name="default_max_comments", scope=SCOPE_PLATFORM, type=TYPE_INT, label="默认每篇评论抓取条数", help="接口无时间排序,只取平台默认排序的前 N 条;N 越大越容易发现新评论。", default=50, minimum=1, maximum=500, ), SettingSpec( name="enable_sub_comments", scope=SCOPE_PLATFORM, type=TYPE_BOOL, label="抓取二级评论", help="请求量显著增加,风控风险更高。", default=False, ), SettingSpec( name="crawl_sleep_sec", scope=SCOPE_PLATFORM, type=TYPE_INT, label="请求间隔(秒)", help="调大更慢但更不容易触发平台限流。各平台风控容忍度不同,故分开配置。", default=_DEFAULT_SLEEP_SEC, minimum=0, maximum=600, ), SettingSpec( name="enable_ip_proxy", scope=SCOPE_PLATFORM, type=TYPE_BOOL, label="启用 IP 代理", default=False, ), SettingSpec( name="proxy_provider", scope=SCOPE_PLATFORM, type=TYPE_STR, label="代理提供方", default="kuaidaili", choices=["kuaidaili", "wandouhttp", "static"], ), SettingSpec( name="proxy_pool_count", scope=SCOPE_PLATFORM, type=TYPE_INT, label="代理 IP 池大小", default=2, minimum=1, maximum=100, ), SettingSpec( name="static_proxy_url", scope=SCOPE_PLATFORM, type=TYPE_STR, label="静态代理地址", help="仅当提供方选择 static 时使用,格式 http://host:port", default="", ), # --- 系统设置 ----------------------------------------------------------- SettingSpec( name="wecom_webhook", scope=SCOPE_SYSTEM, type=TYPE_SECRET, label="企业微信 Webhook", help="企业微信群机器人地址。所有平台共用同一个群,只有开了推送开关的任务才会发消息。", ), SettingSpec( name="active_hours_start", scope=SCOPE_SYSTEM, type=TYPE_INT, label="活跃时段开始(小时)", help="只在此时段内触发定时采集。默认 0–23 即全天;支持跨午夜,如 22–6。", default=0, minimum=0, maximum=23, affects_new_runs=False, ), SettingSpec( name="active_hours_end", scope=SCOPE_SYSTEM, type=TYPE_INT, label="活跃时段结束(小时)", default=23, minimum=0, maximum=23, affects_new_runs=False, ), SettingSpec( name="cdp_enabled", scope=SCOPE_SYSTEM, type=TYPE_BOOL, label="接管已有 Chrome(CDP)", help=( "开启后爬虫不再自己启动浏览器,而是接管本机已开放远程调试端口的 Chrome" "(默认 127.0.0.1:9222),复用它的登录态与扩展。" "服务器部署请开启;本机桌面使用请保持关闭。" ), default=False, ), ] SPECS_BY_NAME = {spec.name: spec for spec in SETTING_SPECS} # Managed by their own endpoints; never writable through the settings API. # Suffix-matched rather than enumerated, because the cookie bookkeeping keys # exist once per platform. _HIDDEN_KEY_SUFFIXES = (".cookie_updated_at", ".cookie_last_ok_at") _HIDDEN_KEYS = {"auth_password_hash", "auth_password_updated_at"} def _is_hidden(key: str) -> bool: return key in _HIDDEN_KEYS or key.endswith(_HIDDEN_KEY_SUFFIXES) class SettingValidationError(ValueError): """Raised for a value the registry will not accept.""" def _coerce(spec: SettingSpec, raw: Any) -> Any: if spec.type == TYPE_SECRET: return str(raw) if raw is not None else "" if spec.type == TYPE_BOOL: if isinstance(raw, bool): return raw text = str(raw).strip().lower() if text in ("1", "true", "yes", "y", "on"): return True if text in ("0", "false", "no", "n", "off", ""): return False raise SettingValidationError(f"{spec.label}: 需要是/否") if spec.type == TYPE_INT: try: value = int(raw) except (TypeError, ValueError): raise SettingValidationError(f"{spec.label}: 需要整数") if spec.minimum is not None and value < spec.minimum: raise SettingValidationError(f"{spec.label}: 不能小于 {spec.minimum}") if spec.maximum is not None and value > spec.maximum: raise SettingValidationError(f"{spec.label}: 不能大于 {spec.maximum}") return value value = str(raw) if raw is not None else "" if spec.choices and value not in spec.choices: raise SettingValidationError(f"{spec.label}: 只能是 {'/'.join(spec.choices)}") return value def _decode(spec: SettingSpec, raw: Optional[str]) -> Any: if raw is None: return spec.default if spec.type == TYPE_BOOL: return raw.strip().lower() in ("1", "true", "yes", "y", "on") if spec.type == TYPE_INT: try: return int(raw) except ValueError: return spec.default return raw def _encode(spec: SettingSpec, value: Any) -> str: if spec.type == TYPE_BOOL: return "true" if value else "false" return str(value) def _describe(spec: SettingSpec, platform: str) -> Dict[str, Any]: return { "key": spec.key(platform), "name": spec.name, "scope": spec.scope, "type": spec.type, "label": spec.label, "help": spec.help, "default": spec.default, "minimum": spec.minimum, "maximum": spec.maximum, "choices": spec.choices, "affects_new_runs": spec.affects_new_runs, } async def get_all(session: AsyncSession, platform: str = PLATFORM_XHS) -> Dict[str, Any]: """Every editable setting for one platform, plus the system-wide ones. Secrets come back masked, never in the clear. """ values: Dict[str, Any] = {} secrets: Dict[str, Any] = {} for spec in SETTING_SPECS: key = spec.key(platform) raw = await get_setting(session, key) if spec.type == TYPE_SECRET: secrets[key] = {"present": bool(raw), "length": len(raw or "")} else: values[key] = _decode(spec, raw) return { "platform": platform, "values": values, "secrets": secrets, "specs": [_describe(spec, platform) for spec in SETTING_SPECS], } def _spec_for_key(key: str, platform: str) -> Optional[SettingSpec]: """Resolve a full key back to its spec, rejecting keys for another platform.""" for spec in SETTING_SPECS: if spec.key(platform) == key: return spec return None async def update( session: AsyncSession, payload: Dict[str, Any], platform: str = PLATFORM_XHS ) -> List[str]: """Apply a partial update. Returns the keys that changed. Only keys present in ``payload`` are touched: a form that omits a secret must not blank it. Keys belonging to a different platform are rejected rather than silently written somewhere unexpected. """ changed: List[str] = [] for key, raw in payload.items(): if _is_hidden(key): continue spec = _spec_for_key(key, platform) if spec is None: raise SettingValidationError(f"未知的设置项:{key}") # An explicit empty string clears a secret -- that is how the UI removes # one. For everything else it is just a value. if spec.type == TYPE_SECRET and raw == "": await delete_setting(session, key) changed.append(key) continue value = _coerce(spec, raw) await set_setting(session, key, _encode(spec, value)) changed.append(key) return changed async def get_value( session: AsyncSession, name: str, platform: str = PLATFORM_XHS, fallback: Any = None, ) -> Any: """Read one typed setting for internal callers (the runner, the scheduler).""" spec = SPECS_BY_NAME.get(name) if spec is None: return fallback raw = await get_setting(session, spec.key(platform)) if raw is None: return spec.default if fallback is None else fallback return _decode(spec, raw) async def defaults(session: AsyncSession, platform: str = PLATFORM_XHS) -> Dict[str, Any]: """Defaults applied when creating a task on this platform. This is what makes the Settings page govern new tasks: the create endpoint falls back to these for anything the caller omits. """ return { "interval_minutes": int( await get_value(session, "default_interval_minutes", platform, 360) ), "max_notes_count": int(await get_value(session, "default_max_notes", platform, 20)), "max_comments_count": int( await get_value(session, "default_max_comments", platform, 50) ), } async def active_hours(session: AsyncSession) -> tuple[int, int]: """The (start, end) hour window for scheduled runs. System-wide.""" start = await get_value(session, "active_hours_start", fallback=0) end = await get_value(session, "active_hours_end", fallback=23) return int(start), int(end)