MediaCrawler/api/monitor/app_settings.py

# -*- coding: utf-8 -*-
# Copyright (c) 2025 [email protected]
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/monitor/app_settings.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
# 1. 不得用于任何商业用途。
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
# 3. 不得进行大规模爬取或对平台造成运营干扰。
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
# 5. 不得用于任何非法或不当的用途。
#
# 详细许可条款请参阅项目根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。

"""Application settings, declared once and rendered from that declaration.

Every setting carries a **scope**, which is the whole reason this is not a flat
list:

* ``platform`` -- each platform keeps its own copy. A cookie obviously differs,
  but so do crawl pacing and proxies: what is safe on one platform is a rate
  limit on another. Stored as ``platform.<p>.<name>``.
* ``system``   -- one value for the whole instance. The notification webhook is
  a single group chat, and the scheduler has a single active-hours window, so
  scoping those per platform would be a fiction.

The registry is the single source of truth: the API returns it and the Settings
page builds its form from it, so adding a setting does not mean editing a
matching list on the frontend.

Two rules carry over from how the cookie and webhook were already handled:

* **Secrets are never returned.** A sensitive key comes back as
  ``{present, length, updated_at}``, never as a value.
* **Update is partial.** Only keys present in the request are written, so a form
  that does not resubmit a secret cannot silently wipe it.
"""

from dataclasses import dataclass
from typing import Any, Dict, List, Optional

from sqlalchemy.ext.asyncio import AsyncSession

from .platforms import PLATFORM_XHS
from .settings import (
    delete_setting,
    get_setting,
    platform_key,
    set_setting,
    system_key,
)

SCOPE_PLATFORM = "platform"
SCOPE_SYSTEM = "system"

TYPE_BOOL = "bool"
TYPE_INT = "int"
TYPE_STR = "str"
TYPE_SECRET = "secret"

# Mirrors config/base_config.py. Nothing is written until the operator changes
# something; an unset value simply means "pass no CLI flag, so the config file's
# value applies".
_DEFAULT_SLEEP_SEC = 2


@dataclass
class SettingSpec:
    name: str
    scope: str
    type: str
    label: str
    help: str = ""
    default: Any = None
    minimum: Optional[int] = None
    maximum: Optional[int] = None
    choices: Optional[List[str]] = None
    affects_new_runs: bool = True

    def key(self, platform: str = PLATFORM_XHS) -> str:
        if self.scope == SCOPE_SYSTEM:
            return system_key(self.name)
        return platform_key(platform, self.name)


SETTING_SPECS: List[SettingSpec] = [
    # --- 平台设置 -----------------------------------------------------------
    SettingSpec(
        name="cookie",
        scope=SCOPE_PLATFORM,
        type=TYPE_SECRET,
        label="登录 Cookie",
        help="定时监控必须持久化登录态。建议先手动登录一次再粘贴 Cookie。",
    ),
    SettingSpec(
        name="default_interval_minutes",
        scope=SCOPE_PLATFORM,
        type=TYPE_INT,
        label="新任务默认采集间隔（分钟）",
        help="仅影响新建任务时的默认值，不会改动已有任务。",
        default=360,
        minimum=30,
        maximum=10080,
    ),
    SettingSpec(
        name="default_max_notes",
        scope=SCOPE_PLATFORM,
        type=TYPE_INT,
        label="默认单轮作品上限",
        default=20,
        minimum=1,
        maximum=500,
    ),
    SettingSpec(
        name="default_max_comments",
        scope=SCOPE_PLATFORM,
        type=TYPE_INT,
        label="默认每篇评论抓取条数",
        help="接口无时间排序，只取平台默认排序的前 N 条；N 越大越容易发现新评论。",
        default=50,
        minimum=1,
        maximum=500,
    ),
    SettingSpec(
        name="enable_sub_comments",
        scope=SCOPE_PLATFORM,
        type=TYPE_BOOL,
        label="抓取二级评论",
        help="请求量显著增加，风控风险更高。",
        default=False,
    ),
    SettingSpec(
        name="crawl_sleep_sec",
        scope=SCOPE_PLATFORM,
        type=TYPE_INT,
        label="请求间隔（秒）",
        help="调大更慢但更不容易触发平台限流。各平台风控容忍度不同，故分开配置。",
        default=_DEFAULT_SLEEP_SEC,
        minimum=0,
        maximum=600,
    ),
    SettingSpec(
        name="enable_ip_proxy",
        scope=SCOPE_PLATFORM,
        type=TYPE_BOOL,
        label="启用 IP 代理",
        default=False,
    ),
    SettingSpec(
        name="proxy_provider",
        scope=SCOPE_PLATFORM,
        type=TYPE_STR,
        label="代理提供方",
        default="kuaidaili",
        choices=["kuaidaili", "wandouhttp", "static"],
    ),
    SettingSpec(
        name="proxy_pool_count",
        scope=SCOPE_PLATFORM,
        type=TYPE_INT,
        label="代理 IP 池大小",
        default=2,
        minimum=1,
        maximum=100,
    ),
    SettingSpec(
        name="static_proxy_url",
        scope=SCOPE_PLATFORM,
        type=TYPE_STR,
        label="静态代理地址",
        help="仅当提供方选择 static 时使用，格式 http://host:port",
        default="",
    ),
    # --- 系统设置 -----------------------------------------------------------
    SettingSpec(
        name="wecom_webhook",
        scope=SCOPE_SYSTEM,
        type=TYPE_SECRET,
        label="企业微信 Webhook",
        help="企业微信群机器人地址。所有平台共用同一个群，只有开了推送开关的任务才会发消息。",
    ),
    SettingSpec(
        name="active_hours_start",
        scope=SCOPE_SYSTEM,
        type=TYPE_INT,
        label="活跃时段开始（小时）",
        help="只在此时段内触发定时采集。默认 0–23 即全天；支持跨午夜，如 22–6。",
        default=0,
        minimum=0,
        maximum=23,
        affects_new_runs=False,
    ),
    SettingSpec(
        name="active_hours_end",
        scope=SCOPE_SYSTEM,
        type=TYPE_INT,
        label="活跃时段结束（小时）",
        default=23,
        minimum=0,
        maximum=23,
        affects_new_runs=False,
    ),
]

SPECS_BY_NAME = {spec.name: spec for spec in SETTING_SPECS}

# Managed by their own endpoints; never writable through the settings API.
# Suffix-matched rather than enumerated, because the cookie bookkeeping keys
# exist once per platform.
_HIDDEN_KEY_SUFFIXES = (".cookie_updated_at", ".cookie_last_ok_at")
_HIDDEN_KEYS = {"auth_password_hash", "auth_password_updated_at"}


def _is_hidden(key: str) -> bool:
    return key in _HIDDEN_KEYS or key.endswith(_HIDDEN_KEY_SUFFIXES)


class SettingValidationError(ValueError):
    """Raised for a value the registry will not accept."""


def _coerce(spec: SettingSpec, raw: Any) -> Any:
    if spec.type == TYPE_SECRET:
        return str(raw) if raw is not None else ""

    if spec.type == TYPE_BOOL:
        if isinstance(raw, bool):
            return raw
        text = str(raw).strip().lower()
        if text in ("1", "true", "yes", "y", "on"):
            return True
        if text in ("0", "false", "no", "n", "off", ""):
            return False
        raise SettingValidationError(f"{spec.label}: 需要是/否")

    if spec.type == TYPE_INT:
        try:
            value = int(raw)
        except (TypeError, ValueError):
            raise SettingValidationError(f"{spec.label}: 需要整数")
        if spec.minimum is not None and value < spec.minimum:
            raise SettingValidationError(f"{spec.label}: 不能小于 {spec.minimum}")
        if spec.maximum is not None and value > spec.maximum:
            raise SettingValidationError(f"{spec.label}: 不能大于 {spec.maximum}")
        return value

    value = str(raw) if raw is not None else ""
    if spec.choices and value not in spec.choices:
        raise SettingValidationError(f"{spec.label}: 只能是 {'/'.join(spec.choices)}")
    return value


def _decode(spec: SettingSpec, raw: Optional[str]) -> Any:
    if raw is None:
        return spec.default
    if spec.type == TYPE_BOOL:
        return raw.strip().lower() in ("1", "true", "yes", "y", "on")
    if spec.type == TYPE_INT:
        try:
            return int(raw)
        except ValueError:
            return spec.default
    return raw


def _encode(spec: SettingSpec, value: Any) -> str:
    if spec.type == TYPE_BOOL:
        return "true" if value else "false"
    return str(value)


def _describe(spec: SettingSpec, platform: str) -> Dict[str, Any]:
    return {
        "key": spec.key(platform),
        "name": spec.name,
        "scope": spec.scope,
        "type": spec.type,
        "label": spec.label,
        "help": spec.help,
        "default": spec.default,
        "minimum": spec.minimum,
        "maximum": spec.maximum,
        "choices": spec.choices,
        "affects_new_runs": spec.affects_new_runs,
    }


async def get_all(session: AsyncSession, platform: str = PLATFORM_XHS) -> Dict[str, Any]:
    """Every editable setting for one platform, plus the system-wide ones.

    Secrets come back masked, never in the clear.
    """
    values: Dict[str, Any] = {}
    secrets: Dict[str, Any] = {}

    for spec in SETTING_SPECS:
        key = spec.key(platform)
        raw = await get_setting(session, key)

        if spec.type == TYPE_SECRET:
            secrets[key] = {"present": bool(raw), "length": len(raw or "")}
        else:
            values[key] = _decode(spec, raw)

    return {
        "platform": platform,
        "values": values,
        "secrets": secrets,
        "specs": [_describe(spec, platform) for spec in SETTING_SPECS],
    }


def _spec_for_key(key: str, platform: str) -> Optional[SettingSpec]:
    """Resolve a full key back to its spec, rejecting keys for another platform."""
    for spec in SETTING_SPECS:
        if spec.key(platform) == key:
            return spec
    return None


async def update(
    session: AsyncSession, payload: Dict[str, Any], platform: str = PLATFORM_XHS
) -> List[str]:
    """Apply a partial update. Returns the keys that changed.

    Only keys present in ``payload`` are touched: a form that omits a secret must
    not blank it. Keys belonging to a different platform are rejected rather than
    silently written somewhere unexpected.
    """
    changed: List[str] = []

    for key, raw in payload.items():
        if _is_hidden(key):
            continue

        spec = _spec_for_key(key, platform)
        if spec is None:
            raise SettingValidationError(f"未知的设置项：{key}")

        # An explicit empty string clears a secret -- that is how the UI removes
        # one. For everything else it is just a value.
        if spec.type == TYPE_SECRET and raw == "":
            await delete_setting(session, key)
            changed.append(key)
            continue

        value = _coerce(spec, raw)
        await set_setting(session, key, _encode(spec, value))
        changed.append(key)

    return changed


async def get_value(
    session: AsyncSession,
    name: str,
    platform: str = PLATFORM_XHS,
    fallback: Any = None,
) -> Any:
    """Read one typed setting for internal callers (the runner, the scheduler)."""
    spec = SPECS_BY_NAME.get(name)
    if spec is None:
        return fallback
    raw = await get_setting(session, spec.key(platform))
    if raw is None:
        return spec.default if fallback is None else fallback
    return _decode(spec, raw)


async def defaults(session: AsyncSession, platform: str = PLATFORM_XHS) -> Dict[str, Any]:
    """Defaults applied when creating a task on this platform.

    This is what makes the Settings page govern new tasks: the create endpoint
    falls back to these for anything the caller omits.
    """
    return {
        "interval_minutes": int(
            await get_value(session, "default_interval_minutes", platform, 360)
        ),
        "max_notes_count": int(await get_value(session, "default_max_notes", platform, 20)),
        "max_comments_count": int(
            await get_value(session, "default_max_comments", platform, 50)
        ),
    }


async def active_hours(session: AsyncSession) -> tuple[int, int]:
    """The (start, end) hour window for scheduled runs. System-wide."""
    start = await get_value(session, "active_hours_start", fallback=0)
    end = await get_value(session, "active_hours_end", fallback=23)
    return int(start), int(end)