butubb
4e60524f37
在上游 MediaCrawler 之上新增一层： - 监控层 api/monitor/ —— 多博主/多笔记的定时采集、指标快照差分、报表、 企业微信通知。每轮采集写入独立目录，差分才成立。 - WebUI 登录鉴权 api/auth.py —— PBKDF2 口令 + 服务端会话，/api 全接口防护。 WebSocket 单独加依赖：BaseHTTPMiddleware 对 ws 作用域直接放行，覆盖不到。 - 全局平台切换 + 能力矩阵 —— 如实区分「爬虫模块支持」与「监控层已接线」， 未接通的平台直接拒绝建任务，而不是静默跑空。 - 监控库改用 MySQL 5.7（可回退 SQLite 供测试）：逐表强制 utf8mb4 （服务端与库默认都是 latin1），启动校验所连 schema 以防写错库， 连接池 recycle + pre_ping 应对 MySQL 的 8 小时空闲断连。 修复上游缺陷： - xhs/core.py: 主页抓取失败会跳掉整个博主，导致一条作品都抓不到， 而那份资料只喂给一个空函数。改为尽力而为，失败不中断。 - xhs/login.py: cookie 登录只注入 web_session，冷启动签名会失败。 新增 INJECT_ALL_COOKIES 开关（默认关闭，原有行为不变）。 - requirements.txt: 补上 websockets。它在上游 pyproject.toml 里有声明、 这里漏了，导致 uvicorn 没有 WebSocket 能力，实时日志流从未工作。 改动过的上游文件清单及合并方式见 UPSTREAM.md。 测试：492 passed（另有 1 个既有的 Windows/gbk 上游测试失败，与本改动无关）
375 lines
14 KiB
Python
375 lines
14 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright (c) 2025 [email protected]
|
|
#
|
|
# This file is part of MediaCrawler project.
|
|
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_settings.py
|
|
# GitHub: https://github.com/NanmiCoder
|
|
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
|
#
|
|
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
|
# 1. 不得用于任何商业用途。
|
|
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
|
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
|
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
|
# 5. 不得用于任何非法或不当的用途。
|
|
#
|
|
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
|
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
|
|
|
"""Unified settings endpoint, and the effect its values actually have."""
|
|
|
|
from datetime import datetime
|
|
|
|
import httpx
|
|
import pytest
|
|
import pytest_asyncio
|
|
|
|
from api.main import app
|
|
from api.monitor import app_settings, db as monitor_db
|
|
from api.monitor import scheduler as scheduler_module
|
|
from api.monitor.scheduler import MonitorScheduler
|
|
from api.monitor.settings import get_setting
|
|
|
|
SECRET_VALUE = "web_session=SUPERSECRET; a1=abc"
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def client(tmp_path):
|
|
monitor_db.set_sqlite_path(tmp_path / "monitor.db")
|
|
await monitor_db.init_db()
|
|
|
|
transport = httpx.ASGITransport(app=app)
|
|
async with httpx.AsyncClient(transport=transport, base_url="http://test") as http_client:
|
|
yield http_client
|
|
|
|
await monitor_db.dispose_engine()
|
|
|
|
|
|
class TestReadSettings:
|
|
@pytest.mark.asyncio
|
|
async def test_returns_values_secrets_and_the_spec(self, client):
|
|
body = (await client.get("/api/settings")).json()
|
|
|
|
assert "values" in body and "secrets" in body and "specs" in body
|
|
# The spec drives the UI form, so every key must be described.
|
|
spec_keys = {spec["key"] for spec in body["specs"]}
|
|
assert "platform.xhs.default_interval_minutes" in spec_keys
|
|
assert "platform.xhs.enable_ip_proxy" in spec_keys
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unset_values_fall_back_to_spec_defaults(self, client):
|
|
values = (await client.get("/api/settings")).json()["values"]
|
|
assert values["platform.xhs.default_interval_minutes"] == 360
|
|
assert values["platform.xhs.enable_ip_proxy"] is False
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_secrets_are_masked_never_returned(self, client):
|
|
await client.put("/api/settings", json={"platform.xhs.cookie": SECRET_VALUE})
|
|
|
|
response = await client.get("/api/settings")
|
|
assert SECRET_VALUE not in response.text
|
|
|
|
secret = response.json()["secrets"]["platform.xhs.cookie"]
|
|
assert secret["present"] is True
|
|
assert secret["length"] == len(SECRET_VALUE)
|
|
|
|
|
|
class TestUpdateSettings:
|
|
@pytest.mark.asyncio
|
|
async def test_partial_update_leaves_other_keys_alone(self, client):
|
|
await client.put(
|
|
"/api/settings",
|
|
json={"platform.xhs.default_interval_minutes": 120, "platform.xhs.cookie": SECRET_VALUE},
|
|
)
|
|
|
|
# A form that only submits the interval must not blank the cookie.
|
|
await client.put("/api/settings", json={"platform.xhs.default_interval_minutes": 240})
|
|
|
|
body = (await client.get("/api/settings")).json()
|
|
assert body["values"]["platform.xhs.default_interval_minutes"] == 240
|
|
assert body["secrets"]["platform.xhs.cookie"]["present"] is True
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_empty_string_clears_a_secret(self, client):
|
|
await client.put("/api/settings", json={"platform.xhs.cookie": SECRET_VALUE})
|
|
await client.put("/api/settings", json={"platform.xhs.cookie": ""})
|
|
|
|
assert (await client.get("/api/settings")).json()["secrets"]["platform.xhs.cookie"][
|
|
"present"
|
|
] is False
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unknown_key_is_rejected(self, client):
|
|
response = await client.put("/api/settings", json={"nope.not.a.setting": 1})
|
|
assert response.status_code == 400
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_out_of_range_is_rejected(self, client):
|
|
response = await client.put(
|
|
"/api/settings", json={"platform.xhs.default_interval_minutes": 1}
|
|
)
|
|
assert response.status_code == 400
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_invalid_choice_is_rejected(self, client):
|
|
response = await client.put("/api/settings", json={"platform.xhs.proxy_provider": "nonsense"})
|
|
assert response.status_code == 400
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_bools_accept_the_ui_shapes(self, client):
|
|
for raw in (True, "true", "1", "yes"):
|
|
response = await client.put("/api/settings", json={"platform.xhs.enable_ip_proxy": raw})
|
|
assert response.status_code == 200
|
|
assert (await client.get("/api/settings")).json()["values"][
|
|
"platform.xhs.enable_ip_proxy"
|
|
] is True
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_password_hash_cannot_be_written_through_this_endpoint(self, client):
|
|
"""It has its own authenticated endpoint; this must not be a back door."""
|
|
await client.put("/api/settings", json={"auth_password_hash": "pbkdf2_sha256$1$a$b"})
|
|
|
|
async with monitor_db.get_session() as session:
|
|
assert await get_setting(session, "auth_password_hash") is None
|
|
|
|
|
|
class TestSettingsActuallyTakeEffect:
|
|
@pytest.mark.asyncio
|
|
async def test_new_tasks_use_the_configured_defaults(self, client):
|
|
await client.put(
|
|
"/api/settings",
|
|
json={
|
|
"platform.xhs.default_interval_minutes": 120,
|
|
"platform.xhs.default_max_notes": 7,
|
|
"platform.xhs.default_max_comments": 33,
|
|
},
|
|
)
|
|
|
|
await client.post(
|
|
"/api/monitor/tasks",
|
|
json={"name": "用默认值", "mode": "creator", "targets": ["5f58bd990000000001003753"]},
|
|
)
|
|
|
|
task = (await client.get("/api/monitor/tasks")).json()["tasks"][0]
|
|
assert task["interval_minutes"] == 120
|
|
assert task["max_notes_count"] == 7
|
|
assert task["max_comments_count"] == 33
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_explicit_values_still_win_over_defaults(self, client):
|
|
await client.put("/api/settings", json={"platform.xhs.default_interval_minutes": 120})
|
|
|
|
await client.post(
|
|
"/api/monitor/tasks",
|
|
json={
|
|
"name": "显式值",
|
|
"mode": "creator",
|
|
"interval_minutes": 720,
|
|
"targets": ["5f58bd990000000001003753"],
|
|
},
|
|
)
|
|
|
|
task = (await client.get("/api/monitor/tasks")).json()["tasks"][0]
|
|
assert task["interval_minutes"] == 720
|
|
|
|
|
|
class TestRunnerAppliesStrategy:
|
|
@pytest.mark.asyncio
|
|
async def test_strategy_settings_reach_the_command(self, client):
|
|
"""Stored settings must actually change how the crawler is invoked."""
|
|
from api.services.crawler_manager import CrawlerManager
|
|
from api.schemas import CrawlerStartRequest, PlatformEnum, CrawlerTypeEnum
|
|
|
|
await client.put(
|
|
"/api/settings",
|
|
json={
|
|
"platform.xhs.crawl_sleep_sec": 7,
|
|
"platform.xhs.enable_sub_comments": True,
|
|
"platform.xhs.enable_ip_proxy": True,
|
|
"platform.xhs.proxy_provider": "static",
|
|
"platform.xhs.proxy_pool_count": 5,
|
|
"platform.xhs.static_proxy_url": "http://127.0.0.1:8888",
|
|
},
|
|
)
|
|
|
|
async with monitor_db.get_session() as session:
|
|
strategy = await scheduler_module.app_settings.get_value(
|
|
session, "crawl_sleep_sec", "xhs", 2
|
|
)
|
|
assert strategy == 7
|
|
|
|
# And the flag builder forwards them when present.
|
|
command = CrawlerManager()._build_command(
|
|
CrawlerStartRequest(
|
|
platform=PlatformEnum.XHS,
|
|
crawler_type=CrawlerTypeEnum.CREATOR,
|
|
creator_ids="abc",
|
|
crawler_max_sleep_sec=7,
|
|
enable_ip_proxy=True,
|
|
ip_proxy_provider_name="static",
|
|
ip_proxy_pool_count=5,
|
|
static_proxy_url="http://127.0.0.1:8888",
|
|
)
|
|
)
|
|
joined = " ".join(command)
|
|
assert "--crawler_max_sleep_sec 7" in joined
|
|
assert "--enable_ip_proxy true" in joined
|
|
assert "--ip_proxy_provider_name static" in joined
|
|
assert "--static_proxy_url http://127.0.0.1:8888" in joined
|
|
|
|
|
|
def _frozen_clock(hour: int):
|
|
"""Stand-in for the datetime class whose now() is pinned to a given hour.
|
|
|
|
Testing an hour window by sleeping is not an option; patching the class the
|
|
scheduler imported is the whole mechanism.
|
|
"""
|
|
|
|
class _Frozen:
|
|
@staticmethod
|
|
def now(tz=None):
|
|
return datetime(2026, 1, 1, hour)
|
|
|
|
return _Frozen
|
|
|
|
|
|
class TestManualCrawlCookieFallback:
|
|
"""The crawl page no longer has its own paste box; it reuses Settings."""
|
|
|
|
@pytest_asyncio.fixture
|
|
async def captured(self, monkeypatch):
|
|
# api.services re-exports the singleton instance, not the module.
|
|
from api.services import crawler_manager
|
|
|
|
seen: dict = {}
|
|
|
|
async def _fake_start(request, extra_args=None):
|
|
seen["cookies"] = request.cookies
|
|
return True
|
|
|
|
monkeypatch.setattr(crawler_manager, "start", _fake_start)
|
|
return seen
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_falls_back_to_the_stored_cookie(self, client, captured):
|
|
await client.put(
|
|
"/api/settings", json={"platform.xhs.cookie": "web_session=stored"}
|
|
)
|
|
|
|
response = await client.post(
|
|
"/api/crawler/start",
|
|
json={
|
|
"platform": "xhs",
|
|
"login_type": "cookie",
|
|
"crawler_type": "creator",
|
|
"creator_ids": "abc",
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
assert captured["cookies"] == "web_session=stored"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_an_explicit_cookie_still_wins(self, client, captured):
|
|
await client.put(
|
|
"/api/settings", json={"platform.xhs.cookie": "web_session=stored"}
|
|
)
|
|
|
|
await client.post(
|
|
"/api/crawler/start",
|
|
json={
|
|
"platform": "xhs",
|
|
"login_type": "cookie",
|
|
"crawler_type": "creator",
|
|
"creator_ids": "abc",
|
|
"cookies": "web_session=explicit",
|
|
},
|
|
)
|
|
|
|
assert captured["cookies"] == "web_session=explicit"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_missing_cookie_is_a_clear_error_not_a_silent_failure(
|
|
self, client, captured
|
|
):
|
|
"""Better a 400 that names the fix than a run that fetches nothing."""
|
|
response = await client.post(
|
|
"/api/crawler/start",
|
|
json={
|
|
"platform": "xhs",
|
|
"login_type": "cookie",
|
|
"crawler_type": "creator",
|
|
"creator_ids": "abc",
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 400
|
|
assert "设置" in response.json()["detail"]
|
|
assert "cookies" not in captured
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_the_cookie_is_read_per_platform(self, client, captured):
|
|
await client.put(
|
|
"/api/settings",
|
|
params={"platform": "xhs"},
|
|
json={"platform.xhs.cookie": "web_session=xhs-only"},
|
|
)
|
|
|
|
# Douyin has no stored cookie, so it must not borrow Xiaohongshu's.
|
|
response = await client.post(
|
|
"/api/crawler/start",
|
|
json={
|
|
"platform": "dy",
|
|
"login_type": "cookie",
|
|
"crawler_type": "creator",
|
|
"creator_ids": "abc",
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 400
|
|
|
|
|
|
class TestActiveHours:
|
|
"""The window gate lives in the scheduler, not the crawler."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_inside_a_daytime_window(self, client, monkeypatch):
|
|
await client.put(
|
|
"/api/settings",
|
|
json={"system.active_hours_start": 8, "system.active_hours_end": 22},
|
|
)
|
|
monkeypatch.setattr(scheduler_module, "datetime", _frozen_clock(12))
|
|
|
|
async with monitor_db.get_session() as session:
|
|
assert await MonitorScheduler()._within_active_hours(session) is True
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_outside_a_daytime_window(self, client, monkeypatch):
|
|
await client.put(
|
|
"/api/settings",
|
|
json={"system.active_hours_start": 8, "system.active_hours_end": 22},
|
|
)
|
|
monkeypatch.setattr(scheduler_module, "datetime", _frozen_clock(3))
|
|
|
|
async with monitor_db.get_session() as session:
|
|
assert await MonitorScheduler()._within_active_hours(session) is False
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_window_wrapping_past_midnight(self, client, monkeypatch):
|
|
await client.put(
|
|
"/api/settings",
|
|
json={"system.active_hours_start": 22, "system.active_hours_end": 6},
|
|
)
|
|
|
|
for hour, expected in ((23, True), (3, True), (12, False)):
|
|
monkeypatch.setattr(scheduler_module, "datetime", _frozen_clock(hour))
|
|
async with monitor_db.get_session() as session:
|
|
assert await MonitorScheduler()._within_active_hours(session) is expected
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_default_window_covers_the_whole_day(self, client, monkeypatch):
|
|
for hour in (0, 12, 23):
|
|
monkeypatch.setattr(scheduler_module, "datetime", _frozen_clock(hour))
|
|
async with monitor_db.get_session() as session:
|
|
assert await MonitorScheduler()._within_active_hours(session) is True
|