butubb
4e60524f37
在上游 MediaCrawler 之上新增一层： - 监控层 api/monitor/ —— 多博主/多笔记的定时采集、指标快照差分、报表、 企业微信通知。每轮采集写入独立目录，差分才成立。 - WebUI 登录鉴权 api/auth.py —— PBKDF2 口令 + 服务端会话，/api 全接口防护。 WebSocket 单独加依赖：BaseHTTPMiddleware 对 ws 作用域直接放行，覆盖不到。 - 全局平台切换 + 能力矩阵 —— 如实区分「爬虫模块支持」与「监控层已接线」， 未接通的平台直接拒绝建任务，而不是静默跑空。 - 监控库改用 MySQL 5.7（可回退 SQLite 供测试）：逐表强制 utf8mb4 （服务端与库默认都是 latin1），启动校验所连 schema 以防写错库， 连接池 recycle + pre_ping 应对 MySQL 的 8 小时空闲断连。 修复上游缺陷： - xhs/core.py: 主页抓取失败会跳掉整个博主，导致一条作品都抓不到， 而那份资料只喂给一个空函数。改为尽力而为，失败不中断。 - xhs/login.py: cookie 登录只注入 web_session，冷启动签名会失败。 新增 INJECT_ALL_COOKIES 开关（默认关闭，原有行为不变）。 - requirements.txt: 补上 websockets。它在上游 pyproject.toml 里有声明、 这里漏了，导致 uvicorn 没有 WebSocket 能力，实时日志流从未工作。 改动过的上游文件清单及合并方式见 UPSTREAM.md。 测试：492 passed（另有 1 个既有的 Windows/gbk 上游测试失败，与本改动无关）
129 lines
4.6 KiB
Python
129 lines
4.6 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright (c) 2025 [email protected]
|
|
#
|
|
# This file is part of MediaCrawler project.
|
|
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_cmd_arg_monitor_flags.py
|
|
# GitHub: https://github.com/NanmiCoder
|
|
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
|
#
|
|
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
|
# 1. 不得用于任何商业用途。
|
|
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
|
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
|
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
|
# 5. 不得用于任何非法或不当的用途。
|
|
#
|
|
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
|
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
|
|
|
"""Tests for the CLI flags added to support unattended monitoring runs.
|
|
|
|
Each flag must default to the existing config value, so a manual crawl that does
|
|
not pass them behaves exactly as before.
|
|
"""
|
|
|
|
import pytest
|
|
|
|
import config
|
|
from cmd_arg.arg import parse_cmd
|
|
|
|
BASE_ARGS = ["--platform", "xhs", "--type", "creator", "--creator_id", "abc123"]
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _isolate_config(monkeypatch):
|
|
monkeypatch.setattr(config, "ENABLE_CDP_MODE", True)
|
|
monkeypatch.setattr(config, "INJECT_ALL_COOKIES", False)
|
|
monkeypatch.setattr(config, "SAVE_LOGIN_STATE", True)
|
|
monkeypatch.setattr(config, "COOKIES", "")
|
|
monkeypatch.setattr(config, "SAVE_DATA_PATH", "")
|
|
monkeypatch.setattr(config, "CRAWLER_MAX_SLEEP_SEC", 2)
|
|
yield
|
|
|
|
|
|
class TestEnableCdpMode:
|
|
"""CDP attaches to the user's desktop Chrome, which cannot work on a server."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_false_disables_cdp(self):
|
|
await parse_cmd([*BASE_ARGS, "--enable_cdp_mode", "false"])
|
|
assert config.ENABLE_CDP_MODE is False
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_defaults_to_config_value(self):
|
|
await parse_cmd(BASE_ARGS)
|
|
assert config.ENABLE_CDP_MODE is True
|
|
|
|
|
|
class TestCookieFlags:
|
|
@pytest.mark.asyncio
|
|
async def test_inject_all_cookies_enables_switch(self):
|
|
await parse_cmd([*BASE_ARGS, "--inject_all_cookies", "true"])
|
|
assert config.INJECT_ALL_COOKIES is True
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_inject_all_cookies_defaults_off(self):
|
|
await parse_cmd(BASE_ARGS)
|
|
assert config.INJECT_ALL_COOKIES is False
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_cookies_file_is_read_into_config(self, tmp_path):
|
|
cookie_file = tmp_path / "cookies.txt"
|
|
cookie_file.write_text("web_session=abc; a1=def", encoding="utf-8")
|
|
|
|
await parse_cmd([*BASE_ARGS, "--cookies_file", str(cookie_file)])
|
|
|
|
assert config.COOKIES == "web_session=abc; a1=def"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_cookies_file_wins_over_inline_cookies(self, tmp_path):
|
|
cookie_file = tmp_path / "cookies.txt"
|
|
cookie_file.write_text("web_session=fromfile", encoding="utf-8")
|
|
|
|
await parse_cmd(
|
|
[*BASE_ARGS, "--cookies", "web_session=inline", "--cookies_file", str(cookie_file)]
|
|
)
|
|
|
|
assert config.COOKIES == "web_session=fromfile"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_missing_cookies_file_is_rejected(self, tmp_path):
|
|
missing = tmp_path / "nope.txt"
|
|
|
|
with pytest.raises(Exception) as excinfo:
|
|
await parse_cmd([*BASE_ARGS, "--cookies_file", str(missing)])
|
|
|
|
# A silently-ignored unreadable cookie file would produce a crawl that
|
|
# returns nothing, which is exactly the failure mode this flag exists
|
|
# to avoid.
|
|
assert "cookies_file" in str(excinfo.value)
|
|
|
|
|
|
class TestSaveDataPath:
|
|
@pytest.mark.asyncio
|
|
async def test_save_data_path_is_applied(self):
|
|
await parse_cmd([*BASE_ARGS, "--save_data_path", "data/monitor_runs/1/2"])
|
|
assert config.SAVE_DATA_PATH == "data/monitor_runs/1/2"
|
|
|
|
|
|
class TestSaveLoginState:
|
|
@pytest.mark.asyncio
|
|
async def test_save_login_state_can_be_disabled(self):
|
|
await parse_cmd([*BASE_ARGS, "--save_login_state", "false"])
|
|
assert config.SAVE_LOGIN_STATE is False
|
|
|
|
|
|
class TestCrawlSleepSec:
|
|
"""Exposed on the Settings page; previously had no CLI flag at all."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_value_is_applied(self):
|
|
await parse_cmd([*BASE_ARGS, "--crawler_max_sleep_sec", "9"])
|
|
assert config.CRAWLER_MAX_SLEEP_SEC == 9
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_defaults_to_config_value(self, monkeypatch):
|
|
monkeypatch.setattr(config, "CRAWLER_MAX_SLEEP_SEC", 4)
|
|
await parse_cmd(BASE_ARGS)
|
|
assert config.CRAWLER_MAX_SLEEP_SEC == 4
|