# -*- coding: utf-8 -*- # Copyright (c) 2025 relakkes@gmail.com # # This file is part of MediaCrawler project. # Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_settings.py # GitHub: https://github.com/NanmiCoder # Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 # # 声明:本代码仅供学习和研究目的使用。使用者应遵守以下原则: # 1. 不得用于任何商业用途。 # 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 # 3. 不得进行大规模爬取或对平台造成运营干扰。 # 4. 应合理控制请求频率,避免给目标平台带来不必要的负担。 # 5. 不得用于任何非法或不当的用途。 # # 详细许可条款请参阅项目根目录下的LICENSE文件。 # 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 """Unified settings endpoint, and the effect its values actually have.""" from datetime import datetime import httpx import pytest import pytest_asyncio from api.main import app from api.monitor import app_settings, db as monitor_db from api.monitor import scheduler as scheduler_module from api.monitor.scheduler import MonitorScheduler from api.monitor.settings import get_setting SECRET_VALUE = "web_session=SUPERSECRET; a1=abc" @pytest_asyncio.fixture async def client(tmp_path): monitor_db.set_sqlite_path(tmp_path / "monitor.db") await monitor_db.init_db() transport = httpx.ASGITransport(app=app) async with httpx.AsyncClient(transport=transport, base_url="http://test") as http_client: yield http_client await monitor_db.dispose_engine() class TestReadSettings: @pytest.mark.asyncio async def test_returns_values_secrets_and_the_spec(self, client): body = (await client.get("/api/settings")).json() assert "values" in body and "secrets" in body and "specs" in body # The spec drives the UI form, so every key must be described. spec_keys = {spec["key"] for spec in body["specs"]} assert "platform.xhs.default_interval_minutes" in spec_keys assert "platform.xhs.enable_ip_proxy" in spec_keys @pytest.mark.asyncio async def test_unset_values_fall_back_to_spec_defaults(self, client): values = (await client.get("/api/settings")).json()["values"] assert values["platform.xhs.default_interval_minutes"] == 360 assert values["platform.xhs.enable_ip_proxy"] is False @pytest.mark.asyncio async def test_secrets_are_masked_never_returned(self, client): await client.put("/api/settings", json={"platform.xhs.cookie": SECRET_VALUE}) response = await client.get("/api/settings") assert SECRET_VALUE not in response.text secret = response.json()["secrets"]["platform.xhs.cookie"] assert secret["present"] is True assert secret["length"] == len(SECRET_VALUE) class TestUpdateSettings: @pytest.mark.asyncio async def test_partial_update_leaves_other_keys_alone(self, client): await client.put( "/api/settings", json={"platform.xhs.default_interval_minutes": 120, "platform.xhs.cookie": SECRET_VALUE}, ) # A form that only submits the interval must not blank the cookie. await client.put("/api/settings", json={"platform.xhs.default_interval_minutes": 240}) body = (await client.get("/api/settings")).json() assert body["values"]["platform.xhs.default_interval_minutes"] == 240 assert body["secrets"]["platform.xhs.cookie"]["present"] is True @pytest.mark.asyncio async def test_empty_string_clears_a_secret(self, client): await client.put("/api/settings", json={"platform.xhs.cookie": SECRET_VALUE}) await client.put("/api/settings", json={"platform.xhs.cookie": ""}) assert (await client.get("/api/settings")).json()["secrets"]["platform.xhs.cookie"][ "present" ] is False @pytest.mark.asyncio async def test_unknown_key_is_rejected(self, client): response = await client.put("/api/settings", json={"nope.not.a.setting": 1}) assert response.status_code == 400 @pytest.mark.asyncio async def test_out_of_range_is_rejected(self, client): response = await client.put( "/api/settings", json={"platform.xhs.default_interval_minutes": 1} ) assert response.status_code == 400 @pytest.mark.asyncio async def test_invalid_choice_is_rejected(self, client): response = await client.put("/api/settings", json={"platform.xhs.proxy_provider": "nonsense"}) assert response.status_code == 400 @pytest.mark.asyncio async def test_bools_accept_the_ui_shapes(self, client): for raw in (True, "true", "1", "yes"): response = await client.put("/api/settings", json={"platform.xhs.enable_ip_proxy": raw}) assert response.status_code == 200 assert (await client.get("/api/settings")).json()["values"][ "platform.xhs.enable_ip_proxy" ] is True @pytest.mark.asyncio async def test_password_hash_cannot_be_written_through_this_endpoint(self, client): """It has its own authenticated endpoint; this must not be a back door.""" await client.put("/api/settings", json={"auth_password_hash": "pbkdf2_sha256$1$a$b"}) async with monitor_db.get_session() as session: assert await get_setting(session, "auth_password_hash") is None class TestSettingsActuallyTakeEffect: @pytest.mark.asyncio async def test_new_tasks_use_the_configured_defaults(self, client): await client.put( "/api/settings", json={ "platform.xhs.default_interval_minutes": 120, "platform.xhs.default_max_notes": 7, "platform.xhs.default_max_comments": 33, }, ) await client.post( "/api/monitor/tasks", json={"name": "用默认值", "mode": "creator", "targets": ["5f58bd990000000001003753"]}, ) task = (await client.get("/api/monitor/tasks")).json()["tasks"][0] assert task["interval_minutes"] == 120 assert task["max_notes_count"] == 7 assert task["max_comments_count"] == 33 @pytest.mark.asyncio async def test_explicit_values_still_win_over_defaults(self, client): await client.put("/api/settings", json={"platform.xhs.default_interval_minutes": 120}) await client.post( "/api/monitor/tasks", json={ "name": "显式值", "mode": "creator", "interval_minutes": 720, "targets": ["5f58bd990000000001003753"], }, ) task = (await client.get("/api/monitor/tasks")).json()["tasks"][0] assert task["interval_minutes"] == 720 class TestRunnerAppliesStrategy: @pytest.mark.asyncio async def test_strategy_settings_reach_the_command(self, client): """Stored settings must actually change how the crawler is invoked.""" from api.services.crawler_manager import CrawlerManager from api.schemas import CrawlerStartRequest, PlatformEnum, CrawlerTypeEnum await client.put( "/api/settings", json={ "platform.xhs.crawl_sleep_sec": 7, "platform.xhs.enable_sub_comments": True, "platform.xhs.enable_ip_proxy": True, "platform.xhs.proxy_provider": "static", "platform.xhs.proxy_pool_count": 5, "platform.xhs.static_proxy_url": "http://127.0.0.1:8888", }, ) async with monitor_db.get_session() as session: strategy = await scheduler_module.app_settings.get_value( session, "crawl_sleep_sec", "xhs", 2 ) assert strategy == 7 # And the flag builder forwards them when present. command = CrawlerManager()._build_command( CrawlerStartRequest( platform=PlatformEnum.XHS, crawler_type=CrawlerTypeEnum.CREATOR, creator_ids="abc", crawler_max_sleep_sec=7, enable_ip_proxy=True, ip_proxy_provider_name="static", ip_proxy_pool_count=5, static_proxy_url="http://127.0.0.1:8888", ) ) joined = " ".join(command) assert "--crawler_max_sleep_sec 7" in joined assert "--enable_ip_proxy true" in joined assert "--ip_proxy_provider_name static" in joined assert "--static_proxy_url http://127.0.0.1:8888" in joined def _frozen_clock(hour: int): """Stand-in for the datetime class whose now() is pinned to a given hour. Testing an hour window by sleeping is not an option; patching the class the scheduler imported is the whole mechanism. """ class _Frozen: @staticmethod def now(tz=None): return datetime(2026, 1, 1, hour) return _Frozen class TestManualCrawlCookieFallback: """The crawl page no longer has its own paste box; it reuses Settings.""" @pytest_asyncio.fixture async def captured(self, monkeypatch): # api.services re-exports the singleton instance, not the module. from api.services import crawler_manager seen: dict = {} async def _fake_start(request, extra_args=None): seen["cookies"] = request.cookies return True monkeypatch.setattr(crawler_manager, "start", _fake_start) return seen @pytest.mark.asyncio async def test_falls_back_to_the_stored_cookie(self, client, captured): await client.put( "/api/settings", json={"platform.xhs.cookie": "web_session=stored"} ) response = await client.post( "/api/crawler/start", json={ "platform": "xhs", "login_type": "cookie", "crawler_type": "creator", "creator_ids": "abc", }, ) assert response.status_code == 200 assert captured["cookies"] == "web_session=stored" @pytest.mark.asyncio async def test_an_explicit_cookie_still_wins(self, client, captured): await client.put( "/api/settings", json={"platform.xhs.cookie": "web_session=stored"} ) await client.post( "/api/crawler/start", json={ "platform": "xhs", "login_type": "cookie", "crawler_type": "creator", "creator_ids": "abc", "cookies": "web_session=explicit", }, ) assert captured["cookies"] == "web_session=explicit" @pytest.mark.asyncio async def test_missing_cookie_is_a_clear_error_not_a_silent_failure( self, client, captured ): """Better a 400 that names the fix than a run that fetches nothing.""" response = await client.post( "/api/crawler/start", json={ "platform": "xhs", "login_type": "cookie", "crawler_type": "creator", "creator_ids": "abc", }, ) assert response.status_code == 400 assert "设置" in response.json()["detail"] assert "cookies" not in captured @pytest.mark.asyncio async def test_the_cookie_is_read_per_platform(self, client, captured): await client.put( "/api/settings", params={"platform": "xhs"}, json={"platform.xhs.cookie": "web_session=xhs-only"}, ) # Douyin has no stored cookie, so it must not borrow Xiaohongshu's. response = await client.post( "/api/crawler/start", json={ "platform": "dy", "login_type": "cookie", "crawler_type": "creator", "creator_ids": "abc", }, ) assert response.status_code == 400 class TestActiveHours: """The window gate lives in the scheduler, not the crawler.""" @pytest.mark.asyncio async def test_inside_a_daytime_window(self, client, monkeypatch): await client.put( "/api/settings", json={"system.active_hours_start": 8, "system.active_hours_end": 22}, ) monkeypatch.setattr(scheduler_module, "datetime", _frozen_clock(12)) async with monitor_db.get_session() as session: assert await MonitorScheduler()._within_active_hours(session) is True @pytest.mark.asyncio async def test_outside_a_daytime_window(self, client, monkeypatch): await client.put( "/api/settings", json={"system.active_hours_start": 8, "system.active_hours_end": 22}, ) monkeypatch.setattr(scheduler_module, "datetime", _frozen_clock(3)) async with monitor_db.get_session() as session: assert await MonitorScheduler()._within_active_hours(session) is False @pytest.mark.asyncio async def test_window_wrapping_past_midnight(self, client, monkeypatch): await client.put( "/api/settings", json={"system.active_hours_start": 22, "system.active_hours_end": 6}, ) for hour, expected in ((23, True), (3, True), (12, False)): monkeypatch.setattr(scheduler_module, "datetime", _frozen_clock(hour)) async with monitor_db.get_session() as session: assert await MonitorScheduler()._within_active_hours(session) is expected @pytest.mark.asyncio async def test_default_window_covers_the_whole_day(self, client, monkeypatch): for hour in (0, 12, 23): monkeypatch.setattr(scheduler_module, "datetime", _frozen_clock(hour)) async with monitor_db.get_session() as session: assert await MonitorScheduler()._within_active_hours(session) is True