feat: 监控面板 / 登录鉴权 / 多平台切换 / MySQL
在上游 MediaCrawler 之上新增一层： - 监控层 api/monitor/ —— 多博主/多笔记的定时采集、指标快照差分、报表、 企业微信通知。每轮采集写入独立目录，差分才成立。 - WebUI 登录鉴权 api/auth.py —— PBKDF2 口令 + 服务端会话，/api 全接口防护。 WebSocket 单独加依赖：BaseHTTPMiddleware 对 ws 作用域直接放行，覆盖不到。 - 全局平台切换 + 能力矩阵 —— 如实区分「爬虫模块支持」与「监控层已接线」， 未接通的平台直接拒绝建任务，而不是静默跑空。 - 监控库改用 MySQL 5.7（可回退 SQLite 供测试）：逐表强制 utf8mb4 （服务端与库默认都是 latin1），启动校验所连 schema 以防写错库， 连接池 recycle + pre_ping 应对 MySQL 的 8 小时空闲断连。 修复上游缺陷： - xhs/core.py: 主页抓取失败会跳掉整个博主，导致一条作品都抓不到， 而那份资料只喂给一个空函数。改为尽力而为，失败不中断。 - xhs/login.py: cookie 登录只注入 web_session，冷启动签名会失败。 新增 INJECT_ALL_COOKIES 开关（默认关闭，原有行为不变）。 - requirements.txt: 补上 websockets。它在上游 pyproject.toml 里有声明、 这里漏了，导致 uvicorn 没有 WebSocket 能力，实时日志流从未工作。 改动过的上游文件清单及合并方式见 UPSTREAM.md。 测试：492 passed（另有 1 个既有的 Windows/gbk 上游测试失败，与本改动无关）
This commit is contained in:
@@ -96,3 +96,32 @@ def sample_xhs_creator():
|
||||
"interaction": 50000,
|
||||
"tag_list": '{"profession": "Designer", "interest": "Photography"}'
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _bypass_auth_for_non_auth_suites(request):
|
||||
"""Skip API authentication for suites that are not about authentication.
|
||||
|
||||
Adding auth to every /api route breaks any test that speaks HTTP, so those
|
||||
suites override the dependency here. This uses FastAPI's own
|
||||
``dependency_overrides`` mechanism rather than a production-visible
|
||||
"test mode" switch, which could be shipped enabled by accident.
|
||||
|
||||
``tests/test_auth.py`` is deliberately excluded: it must exercise the real
|
||||
enforcement path, including the route-enumeration guard that asserts every
|
||||
other /api route really does return 401.
|
||||
"""
|
||||
if request.node.fspath.basename == "test_auth.py":
|
||||
yield
|
||||
return
|
||||
|
||||
from api.auth import require_auth, require_ws_auth
|
||||
from api.main import app
|
||||
|
||||
app.dependency_overrides[require_auth] = lambda: None
|
||||
app.dependency_overrides[require_ws_auth] = lambda: None
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
app.dependency_overrides.pop(require_auth, None)
|
||||
app.dependency_overrides.pop(require_ws_auth, None)
|
||||
|
||||
@@ -0,0 +1,503 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2025 [email protected]
|
||||
#
|
||||
# This file is part of MediaCrawler project.
|
||||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_auth.py
|
||||
# GitHub: https://github.com/NanmiCoder
|
||||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||
#
|
||||
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
||||
# 5. 不得用于任何非法或不当的用途。
|
||||
#
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
"""Tests for WebUI authentication.
|
||||
|
||||
Deliberately does NOT install ``app.dependency_overrides``: the point of this
|
||||
file is to exercise the real enforcement path. Other suites override
|
||||
``require_auth`` so they can keep testing their own concerns.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from fastapi import WebSocketException
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from api import auth
|
||||
from api.main import app
|
||||
from api.monitor import db as monitor_db
|
||||
from api.monitor.models import AuthSession
|
||||
|
||||
PASSWORD = "correct-horse-battery"
|
||||
|
||||
# Captured at import, i.e. before the autouse fixture patches the module global,
|
||||
# so the guard test below checks the value that actually ships.
|
||||
REAL_PBKDF2_ITERATIONS = auth.PBKDF2_ITERATIONS
|
||||
|
||||
# Every /api route that is allowed to answer without a session.
|
||||
EXEMPT_PATHS = {"/api/health", "/api/auth/login", "/api/auth/logout"}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def cheap_hashing(monkeypatch):
|
||||
"""600k iterations is right in production and unusable in a test suite.
|
||||
|
||||
hash_password() resolves the count at call time precisely so this works.
|
||||
"""
|
||||
monkeypatch.setattr(auth, "PBKDF2_ITERATIONS", 1_000)
|
||||
monkeypatch.delenv("MC_PASSWORD", raising=False)
|
||||
auth.reset_throttle_state()
|
||||
yield
|
||||
auth.reset_throttle_state()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def db(tmp_path):
|
||||
monitor_db.set_sqlite_path(tmp_path / "monitor.db")
|
||||
await monitor_db.init_db()
|
||||
yield monitor_db
|
||||
await monitor_db.dispose_engine()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(db):
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as http_client:
|
||||
yield http_client
|
||||
|
||||
|
||||
async def _seed_password(password: str = PASSWORD) -> None:
|
||||
async with monitor_db.get_session() as session:
|
||||
await auth.set_password(session, password)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Password hashing
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
class TestPasswordHashing:
|
||||
def test_iteration_count_has_not_been_lowered(self):
|
||||
"""Guard: someone trimming this for speed would weaken every install."""
|
||||
assert REAL_PBKDF2_ITERATIONS >= 600_000
|
||||
|
||||
def test_round_trip(self):
|
||||
stored = auth.hash_password(PASSWORD)
|
||||
assert auth._verify_password_sync(PASSWORD, stored) is True
|
||||
|
||||
def test_wrong_password_rejected(self):
|
||||
stored = auth.hash_password(PASSWORD)
|
||||
assert auth._verify_password_sync("wrong", stored) is False
|
||||
|
||||
def test_same_password_hashes_differently(self):
|
||||
"""A fixed salt would let one rainbow table crack every install."""
|
||||
assert auth.hash_password(PASSWORD) != auth.hash_password(PASSWORD)
|
||||
|
||||
def test_format_is_self_describing(self):
|
||||
algo, iterations, salt, digest = auth.hash_password(PASSWORD).split("$")
|
||||
assert algo == "pbkdf2_sha256"
|
||||
assert int(iterations) == auth.PBKDF2_ITERATIONS
|
||||
assert salt and digest
|
||||
|
||||
@pytest.mark.parametrize("stored", ["", "garbage", "md5$1$a$b", "pbkdf2_sha256$x$a$b"])
|
||||
def test_malformed_stored_hash_is_rejected_not_raised(self, stored):
|
||||
assert auth._verify_password_sync(PASSWORD, stored) is False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Credentials
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
class TestCredentials:
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_password_against_stored_hash(self, db):
|
||||
await _seed_password()
|
||||
async with monitor_db.get_session() as session:
|
||||
assert await auth.check_password(session, PASSWORD) is True
|
||||
assert await auth.check_password(session, "nope") is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_password_configured_denies_everything(self, db):
|
||||
"""An unset credential must not mean "open"."""
|
||||
async with monitor_db.get_session() as session:
|
||||
assert await auth.check_password(session, "") is False
|
||||
assert await auth.check_password(session, PASSWORD) is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_env_override_wins_and_is_not_persisted(self, db, monkeypatch):
|
||||
"""The documented way back in after forgetting the password."""
|
||||
await _seed_password("stored-password")
|
||||
monkeypatch.setenv("MC_PASSWORD", "env-password")
|
||||
|
||||
async with monitor_db.get_session() as session:
|
||||
assert await auth.check_password(session, "env-password") is True
|
||||
assert await auth.check_password(session, "stored-password") is False
|
||||
|
||||
# Override must never be written to disk.
|
||||
assert await auth.current_password_hash(session) != ""
|
||||
assert "env-password" not in (await auth.current_password_hash(session))
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_first_run_generates_a_credential(self, db, monkeypatch):
|
||||
monkeypatch.delenv("MC_PASSWORD", raising=False)
|
||||
|
||||
generated = await auth.ensure_initial_credential()
|
||||
assert generated
|
||||
|
||||
# Second call is a no-op.
|
||||
assert await auth.ensure_initial_credential() is None
|
||||
|
||||
async with monitor_db.get_session() as session:
|
||||
assert await auth.check_password(session, generated) is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_first_run_defers_to_env_password(self, db, monkeypatch):
|
||||
monkeypatch.setenv("MC_PASSWORD", "env-password")
|
||||
|
||||
assert await auth.ensure_initial_credential() is None
|
||||
|
||||
async with monitor_db.get_session() as session:
|
||||
assert await auth.current_password_hash(session) == ""
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Sessions
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
class TestSessions:
|
||||
@pytest.mark.asyncio
|
||||
async def test_round_trip(self, db):
|
||||
async with monitor_db.get_session() as session:
|
||||
token, expires_at = await auth.create_session(session)
|
||||
|
||||
async with monitor_db.get_session() as session:
|
||||
assert await auth.resolve_session(session, token) is not None
|
||||
assert expires_at > 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_only_the_hash_is_stored(self, db):
|
||||
"""A database leak must not hand over live sessions."""
|
||||
async with monitor_db.get_session() as session:
|
||||
token, _ = await auth.create_session(session)
|
||||
|
||||
async with monitor_db.get_session() as session:
|
||||
stored = (await session.scalars(select(AuthSession.token_hash))).all()
|
||||
assert token not in stored
|
||||
assert auth._hash_token(token) in stored
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_expired_session_is_rejected_and_removed(self, db):
|
||||
async with monitor_db.get_session() as session:
|
||||
token, _ = await auth.create_session(session)
|
||||
row = await session.get(AuthSession, auth._hash_token(token))
|
||||
row.expires_at = 1 # long past
|
||||
|
||||
async with monitor_db.get_session() as session:
|
||||
assert await auth.resolve_session(session, token) is None
|
||||
|
||||
# Fresh session: the identity map in the one above still holds the
|
||||
# pending-delete object, so it would answer as if the row were present.
|
||||
async with monitor_db.get_session() as session:
|
||||
assert await session.get(AuthSession, auth._hash_token(token)) is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_token_is_rejected(self, db):
|
||||
async with monitor_db.get_session() as session:
|
||||
assert await auth.resolve_session(session, "never-issued") is None
|
||||
assert await auth.resolve_session(session, "") is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_logout_revokes_only_that_session(self, db):
|
||||
async with monitor_db.get_session() as session:
|
||||
first, _ = await auth.create_session(session)
|
||||
second, _ = await auth.create_session(session)
|
||||
|
||||
async with monitor_db.get_session() as session:
|
||||
await auth.revoke_session(session, first)
|
||||
|
||||
async with monitor_db.get_session() as session:
|
||||
assert await auth.resolve_session(session, first) is None
|
||||
assert await auth.resolve_session(session, second) is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_revoke_all_clears_every_session(self, db):
|
||||
async with monitor_db.get_session() as session:
|
||||
await auth.create_session(session)
|
||||
await auth.create_session(session)
|
||||
|
||||
async with monitor_db.get_session() as session:
|
||||
removed = await auth.revoke_all_sessions(session)
|
||||
assert removed == 2
|
||||
|
||||
async with monitor_db.get_session() as session:
|
||||
assert await session.scalar(select(func.count()).select_from(AuthSession)) == 0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Throttle
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
class TestThrottle:
|
||||
@pytest.mark.asyncio
|
||||
async def test_below_threshold_is_not_throttled(self, db):
|
||||
key = "1.2.3.4"
|
||||
for _ in range(auth.THROTTLE_THRESHOLD - 1):
|
||||
await auth.record_failure(key)
|
||||
assert await auth.retry_after_seconds(key) == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lockout_after_repeated_failures(self, db):
|
||||
key = "1.2.3.4"
|
||||
for _ in range(auth.THROTTLE_THRESHOLD):
|
||||
await auth.record_failure(key)
|
||||
assert await auth.retry_after_seconds(key) > 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_success_clears_failures(self, db):
|
||||
key = "1.2.3.4"
|
||||
for _ in range(auth.THROTTLE_THRESHOLD):
|
||||
await auth.record_failure(key)
|
||||
await auth.clear_failures(key)
|
||||
assert await auth.retry_after_seconds(key) == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failures_age_out_of_the_window(self, db, monkeypatch):
|
||||
"""Driven by a fake clock rather than sleeping 15 minutes."""
|
||||
key = "1.2.3.4"
|
||||
clock = {"now": 1000.0}
|
||||
monkeypatch.setattr(auth, "_now", lambda: clock["now"])
|
||||
|
||||
for _ in range(auth.THROTTLE_THRESHOLD):
|
||||
await auth.record_failure(key)
|
||||
assert await auth.retry_after_seconds(key) > 0
|
||||
|
||||
clock["now"] += auth.THROTTLE_WINDOW_SECONDS + 1
|
||||
assert await auth.retry_after_seconds(key) == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_keys_are_independent(self, db):
|
||||
for _ in range(auth.THROTTLE_THRESHOLD):
|
||||
await auth.record_failure("attacker")
|
||||
assert await auth.retry_after_seconds("attacker") > 0
|
||||
assert await auth.retry_after_seconds("innocent") == 0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# HTTP enforcement — the acceptance criteria
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
class TestEnforcement:
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_is_reachable_without_a_session(self, client):
|
||||
assert (await client.get("/api/health")).status_code == 200
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_protected_endpoint_returns_401_without_a_session(self, client):
|
||||
response = await client.get("/api/monitor/tasks")
|
||||
assert response.status_code == 401
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrong_password_is_401_and_generic(self, client):
|
||||
await _seed_password()
|
||||
response = await client.post("/api/auth/login", json={"password": "wrong"})
|
||||
assert response.status_code == 401
|
||||
# Must not reveal whether a password is even configured.
|
||||
assert response.json()["detail"] == auth.INVALID_CREDENTIALS
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_login_unlocks_the_api(self, client):
|
||||
await _seed_password()
|
||||
|
||||
login = await client.post("/api/auth/login", json={"password": PASSWORD})
|
||||
assert login.status_code == 200
|
||||
assert auth.SESSION_COOKIE_NAME in client.cookies
|
||||
|
||||
assert (await client.get("/api/monitor/tasks")).status_code == 200
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cookie_is_httponly_and_lax(self, client):
|
||||
await _seed_password()
|
||||
login = await client.post("/api/auth/login", json={"password": PASSWORD})
|
||||
raw = login.headers["set-cookie"].lower()
|
||||
assert "httponly" in raw
|
||||
assert "samesite=lax" in raw
|
||||
# Secure must be OFF by default: the LAN bind is plain HTTP and a Secure
|
||||
# cookie is silently dropped there, looping the login page.
|
||||
assert "secure" not in raw
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bearer_token_also_works(self, client):
|
||||
"""Scripts and curl cannot use a cookie jar conveniently."""
|
||||
await _seed_password()
|
||||
login = await client.post("/api/auth/login", json={"password": PASSWORD})
|
||||
token = login.cookies[auth.SESSION_COOKIE_NAME]
|
||||
|
||||
async with httpx.AsyncClient(
|
||||
transport=httpx.ASGITransport(app=app), base_url="http://test"
|
||||
) as bare:
|
||||
bare.headers["Authorization"] = f"Bearer {token}"
|
||||
assert (await bare.get("/api/monitor/tasks")).status_code == 200
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tampered_token_is_rejected(self, client):
|
||||
await _seed_password()
|
||||
await client.post("/api/auth/login", json={"password": PASSWORD})
|
||||
client.cookies.set(auth.SESSION_COOKIE_NAME, "not-a-real-token")
|
||||
assert (await client.get("/api/monitor/tasks")).status_code == 401
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_logout_invalidates_the_session(self, client):
|
||||
await _seed_password()
|
||||
await client.post("/api/auth/login", json={"password": PASSWORD})
|
||||
assert (await client.get("/api/monitor/tasks")).status_code == 200
|
||||
|
||||
assert (await client.post("/api/auth/logout")).status_code == 200
|
||||
assert (await client.get("/api/monitor/tasks")).status_code == 401
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_me_reports_401_when_logged_out(self, client):
|
||||
await _seed_password()
|
||||
assert (await client.get("/api/auth/me")).status_code == 401
|
||||
|
||||
await client.post("/api/auth/login", json={"password": PASSWORD})
|
||||
me = await client.get("/api/auth/me")
|
||||
assert me.status_code == 200
|
||||
assert me.json()["authenticated"] is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_password_change_evicts_other_devices(self, client):
|
||||
await _seed_password()
|
||||
|
||||
# A second "device" holds its own session.
|
||||
login = await client.post("/api/auth/login", json={"password": PASSWORD})
|
||||
other_token = login.cookies[auth.SESSION_COOKIE_NAME]
|
||||
|
||||
changed = await client.post(
|
||||
"/api/auth/password",
|
||||
json={"current": PASSWORD, "new": "brand-new-password"},
|
||||
)
|
||||
assert changed.status_code == 200
|
||||
|
||||
# The old token is dead.
|
||||
async with httpx.AsyncClient(
|
||||
transport=httpx.ASGITransport(app=app), base_url="http://test"
|
||||
) as other:
|
||||
other.cookies.set(auth.SESSION_COOKIE_NAME, other_token)
|
||||
assert (await other.get("/api/monitor/tasks")).status_code == 401
|
||||
|
||||
# ...and the caller is still logged in.
|
||||
assert (await client.get("/api/monitor/tasks")).status_code == 200
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_password_change_requires_the_current_password(self, client):
|
||||
await _seed_password()
|
||||
await client.post("/api/auth/login", json={"password": PASSWORD})
|
||||
|
||||
response = await client.post(
|
||||
"/api/auth/password", json={"current": "wrong", "new": "whatever-new"}
|
||||
)
|
||||
assert response.status_code == 401
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_repeated_failures_get_throttled(self, client):
|
||||
await _seed_password()
|
||||
for _ in range(auth.THROTTLE_THRESHOLD):
|
||||
await client.post("/api/auth/login", json={"password": "wrong"})
|
||||
|
||||
blocked = await client.post("/api/auth/login", json={"password": PASSWORD})
|
||||
assert blocked.status_code == 429
|
||||
assert "retry-after" in {k.lower() for k in blocked.headers}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_docs_are_not_exposed(self, client):
|
||||
for path in ("/docs", "/redoc", "/openapi.json"):
|
||||
assert (await client.get(path)).status_code == 404
|
||||
|
||||
|
||||
class TestEveryRouteIsGuarded:
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_api_route_is_accidentally_open(self, client):
|
||||
"""The guard that stops the next endpoint from shipping unauthenticated."""
|
||||
unguarded = []
|
||||
|
||||
for route in app.routes:
|
||||
path = getattr(route, "path", "")
|
||||
methods = getattr(route, "methods", None)
|
||||
if not path.startswith("/api") or not methods or path in EXEMPT_PATHS:
|
||||
continue
|
||||
|
||||
# Substitute dummy values for path params so we reach the auth check
|
||||
# rather than a 404/422 on the parameter itself.
|
||||
concrete = "/".join(
|
||||
"1" if segment.startswith("{") else segment for segment in path.split("/")
|
||||
)
|
||||
|
||||
for method in methods - {"HEAD", "OPTIONS"}:
|
||||
response = await client.request(method, concrete, json={})
|
||||
if response.status_code != 401:
|
||||
unguarded.append(f"{method} {path} -> {response.status_code}")
|
||||
|
||||
assert not unguarded, f"以下 /api 路由未受鉴权保护：{unguarded}"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# WebSocket enforcement
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
class _FakeWebSocket:
|
||||
"""Only `.cookies` is read by require_ws_auth."""
|
||||
|
||||
def __init__(self, cookies):
|
||||
self.cookies = cookies
|
||||
|
||||
|
||||
class TestWebSocketAuth:
|
||||
"""Guarding websockets needs its own mechanism: BaseHTTPMiddleware returns
|
||||
early for non-http scopes, and HTTP router dependencies never run for them.
|
||||
Without this the live crawl log stream would be wide open.
|
||||
"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_cookie_is_rejected(self, db):
|
||||
with pytest.raises(WebSocketException) as excinfo:
|
||||
await auth.require_ws_auth(_FakeWebSocket({}))
|
||||
assert excinfo.value.code == 1008
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_valid_cookie_is_accepted(self, db):
|
||||
async with monitor_db.get_session() as session:
|
||||
token, _ = await auth.create_session(session)
|
||||
|
||||
# No exception means accepted.
|
||||
await auth.require_ws_auth(_FakeWebSocket({auth.SESSION_COOKIE_NAME: token}))
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_cookie_is_rejected(self, db):
|
||||
with pytest.raises(WebSocketException):
|
||||
await auth.require_ws_auth(_FakeWebSocket({auth.SESSION_COOKIE_NAME: "bogus"}))
|
||||
|
||||
def test_every_websocket_route_carries_the_guard(self):
|
||||
guarded = {
|
||||
route.path
|
||||
for route in app.routes
|
||||
if route.__class__.__name__ == "APIWebSocketRoute"
|
||||
and any(
|
||||
getattr(dep.dependency, "__name__", "") == "require_ws_auth"
|
||||
for dep in (route.dependencies or [])
|
||||
)
|
||||
}
|
||||
every_ws = {
|
||||
route.path
|
||||
for route in app.routes
|
||||
if route.__class__.__name__ == "APIWebSocketRoute"
|
||||
}
|
||||
assert every_ws, "expected at least one websocket route"
|
||||
assert every_ws == guarded
|
||||
@@ -0,0 +1,128 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2025 [email protected]
|
||||
#
|
||||
# This file is part of MediaCrawler project.
|
||||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_cmd_arg_monitor_flags.py
|
||||
# GitHub: https://github.com/NanmiCoder
|
||||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||
#
|
||||
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
||||
# 5. 不得用于任何非法或不当的用途。
|
||||
#
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
"""Tests for the CLI flags added to support unattended monitoring runs.
|
||||
|
||||
Each flag must default to the existing config value, so a manual crawl that does
|
||||
not pass them behaves exactly as before.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
import config
|
||||
from cmd_arg.arg import parse_cmd
|
||||
|
||||
BASE_ARGS = ["--platform", "xhs", "--type", "creator", "--creator_id", "abc123"]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolate_config(monkeypatch):
|
||||
monkeypatch.setattr(config, "ENABLE_CDP_MODE", True)
|
||||
monkeypatch.setattr(config, "INJECT_ALL_COOKIES", False)
|
||||
monkeypatch.setattr(config, "SAVE_LOGIN_STATE", True)
|
||||
monkeypatch.setattr(config, "COOKIES", "")
|
||||
monkeypatch.setattr(config, "SAVE_DATA_PATH", "")
|
||||
monkeypatch.setattr(config, "CRAWLER_MAX_SLEEP_SEC", 2)
|
||||
yield
|
||||
|
||||
|
||||
class TestEnableCdpMode:
|
||||
"""CDP attaches to the user's desktop Chrome, which cannot work on a server."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_false_disables_cdp(self):
|
||||
await parse_cmd([*BASE_ARGS, "--enable_cdp_mode", "false"])
|
||||
assert config.ENABLE_CDP_MODE is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_defaults_to_config_value(self):
|
||||
await parse_cmd(BASE_ARGS)
|
||||
assert config.ENABLE_CDP_MODE is True
|
||||
|
||||
|
||||
class TestCookieFlags:
|
||||
@pytest.mark.asyncio
|
||||
async def test_inject_all_cookies_enables_switch(self):
|
||||
await parse_cmd([*BASE_ARGS, "--inject_all_cookies", "true"])
|
||||
assert config.INJECT_ALL_COOKIES is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_inject_all_cookies_defaults_off(self):
|
||||
await parse_cmd(BASE_ARGS)
|
||||
assert config.INJECT_ALL_COOKIES is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cookies_file_is_read_into_config(self, tmp_path):
|
||||
cookie_file = tmp_path / "cookies.txt"
|
||||
cookie_file.write_text("web_session=abc; a1=def", encoding="utf-8")
|
||||
|
||||
await parse_cmd([*BASE_ARGS, "--cookies_file", str(cookie_file)])
|
||||
|
||||
assert config.COOKIES == "web_session=abc; a1=def"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cookies_file_wins_over_inline_cookies(self, tmp_path):
|
||||
cookie_file = tmp_path / "cookies.txt"
|
||||
cookie_file.write_text("web_session=fromfile", encoding="utf-8")
|
||||
|
||||
await parse_cmd(
|
||||
[*BASE_ARGS, "--cookies", "web_session=inline", "--cookies_file", str(cookie_file)]
|
||||
)
|
||||
|
||||
assert config.COOKIES == "web_session=fromfile"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_cookies_file_is_rejected(self, tmp_path):
|
||||
missing = tmp_path / "nope.txt"
|
||||
|
||||
with pytest.raises(Exception) as excinfo:
|
||||
await parse_cmd([*BASE_ARGS, "--cookies_file", str(missing)])
|
||||
|
||||
# A silently-ignored unreadable cookie file would produce a crawl that
|
||||
# returns nothing, which is exactly the failure mode this flag exists
|
||||
# to avoid.
|
||||
assert "cookies_file" in str(excinfo.value)
|
||||
|
||||
|
||||
class TestSaveDataPath:
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_data_path_is_applied(self):
|
||||
await parse_cmd([*BASE_ARGS, "--save_data_path", "data/monitor_runs/1/2"])
|
||||
assert config.SAVE_DATA_PATH == "data/monitor_runs/1/2"
|
||||
|
||||
|
||||
class TestSaveLoginState:
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_login_state_can_be_disabled(self):
|
||||
await parse_cmd([*BASE_ARGS, "--save_login_state", "false"])
|
||||
assert config.SAVE_LOGIN_STATE is False
|
||||
|
||||
|
||||
class TestCrawlSleepSec:
|
||||
"""Exposed on the Settings page; previously had no CLI flag at all."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_value_is_applied(self):
|
||||
await parse_cmd([*BASE_ARGS, "--crawler_max_sleep_sec", "9"])
|
||||
assert config.CRAWLER_MAX_SLEEP_SEC == 9
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_defaults_to_config_value(self, monkeypatch):
|
||||
monkeypatch.setattr(config, "CRAWLER_MAX_SLEEP_SEC", 4)
|
||||
await parse_cmd(BASE_ARGS)
|
||||
assert config.CRAWLER_MAX_SLEEP_SEC == 4
|
||||
@@ -0,0 +1,67 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2025 [email protected]
|
||||
#
|
||||
# This file is part of MediaCrawler project.
|
||||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_interpreter.py
|
||||
# GitHub: https://github.com/NanmiCoder
|
||||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||
#
|
||||
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
||||
# 5. 不得用于任何非法或不当的用途。
|
||||
#
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
"""Tests for the subprocess interpreter resolver."""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from api.services.interpreter import (
|
||||
describe_interpreter,
|
||||
resolve_python_cmd,
|
||||
venv_python_path,
|
||||
)
|
||||
|
||||
|
||||
def _make_venv(root: Path) -> Path:
|
||||
"""Create a fake venv layout and return the expected python path."""
|
||||
exe = venv_python_path(root)
|
||||
exe.parent.mkdir(parents=True, exist_ok=True)
|
||||
exe.write_text("", encoding="utf-8")
|
||||
return exe
|
||||
|
||||
|
||||
def test_prefers_uv_when_available(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr("shutil.which", lambda name: "/usr/bin/uv" if name == "uv" else None)
|
||||
_make_venv(tmp_path)
|
||||
|
||||
# uv wins even when a venv exists, matching the upstream documented workflow.
|
||||
assert resolve_python_cmd(tmp_path) == ["uv", "run", "python"]
|
||||
|
||||
|
||||
def test_falls_back_to_project_venv(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr("shutil.which", lambda name: None)
|
||||
exe = _make_venv(tmp_path)
|
||||
|
||||
assert resolve_python_cmd(tmp_path) == [str(exe)]
|
||||
|
||||
|
||||
def test_falls_back_to_current_interpreter(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr("shutil.which", lambda name: None)
|
||||
|
||||
# No uv, no venv anywhere under the given root.
|
||||
assert resolve_python_cmd(tmp_path) == [sys.executable]
|
||||
|
||||
|
||||
def test_describe_is_human_readable(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr("shutil.which", lambda name: None)
|
||||
_make_venv(tmp_path)
|
||||
assert "virtualenv" in describe_interpreter(tmp_path)
|
||||
|
||||
monkeypatch.setattr("shutil.which", lambda name: "/usr/bin/uv" if name == "uv" else None)
|
||||
assert describe_interpreter(tmp_path) == "uv run python"
|
||||
@@ -0,0 +1,208 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2025 [email protected]
|
||||
#
|
||||
# This file is part of MediaCrawler project.
|
||||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_monitor_api.py
|
||||
# GitHub: https://github.com/NanmiCoder
|
||||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||
#
|
||||
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
||||
# 5. 不得用于任何非法或不当的用途。
|
||||
#
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
"""API-level tests for the monitoring endpoints.
|
||||
|
||||
Run against an ASGI transport with a temporary database, so no server, network
|
||||
or login is required. Lifespan is deliberately not exercised: it would start the
|
||||
scheduler, and these tests only cover routing, validation and persistence.
|
||||
"""
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from api.main import app
|
||||
from api.monitor import db as monitor_db
|
||||
from api.monitor.service import TargetParseError, parse_target_input
|
||||
|
||||
CREATOR_URL = (
|
||||
"https://www.xiaohongshu.com/user/profile/5f58bd990000000001003753"
|
||||
"?xsec_token=ABYVg1evluJZZzpMX-VWzchxQ1qSNVW3r-jOEnKqMcgZw=&xsec_source=pc_search"
|
||||
)
|
||||
NOTE_URL = "https://www.xiaohongshu.com/explore/6aa3d827000000002802c5c8?xsec_token=TOKEN&xsec_source=pc_search"
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(tmp_path):
|
||||
monitor_db.set_sqlite_path(tmp_path / "monitor.db")
|
||||
await monitor_db.init_db()
|
||||
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as http_client:
|
||||
yield http_client
|
||||
|
||||
await monitor_db.dispose_engine()
|
||||
|
||||
|
||||
class TestParseTargetInput:
|
||||
def test_full_url_splits_id_from_token(self):
|
||||
"""The id is the stable key; the token is a refreshable credential."""
|
||||
parsed = parse_target_input(CREATOR_URL, "creator")
|
||||
assert parsed["external_id"] == "5f58bd990000000001003753"
|
||||
assert parsed["xsec_token"].startswith("ABYVg1evluJZZzpMX")
|
||||
assert parsed["xsec_source"] == "pc_search"
|
||||
|
||||
def test_bare_id_is_accepted(self):
|
||||
parsed = parse_target_input("5f58bd990000000001003753", "creator")
|
||||
assert parsed["external_id"] == "5f58bd990000000001003753"
|
||||
assert parsed["xsec_token"] == ""
|
||||
|
||||
def test_note_url_without_token_still_parses(self):
|
||||
parsed = parse_target_input(
|
||||
"https://www.xiaohongshu.com/explore/6aa3d827000000002802c5c8", "note"
|
||||
)
|
||||
assert parsed["external_id"] == "6aa3d827000000002802c5c8"
|
||||
assert parsed["xsec_token"] == ""
|
||||
|
||||
def test_creator_url_rejected_in_note_mode(self):
|
||||
with pytest.raises(TargetParseError):
|
||||
parse_target_input(CREATOR_URL, "note")
|
||||
|
||||
def test_garbage_is_rejected(self):
|
||||
with pytest.raises(TargetParseError):
|
||||
parse_target_input("not a url at all !!", "creator")
|
||||
|
||||
|
||||
class TestTaskCrud:
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_and_list_task(self, client):
|
||||
response = await client.post(
|
||||
"/api/monitor/tasks",
|
||||
json={
|
||||
"name": "网文作者监控",
|
||||
"mode": "creator",
|
||||
"interval_minutes": 120,
|
||||
"targets": [CREATOR_URL, "5f58bd990000000001003754"],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 201
|
||||
task_id = response.json()["id"]
|
||||
|
||||
listing = await client.get("/api/monitor/tasks")
|
||||
assert listing.status_code == 200
|
||||
tasks = listing.json()["tasks"]
|
||||
assert len(tasks) == 1
|
||||
assert tasks[0]["id"] == task_id
|
||||
assert tasks[0]["target_count"] == 2
|
||||
# next_run_at is persisted so the schedule survives a restart.
|
||||
assert tasks[0]["next_run_at"] is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplicate_targets_are_deduplicated(self, client):
|
||||
response = await client.post(
|
||||
"/api/monitor/tasks",
|
||||
json={
|
||||
"name": "dedup",
|
||||
"mode": "creator",
|
||||
"targets": [CREATOR_URL, CREATOR_URL],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 201
|
||||
|
||||
listing = await client.get("/api/monitor/tasks")
|
||||
assert listing.json()["tasks"][0]["target_count"] == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_target_returns_400(self, client):
|
||||
response = await client.post(
|
||||
"/api/monitor/tasks",
|
||||
json={"name": "bad", "mode": "creator", "targets": ["!!! nonsense !!!"]},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interval_floor_is_enforced(self, client):
|
||||
"""A tight poll loop is the pattern that triggers platform rate limits."""
|
||||
response = await client.post(
|
||||
"/api/monitor/tasks",
|
||||
json={"name": "too fast", "mode": "creator", "interval_minutes": 1, "targets": [CREATOR_URL]},
|
||||
)
|
||||
assert response.status_code == 422
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_and_delete(self, client):
|
||||
created = await client.post(
|
||||
"/api/monitor/tasks",
|
||||
json={"name": "t", "mode": "note", "targets": [NOTE_URL]},
|
||||
)
|
||||
task_id = created.json()["id"]
|
||||
|
||||
patched = await client.patch(f"/api/monitor/tasks/{task_id}", json={"enabled": False})
|
||||
assert patched.status_code == 200
|
||||
listing = await client.get("/api/monitor/tasks")
|
||||
assert listing.json()["tasks"][0]["enabled"] is False
|
||||
|
||||
deleted = await client.delete(f"/api/monitor/tasks/{task_id}")
|
||||
assert deleted.status_code == 200
|
||||
assert (await client.get("/api/monitor/tasks")).json()["tasks"] == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_now_on_missing_task_is_404(self, client):
|
||||
response = await client.post("/api/monitor/tasks/9999/run")
|
||||
assert response.status_code == 404
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_history_starts_empty(self, client):
|
||||
created = await client.post(
|
||||
"/api/monitor/tasks",
|
||||
json={"name": "t", "mode": "creator", "targets": [CREATOR_URL]},
|
||||
)
|
||||
task_id = created.json()["id"]
|
||||
runs = await client.get(f"/api/monitor/tasks/{task_id}/runs")
|
||||
assert runs.status_code == 200
|
||||
assert runs.json()["runs"] == []
|
||||
|
||||
|
||||
class TestCookieEndpoints:
|
||||
@pytest.mark.asyncio
|
||||
async def test_cookie_value_is_never_returned(self, client):
|
||||
"""The GET must expose health only, never the credential."""
|
||||
secret = "web_session=SUPERSECRETVALUE; a1=abc123"
|
||||
saved = await client.post("/api/monitor/cookie", json={"cookie": secret})
|
||||
assert saved.status_code == 200
|
||||
|
||||
status_response = await client.get("/api/monitor/cookie")
|
||||
assert status_response.status_code == 200
|
||||
body = status_response.json()
|
||||
|
||||
assert body["present"] is True
|
||||
assert body["length"] == len(secret)
|
||||
assert "SUPERSECRETVALUE" not in status_response.text
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cookie_initially_absent_and_clearable(self, client):
|
||||
assert (await client.get("/api/monitor/cookie")).json()["present"] is False
|
||||
|
||||
await client.post("/api/monitor/cookie", json={"cookie": "web_session=x"})
|
||||
assert (await client.get("/api/monitor/cookie")).json()["present"] is True
|
||||
|
||||
await client.delete("/api/monitor/cookie")
|
||||
assert (await client.get("/api/monitor/cookie")).json()["present"] is False
|
||||
|
||||
|
||||
class TestDashboardQueries:
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_dashboard_shapes(self, client):
|
||||
assert (await client.get("/api/monitor/notes")).json()["notes"] == []
|
||||
assert (await client.get("/api/monitor/comments")).json()["comments"] == []
|
||||
assert (await client.get("/api/monitor/events")).json()["events"] == []
|
||||
|
||||
overview = (await client.get("/api/monitor/overview")).json()
|
||||
assert overview["tasks"] == 0
|
||||
assert overview["notes"] == 0
|
||||
@@ -0,0 +1,232 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2025 [email protected]
|
||||
#
|
||||
# This file is part of MediaCrawler project.
|
||||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_monitor_comments.py
|
||||
# GitHub: https://github.com/NanmiCoder
|
||||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||
#
|
||||
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
||||
# 5. 不得用于任何非法或不当的用途。
|
||||
#
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
"""Comment note-association, grouping, and the export endpoint."""
|
||||
|
||||
import csv
|
||||
import io
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from api.main import app
|
||||
from api.monitor import db as monitor_db
|
||||
from api.monitor.models import (
|
||||
MODE_CREATOR,
|
||||
MonitorComment,
|
||||
MonitorNote,
|
||||
MonitorTask,
|
||||
)
|
||||
|
||||
TASK_NAME = "评论归属测试"
|
||||
|
||||
|
||||
async def _seed():
|
||||
"""Two works; three comments on the first, one on the second."""
|
||||
async with monitor_db.get_session() as session:
|
||||
task = MonitorTask(
|
||||
name=TASK_NAME, platform="xhs", mode=MODE_CREATOR, enabled=True,
|
||||
interval_minutes=60, max_notes_count=20, enable_comments=True,
|
||||
max_comments_count=50, run_timeout_seconds=3600,
|
||||
notify_enabled=False, created_at=0, updated_at=0,
|
||||
)
|
||||
session.add(task)
|
||||
await session.flush()
|
||||
|
||||
for note_id, title in (("note-a", "作品甲"), ("note-b", "作品乙")):
|
||||
session.add(
|
||||
MonitorNote(
|
||||
task_id=task.id, note_id=note_id, title=title,
|
||||
note_url=f"https://www.xiaohongshu.com/explore/{note_id}",
|
||||
cover=f"https://img/{note_id}.jpg", creator_hash="h",
|
||||
source_kind="video", published_at=None,
|
||||
first_seen_run_id=1, first_seen_at=1_700_000_000_000,
|
||||
last_seen_run_id=1, last_seen_at=1_700_000_000_000,
|
||||
)
|
||||
)
|
||||
|
||||
# note-a has three comments, note-b has one.
|
||||
plan = [
|
||||
("c1", "note-a", 1_700_000_001_000),
|
||||
("c2", "note-a", 1_700_000_002_000),
|
||||
("c3", "note-a", 1_700_000_003_000),
|
||||
("c4", "note-b", 1_700_000_004_000),
|
||||
]
|
||||
for comment_id, note_id, seen_at in plan:
|
||||
session.add(
|
||||
MonitorComment(
|
||||
task_id=task.id, note_id=note_id, comment_id=comment_id,
|
||||
content=f"内容-{comment_id}", nickname="u***r", creator_hash="h",
|
||||
create_time=seen_at, like_count=1, sub_comment_count=0,
|
||||
parent_comment_id="", first_seen_run_id=1, first_seen_at=seen_at,
|
||||
)
|
||||
)
|
||||
return task.id
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(tmp_path):
|
||||
monitor_db.set_sqlite_path(tmp_path / "monitor.db")
|
||||
await monitor_db.init_db()
|
||||
await _seed()
|
||||
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as http_client:
|
||||
yield http_client
|
||||
|
||||
await monitor_db.dispose_engine()
|
||||
|
||||
|
||||
class TestCommentsCarryTheirNote:
|
||||
@pytest.mark.asyncio
|
||||
async def test_each_comment_names_its_work(self, client):
|
||||
"""A bare note_id is unreadable -- the title is the whole point."""
|
||||
response = await client.get("/api/monitor/comments")
|
||||
assert response.status_code == 200
|
||||
|
||||
comments = response.json()["comments"]
|
||||
assert len(comments) == 4
|
||||
|
||||
by_id = {c["comment_id"]: c for c in comments}
|
||||
assert by_id["c1"]["note_title"] == "作品甲"
|
||||
assert by_id["c1"]["note_url"].endswith("note-a")
|
||||
assert by_id["c1"]["note_cover"].endswith("note-a.jpg")
|
||||
assert by_id["c4"]["note_title"] == "作品乙"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_note_id_filters_the_stream(self, client):
|
||||
response = await client.get("/api/monitor/comments", params={"note_id": "note-a"})
|
||||
comments = response.json()["comments"]
|
||||
assert {c["comment_id"] for c in comments} == {"c1", "c2", "c3"}
|
||||
|
||||
|
||||
class TestGroupByNote:
|
||||
@pytest.mark.asyncio
|
||||
async def test_groups_bucket_by_work(self, client):
|
||||
response = await client.get("/api/monitor/comments", params={"group_by": "note"})
|
||||
body = response.json()
|
||||
|
||||
assert "groups" in body
|
||||
assert body["total"] == 4
|
||||
|
||||
groups = {g["note_id"]: g for g in body["groups"]}
|
||||
assert set(groups) == {"note-a", "note-b"}
|
||||
assert len(groups["note-a"]["comments"]) == 3
|
||||
assert len(groups["note-b"]["comments"]) == 1
|
||||
assert groups["note-a"]["note_title"] == "作品甲"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_newest_group_comes_first(self, client):
|
||||
"""The UI expands the first group by default, so it must be the newest."""
|
||||
response = await client.get("/api/monitor/comments", params={"group_by": "note"})
|
||||
groups = response.json()["groups"]
|
||||
# note-b's only comment is the most recent overall.
|
||||
assert groups[0]["note_id"] == "note-b"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flat_shape_is_unchanged_without_the_flag(self, client):
|
||||
body = (await client.get("/api/monitor/comments")).json()
|
||||
assert "comments" in body and "groups" not in body
|
||||
|
||||
|
||||
class TestCommentNoteFilterOptions:
|
||||
@pytest.mark.asyncio
|
||||
async def test_options_carry_counts_and_titles(self, client):
|
||||
response = await client.get("/api/monitor/comment-notes")
|
||||
assert response.status_code == 200
|
||||
|
||||
notes = {n["note_id"]: n for n in response.json()["notes"]}
|
||||
assert notes["note-a"]["comment_count"] == 3
|
||||
assert notes["note-b"]["comment_count"] == 1
|
||||
assert notes["note-a"]["note_title"] == "作品甲"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scoped_to_a_task(self, client):
|
||||
tasks = (await client.get("/api/monitor/tasks")).json()["tasks"]
|
||||
task_id = tasks[0]["id"]
|
||||
|
||||
scoped = await client.get("/api/monitor/comment-notes", params={"task_id": task_id})
|
||||
assert len(scoped.json()["notes"]) == 2
|
||||
|
||||
# A task with no comments yields an empty list, not an error.
|
||||
other = await client.get("/api/monitor/comment-notes", params={"task_id": 9999})
|
||||
assert other.json()["notes"] == []
|
||||
|
||||
|
||||
class TestExport:
|
||||
@pytest.mark.asyncio
|
||||
async def test_csv_has_a_bom_so_excel_does_not_mangle_chinese(self, client):
|
||||
response = await client.get("/api/monitor/export", params={"kind": "comments"})
|
||||
assert response.status_code == 200
|
||||
assert response.content.startswith(b"\xef\xbb\xbf")
|
||||
assert "attachment" in response.headers["content-disposition"]
|
||||
|
||||
text = response.content.decode("utf-8-sig")
|
||||
rows = list(csv.DictReader(io.StringIO(text)))
|
||||
assert len(rows) == 4
|
||||
assert rows[0]["所属作品"] in ("作品甲", "作品乙")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_notes_export(self, client):
|
||||
response = await client.get(
|
||||
"/api/monitor/export", params={"kind": "notes", "format": "csv"}
|
||||
)
|
||||
rows = list(csv.DictReader(io.StringIO(response.content.decode("utf-8-sig"))))
|
||||
assert {r["作品ID"] for r in rows} == {"note-a", "note-b"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_xlsx_is_a_readable_workbook(self, client):
|
||||
from openpyxl import load_workbook
|
||||
|
||||
response = await client.get(
|
||||
"/api/monitor/export", params={"kind": "comments", "format": "xlsx"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
workbook = load_workbook(io.BytesIO(response.content))
|
||||
sheet = workbook.active
|
||||
assert sheet.max_row == 5 # header + four comments
|
||||
assert sheet.cell(row=1, column=1).value == "所属作品"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_report_export(self, client):
|
||||
response = await client.get(
|
||||
"/api/monitor/export",
|
||||
params={"kind": "report", "days": 3},
|
||||
)
|
||||
rows = list(csv.DictReader(io.StringIO(response.content.decode("utf-8-sig"))))
|
||||
assert len(rows) == 3
|
||||
assert "日期" in rows[0]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_kind_and_format_are_rejected(self, client):
|
||||
assert (
|
||||
await client.get("/api/monitor/export", params={"kind": "nope"})
|
||||
).status_code == 400
|
||||
assert (
|
||||
await client.get("/api/monitor/export", params={"kind": "notes", "format": "pdf"})
|
||||
).status_code == 400
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_selection_is_a_404_not_an_empty_file(self, client):
|
||||
"""An empty download looks like a bug; say so instead."""
|
||||
response = await client.get(
|
||||
"/api/monitor/export", params={"kind": "comments", "note_id": "no-such-note"}
|
||||
)
|
||||
assert response.status_code == 404
|
||||
@@ -0,0 +1,527 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2025 [email protected]
|
||||
#
|
||||
# This file is part of MediaCrawler project.
|
||||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_monitor_ingest.py
|
||||
# GitHub: https://github.com/NanmiCoder
|
||||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||
#
|
||||
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
||||
# 5. 不得用于任何非法或不当的用途。
|
||||
#
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
"""Offline tests for the monitoring ingest/diff layer.
|
||||
|
||||
These run without network, browser or login and cover the correctness caveats
|
||||
that matter most: baseline suppression, count parsing, NULL-vs-zero, the
|
||||
posted/seen comment split, idempotency, and the silent-cookie-failure signal.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from tools.time_util import get_current_timestamp
|
||||
|
||||
from api.monitor.ingest import describe_exit_code, ingest_run, parse_count
|
||||
from api.monitor.models import (
|
||||
EVENT_AUTH_FAILURE,
|
||||
EVENT_METRIC_DELTA,
|
||||
EVENT_NEW_COMMENT_POSTED,
|
||||
EVENT_NEW_COMMENT_SEEN,
|
||||
EVENT_NEW_NOTE,
|
||||
EVENT_NO_DATA,
|
||||
EVENT_RUN_FAILED,
|
||||
MODE_CREATOR,
|
||||
MonitorBase,
|
||||
MonitorEvent,
|
||||
MonitorNote,
|
||||
MonitorNoteMetric,
|
||||
MonitorRun,
|
||||
MonitorTask,
|
||||
RUN_FAILED,
|
||||
RUN_PARTIAL,
|
||||
RUN_SUCCESS,
|
||||
)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def db():
|
||||
"""An isolated in-memory monitoring database."""
|
||||
engine = create_async_engine("sqlite+aiosqlite://", poolclass=StaticPool)
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(MonitorBase.metadata.create_all)
|
||||
|
||||
factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
||||
async with factory() as db_session:
|
||||
yield db_session
|
||||
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def _make_task(db: AsyncSession, **overrides) -> MonitorTask:
|
||||
defaults = dict(
|
||||
name="test task",
|
||||
platform="xhs",
|
||||
mode=MODE_CREATOR,
|
||||
enabled=True,
|
||||
interval_minutes=60,
|
||||
max_notes_count=20,
|
||||
enable_comments=True,
|
||||
max_comments_count=50,
|
||||
run_timeout_seconds=3600,
|
||||
created_at=0,
|
||||
updated_at=0,
|
||||
)
|
||||
defaults.update(overrides)
|
||||
task = MonitorTask(**defaults)
|
||||
db.add(task)
|
||||
await db.flush()
|
||||
return task
|
||||
|
||||
|
||||
async def _make_run(
|
||||
db: AsyncSession,
|
||||
task: MonitorTask,
|
||||
started_at: int,
|
||||
exit_code: Optional[int] = 0,
|
||||
) -> MonitorRun:
|
||||
run = MonitorRun(
|
||||
task_id=task.id,
|
||||
trigger="manual",
|
||||
status=RUN_SUCCESS,
|
||||
phase=task.mode,
|
||||
save_data_path="",
|
||||
queued_at=started_at,
|
||||
not_before=0,
|
||||
started_at=started_at,
|
||||
exit_code=exit_code,
|
||||
)
|
||||
db.add(run)
|
||||
await db.flush()
|
||||
return run
|
||||
|
||||
|
||||
def _write_run_dir(
|
||||
root: Path,
|
||||
notes: List[Dict[str, Any]],
|
||||
comments: Optional[List[Dict[str, Any]]] = None,
|
||||
) -> Path:
|
||||
"""Write a run's jsonl output in the crawler's own layout."""
|
||||
jsonl_dir = root / "xhs" / "jsonl"
|
||||
jsonl_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
contents = jsonl_dir / "creator_contents_2026-01-01.jsonl"
|
||||
contents.write_text(
|
||||
"\n".join(json.dumps(n, ensure_ascii=False) for n in notes),
|
||||
encoding="utf-8",
|
||||
)
|
||||
if comments is not None:
|
||||
comment_file = jsonl_dir / "creator_comments_2026-01-01.jsonl"
|
||||
comment_file.write_text(
|
||||
"\n".join(json.dumps(c, ensure_ascii=False) for c in comments),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return root
|
||||
|
||||
|
||||
def _note(note_id: str, liked: Any = "10", **extra) -> Dict[str, Any]:
|
||||
record = {
|
||||
"note_id": note_id,
|
||||
"title": f"title-{note_id}",
|
||||
"note_url": f"https://www.xiaohongshu.com/explore/{note_id}",
|
||||
"image_list": "https://img/cover.jpg",
|
||||
"creator_hash": "hash",
|
||||
"time": 1700000000000,
|
||||
"liked_count": liked,
|
||||
"comment_count": "1",
|
||||
"collected_count": "1",
|
||||
"share_count": "1",
|
||||
}
|
||||
record.update(extra)
|
||||
return record
|
||||
|
||||
|
||||
def _comment(comment_id: str, note_id: str, create_time: int, **extra) -> Dict[str, Any]:
|
||||
record = {
|
||||
"comment_id": comment_id,
|
||||
"note_id": note_id,
|
||||
"content": f"content-{comment_id}",
|
||||
"nickname": "u***r",
|
||||
"creator_hash": "hash",
|
||||
"create_time": create_time,
|
||||
"like_count": "0",
|
||||
"sub_comment_count": 0,
|
||||
"parent_comment_id": "",
|
||||
}
|
||||
record.update(extra)
|
||||
return record
|
||||
|
||||
|
||||
async def _events(db: AsyncSession, event_type: Optional[str] = None) -> List[MonitorEvent]:
|
||||
stmt = select(MonitorEvent)
|
||||
if event_type:
|
||||
stmt = stmt.where(MonitorEvent.type == event_type)
|
||||
return list((await db.scalars(stmt)).all())
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# parse_count
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
class TestParseCount:
|
||||
@pytest.mark.parametrize(
|
||||
"raw,expected",
|
||||
[
|
||||
("1234", 1234),
|
||||
("1.2万", 12000),
|
||||
("1.2w", 12000),
|
||||
("3亿", 300000000),
|
||||
("1,234", 1234),
|
||||
(42, 42),
|
||||
],
|
||||
)
|
||||
def test_parses_platform_formats(self, raw, expected):
|
||||
assert parse_count(raw) == expected
|
||||
|
||||
@pytest.mark.parametrize("raw", ["", None, "暂无", "-", "abc", True])
|
||||
def test_unparseable_values_return_none(self, raw):
|
||||
assert parse_count(raw) is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Exit codes
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
class TestExitCodeStorage:
|
||||
"""Guards a bug that only showed up when the data moved to MySQL.
|
||||
|
||||
Windows reports process failures as unsigned 32-bit NTSTATUS values
|
||||
(0xC0000142 = 3221225794). That overflows MySQL's signed INT, while SQLite's
|
||||
dynamic typing accepted it happily -- so the column silently worked until a
|
||||
real migration hit it with real data.
|
||||
"""
|
||||
|
||||
def test_column_is_bigint_not_int(self):
|
||||
from sqlalchemy import BigInteger
|
||||
|
||||
from api.monitor.models import MonitorRun
|
||||
|
||||
column_type = MonitorRun.__table__.c.exit_code.type
|
||||
assert isinstance(column_type, BigInteger), (
|
||||
f"exit_code must be BigInteger to hold unsigned 32-bit codes, got {column_type!r}"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_ntstatus_value_round_trips(self, db):
|
||||
task = await _make_task(db)
|
||||
run = await _make_run(db, task, started_at=1000, exit_code=3221225794)
|
||||
await db.commit()
|
||||
|
||||
stored = await db.scalar(
|
||||
select(MonitorRun.exit_code).where(MonitorRun.id == run.id)
|
||||
)
|
||||
assert stored == 3221225794
|
||||
|
||||
|
||||
class TestDescribeExitCode:
|
||||
def test_windows_status_code_is_decoded(self):
|
||||
"""3221225794 is 0xC0000142, which is meaningless without decoding."""
|
||||
message = describe_exit_code(3221225794)
|
||||
assert "0xC0000142" in message
|
||||
assert "DLL_INIT_FAILED" in message
|
||||
|
||||
def test_negative_signed_form_is_also_decoded(self):
|
||||
# Python may hand back the signed form depending on how it was launched.
|
||||
assert "0xC0000142" in describe_exit_code(-1073741502)
|
||||
|
||||
def test_unknown_code_degrades_to_the_raw_number(self):
|
||||
assert describe_exit_code(1) == "Crawler exited with code 1"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Notes
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
class TestNoteIngest:
|
||||
@pytest.mark.asyncio
|
||||
async def test_baseline_run_emits_no_new_note_events(self, db, tmp_path):
|
||||
task = await _make_task(db)
|
||||
run = await _make_run(db, task, started_at=1000)
|
||||
_write_run_dir(tmp_path, [_note("n1"), _note("n2")], comments=[])
|
||||
|
||||
result = await ingest_run(db, run, task, tmp_path)
|
||||
|
||||
assert result.status == RUN_SUCCESS
|
||||
assert result.is_baseline is True
|
||||
assert result.new_notes == 2
|
||||
# Everything is "new" on the first run; emitting that would be pure noise.
|
||||
assert await _events(db, EVENT_NEW_NOTE) == []
|
||||
assert len(list((await db.scalars(select(MonitorNote))).all())) == 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_empty_run_does_not_establish_a_baseline(self, db, tmp_path):
|
||||
"""A run that fetched nothing observed nothing, so it is not a baseline.
|
||||
|
||||
Otherwise the first crawl that actually works reports every work as
|
||||
newly discovered.
|
||||
"""
|
||||
task = await _make_task(db)
|
||||
|
||||
empty_run = await _make_run(db, task, started_at=1000)
|
||||
(tmp_path / "empty").mkdir(parents=True, exist_ok=True)
|
||||
await ingest_run(db, empty_run, task, tmp_path / "empty")
|
||||
|
||||
real_run = await _make_run(db, task, started_at=2000)
|
||||
result = await ingest_run(
|
||||
db, real_run, task, _write_run_dir(tmp_path / "ok", [_note("n1")], comments=[])
|
||||
)
|
||||
|
||||
assert result.is_baseline is True
|
||||
assert await _events(db, EVENT_NEW_NOTE) == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_second_run_reports_only_the_added_note(self, db, tmp_path):
|
||||
task = await _make_task(db)
|
||||
|
||||
first_dir = _write_run_dir(tmp_path / "run1", [_note("n1")], comments=[])
|
||||
run1 = await _make_run(db, task, started_at=1000)
|
||||
await ingest_run(db, run1, task, first_dir)
|
||||
|
||||
second_dir = _write_run_dir(tmp_path / "run2", [_note("n1"), _note("n2")], comments=[])
|
||||
run2 = await _make_run(db, task, started_at=2000)
|
||||
result = await ingest_run(db, run2, task, second_dir)
|
||||
|
||||
assert result.is_baseline is False
|
||||
assert result.new_notes == 1
|
||||
|
||||
events = await _events(db, EVENT_NEW_NOTE)
|
||||
assert len(events) == 1
|
||||
assert events[0].target_id == "n2"
|
||||
assert events[0].run_id == run2.id
|
||||
|
||||
|
||||
class TestMetricSnapshots:
|
||||
@pytest.mark.asyncio
|
||||
async def test_delta_event_emitted_when_like_count_changes(self, db, tmp_path):
|
||||
task = await _make_task(db)
|
||||
|
||||
run1 = await _make_run(db, task, started_at=1000)
|
||||
await ingest_run(db, run1, task, _write_run_dir(tmp_path / "r1", [_note("n1", "100")], comments=[]))
|
||||
|
||||
run2 = await _make_run(db, task, started_at=2000)
|
||||
await ingest_run(db, run2, task, _write_run_dir(tmp_path / "r2", [_note("n1", "150")], comments=[]))
|
||||
|
||||
events = await _events(db, EVENT_METRIC_DELTA)
|
||||
assert len(events) == 1
|
||||
|
||||
payload = json.loads(events[0].payload_json)
|
||||
assert payload["deltas"]["liked_count"] == {"from": 100, "to": 150, "delta": 50}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_delta_when_nothing_changed(self, db, tmp_path):
|
||||
task = await _make_task(db)
|
||||
run1 = await _make_run(db, task, started_at=1000)
|
||||
await ingest_run(db, run1, task, _write_run_dir(tmp_path / "r1", [_note("n1", "100")], comments=[]))
|
||||
run2 = await _make_run(db, task, started_at=2000)
|
||||
await ingest_run(db, run2, task, _write_run_dir(tmp_path / "r2", [_note("n1", "100")], comments=[]))
|
||||
|
||||
assert await _events(db, EVENT_METRIC_DELTA) == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unparseable_count_is_null_not_zero(self, db, tmp_path):
|
||||
task = await _make_task(db)
|
||||
run = await _make_run(db, task, started_at=1000)
|
||||
await ingest_run(db, run, task, _write_run_dir(tmp_path, [_note("n1", "暂无")], comments=[]))
|
||||
|
||||
metric = await db.scalar(select(MonitorNoteMetric).where(MonitorNoteMetric.note_id == "n1"))
|
||||
# Zero would forge a large negative delta on the next comparison.
|
||||
assert metric.liked_count is None
|
||||
assert metric.raw_liked_count == "暂无"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_delta_when_previous_value_was_unparseable(self, db, tmp_path):
|
||||
task = await _make_task(db)
|
||||
run1 = await _make_run(db, task, started_at=1000)
|
||||
await ingest_run(db, run1, task, _write_run_dir(tmp_path / "r1", [_note("n1", "暂无")], comments=[]))
|
||||
run2 = await _make_run(db, task, started_at=2000)
|
||||
await ingest_run(db, run2, task, _write_run_dir(tmp_path / "r2", [_note("n1", "50")], comments=[]))
|
||||
|
||||
assert await _events(db, EVENT_METRIC_DELTA) == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metric_snapshot_survives_across_runs(self, db, tmp_path):
|
||||
"""The crawler's own DB store overwrites metrics; ours must not."""
|
||||
task = await _make_task(db)
|
||||
for index, liked in enumerate(["100", "150", "300"]):
|
||||
run = await _make_run(db, task, started_at=1000 * (index + 1))
|
||||
await ingest_run(
|
||||
db, run, task, _write_run_dir(tmp_path / f"r{index}", [_note("n1", liked)], comments=[])
|
||||
)
|
||||
|
||||
snapshots = list(
|
||||
(
|
||||
await db.scalars(
|
||||
select(MonitorNoteMetric)
|
||||
.where(MonitorNoteMetric.note_id == "n1")
|
||||
.order_by(MonitorNoteMetric.run_id)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
assert [s.liked_count for s in snapshots] == [100, 150, 300]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Comments
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
class TestCommentIngest:
|
||||
@pytest.mark.asyncio
|
||||
async def test_posted_vs_seen_split_by_create_time(self, db, tmp_path):
|
||||
task = await _make_task(db)
|
||||
|
||||
# Baseline establishes the seen-set; no events on the first run.
|
||||
run1 = await _make_run(db, task, started_at=1000)
|
||||
await ingest_run(
|
||||
db, run1, task,
|
||||
_write_run_dir(tmp_path / "r1", [_note("n1")], comments=[_comment("c1", "n1", create_time=500)]),
|
||||
)
|
||||
assert await _events(db, EVENT_NEW_COMMENT_POSTED) == []
|
||||
|
||||
# c2 was published after run1 started -> genuinely new.
|
||||
# c3 is old but only just surfaced in the top-N window -> seen, not posted.
|
||||
run2 = await _make_run(db, task, started_at=2000)
|
||||
await ingest_run(
|
||||
db, run2, task,
|
||||
_write_run_dir(
|
||||
tmp_path / "r2",
|
||||
[_note("n1")],
|
||||
comments=[
|
||||
_comment("c1", "n1", create_time=500),
|
||||
_comment("c2", "n1", create_time=2500),
|
||||
_comment("c3", "n1", create_time=100),
|
||||
],
|
||||
),
|
||||
)
|
||||
|
||||
posted = await _events(db, EVENT_NEW_COMMENT_POSTED)
|
||||
seen = await _events(db, EVENT_NEW_COMMENT_SEEN)
|
||||
assert len(posted) == 1
|
||||
assert json.loads(posted[0].payload_json)["comment_id"] == "c2"
|
||||
assert len(seen) == 1
|
||||
assert json.loads(seen[0].payload_json)["comment_id"] == "c3"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_comments_not_ingested_when_disabled(self, db, tmp_path):
|
||||
task = await _make_task(db, enable_comments=False)
|
||||
run = await _make_run(db, task, started_at=1000)
|
||||
result = await ingest_run(
|
||||
db, run, task,
|
||||
_write_run_dir(tmp_path, [_note("n1")], comments=[_comment("c1", "n1", 500)]),
|
||||
)
|
||||
assert result.new_comments == 0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Failure handling
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
class TestFailureHandling:
|
||||
@pytest.mark.asyncio
|
||||
async def test_nonzero_exit_is_a_failure(self, db, tmp_path):
|
||||
task = await _make_task(db)
|
||||
run = await _make_run(db, task, started_at=1000, exit_code=1)
|
||||
_write_run_dir(tmp_path, [_note("n1")], comments=[])
|
||||
|
||||
result = await ingest_run(db, run, task, tmp_path)
|
||||
|
||||
assert result.status == RUN_FAILED
|
||||
assert len(await _events(db, EVENT_RUN_FAILED)) == 1
|
||||
# A crashed run must not touch the seen-set.
|
||||
assert await db.scalar(select(MonitorNote.id)) is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_zero_notes_with_exit_zero_is_a_suspected_auth_failure(self, db, tmp_path):
|
||||
"""The silent-cookie-failure signature: exit 0 but nothing fetched.
|
||||
|
||||
A real bad-cookie run writes no output file at all, which is why the
|
||||
exit code has to be checked before the files are.
|
||||
"""
|
||||
task = await _make_task(db)
|
||||
run = await _make_run(db, task, started_at=1000, exit_code=0)
|
||||
tmp_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
result = await ingest_run(db, run, task, tmp_path)
|
||||
|
||||
assert result.status == RUN_PARTIAL
|
||||
assert len(await _events(db, EVENT_AUTH_FAILURE)) == 1
|
||||
assert await _events(db, EVENT_RUN_FAILED) == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_data_is_not_blamed_on_the_cookie_when_a_sibling_succeeded(
|
||||
self, db, tmp_path
|
||||
):
|
||||
"""A task that just worked proves the login is fine; do not cry wolf."""
|
||||
healthy = await _make_task(db, name="healthy")
|
||||
healthy_run = await _make_run(db, healthy, started_at=get_current_timestamp())
|
||||
await ingest_run(
|
||||
db, healthy_run, healthy,
|
||||
_write_run_dir(tmp_path / "ok", [_note("n1")], comments=[]),
|
||||
)
|
||||
|
||||
task = await _make_task(db, name="suspect")
|
||||
run = await _make_run(db, task, started_at=get_current_timestamp())
|
||||
(tmp_path / "empty").mkdir(parents=True, exist_ok=True)
|
||||
result = await ingest_run(db, run, task, tmp_path / "empty")
|
||||
|
||||
assert result.status == RUN_PARTIAL
|
||||
assert await _events(db, EVENT_NO_DATA) != []
|
||||
assert await _events(db, EVENT_AUTH_FAILURE) == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_contents_file_is_also_an_auth_failure(self, db, tmp_path):
|
||||
task = await _make_task(db)
|
||||
run = await _make_run(db, task, started_at=1000, exit_code=0)
|
||||
_write_run_dir(tmp_path, [], comments=[])
|
||||
|
||||
result = await ingest_run(db, run, task, tmp_path)
|
||||
|
||||
assert result.status == RUN_PARTIAL
|
||||
assert len(await _events(db, EVENT_AUTH_FAILURE)) == 1
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Idempotency
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
class TestIdempotency:
|
||||
@pytest.mark.asyncio
|
||||
async def test_reingesting_the_same_data_adds_nothing(self, db, tmp_path):
|
||||
task = await _make_task(db)
|
||||
run_dir = _write_run_dir(
|
||||
tmp_path, [_note("n1"), _note("n2")], comments=[_comment("c1", "n1", 500)]
|
||||
)
|
||||
|
||||
run1 = await _make_run(db, task, started_at=1000)
|
||||
await ingest_run(db, run1, task, run_dir)
|
||||
notes_after_first = len(list((await db.scalars(select(MonitorNote))).all()))
|
||||
|
||||
# A retry of the same crawl content must not duplicate rows or events.
|
||||
run2 = await _make_run(db, task, started_at=2000)
|
||||
result = await ingest_run(db, run2, task, run_dir)
|
||||
|
||||
assert result.new_notes == 0
|
||||
assert result.new_comments == 0
|
||||
assert len(list((await db.scalars(select(MonitorNote))).all())) == notes_after_first
|
||||
@@ -0,0 +1,315 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2025 [email protected]
|
||||
#
|
||||
# This file is part of MediaCrawler project.
|
||||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_monitor_notify.py
|
||||
# GitHub: https://github.com/NanmiCoder
|
||||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||
#
|
||||
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
||||
# 5. 不得用于任何非法或不当的用途。
|
||||
#
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
"""Tests for the WeCom notification layer.
|
||||
|
||||
The webhook is stubbed, so nothing here touches the network.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from api.monitor import notify
|
||||
from api.monitor.models import (
|
||||
EVENT_AUTH_FAILURE,
|
||||
EVENT_METRIC_DELTA,
|
||||
EVENT_NEW_NOTE,
|
||||
EVENT_NEW_COMMENT_POSTED,
|
||||
MODE_CREATOR,
|
||||
SETTING_WECOM_WEBHOOK,
|
||||
MonitorBase,
|
||||
MonitorEvent,
|
||||
MonitorRun,
|
||||
MonitorTask,
|
||||
RUN_SUCCESS,
|
||||
)
|
||||
from api.monitor.settings import set_setting
|
||||
|
||||
WEBHOOK = "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=abc123"
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def db():
|
||||
engine = create_async_engine("sqlite+aiosqlite://", poolclass=StaticPool)
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(MonitorBase.metadata.create_all)
|
||||
factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
||||
async with factory() as session:
|
||||
yield session
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def _seed(db: AsyncSession, notify_enabled: bool = True):
|
||||
task = MonitorTask(
|
||||
name="竞品监控", platform="xhs", mode=MODE_CREATOR, enabled=True,
|
||||
interval_minutes=60, max_notes_count=20, enable_comments=True,
|
||||
max_comments_count=50, run_timeout_seconds=3600,
|
||||
notify_enabled=notify_enabled, created_at=0, updated_at=0,
|
||||
)
|
||||
db.add(task)
|
||||
await db.flush()
|
||||
|
||||
run = MonitorRun(
|
||||
task_id=task.id, trigger="scheduled", status=RUN_SUCCESS, phase=MODE_CREATOR,
|
||||
save_data_path="", queued_at=0, not_before=0, max_comments_count=50,
|
||||
)
|
||||
db.add(run)
|
||||
await db.flush()
|
||||
return task, run
|
||||
|
||||
|
||||
def _add_event(db, task, run, event_type, title, payload=None, severity="info"):
|
||||
db.add(
|
||||
MonitorEvent(
|
||||
task_id=task.id, run_id=run.id, type=event_type, severity=severity,
|
||||
target_kind="note", target_id="note-1", title=title,
|
||||
payload_json=json.dumps(payload or {}, ensure_ascii=False), created_at=0,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Message building
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
class TestBuildRunMessage:
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_notifiable_events_means_no_message(self, db):
|
||||
task, run = await _seed(db)
|
||||
# Metric deltas are not something anyone wants pushed.
|
||||
_add_event(db, task, run, EVENT_METRIC_DELTA, "点赞 10→20")
|
||||
_add_event(db, task, run, EVENT_NEW_COMMENT_POSTED, "新评论")
|
||||
await db.flush()
|
||||
|
||||
assert await notify.build_run_message(db, task, run) is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_notes_are_listed_with_links(self, db):
|
||||
task, run = await _seed(db)
|
||||
_add_event(
|
||||
db, task, run, EVENT_NEW_NOTE, "新作品：标题A",
|
||||
payload={"note_id": "abc123", "title": "标题A"},
|
||||
)
|
||||
await db.flush()
|
||||
|
||||
message = await notify.build_run_message(db, task, run)
|
||||
|
||||
assert "竞品监控" in message
|
||||
assert "新增作品 **1** 篇" in message
|
||||
assert "标题A" in message
|
||||
assert "https://www.xiaohongshu.com/explore/abc123" in message
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_long_note_lists_are_truncated(self, db):
|
||||
"""A first run can find dozens; a wall of text is worse than a count."""
|
||||
task, run = await _seed(db)
|
||||
for index in range(14):
|
||||
_add_event(
|
||||
db, task, run, EVENT_NEW_NOTE, f"新作品：{index}",
|
||||
payload={"note_id": f"n{index}", "title": f"标题{index}"},
|
||||
)
|
||||
await db.flush()
|
||||
|
||||
message = await notify.build_run_message(db, task, run)
|
||||
|
||||
assert "新增作品 **14** 篇" in message
|
||||
assert "标题0" in message
|
||||
assert "标题13" not in message
|
||||
assert "等共 14 篇" in message
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_is_reported_as_a_warning(self, db):
|
||||
task, run = await _seed(db)
|
||||
_add_event(
|
||||
db, task, run, EVENT_AUTH_FAILURE,
|
||||
"疑似登录态失效：本次未抓到任何作品", severity="error",
|
||||
)
|
||||
await db.flush()
|
||||
|
||||
message = await notify.build_run_message(db, task, run)
|
||||
|
||||
assert "异常" in message
|
||||
assert "登录态失效" in message
|
||||
assert notify._COLOR_WARNING in message
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_baseline_runs_say_so(self, db):
|
||||
task, run = await _seed(db)
|
||||
run.is_baseline = True
|
||||
_add_event(db, task, run, EVENT_NEW_NOTE, "新作品", payload={"note_id": "x", "title": "t"})
|
||||
await db.flush()
|
||||
|
||||
message = await notify.build_run_message(db, task, run)
|
||||
|
||||
assert "基线" in message
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# notify_run gating
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
class TestNotifyRunGating:
|
||||
@pytest.mark.asyncio
|
||||
async def test_disabled_task_is_skipped(self, db, monkeypatch):
|
||||
task, run = await _seed(db, notify_enabled=False)
|
||||
_add_event(db, task, run, EVENT_NEW_NOTE, "新作品", payload={"note_id": "x", "title": "t"})
|
||||
await set_setting(db, SETTING_WECOM_WEBHOOK, WEBHOOK)
|
||||
await db.flush()
|
||||
|
||||
called = []
|
||||
monkeypatch.setattr(notify, "send_wecom", lambda *a, **k: called.append(a) or _ok())
|
||||
|
||||
assert await notify.notify_run(db, task, run) is None
|
||||
assert called == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_webhook_is_skipped(self, db, monkeypatch):
|
||||
task, run = await _seed(db, notify_enabled=True)
|
||||
_add_event(db, task, run, EVENT_NEW_NOTE, "新作品", payload={"note_id": "x", "title": "t"})
|
||||
await db.flush()
|
||||
|
||||
called = []
|
||||
monkeypatch.setattr(notify, "send_wecom", lambda *a, **k: called.append(a) or _ok())
|
||||
|
||||
assert await notify.notify_run(db, task, run) is None
|
||||
assert called == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_successful_push_records_the_timestamp(self, db, monkeypatch):
|
||||
task, run = await _seed(db, notify_enabled=True)
|
||||
_add_event(db, task, run, EVENT_NEW_NOTE, "新作品", payload={"note_id": "x", "title": "t"})
|
||||
await set_setting(db, SETTING_WECOM_WEBHOOK, WEBHOOK)
|
||||
await db.flush()
|
||||
|
||||
monkeypatch.setattr(notify, "send_wecom", lambda *a, **k: _ok())
|
||||
|
||||
message = await notify.notify_run(db, task, run)
|
||||
|
||||
assert message is not None
|
||||
# Lets the UI answer "why did I not get a push for this run?".
|
||||
assert task.last_notified_at is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_push_failure_never_raises(self, db, monkeypatch):
|
||||
"""A broken webhook must not take down the crawl that just succeeded."""
|
||||
task, run = await _seed(db, notify_enabled=True)
|
||||
_add_event(db, task, run, EVENT_NEW_NOTE, "新作品", payload={"note_id": "x", "title": "t"})
|
||||
await set_setting(db, SETTING_WECOM_WEBHOOK, WEBHOOK)
|
||||
await db.flush()
|
||||
|
||||
async def _boom(*args, **kwargs):
|
||||
raise RuntimeError("network exploded")
|
||||
|
||||
monkeypatch.setattr(notify, "send_wecom", _boom)
|
||||
|
||||
assert await notify.notify_run(db, task, run) is None
|
||||
|
||||
|
||||
async def _ok():
|
||||
return True, "发送成功"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# send_wecom
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, payload):
|
||||
self._payload = payload
|
||||
|
||||
def raise_for_status(self):
|
||||
return None
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
|
||||
class _FakeClient:
|
||||
"""Captures the request and replays a canned WeCom reply."""
|
||||
|
||||
last_payload = None
|
||||
|
||||
def __init__(self, reply=None, error=None):
|
||||
self._reply = reply if reply is not None else {"errcode": 0, "errmsg": "ok"}
|
||||
self._error = error
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
return self
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc):
|
||||
return False
|
||||
|
||||
async def post(self, url, json=None):
|
||||
if self._error:
|
||||
raise self._error
|
||||
type(self).last_payload = json
|
||||
return _FakeResponse(self._reply)
|
||||
|
||||
|
||||
class TestSendWecom:
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_url_is_reported(self):
|
||||
ok, detail = await notify.send_wecom("", "hi")
|
||||
assert ok is False
|
||||
assert "未配置" in detail
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_success(self, monkeypatch):
|
||||
monkeypatch.setattr(notify.httpx, "AsyncClient", _FakeClient())
|
||||
|
||||
ok, detail = await notify.send_wecom(WEBHOOK, "**标题**\n> 内容")
|
||||
|
||||
assert ok is True
|
||||
assert detail == "发送成功"
|
||||
# WeCom expects a markdown message envelope.
|
||||
assert _FakeClient.last_payload["msgtype"] == "markdown"
|
||||
assert _FakeClient.last_payload["markdown"]["content"] == "**标题**\n> 内容"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_nonzero_errcode_is_a_failure(self, monkeypatch):
|
||||
"""WeCom answers HTTP 200 even when it rejects the message."""
|
||||
monkeypatch.setattr(
|
||||
notify.httpx, "AsyncClient",
|
||||
_FakeClient(reply={"errcode": 93000, "errmsg": "invalid webhook url"}),
|
||||
)
|
||||
|
||||
ok, detail = await notify.send_wecom(WEBHOOK, "hi")
|
||||
|
||||
assert ok is False
|
||||
assert "93000" in detail
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_network_error_is_returned_not_raised(self, monkeypatch):
|
||||
import httpx
|
||||
|
||||
monkeypatch.setattr(
|
||||
notify.httpx, "AsyncClient",
|
||||
_FakeClient(error=httpx.ConnectError("boom")),
|
||||
)
|
||||
|
||||
ok, detail = await notify.send_wecom(WEBHOOK, "hi")
|
||||
|
||||
assert ok is False
|
||||
assert "请求失败" in detail
|
||||
@@ -0,0 +1,304 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2025 [email protected]
|
||||
#
|
||||
# This file is part of MediaCrawler project.
|
||||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_monitor_report.py
|
||||
# GitHub: https://github.com/NanmiCoder
|
||||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||
#
|
||||
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
||||
# 5. 不得用于任何非法或不当的用途。
|
||||
#
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
"""Tests for the cross-task report aggregation.
|
||||
|
||||
The interaction delta is the part that is easy to get subtly wrong, so it is
|
||||
covered directly against the pure aggregation function.
|
||||
"""
|
||||
|
||||
from datetime import date, datetime
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from api.monitor.models import (
|
||||
MODE_CREATOR,
|
||||
MonitorBase,
|
||||
MonitorComment,
|
||||
MonitorNote,
|
||||
MonitorNoteMetric,
|
||||
MonitorTask,
|
||||
)
|
||||
from api.monitor.report import build_report, compute_daily_rows, day_bounds, iter_days
|
||||
|
||||
|
||||
def _ms(year: int, month: int, day: int, hour: int = 12) -> int:
|
||||
return int(datetime(year, month, day, hour).timestamp() * 1000)
|
||||
|
||||
|
||||
def _metrics(liked=0, comment=0, collected=0, share=0):
|
||||
"""All four metrics default to parsed values; pass None to simulate a
|
||||
platform value we could not parse."""
|
||||
return {
|
||||
"liked_count": liked,
|
||||
"comment_count": comment,
|
||||
"collected_count": collected,
|
||||
"share_count": share,
|
||||
}
|
||||
|
||||
|
||||
class TestDayHelpers:
|
||||
def test_day_bounds_cover_the_whole_local_day(self):
|
||||
start, end = day_bounds(date(2026, 1, 10))
|
||||
assert start < _ms(2026, 1, 10, 0) or start == _ms(2026, 1, 10, 0)
|
||||
assert end > _ms(2026, 1, 10, 23)
|
||||
|
||||
def test_iter_days_is_inclusive(self):
|
||||
days = iter_days(date(2026, 1, 10), date(2026, 1, 12))
|
||||
assert days == [date(2026, 1, 10), date(2026, 1, 11), date(2026, 1, 12)]
|
||||
|
||||
|
||||
class TestInteractionDelta:
|
||||
def test_note_first_seen_counts_all_of_its_value(self):
|
||||
"""A brand-new note has no earlier baseline, so it starts from zero."""
|
||||
day = date(2026, 1, 10)
|
||||
series = {"n1": [(_ms(2026, 1, 10, 10), _metrics(liked=100, comment=5))]}
|
||||
|
||||
rows = compute_daily_rows(series, {}, {}, [day])
|
||||
|
||||
assert rows[0]["liked_count_delta"] == 100
|
||||
assert rows[0]["comment_count_delta"] == 5
|
||||
|
||||
def test_growth_is_split_across_days(self):
|
||||
series = {
|
||||
"n1": [
|
||||
(_ms(2026, 1, 10, 10), _metrics(liked=100)),
|
||||
(_ms(2026, 1, 11, 10), _metrics(liked=300)),
|
||||
]
|
||||
}
|
||||
|
||||
rows = compute_daily_rows(series, {}, {}, [date(2026, 1, 10), date(2026, 1, 11)])
|
||||
|
||||
# Day 1: 0 -> 100. Day 2: 100 -> 300.
|
||||
assert [row["liked_count_delta"] for row in rows] == [100, 200]
|
||||
|
||||
def test_day_without_a_snapshot_reports_no_growth(self):
|
||||
series = {
|
||||
"n1": [
|
||||
(_ms(2026, 1, 10, 10), _metrics(liked=100)),
|
||||
(_ms(2026, 1, 12, 10), _metrics(liked=400)),
|
||||
]
|
||||
}
|
||||
days = [date(2026, 1, 10), date(2026, 1, 11), date(2026, 1, 12)]
|
||||
|
||||
rows = compute_daily_rows(series, {}, {}, days)
|
||||
|
||||
# The note was not crawled on the 11th, so nothing is claimed for it.
|
||||
assert [row["liked_count_delta"] for row in rows] == [100, 0, 300]
|
||||
|
||||
def test_deltas_aggregate_across_notes(self):
|
||||
series = {
|
||||
"n1": [
|
||||
(_ms(2026, 1, 10, 10), _metrics(liked=100)),
|
||||
(_ms(2026, 1, 11, 10), _metrics(liked=150)),
|
||||
],
|
||||
"n2": [
|
||||
(_ms(2026, 1, 10, 10), _metrics(liked=10)),
|
||||
(_ms(2026, 1, 11, 10), _metrics(liked=40)),
|
||||
],
|
||||
}
|
||||
|
||||
rows = compute_daily_rows(series, {}, {}, [date(2026, 1, 10), date(2026, 1, 11)])
|
||||
|
||||
assert [row["liked_count_delta"] for row in rows] == [110, 80]
|
||||
|
||||
def test_unparseable_metric_names_the_offending_field(self):
|
||||
"""A NULL count makes the delta unknown; it must not be reported as 0."""
|
||||
series = {
|
||||
"n1": [
|
||||
(_ms(2026, 1, 10, 10), _metrics(liked=100, comment=None)),
|
||||
(_ms(2026, 1, 11, 10), _metrics(liked=200, comment=None)),
|
||||
]
|
||||
}
|
||||
|
||||
rows = compute_daily_rows(series, {}, {}, [date(2026, 1, 11)])
|
||||
|
||||
# Naming the field is actionable; a bare boolean is not.
|
||||
assert rows[0]["partial_metrics"] == ["comment_count"]
|
||||
# The parseable metric is still summed correctly.
|
||||
assert rows[0]["liked_count_delta"] == 100
|
||||
|
||||
def test_unknown_value_only_taints_the_days_it_touches(self):
|
||||
series = {
|
||||
"n1": [
|
||||
(_ms(2026, 1, 10, 10), _metrics(liked=None)),
|
||||
(_ms(2026, 1, 11, 10), _metrics(liked=50)),
|
||||
(_ms(2026, 1, 12, 10), _metrics(liked=90)),
|
||||
]
|
||||
}
|
||||
days = [date(2026, 1, 10), date(2026, 1, 11), date(2026, 1, 12)]
|
||||
|
||||
rows = compute_daily_rows(series, {}, {}, days)
|
||||
|
||||
# Day 12 compares two known values, so it is clean.
|
||||
assert [row["partial_metrics"] for row in rows] == [
|
||||
["liked_count"],
|
||||
["liked_count"],
|
||||
[],
|
||||
]
|
||||
assert rows[2]["liked_count_delta"] == 40
|
||||
|
||||
def test_new_content_counts_come_from_the_day_maps(self):
|
||||
rows = compute_daily_rows(
|
||||
{},
|
||||
{date(2026, 1, 10): 3},
|
||||
{date(2026, 1, 10): 7},
|
||||
[date(2026, 1, 10), date(2026, 1, 11)],
|
||||
)
|
||||
|
||||
assert rows[0]["new_notes"] == 3
|
||||
assert rows[0]["new_comments"] == 7
|
||||
assert rows[1]["new_notes"] == 0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# DB-backed report + task filtering
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def db():
|
||||
engine = create_async_engine("sqlite+aiosqlite://", poolclass=StaticPool)
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(MonitorBase.metadata.create_all)
|
||||
factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
||||
async with factory() as session:
|
||||
yield session
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def _seed_task(db: AsyncSession, name: str) -> MonitorTask:
|
||||
task = MonitorTask(
|
||||
name=name, platform="xhs", mode=MODE_CREATOR, enabled=True,
|
||||
interval_minutes=60, max_notes_count=20, enable_comments=True,
|
||||
max_comments_count=50, run_timeout_seconds=3600,
|
||||
notify_enabled=False, created_at=0, updated_at=0,
|
||||
)
|
||||
db.add(task)
|
||||
await db.flush()
|
||||
return task
|
||||
|
||||
|
||||
async def _seed_note_with_metrics(
|
||||
db: AsyncSession, task: MonitorTask, note_id: str, samples
|
||||
) -> None:
|
||||
db.add(
|
||||
MonitorNote(
|
||||
task_id=task.id, note_id=note_id, title=note_id, note_url="",
|
||||
cover="", creator_hash="", source_kind="", published_at=None,
|
||||
first_seen_run_id=1, first_seen_at=samples[0][0],
|
||||
last_seen_run_id=len(samples), last_seen_at=samples[-1][0],
|
||||
)
|
||||
)
|
||||
for run_id, (ts, liked) in enumerate(samples, start=1):
|
||||
db.add(
|
||||
MonitorNoteMetric(
|
||||
task_id=task.id, note_id=note_id, run_id=run_id, captured_at=ts,
|
||||
liked_count=liked, comment_count=0, collected_count=0, share_count=0,
|
||||
raw_liked_count=str(liked), raw_comment_count="0",
|
||||
raw_collected_count="0", raw_share_count="0",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class TestBuildReport:
|
||||
@pytest.mark.asyncio
|
||||
async def test_totals_and_rows(self, db):
|
||||
task = await _seed_task(db, "t1")
|
||||
await _seed_note_with_metrics(
|
||||
db, task, "n1",
|
||||
[(_ms(2026, 1, 10, 10), 100), (_ms(2026, 1, 11, 10), 250)],
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
result = await build_report(db, [task.id], date(2026, 1, 10), date(2026, 1, 11))
|
||||
|
||||
assert result["totals"]["liked_count_delta"] == 250
|
||||
assert len(result["rows"]) == 2
|
||||
assert result["note_count"] == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_selection_isolates_the_report(self, db):
|
||||
"""The whole point: a report for a chosen subset must exclude the rest."""
|
||||
kept = await _seed_task(db, "kept")
|
||||
other = await _seed_task(db, "other")
|
||||
await _seed_note_with_metrics(db, kept, "n1", [(_ms(2026, 1, 10, 10), 100)])
|
||||
await _seed_note_with_metrics(db, other, "n2", [(_ms(2026, 1, 10, 10), 999)])
|
||||
await db.commit()
|
||||
|
||||
only_kept = await build_report(db, [kept.id], date(2026, 1, 10), date(2026, 1, 10))
|
||||
assert only_kept["totals"]["liked_count_delta"] == 100
|
||||
assert only_kept["note_count"] == 1
|
||||
|
||||
both = await build_report(db, [kept.id, other.id], date(2026, 1, 10), date(2026, 1, 10))
|
||||
assert both["totals"]["liked_count_delta"] == 1099
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_task_filter_covers_everything(self, db):
|
||||
first = await _seed_task(db, "a")
|
||||
second = await _seed_task(db, "b")
|
||||
await _seed_note_with_metrics(db, first, "n1", [(_ms(2026, 1, 10, 10), 10)])
|
||||
await _seed_note_with_metrics(db, second, "n2", [(_ms(2026, 1, 10, 10), 20)])
|
||||
await db.commit()
|
||||
|
||||
result = await build_report(db, None, date(2026, 1, 10), date(2026, 1, 10))
|
||||
|
||||
assert result["totals"]["liked_count_delta"] == 30
|
||||
assert result["task_ids"] is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_baseline_from_before_the_range_is_used(self, db):
|
||||
"""Growth is measured against the last value before the window opens."""
|
||||
task = await _seed_task(db, "t")
|
||||
await _seed_note_with_metrics(
|
||||
db, task, "n1",
|
||||
[(_ms(2026, 1, 5, 10), 1000), (_ms(2026, 1, 10, 10), 1050)],
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
# Report only for the 10th: the delta must be 50, not 1050.
|
||||
result = await build_report(db, [task.id], date(2026, 1, 10), date(2026, 1, 10))
|
||||
|
||||
assert result["totals"]["liked_count_delta"] == 50
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_range_returns_zeroed_rows(self, db):
|
||||
result = await build_report(db, None, date(2026, 2, 1), date(2026, 2, 3))
|
||||
|
||||
assert len(result["rows"]) == 3
|
||||
assert result["totals"]["liked_count_delta"] == 0
|
||||
assert result["totals"]["new_notes"] == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_comments_are_counted_by_first_seen_day(self, db):
|
||||
task = await _seed_task(db, "t")
|
||||
db.add(
|
||||
MonitorComment(
|
||||
task_id=task.id, note_id="n1", comment_id="c1", content="x",
|
||||
nickname="u", creator_hash="h", create_time=_ms(2026, 1, 9),
|
||||
like_count=0, sub_comment_count=0, parent_comment_id="",
|
||||
first_seen_run_id=1, first_seen_at=_ms(2026, 1, 10, 10),
|
||||
)
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
result = await build_report(db, [task.id], date(2026, 1, 10), date(2026, 1, 10))
|
||||
|
||||
assert result["totals"]["new_comments"] == 1
|
||||
@@ -0,0 +1,266 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2025 [email protected]
|
||||
#
|
||||
# This file is part of MediaCrawler project.
|
||||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_monitor_scheduler.py
|
||||
# GitHub: https://github.com/NanmiCoder
|
||||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||
#
|
||||
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
||||
# 5. 不得用于任何非法或不当的用途。
|
||||
#
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
"""Tests for the monitor scheduler's firing, deferral and recovery rules."""
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy import select
|
||||
|
||||
from api.monitor import db as monitor_db
|
||||
from api.monitor import scheduler as scheduler_module
|
||||
from api.monitor.models import (
|
||||
MODE_CREATOR,
|
||||
MonitorRun,
|
||||
MonitorTarget,
|
||||
MonitorTask,
|
||||
RUN_INTERRUPTED,
|
||||
RUN_RUNNING,
|
||||
RUN_SUCCESS,
|
||||
)
|
||||
from api.monitor.scheduler import MonitorScheduler
|
||||
from api.monitor.settings import set_cookie
|
||||
from tools.time_util import get_current_timestamp
|
||||
|
||||
MS_PER_MINUTE = 60_000
|
||||
|
||||
|
||||
class FakeCrawlerManager:
|
||||
"""Stands in for the global subprocess singleton."""
|
||||
|
||||
def __init__(self, busy: bool = False) -> None:
|
||||
self.busy = busy
|
||||
|
||||
def is_busy(self) -> bool:
|
||||
return self.busy
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def db(tmp_path):
|
||||
monitor_db.set_sqlite_path(tmp_path / "monitor.db")
|
||||
await monitor_db.init_db()
|
||||
async with monitor_db.get_session() as session:
|
||||
await set_cookie(session, "web_session=test")
|
||||
yield monitor_db
|
||||
await monitor_db.dispose_engine()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def executed(monkeypatch):
|
||||
"""Record execute_task calls instead of launching a real crawl."""
|
||||
calls: list[tuple[int, str]] = []
|
||||
|
||||
async def _fake_execute(task_id: int, trigger: str = "manual"):
|
||||
calls.append((task_id, trigger))
|
||||
|
||||
monkeypatch.setattr(scheduler_module, "execute_task", _fake_execute)
|
||||
return calls
|
||||
|
||||
|
||||
async def _make_task(next_run_at, enabled: bool = True, interval: int = 60) -> int:
|
||||
async with monitor_db.get_session() as session:
|
||||
now = get_current_timestamp()
|
||||
task = MonitorTask(
|
||||
name="t",
|
||||
platform="xhs",
|
||||
mode=MODE_CREATOR,
|
||||
enabled=enabled,
|
||||
interval_minutes=interval,
|
||||
max_notes_count=20,
|
||||
enable_comments=True,
|
||||
max_comments_count=50,
|
||||
run_timeout_seconds=3600,
|
||||
next_run_at=next_run_at,
|
||||
last_status="idle",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(task)
|
||||
await session.flush()
|
||||
session.add(
|
||||
MonitorTarget(
|
||||
task_id=task.id,
|
||||
kind=MODE_CREATOR,
|
||||
external_id="abc123",
|
||||
xsec_token="",
|
||||
xsec_source="",
|
||||
raw_value="abc123",
|
||||
label="abc123",
|
||||
enabled=True,
|
||||
created_at=now,
|
||||
)
|
||||
)
|
||||
return task.id
|
||||
|
||||
|
||||
async def _get_task(task_id: int) -> MonitorTask:
|
||||
async with monitor_db.get_session() as session:
|
||||
return await session.get(MonitorTask, task_id)
|
||||
|
||||
|
||||
class TestFiring:
|
||||
@pytest.mark.asyncio
|
||||
async def test_due_task_runs_and_advances(self, monkeypatch, db, executed):
|
||||
monkeypatch.setattr(scheduler_module, "crawler_manager", FakeCrawlerManager(busy=False))
|
||||
past = get_current_timestamp() - MS_PER_MINUTE
|
||||
task_id = await _make_task(past)
|
||||
|
||||
await MonitorScheduler().tick()
|
||||
|
||||
assert executed == [(task_id, "scheduled")]
|
||||
task = await _get_task(task_id)
|
||||
# Fixed-delay: the next fire is measured from now, not from the missed slot.
|
||||
assert task.next_run_at > get_current_timestamp()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_future_task_does_not_run(self, monkeypatch, db, executed):
|
||||
monkeypatch.setattr(scheduler_module, "crawler_manager", FakeCrawlerManager(busy=False))
|
||||
await _make_task(get_current_timestamp() + 10 * MS_PER_MINUTE)
|
||||
|
||||
await MonitorScheduler().tick()
|
||||
|
||||
assert executed == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disabled_task_does_not_run(self, monkeypatch, db, executed):
|
||||
monkeypatch.setattr(scheduler_module, "crawler_manager", FakeCrawlerManager(busy=False))
|
||||
await _make_task(get_current_timestamp() - MS_PER_MINUTE, enabled=False)
|
||||
|
||||
await MonitorScheduler().tick()
|
||||
|
||||
assert executed == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_long_outage_coalesces_into_one_run(self, monkeypatch, db, executed):
|
||||
"""A missed schedule fires once, not once per missed interval."""
|
||||
monkeypatch.setattr(scheduler_module, "crawler_manager", FakeCrawlerManager(busy=False))
|
||||
# Due two days ago on a 1-hour interval.
|
||||
await _make_task(get_current_timestamp() - 48 * 60 * MS_PER_MINUTE)
|
||||
|
||||
scheduler = MonitorScheduler()
|
||||
await scheduler.tick()
|
||||
await scheduler.tick()
|
||||
|
||||
assert len(executed) == 1
|
||||
|
||||
|
||||
class TestDeferral:
|
||||
@pytest.mark.asyncio
|
||||
async def test_busy_crawler_defers_without_advancing(self, monkeypatch, db, executed):
|
||||
"""A manual crawl must not consume the monitor task's slot or lose it."""
|
||||
monkeypatch.setattr(scheduler_module, "crawler_manager", FakeCrawlerManager(busy=True))
|
||||
due_at = get_current_timestamp() - MS_PER_MINUTE
|
||||
task_id = await _make_task(due_at)
|
||||
|
||||
await MonitorScheduler().tick()
|
||||
|
||||
assert executed == []
|
||||
task = await _get_task(task_id)
|
||||
# Still due, so the next free tick picks it up rather than skipping a cycle.
|
||||
assert task.next_run_at == due_at
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deferred_task_runs_once_crawler_frees_up(self, monkeypatch, db, executed):
|
||||
fake = FakeCrawlerManager(busy=True)
|
||||
monkeypatch.setattr(scheduler_module, "crawler_manager", fake)
|
||||
task_id = await _make_task(get_current_timestamp() - MS_PER_MINUTE)
|
||||
|
||||
scheduler = MonitorScheduler()
|
||||
await scheduler.tick()
|
||||
assert executed == []
|
||||
|
||||
fake.busy = False
|
||||
await scheduler.tick()
|
||||
assert executed == [(task_id, "scheduled")]
|
||||
|
||||
|
||||
class TestCookieGuard:
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_cookie_blocks_run_and_keeps_task_due(self, monkeypatch, db, executed):
|
||||
"""Without a cookie every run would be an auth failure; skip instead."""
|
||||
monkeypatch.setattr(scheduler_module, "crawler_manager", FakeCrawlerManager(busy=False))
|
||||
async with monitor_db.get_session() as session:
|
||||
from api.monitor.settings import cookie_key, delete_setting
|
||||
|
||||
await delete_setting(session, cookie_key("xhs"))
|
||||
|
||||
due_at = get_current_timestamp() - MS_PER_MINUTE
|
||||
task_id = await _make_task(due_at)
|
||||
|
||||
await MonitorScheduler().tick()
|
||||
|
||||
assert executed == []
|
||||
task = await _get_task(task_id)
|
||||
# Left due so it starts working the moment a cookie is pasted.
|
||||
assert task.next_run_at == due_at
|
||||
|
||||
|
||||
class TestRecovery:
|
||||
@pytest.mark.asyncio
|
||||
async def test_running_runs_are_marked_interrupted(self, db):
|
||||
"""A run left 'running' cannot be alive -- its process died with the server."""
|
||||
async with monitor_db.get_session() as session:
|
||||
now = get_current_timestamp()
|
||||
task = MonitorTask(
|
||||
name="t", platform="xhs", mode=MODE_CREATOR, enabled=True,
|
||||
interval_minutes=60, max_notes_count=20, enable_comments=True,
|
||||
max_comments_count=50, run_timeout_seconds=3600,
|
||||
next_run_at=now, last_status="running", created_at=now, updated_at=now,
|
||||
)
|
||||
session.add(task)
|
||||
await session.flush()
|
||||
session.add(
|
||||
MonitorRun(
|
||||
task_id=task.id, trigger="scheduled", status=RUN_RUNNING,
|
||||
phase=MODE_CREATOR, save_data_path="", queued_at=now, not_before=0,
|
||||
started_at=now, max_comments_count=50,
|
||||
)
|
||||
)
|
||||
|
||||
await MonitorScheduler().recover()
|
||||
|
||||
async with monitor_db.get_session() as session:
|
||||
run = await session.scalar(select(MonitorRun))
|
||||
assert run.status == RUN_INTERRUPTED
|
||||
assert run.finished_at is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completed_runs_are_left_alone(self, db):
|
||||
async with monitor_db.get_session() as session:
|
||||
now = get_current_timestamp()
|
||||
task = MonitorTask(
|
||||
name="t", platform="xhs", mode=MODE_CREATOR, enabled=True,
|
||||
interval_minutes=60, max_notes_count=20, enable_comments=True,
|
||||
max_comments_count=50, run_timeout_seconds=3600,
|
||||
next_run_at=now, last_status="success", created_at=now, updated_at=now,
|
||||
)
|
||||
session.add(task)
|
||||
await session.flush()
|
||||
session.add(
|
||||
MonitorRun(
|
||||
task_id=task.id, trigger="scheduled", status=RUN_SUCCESS,
|
||||
phase=MODE_CREATOR, save_data_path="", queued_at=now, not_before=0,
|
||||
started_at=now, finished_at=now, max_comments_count=50,
|
||||
)
|
||||
)
|
||||
|
||||
await MonitorScheduler().recover()
|
||||
|
||||
async with monitor_db.get_session() as session:
|
||||
run = await session.scalar(select(MonitorRun))
|
||||
assert run.status == RUN_SUCCESS
|
||||
@@ -0,0 +1,302 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2025 [email protected]
|
||||
#
|
||||
# This file is part of MediaCrawler project.
|
||||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_platforms.py
|
||||
# GitHub: https://github.com/NanmiCoder
|
||||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||
#
|
||||
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
||||
# 5. 不得用于任何非法或不当的用途。
|
||||
#
|
||||
# 详细许可条款请参阅根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
"""Platform capability matrix, platform scoping, and per-platform settings."""
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy import text
|
||||
|
||||
from api.main import app
|
||||
from api.monitor import db as monitor_db
|
||||
from api.monitor import platforms
|
||||
from api.monitor.models import MonitorTask
|
||||
|
||||
XHS_TARGET = "5f58bd990000000001003753"
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(tmp_path):
|
||||
monitor_db.set_sqlite_path(tmp_path / "monitor.db")
|
||||
await monitor_db.init_db()
|
||||
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as http_client:
|
||||
yield http_client
|
||||
|
||||
await monitor_db.dispose_engine()
|
||||
|
||||
|
||||
class TestCapabilityMatrix:
|
||||
@pytest.mark.asyncio
|
||||
async def test_matrix_is_exposed_to_the_ui(self, client):
|
||||
body = (await client.get("/api/config/platforms")).json()
|
||||
by_value = {p["value"]: p for p in body["platforms"]}
|
||||
|
||||
assert set(by_value) == {"xhs", "dy", "ks", "bili", "wb", "tieba", "zhihu"}
|
||||
# Every entry must say whether monitoring is actually wired up -- this is
|
||||
# what stops the UI offering a platform that can never produce data.
|
||||
assert all("monitor_wired" in p for p in body["platforms"])
|
||||
assert by_value["xhs"]["monitor_wired"] is True
|
||||
assert by_value["dy"]["monitor_wired"] is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metrics_are_per_platform_and_labelled(self, client):
|
||||
body = (await client.get("/api/config/platforms")).json()
|
||||
by_value = {p["value"]: p for p in body["platforms"]}
|
||||
|
||||
# Bilibili has play count and danmaku; Xiaohongshu has neither.
|
||||
assert "video_play_count" in by_value["bili"]["metrics"]
|
||||
assert "video_danmaku" in by_value["bili"]["metrics"]
|
||||
assert "video_play_count" not in by_value["xhs"]["metrics"]
|
||||
|
||||
# Every metric shown to a user must have a human label.
|
||||
for capability in body["platforms"]:
|
||||
for metric in capability["metrics"]:
|
||||
assert capability["metric_labels"][metric]
|
||||
|
||||
def test_unknown_platform_is_not_monitor_wired(self):
|
||||
assert platforms.is_known("xhs") is True
|
||||
assert platforms.is_known("myspace") is False
|
||||
assert platforms.is_monitor_wired("myspace") is False
|
||||
|
||||
|
||||
class TestTaskCreationGuard:
|
||||
@pytest.mark.asyncio
|
||||
async def test_unwired_platform_is_rejected_with_an_explanation(self, client):
|
||||
"""Accepting it would create a task that silently never produces data."""
|
||||
response = await client.post(
|
||||
"/api/monitor/tasks",
|
||||
json={"name": "抖音任务", "mode": "creator", "platform": "dy", "targets": ["x"]},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
detail = response.json()["detail"]
|
||||
assert "抖音" in detail
|
||||
assert "尚未接通" in detail
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_platform_is_rejected(self, client):
|
||||
response = await client.post(
|
||||
"/api/monitor/tasks",
|
||||
json={"name": "x", "mode": "creator", "platform": "myspace", "targets": ["x"]},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_task_row_is_created_when_rejected(self, client):
|
||||
await client.post(
|
||||
"/api/monitor/tasks",
|
||||
json={"name": "抖音任务", "mode": "creator", "platform": "dy", "targets": ["x"]},
|
||||
)
|
||||
assert (await client.get("/api/monitor/tasks")).json()["tasks"] == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_xhs_still_works_and_is_the_default(self, client):
|
||||
explicit = await client.post(
|
||||
"/api/monitor/tasks",
|
||||
json={"name": "显式", "mode": "creator", "platform": "xhs", "targets": [XHS_TARGET]},
|
||||
)
|
||||
assert explicit.status_code == 201
|
||||
|
||||
defaulted = await client.post(
|
||||
"/api/monitor/tasks",
|
||||
json={"name": "默认", "mode": "creator", "targets": [XHS_TARGET]},
|
||||
)
|
||||
assert defaulted.status_code == 201
|
||||
|
||||
tasks = (await client.get("/api/monitor/tasks")).json()["tasks"]
|
||||
assert {t["platform"] for t in tasks} == {"xhs"}
|
||||
|
||||
|
||||
class TestPlatformScoping:
|
||||
async def _seed_two_platforms(self, client):
|
||||
"""One real XHS task plus a Douyin task inserted directly, since the API
|
||||
refuses to create the latter."""
|
||||
await client.post(
|
||||
"/api/monitor/tasks",
|
||||
json={"name": "小红书任务", "mode": "creator", "targets": [XHS_TARGET]},
|
||||
)
|
||||
async with monitor_db.get_session() as session:
|
||||
session.add(
|
||||
MonitorTask(
|
||||
name="抖音任务", platform="dy", mode="creator", enabled=True,
|
||||
interval_minutes=60, max_notes_count=20, enable_comments=True,
|
||||
max_comments_count=50, run_timeout_seconds=3600,
|
||||
notify_enabled=False, created_at=0, updated_at=0,
|
||||
)
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tasks_are_filtered_by_platform(self, client):
|
||||
await self._seed_two_platforms(client)
|
||||
|
||||
all_tasks = (await client.get("/api/monitor/tasks")).json()["tasks"]
|
||||
assert len(all_tasks) == 2
|
||||
|
||||
xhs_only = (await client.get("/api/monitor/tasks", params={"platform": "xhs"})).json()
|
||||
assert [t["name"] for t in xhs_only["tasks"]] == ["小红书任务"]
|
||||
|
||||
dy_only = (await client.get("/api/monitor/tasks", params={"platform": "dy"})).json()
|
||||
assert [t["name"] for t in dy_only["tasks"]] == ["抖音任务"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_overview_is_scoped(self, client):
|
||||
await self._seed_two_platforms(client)
|
||||
|
||||
assert (await client.get("/api/monitor/overview")).json()["tasks"] == 2
|
||||
assert (
|
||||
await client.get("/api/monitor/overview", params={"platform": "xhs"})
|
||||
).json()["tasks"] == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_platform_with_no_tasks_yields_empty_not_everything(self, client):
|
||||
"""An empty task set must not degrade into "no filter"."""
|
||||
await self._seed_two_platforms(client)
|
||||
|
||||
body = (await client.get("/api/monitor/notes", params={"platform": "bili"})).json()
|
||||
assert body["notes"] == []
|
||||
|
||||
report = (
|
||||
await client.get("/api/monitor/report", params={"platform": "bili"})
|
||||
).json()
|
||||
assert report["totals"]["liked_count_delta"] == 0
|
||||
assert report["note_count"] == 0
|
||||
|
||||
|
||||
class TestPerPlatformSettings:
|
||||
@pytest.mark.asyncio
|
||||
async def test_each_platform_keeps_its_own_values(self, client):
|
||||
await client.put(
|
||||
"/api/settings",
|
||||
params={"platform": "xhs"},
|
||||
json={"platform.xhs.crawl_sleep_sec": 3},
|
||||
)
|
||||
await client.put(
|
||||
"/api/settings",
|
||||
params={"platform": "dy"},
|
||||
json={"platform.dy.crawl_sleep_sec": 9},
|
||||
)
|
||||
|
||||
xhs = (await client.get("/api/settings", params={"platform": "xhs"})).json()
|
||||
dy = (await client.get("/api/settings", params={"platform": "dy"})).json()
|
||||
|
||||
assert xhs["values"]["platform.xhs.crawl_sleep_sec"] == 3
|
||||
assert dy["values"]["platform.dy.crawl_sleep_sec"] == 9
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_system_settings_are_shared_across_platforms(self, client):
|
||||
await client.put(
|
||||
"/api/settings",
|
||||
params={"platform": "xhs"},
|
||||
json={"system.active_hours_start": 8},
|
||||
)
|
||||
|
||||
dy = (await client.get("/api/settings", params={"platform": "dy"})).json()
|
||||
assert dy["values"]["system.active_hours_start"] == 8
|
||||
# ...and the system specs are present in every platform's response.
|
||||
assert "system.active_hours_end" in dy["values"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_key_for_another_platform_is_rejected(self, client):
|
||||
"""Writing xhs's key while scoped to dy would land somewhere unexpected."""
|
||||
response = await client.put(
|
||||
"/api/settings",
|
||||
params={"platform": "dy"},
|
||||
json={"platform.xhs.crawl_sleep_sec": 5},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cookies_are_per_platform(self, client):
|
||||
await client.post(
|
||||
"/api/monitor/cookie",
|
||||
params={"platform": "xhs"},
|
||||
json={"cookie": "web_session=xhs-secret"},
|
||||
)
|
||||
|
||||
xhs = (await client.get("/api/monitor/cookie", params={"platform": "xhs"})).json()
|
||||
dy = (await client.get("/api/monitor/cookie", params={"platform": "dy"})).json()
|
||||
|
||||
assert xhs["present"] is True
|
||||
assert dy["present"] is False
|
||||
|
||||
# The old endpoint still defaults to Xiaohongshu.
|
||||
assert (await client.get("/api/monitor/cookie")).json()["present"] is True
|
||||
|
||||
|
||||
class TestLegacyKeyMigration:
|
||||
@pytest.mark.asyncio
|
||||
async def test_old_flat_keys_are_moved_to_the_new_namespace(self, tmp_path):
|
||||
"""Existing installs must not lose their cookie on upgrade."""
|
||||
monitor_db.set_sqlite_path(tmp_path / "monitor.db")
|
||||
await monitor_db.init_db()
|
||||
|
||||
async with monitor_db.get_engine().begin() as conn:
|
||||
await conn.execute(
|
||||
text(
|
||||
"INSERT INTO monitor_setting (key, value, updated_at) "
|
||||
"VALUES ('xhs_cookie', 'web_session=legacy', 1)"
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"INSERT INTO monitor_setting (key, value, updated_at) "
|
||||
"VALUES ('wecom_webhook', 'https://qyapi.weixin.qq.com/x', 1)"
|
||||
)
|
||||
)
|
||||
|
||||
# Re-running init performs the rename.
|
||||
await monitor_db.init_db()
|
||||
|
||||
async with monitor_db.get_engine().begin() as conn:
|
||||
rows = dict(
|
||||
(await conn.execute(text("SELECT key, value FROM monitor_setting"))).all()
|
||||
)
|
||||
|
||||
assert rows.get("platform.xhs.cookie") == "web_session=legacy"
|
||||
assert rows.get("system.wecom_webhook") == "https://qyapi.weixin.qq.com/x"
|
||||
assert "xhs_cookie" not in rows
|
||||
assert "wecom_webhook" not in rows
|
||||
|
||||
await monitor_db.dispose_engine()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_migration_is_idempotent_and_keeps_the_newer_value(self, tmp_path):
|
||||
monitor_db.set_sqlite_path(tmp_path / "monitor.db")
|
||||
await monitor_db.init_db()
|
||||
|
||||
async with monitor_db.get_engine().begin() as conn:
|
||||
await conn.execute(
|
||||
text(
|
||||
"INSERT INTO monitor_setting (key, value, updated_at) VALUES "
|
||||
"('platform.xhs.cookie', 'current', 2), ('xhs_cookie', 'stale', 1)"
|
||||
)
|
||||
)
|
||||
|
||||
await monitor_db.init_db()
|
||||
|
||||
async with monitor_db.get_engine().begin() as conn:
|
||||
rows = dict(
|
||||
(await conn.execute(text("SELECT key, value FROM monitor_setting"))).all()
|
||||
)
|
||||
|
||||
assert rows.get("platform.xhs.cookie") == "current"
|
||||
assert "xhs_cookie" not in rows
|
||||
|
||||
await monitor_db.dispose_engine()
|
||||
@@ -0,0 +1,374 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2025 [email protected]
|
||||
#
|
||||
# This file is part of MediaCrawler project.
|
||||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_settings.py
|
||||
# GitHub: https://github.com/NanmiCoder
|
||||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||
#
|
||||
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
||||
# 5. 不得用于任何非法或不当的用途。
|
||||
#
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
"""Unified settings endpoint, and the effect its values actually have."""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from api.main import app
|
||||
from api.monitor import app_settings, db as monitor_db
|
||||
from api.monitor import scheduler as scheduler_module
|
||||
from api.monitor.scheduler import MonitorScheduler
|
||||
from api.monitor.settings import get_setting
|
||||
|
||||
SECRET_VALUE = "web_session=SUPERSECRET; a1=abc"
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(tmp_path):
|
||||
monitor_db.set_sqlite_path(tmp_path / "monitor.db")
|
||||
await monitor_db.init_db()
|
||||
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as http_client:
|
||||
yield http_client
|
||||
|
||||
await monitor_db.dispose_engine()
|
||||
|
||||
|
||||
class TestReadSettings:
|
||||
@pytest.mark.asyncio
|
||||
async def test_returns_values_secrets_and_the_spec(self, client):
|
||||
body = (await client.get("/api/settings")).json()
|
||||
|
||||
assert "values" in body and "secrets" in body and "specs" in body
|
||||
# The spec drives the UI form, so every key must be described.
|
||||
spec_keys = {spec["key"] for spec in body["specs"]}
|
||||
assert "platform.xhs.default_interval_minutes" in spec_keys
|
||||
assert "platform.xhs.enable_ip_proxy" in spec_keys
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unset_values_fall_back_to_spec_defaults(self, client):
|
||||
values = (await client.get("/api/settings")).json()["values"]
|
||||
assert values["platform.xhs.default_interval_minutes"] == 360
|
||||
assert values["platform.xhs.enable_ip_proxy"] is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_secrets_are_masked_never_returned(self, client):
|
||||
await client.put("/api/settings", json={"platform.xhs.cookie": SECRET_VALUE})
|
||||
|
||||
response = await client.get("/api/settings")
|
||||
assert SECRET_VALUE not in response.text
|
||||
|
||||
secret = response.json()["secrets"]["platform.xhs.cookie"]
|
||||
assert secret["present"] is True
|
||||
assert secret["length"] == len(SECRET_VALUE)
|
||||
|
||||
|
||||
class TestUpdateSettings:
|
||||
@pytest.mark.asyncio
|
||||
async def test_partial_update_leaves_other_keys_alone(self, client):
|
||||
await client.put(
|
||||
"/api/settings",
|
||||
json={"platform.xhs.default_interval_minutes": 120, "platform.xhs.cookie": SECRET_VALUE},
|
||||
)
|
||||
|
||||
# A form that only submits the interval must not blank the cookie.
|
||||
await client.put("/api/settings", json={"platform.xhs.default_interval_minutes": 240})
|
||||
|
||||
body = (await client.get("/api/settings")).json()
|
||||
assert body["values"]["platform.xhs.default_interval_minutes"] == 240
|
||||
assert body["secrets"]["platform.xhs.cookie"]["present"] is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_string_clears_a_secret(self, client):
|
||||
await client.put("/api/settings", json={"platform.xhs.cookie": SECRET_VALUE})
|
||||
await client.put("/api/settings", json={"platform.xhs.cookie": ""})
|
||||
|
||||
assert (await client.get("/api/settings")).json()["secrets"]["platform.xhs.cookie"][
|
||||
"present"
|
||||
] is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_key_is_rejected(self, client):
|
||||
response = await client.put("/api/settings", json={"nope.not.a.setting": 1})
|
||||
assert response.status_code == 400
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_out_of_range_is_rejected(self, client):
|
||||
response = await client.put(
|
||||
"/api/settings", json={"platform.xhs.default_interval_minutes": 1}
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_choice_is_rejected(self, client):
|
||||
response = await client.put("/api/settings", json={"platform.xhs.proxy_provider": "nonsense"})
|
||||
assert response.status_code == 400
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bools_accept_the_ui_shapes(self, client):
|
||||
for raw in (True, "true", "1", "yes"):
|
||||
response = await client.put("/api/settings", json={"platform.xhs.enable_ip_proxy": raw})
|
||||
assert response.status_code == 200
|
||||
assert (await client.get("/api/settings")).json()["values"][
|
||||
"platform.xhs.enable_ip_proxy"
|
||||
] is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_password_hash_cannot_be_written_through_this_endpoint(self, client):
|
||||
"""It has its own authenticated endpoint; this must not be a back door."""
|
||||
await client.put("/api/settings", json={"auth_password_hash": "pbkdf2_sha256$1$a$b"})
|
||||
|
||||
async with monitor_db.get_session() as session:
|
||||
assert await get_setting(session, "auth_password_hash") is None
|
||||
|
||||
|
||||
class TestSettingsActuallyTakeEffect:
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_tasks_use_the_configured_defaults(self, client):
|
||||
await client.put(
|
||||
"/api/settings",
|
||||
json={
|
||||
"platform.xhs.default_interval_minutes": 120,
|
||||
"platform.xhs.default_max_notes": 7,
|
||||
"platform.xhs.default_max_comments": 33,
|
||||
},
|
||||
)
|
||||
|
||||
await client.post(
|
||||
"/api/monitor/tasks",
|
||||
json={"name": "用默认值", "mode": "creator", "targets": ["5f58bd990000000001003753"]},
|
||||
)
|
||||
|
||||
task = (await client.get("/api/monitor/tasks")).json()["tasks"][0]
|
||||
assert task["interval_minutes"] == 120
|
||||
assert task["max_notes_count"] == 7
|
||||
assert task["max_comments_count"] == 33
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_explicit_values_still_win_over_defaults(self, client):
|
||||
await client.put("/api/settings", json={"platform.xhs.default_interval_minutes": 120})
|
||||
|
||||
await client.post(
|
||||
"/api/monitor/tasks",
|
||||
json={
|
||||
"name": "显式值",
|
||||
"mode": "creator",
|
||||
"interval_minutes": 720,
|
||||
"targets": ["5f58bd990000000001003753"],
|
||||
},
|
||||
)
|
||||
|
||||
task = (await client.get("/api/monitor/tasks")).json()["tasks"][0]
|
||||
assert task["interval_minutes"] == 720
|
||||
|
||||
|
||||
class TestRunnerAppliesStrategy:
|
||||
@pytest.mark.asyncio
|
||||
async def test_strategy_settings_reach_the_command(self, client):
|
||||
"""Stored settings must actually change how the crawler is invoked."""
|
||||
from api.services.crawler_manager import CrawlerManager
|
||||
from api.schemas import CrawlerStartRequest, PlatformEnum, CrawlerTypeEnum
|
||||
|
||||
await client.put(
|
||||
"/api/settings",
|
||||
json={
|
||||
"platform.xhs.crawl_sleep_sec": 7,
|
||||
"platform.xhs.enable_sub_comments": True,
|
||||
"platform.xhs.enable_ip_proxy": True,
|
||||
"platform.xhs.proxy_provider": "static",
|
||||
"platform.xhs.proxy_pool_count": 5,
|
||||
"platform.xhs.static_proxy_url": "http://127.0.0.1:8888",
|
||||
},
|
||||
)
|
||||
|
||||
async with monitor_db.get_session() as session:
|
||||
strategy = await scheduler_module.app_settings.get_value(
|
||||
session, "crawl_sleep_sec", "xhs", 2
|
||||
)
|
||||
assert strategy == 7
|
||||
|
||||
# And the flag builder forwards them when present.
|
||||
command = CrawlerManager()._build_command(
|
||||
CrawlerStartRequest(
|
||||
platform=PlatformEnum.XHS,
|
||||
crawler_type=CrawlerTypeEnum.CREATOR,
|
||||
creator_ids="abc",
|
||||
crawler_max_sleep_sec=7,
|
||||
enable_ip_proxy=True,
|
||||
ip_proxy_provider_name="static",
|
||||
ip_proxy_pool_count=5,
|
||||
static_proxy_url="http://127.0.0.1:8888",
|
||||
)
|
||||
)
|
||||
joined = " ".join(command)
|
||||
assert "--crawler_max_sleep_sec 7" in joined
|
||||
assert "--enable_ip_proxy true" in joined
|
||||
assert "--ip_proxy_provider_name static" in joined
|
||||
assert "--static_proxy_url http://127.0.0.1:8888" in joined
|
||||
|
||||
|
||||
def _frozen_clock(hour: int):
|
||||
"""Stand-in for the datetime class whose now() is pinned to a given hour.
|
||||
|
||||
Testing an hour window by sleeping is not an option; patching the class the
|
||||
scheduler imported is the whole mechanism.
|
||||
"""
|
||||
|
||||
class _Frozen:
|
||||
@staticmethod
|
||||
def now(tz=None):
|
||||
return datetime(2026, 1, 1, hour)
|
||||
|
||||
return _Frozen
|
||||
|
||||
|
||||
class TestManualCrawlCookieFallback:
|
||||
"""The crawl page no longer has its own paste box; it reuses Settings."""
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def captured(self, monkeypatch):
|
||||
# api.services re-exports the singleton instance, not the module.
|
||||
from api.services import crawler_manager
|
||||
|
||||
seen: dict = {}
|
||||
|
||||
async def _fake_start(request, extra_args=None):
|
||||
seen["cookies"] = request.cookies
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(crawler_manager, "start", _fake_start)
|
||||
return seen
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_falls_back_to_the_stored_cookie(self, client, captured):
|
||||
await client.put(
|
||||
"/api/settings", json={"platform.xhs.cookie": "web_session=stored"}
|
||||
)
|
||||
|
||||
response = await client.post(
|
||||
"/api/crawler/start",
|
||||
json={
|
||||
"platform": "xhs",
|
||||
"login_type": "cookie",
|
||||
"crawler_type": "creator",
|
||||
"creator_ids": "abc",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert captured["cookies"] == "web_session=stored"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_explicit_cookie_still_wins(self, client, captured):
|
||||
await client.put(
|
||||
"/api/settings", json={"platform.xhs.cookie": "web_session=stored"}
|
||||
)
|
||||
|
||||
await client.post(
|
||||
"/api/crawler/start",
|
||||
json={
|
||||
"platform": "xhs",
|
||||
"login_type": "cookie",
|
||||
"crawler_type": "creator",
|
||||
"creator_ids": "abc",
|
||||
"cookies": "web_session=explicit",
|
||||
},
|
||||
)
|
||||
|
||||
assert captured["cookies"] == "web_session=explicit"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_cookie_is_a_clear_error_not_a_silent_failure(
|
||||
self, client, captured
|
||||
):
|
||||
"""Better a 400 that names the fix than a run that fetches nothing."""
|
||||
response = await client.post(
|
||||
"/api/crawler/start",
|
||||
json={
|
||||
"platform": "xhs",
|
||||
"login_type": "cookie",
|
||||
"crawler_type": "creator",
|
||||
"creator_ids": "abc",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert "设置" in response.json()["detail"]
|
||||
assert "cookies" not in captured
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_cookie_is_read_per_platform(self, client, captured):
|
||||
await client.put(
|
||||
"/api/settings",
|
||||
params={"platform": "xhs"},
|
||||
json={"platform.xhs.cookie": "web_session=xhs-only"},
|
||||
)
|
||||
|
||||
# Douyin has no stored cookie, so it must not borrow Xiaohongshu's.
|
||||
response = await client.post(
|
||||
"/api/crawler/start",
|
||||
json={
|
||||
"platform": "dy",
|
||||
"login_type": "cookie",
|
||||
"crawler_type": "creator",
|
||||
"creator_ids": "abc",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
class TestActiveHours:
|
||||
"""The window gate lives in the scheduler, not the crawler."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_inside_a_daytime_window(self, client, monkeypatch):
|
||||
await client.put(
|
||||
"/api/settings",
|
||||
json={"system.active_hours_start": 8, "system.active_hours_end": 22},
|
||||
)
|
||||
monkeypatch.setattr(scheduler_module, "datetime", _frozen_clock(12))
|
||||
|
||||
async with monitor_db.get_session() as session:
|
||||
assert await MonitorScheduler()._within_active_hours(session) is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_outside_a_daytime_window(self, client, monkeypatch):
|
||||
await client.put(
|
||||
"/api/settings",
|
||||
json={"system.active_hours_start": 8, "system.active_hours_end": 22},
|
||||
)
|
||||
monkeypatch.setattr(scheduler_module, "datetime", _frozen_clock(3))
|
||||
|
||||
async with monitor_db.get_session() as session:
|
||||
assert await MonitorScheduler()._within_active_hours(session) is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_window_wrapping_past_midnight(self, client, monkeypatch):
|
||||
await client.put(
|
||||
"/api/settings",
|
||||
json={"system.active_hours_start": 22, "system.active_hours_end": 6},
|
||||
)
|
||||
|
||||
for hour, expected in ((23, True), (3, True), (12, False)):
|
||||
monkeypatch.setattr(scheduler_module, "datetime", _frozen_clock(hour))
|
||||
async with monitor_db.get_session() as session:
|
||||
assert await MonitorScheduler()._within_active_hours(session) is expected
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_window_covers_the_whole_day(self, client, monkeypatch):
|
||||
for hour in (0, 12, 23):
|
||||
monkeypatch.setattr(scheduler_module, "datetime", _frozen_clock(hour))
|
||||
async with monitor_db.get_session() as session:
|
||||
assert await MonitorScheduler()._within_active_hours(session) is True
|
||||
Reference in New Issue
Block a user