butubb
4e60524f37
在上游 MediaCrawler 之上新增一层： - 监控层 api/monitor/ —— 多博主/多笔记的定时采集、指标快照差分、报表、 企业微信通知。每轮采集写入独立目录，差分才成立。 - WebUI 登录鉴权 api/auth.py —— PBKDF2 口令 + 服务端会话，/api 全接口防护。 WebSocket 单独加依赖：BaseHTTPMiddleware 对 ws 作用域直接放行，覆盖不到。 - 全局平台切换 + 能力矩阵 —— 如实区分「爬虫模块支持」与「监控层已接线」， 未接通的平台直接拒绝建任务，而不是静默跑空。 - 监控库改用 MySQL 5.7（可回退 SQLite 供测试）：逐表强制 utf8mb4 （服务端与库默认都是 latin1），启动校验所连 schema 以防写错库， 连接池 recycle + pre_ping 应对 MySQL 的 8 小时空闲断连。 修复上游缺陷： - xhs/core.py: 主页抓取失败会跳掉整个博主，导致一条作品都抓不到， 而那份资料只喂给一个空函数。改为尽力而为，失败不中断。 - xhs/login.py: cookie 登录只注入 web_session，冷启动签名会失败。 新增 INJECT_ALL_COOKIES 开关（默认关闭，原有行为不变）。 - requirements.txt: 补上 websockets。它在上游 pyproject.toml 里有声明、 这里漏了，导致 uvicorn 没有 WebSocket 能力，实时日志流从未工作。 改动过的上游文件清单及合并方式见 UPSTREAM.md。 测试：492 passed（另有 1 个既有的 Windows/gbk 上游测试失败，与本改动无关）
305 lines
11 KiB
Python
305 lines
11 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright (c) 2025 [email protected]
|
|
#
|
|
# This file is part of MediaCrawler project.
|
|
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_monitor_report.py
|
|
# GitHub: https://github.com/NanmiCoder
|
|
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
|
#
|
|
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
|
# 1. 不得用于任何商业用途。
|
|
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
|
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
|
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
|
# 5. 不得用于任何非法或不当的用途。
|
|
#
|
|
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
|
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
|
|
|
"""Tests for the cross-task report aggregation.
|
|
|
|
The interaction delta is the part that is easy to get subtly wrong, so it is
|
|
covered directly against the pure aggregation function.
|
|
"""
|
|
|
|
from datetime import date, datetime
|
|
|
|
import pytest
|
|
import pytest_asyncio
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
|
from sqlalchemy.pool import StaticPool
|
|
|
|
from api.monitor.models import (
|
|
MODE_CREATOR,
|
|
MonitorBase,
|
|
MonitorComment,
|
|
MonitorNote,
|
|
MonitorNoteMetric,
|
|
MonitorTask,
|
|
)
|
|
from api.monitor.report import build_report, compute_daily_rows, day_bounds, iter_days
|
|
|
|
|
|
def _ms(year: int, month: int, day: int, hour: int = 12) -> int:
|
|
return int(datetime(year, month, day, hour).timestamp() * 1000)
|
|
|
|
|
|
def _metrics(liked=0, comment=0, collected=0, share=0):
|
|
"""All four metrics default to parsed values; pass None to simulate a
|
|
platform value we could not parse."""
|
|
return {
|
|
"liked_count": liked,
|
|
"comment_count": comment,
|
|
"collected_count": collected,
|
|
"share_count": share,
|
|
}
|
|
|
|
|
|
class TestDayHelpers:
|
|
def test_day_bounds_cover_the_whole_local_day(self):
|
|
start, end = day_bounds(date(2026, 1, 10))
|
|
assert start < _ms(2026, 1, 10, 0) or start == _ms(2026, 1, 10, 0)
|
|
assert end > _ms(2026, 1, 10, 23)
|
|
|
|
def test_iter_days_is_inclusive(self):
|
|
days = iter_days(date(2026, 1, 10), date(2026, 1, 12))
|
|
assert days == [date(2026, 1, 10), date(2026, 1, 11), date(2026, 1, 12)]
|
|
|
|
|
|
class TestInteractionDelta:
|
|
def test_note_first_seen_counts_all_of_its_value(self):
|
|
"""A brand-new note has no earlier baseline, so it starts from zero."""
|
|
day = date(2026, 1, 10)
|
|
series = {"n1": [(_ms(2026, 1, 10, 10), _metrics(liked=100, comment=5))]}
|
|
|
|
rows = compute_daily_rows(series, {}, {}, [day])
|
|
|
|
assert rows[0]["liked_count_delta"] == 100
|
|
assert rows[0]["comment_count_delta"] == 5
|
|
|
|
def test_growth_is_split_across_days(self):
|
|
series = {
|
|
"n1": [
|
|
(_ms(2026, 1, 10, 10), _metrics(liked=100)),
|
|
(_ms(2026, 1, 11, 10), _metrics(liked=300)),
|
|
]
|
|
}
|
|
|
|
rows = compute_daily_rows(series, {}, {}, [date(2026, 1, 10), date(2026, 1, 11)])
|
|
|
|
# Day 1: 0 -> 100. Day 2: 100 -> 300.
|
|
assert [row["liked_count_delta"] for row in rows] == [100, 200]
|
|
|
|
def test_day_without_a_snapshot_reports_no_growth(self):
|
|
series = {
|
|
"n1": [
|
|
(_ms(2026, 1, 10, 10), _metrics(liked=100)),
|
|
(_ms(2026, 1, 12, 10), _metrics(liked=400)),
|
|
]
|
|
}
|
|
days = [date(2026, 1, 10), date(2026, 1, 11), date(2026, 1, 12)]
|
|
|
|
rows = compute_daily_rows(series, {}, {}, days)
|
|
|
|
# The note was not crawled on the 11th, so nothing is claimed for it.
|
|
assert [row["liked_count_delta"] for row in rows] == [100, 0, 300]
|
|
|
|
def test_deltas_aggregate_across_notes(self):
|
|
series = {
|
|
"n1": [
|
|
(_ms(2026, 1, 10, 10), _metrics(liked=100)),
|
|
(_ms(2026, 1, 11, 10), _metrics(liked=150)),
|
|
],
|
|
"n2": [
|
|
(_ms(2026, 1, 10, 10), _metrics(liked=10)),
|
|
(_ms(2026, 1, 11, 10), _metrics(liked=40)),
|
|
],
|
|
}
|
|
|
|
rows = compute_daily_rows(series, {}, {}, [date(2026, 1, 10), date(2026, 1, 11)])
|
|
|
|
assert [row["liked_count_delta"] for row in rows] == [110, 80]
|
|
|
|
def test_unparseable_metric_names_the_offending_field(self):
|
|
"""A NULL count makes the delta unknown; it must not be reported as 0."""
|
|
series = {
|
|
"n1": [
|
|
(_ms(2026, 1, 10, 10), _metrics(liked=100, comment=None)),
|
|
(_ms(2026, 1, 11, 10), _metrics(liked=200, comment=None)),
|
|
]
|
|
}
|
|
|
|
rows = compute_daily_rows(series, {}, {}, [date(2026, 1, 11)])
|
|
|
|
# Naming the field is actionable; a bare boolean is not.
|
|
assert rows[0]["partial_metrics"] == ["comment_count"]
|
|
# The parseable metric is still summed correctly.
|
|
assert rows[0]["liked_count_delta"] == 100
|
|
|
|
def test_unknown_value_only_taints_the_days_it_touches(self):
|
|
series = {
|
|
"n1": [
|
|
(_ms(2026, 1, 10, 10), _metrics(liked=None)),
|
|
(_ms(2026, 1, 11, 10), _metrics(liked=50)),
|
|
(_ms(2026, 1, 12, 10), _metrics(liked=90)),
|
|
]
|
|
}
|
|
days = [date(2026, 1, 10), date(2026, 1, 11), date(2026, 1, 12)]
|
|
|
|
rows = compute_daily_rows(series, {}, {}, days)
|
|
|
|
# Day 12 compares two known values, so it is clean.
|
|
assert [row["partial_metrics"] for row in rows] == [
|
|
["liked_count"],
|
|
["liked_count"],
|
|
[],
|
|
]
|
|
assert rows[2]["liked_count_delta"] == 40
|
|
|
|
def test_new_content_counts_come_from_the_day_maps(self):
|
|
rows = compute_daily_rows(
|
|
{},
|
|
{date(2026, 1, 10): 3},
|
|
{date(2026, 1, 10): 7},
|
|
[date(2026, 1, 10), date(2026, 1, 11)],
|
|
)
|
|
|
|
assert rows[0]["new_notes"] == 3
|
|
assert rows[0]["new_comments"] == 7
|
|
assert rows[1]["new_notes"] == 0
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# DB-backed report + task filtering
|
|
# --------------------------------------------------------------------------
|
|
|
|
@pytest_asyncio.fixture
|
|
async def db():
|
|
engine = create_async_engine("sqlite+aiosqlite://", poolclass=StaticPool)
|
|
async with engine.begin() as conn:
|
|
await conn.run_sync(MonitorBase.metadata.create_all)
|
|
factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
async with factory() as session:
|
|
yield session
|
|
await engine.dispose()
|
|
|
|
|
|
async def _seed_task(db: AsyncSession, name: str) -> MonitorTask:
|
|
task = MonitorTask(
|
|
name=name, platform="xhs", mode=MODE_CREATOR, enabled=True,
|
|
interval_minutes=60, max_notes_count=20, enable_comments=True,
|
|
max_comments_count=50, run_timeout_seconds=3600,
|
|
notify_enabled=False, created_at=0, updated_at=0,
|
|
)
|
|
db.add(task)
|
|
await db.flush()
|
|
return task
|
|
|
|
|
|
async def _seed_note_with_metrics(
|
|
db: AsyncSession, task: MonitorTask, note_id: str, samples
|
|
) -> None:
|
|
db.add(
|
|
MonitorNote(
|
|
task_id=task.id, note_id=note_id, title=note_id, note_url="",
|
|
cover="", creator_hash="", source_kind="", published_at=None,
|
|
first_seen_run_id=1, first_seen_at=samples[0][0],
|
|
last_seen_run_id=len(samples), last_seen_at=samples[-1][0],
|
|
)
|
|
)
|
|
for run_id, (ts, liked) in enumerate(samples, start=1):
|
|
db.add(
|
|
MonitorNoteMetric(
|
|
task_id=task.id, note_id=note_id, run_id=run_id, captured_at=ts,
|
|
liked_count=liked, comment_count=0, collected_count=0, share_count=0,
|
|
raw_liked_count=str(liked), raw_comment_count="0",
|
|
raw_collected_count="0", raw_share_count="0",
|
|
)
|
|
)
|
|
|
|
|
|
class TestBuildReport:
|
|
@pytest.mark.asyncio
|
|
async def test_totals_and_rows(self, db):
|
|
task = await _seed_task(db, "t1")
|
|
await _seed_note_with_metrics(
|
|
db, task, "n1",
|
|
[(_ms(2026, 1, 10, 10), 100), (_ms(2026, 1, 11, 10), 250)],
|
|
)
|
|
await db.commit()
|
|
|
|
result = await build_report(db, [task.id], date(2026, 1, 10), date(2026, 1, 11))
|
|
|
|
assert result["totals"]["liked_count_delta"] == 250
|
|
assert len(result["rows"]) == 2
|
|
assert result["note_count"] == 1
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_task_selection_isolates_the_report(self, db):
|
|
"""The whole point: a report for a chosen subset must exclude the rest."""
|
|
kept = await _seed_task(db, "kept")
|
|
other = await _seed_task(db, "other")
|
|
await _seed_note_with_metrics(db, kept, "n1", [(_ms(2026, 1, 10, 10), 100)])
|
|
await _seed_note_with_metrics(db, other, "n2", [(_ms(2026, 1, 10, 10), 999)])
|
|
await db.commit()
|
|
|
|
only_kept = await build_report(db, [kept.id], date(2026, 1, 10), date(2026, 1, 10))
|
|
assert only_kept["totals"]["liked_count_delta"] == 100
|
|
assert only_kept["note_count"] == 1
|
|
|
|
both = await build_report(db, [kept.id, other.id], date(2026, 1, 10), date(2026, 1, 10))
|
|
assert both["totals"]["liked_count_delta"] == 1099
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_no_task_filter_covers_everything(self, db):
|
|
first = await _seed_task(db, "a")
|
|
second = await _seed_task(db, "b")
|
|
await _seed_note_with_metrics(db, first, "n1", [(_ms(2026, 1, 10, 10), 10)])
|
|
await _seed_note_with_metrics(db, second, "n2", [(_ms(2026, 1, 10, 10), 20)])
|
|
await db.commit()
|
|
|
|
result = await build_report(db, None, date(2026, 1, 10), date(2026, 1, 10))
|
|
|
|
assert result["totals"]["liked_count_delta"] == 30
|
|
assert result["task_ids"] is None
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_baseline_from_before_the_range_is_used(self, db):
|
|
"""Growth is measured against the last value before the window opens."""
|
|
task = await _seed_task(db, "t")
|
|
await _seed_note_with_metrics(
|
|
db, task, "n1",
|
|
[(_ms(2026, 1, 5, 10), 1000), (_ms(2026, 1, 10, 10), 1050)],
|
|
)
|
|
await db.commit()
|
|
|
|
# Report only for the 10th: the delta must be 50, not 1050.
|
|
result = await build_report(db, [task.id], date(2026, 1, 10), date(2026, 1, 10))
|
|
|
|
assert result["totals"]["liked_count_delta"] == 50
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_empty_range_returns_zeroed_rows(self, db):
|
|
result = await build_report(db, None, date(2026, 2, 1), date(2026, 2, 3))
|
|
|
|
assert len(result["rows"]) == 3
|
|
assert result["totals"]["liked_count_delta"] == 0
|
|
assert result["totals"]["new_notes"] == 0
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_new_comments_are_counted_by_first_seen_day(self, db):
|
|
task = await _seed_task(db, "t")
|
|
db.add(
|
|
MonitorComment(
|
|
task_id=task.id, note_id="n1", comment_id="c1", content="x",
|
|
nickname="u", creator_hash="h", create_time=_ms(2026, 1, 9),
|
|
like_count=0, sub_comment_count=0, parent_comment_id="",
|
|
first_seen_run_id=1, first_seen_at=_ms(2026, 1, 10, 10),
|
|
)
|
|
)
|
|
await db.commit()
|
|
|
|
result = await build_report(db, [task.id], date(2026, 1, 10), date(2026, 1, 10))
|
|
|
|
assert result["totals"]["new_comments"] == 1
|