MediaCrawler/tests/test_platforms.py

# -*- coding: utf-8 -*-
# Copyright (c) 2025 [email protected]
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_platforms.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
# 1. 不得用于任何商业用途。
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
# 3. 不得进行大规模爬取或对平台造成运营干扰。
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
# 5. 不得用于任何非法或不当的用途。
#
# 详细许可条款请参阅根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。

"""Platform capability matrix, platform scoping, and per-platform settings."""

import httpx
import pytest
import pytest_asyncio
from sqlalchemy import select, text

from tools.time_util import get_current_timestamp

from api.main import app
from api.monitor import adapters
from api.monitor import db as monitor_db
from api.monitor import platforms
from api.monitor.models import MonitorNote, MonitorNoteMetric, MonitorTask

XHS_TARGET = "5f58bd990000000001003753"


@pytest_asyncio.fixture
async def client(tmp_path):
    monitor_db.set_sqlite_path(tmp_path / "monitor.db")
    await monitor_db.init_db()

    transport = httpx.ASGITransport(app=app)
    async with httpx.AsyncClient(transport=transport, base_url="http://test") as http_client:
        yield http_client

    await monitor_db.dispose_engine()


async def _seed_note_with_one_snapshot(task_name: str) -> None:
    """给某个任务塞一条作品和一次指标快照。

    过滤类测试**必须有真数据**才有意义 —— 库里空着的话，过滤有没有生效结果都是 0，
    测试就变成了空跑（这个坑踩过一次：一个报表串数据的 bug 因此没被拦住）。
    """
    async with monitor_db.get_session() as session:
        task = await session.scalar(select(MonitorTask).where(MonitorTask.name == task_name))
        now = get_current_timestamp()
        session.add(
            MonitorNote(
                task_id=task.id, note_id="seed-note", title="seed", note_url="",
                cover="", creator_hash="", source_kind="", published_at=None,
                first_seen_run_id=1, first_seen_at=now,
                last_seen_run_id=1, last_seen_at=now,
            )
        )
        session.add(
            MonitorNoteMetric(
                task_id=task.id, note_id="seed-note", run_id=1, captured_at=now,
                liked_count=42, comment_count=0, collected_count=0, share_count=0,
                raw_liked_count="42", raw_comment_count="0",
                raw_collected_count="0", raw_share_count="0",
            )
        )


class TestCapabilityMatrix:
    @pytest.mark.asyncio
    async def test_matrix_is_exposed_to_the_ui(self, client):
        body = (await client.get("/api/config/platforms")).json()
        by_value = {p["value"]: p for p in body["platforms"]}

        assert set(by_value) == {"xhs", "dy", "ks", "bili", "wb", "tieba", "zhihu"}
        # Every entry must say whether monitoring is actually wired up -- this is
        # what stops the UI offering a platform that can never produce data.
        assert all("monitor_wired" in p for p in body["platforms"])
        assert by_value["xhs"]["monitor_wired"] is True
        assert by_value["dy"]["monitor_wired"] is True

    def test_every_wired_platform_has_an_adapter(self):
        """能力矩阵说「接通了」，就必须真的有一套适配管子。

        两个注册表（platforms.PLATFORM_CAPABILITIES 与 adapters.ADAPTERS）分开是有意的
        —— 前者是给前端看的能力描述，后者是爬虫的管道细节。代价是它们可能漂移，
        所以在这里钉一条：凡声明接通的，必须能找到适配器。
        """
        for platform in platforms.all_platforms():
            if platforms.is_monitor_wired(platform):
                assert adapters.has_adapter(platform), f"{platform} 声明接通但没有适配器"

    @pytest.mark.asyncio
    async def test_target_hints_are_exposed_for_wired_platforms(self, client):
        """前端的目标输入框拿它做 placeholder —— 让用户看到本平台该粘什么样的链接。"""
        body = (await client.get("/api/config/platforms")).json()
        by_value = {p["value"]: p for p in body["platforms"]}

        assert "douyin.com/user/" in by_value["dy"]["target_hints"]["creator"]
        assert "douyin.com/video/" in by_value["dy"]["target_hints"]["note"]
        assert "xiaohongshu.com" in by_value["xhs"]["target_hints"]["creator"]

    @pytest.mark.asyncio
    async def test_metrics_are_per_platform_and_labelled(self, client):
        body = (await client.get("/api/config/platforms")).json()
        by_value = {p["value"]: p for p in body["platforms"]}

        # Bilibili has play count and danmaku; Xiaohongshu has neither.
        assert "video_play_count" in by_value["bili"]["metrics"]
        assert "video_danmaku" in by_value["bili"]["metrics"]
        assert "video_play_count" not in by_value["xhs"]["metrics"]

        # Every metric shown to a user must have a human label.
        for capability in body["platforms"]:
            for metric in capability["metrics"]:
                assert capability["metric_labels"][metric]

    def test_unknown_platform_is_not_monitor_wired(self):
        assert platforms.is_known("xhs") is True
        assert platforms.is_known("myspace") is False
        assert platforms.is_monitor_wired("myspace") is False


class TestTaskCreationGuard:
    @pytest.mark.asyncio
    async def test_unwired_platform_is_rejected_with_an_explanation(self, client):
        """Accepting it would create a task that silently never produces data.

        用 B站 而不是抖音：抖音现在接通了，不再是「已知但未接通」的例子。
        """
        response = await client.post(
            "/api/monitor/tasks",
            json={"name": "B站任务", "mode": "creator", "platform": "bili", "targets": ["x"]},
        )
        assert response.status_code == 400
        detail = response.json()["detail"]
        assert "B站" in detail
        assert "尚未接通" in detail

    @pytest.mark.asyncio
    async def test_unknown_platform_is_rejected(self, client):
        response = await client.post(
            "/api/monitor/tasks",
            json={"name": "x", "mode": "creator", "platform": "myspace", "targets": ["x"]},
        )
        assert response.status_code == 400

    @pytest.mark.asyncio
    async def test_no_task_row_is_created_when_rejected(self, client):
        await client.post(
            "/api/monitor/tasks",
            json={"name": "B站任务", "mode": "creator", "platform": "bili", "targets": ["x"]},
        )
        assert (await client.get("/api/monitor/tasks")).json()["tasks"] == []

    @pytest.mark.asyncio
    async def test_xhs_still_works_and_is_the_default(self, client):
        explicit = await client.post(
            "/api/monitor/tasks",
            json={"name": "显式", "mode": "creator", "platform": "xhs", "targets": [XHS_TARGET]},
        )
        assert explicit.status_code == 201

        defaulted = await client.post(
            "/api/monitor/tasks",
            json={"name": "默认", "mode": "creator", "targets": [XHS_TARGET]},
        )
        assert defaulted.status_code == 201

        tasks = (await client.get("/api/monitor/tasks")).json()["tasks"]
        assert {t["platform"] for t in tasks} == {"xhs"}


class TestPlatformScoping:
    async def _seed_two_platforms(self, client):
        """One XHS task created through the API, plus a Douyin task inserted
        directly so its fields can be pinned exactly."""
        await client.post(
            "/api/monitor/tasks",
            json={"name": "小红书任务", "mode": "creator", "targets": [XHS_TARGET]},
        )
        async with monitor_db.get_session() as session:
            session.add(
                MonitorTask(
                    name="抖音任务", platform="dy", mode="creator", enabled=True,
                    interval_minutes=60, max_notes_count=20, enable_comments=True,
                    max_comments_count=50, run_timeout_seconds=3600,
                    notify_enabled=False, created_at=0, updated_at=0,
                )
            )

    @pytest.mark.asyncio
    async def test_tasks_are_filtered_by_platform(self, client):
        await self._seed_two_platforms(client)

        all_tasks = (await client.get("/api/monitor/tasks")).json()["tasks"]
        assert len(all_tasks) == 2

        xhs_only = (await client.get("/api/monitor/tasks", params={"platform": "xhs"})).json()
        assert [t["name"] for t in xhs_only["tasks"]] == ["小红书任务"]

        dy_only = (await client.get("/api/monitor/tasks", params={"platform": "dy"})).json()
        assert [t["name"] for t in dy_only["tasks"]] == ["抖音任务"]

    @pytest.mark.asyncio
    async def test_overview_is_scoped(self, client):
        await self._seed_two_platforms(client)

        assert (await client.get("/api/monitor/overview")).json()["tasks"] == 2
        assert (
            await client.get("/api/monitor/overview", params={"platform": "xhs"})
        ).json()["tasks"] == 1

    @pytest.mark.asyncio
    async def test_a_platform_with_no_tasks_yields_empty_not_everything(self, client):
        """空的任务集合不能退化成「不加过滤」。

        **这里必须真的有数据。** 没有数据时，过滤生效与否结果都是 0 —— 这条测试原先
        就栽在这个空跑上，所以没能拦下一个报表串数据的 bug（切到抖音，报表里却出现
        小红书的数据）。最后那段「小红书自己的报表看得到」就是为了证明这些数据确实
        存在、上面那两个 0 是过滤出来的。
        """
        await self._seed_two_platforms(client)
        await _seed_note_with_one_snapshot("小红书任务")

        body = (await client.get("/api/monitor/notes", params={"platform": "bili"})).json()
        assert body["notes"] == []

        report = (
            await client.get("/api/monitor/report", params={"platform": "bili"})
        ).json()
        assert report["totals"]["liked_count_delta"] == 0
        assert report["note_count"] == 0

        xhs = (
            await client.get("/api/monitor/report", params={"platform": "xhs"})
        ).json()
        assert xhs["note_count"] == 1


class TestPerPlatformSettings:
    @pytest.mark.asyncio
    async def test_each_platform_keeps_its_own_values(self, client):
        await client.put(
            "/api/settings",
            params={"platform": "xhs"},
            json={"platform.xhs.crawl_sleep_sec": 3},
        )
        await client.put(
            "/api/settings",
            params={"platform": "dy"},
            json={"platform.dy.crawl_sleep_sec": 9},
        )

        xhs = (await client.get("/api/settings", params={"platform": "xhs"})).json()
        dy = (await client.get("/api/settings", params={"platform": "dy"})).json()

        assert xhs["values"]["platform.xhs.crawl_sleep_sec"] == 3
        assert dy["values"]["platform.dy.crawl_sleep_sec"] == 9

    @pytest.mark.asyncio
    async def test_system_settings_are_shared_across_platforms(self, client):
        await client.put(
            "/api/settings",
            params={"platform": "xhs"},
            json={"system.active_hours_start": 8},
        )

        dy = (await client.get("/api/settings", params={"platform": "dy"})).json()
        assert dy["values"]["system.active_hours_start"] == 8
        # ...and the system specs are present in every platform's response.
        assert "system.active_hours_end" in dy["values"]

    @pytest.mark.asyncio
    async def test_a_key_for_another_platform_is_rejected(self, client):
        """Writing xhs's key while scoped to dy would land somewhere unexpected."""
        response = await client.put(
            "/api/settings",
            params={"platform": "dy"},
            json={"platform.xhs.crawl_sleep_sec": 5},
        )
        assert response.status_code == 400

    @pytest.mark.asyncio
    async def test_cookies_are_per_platform(self, client):
        await client.post(
            "/api/monitor/cookie",
            params={"platform": "xhs"},
            json={"cookie": "web_session=xhs-secret"},
        )

        xhs = (await client.get("/api/monitor/cookie", params={"platform": "xhs"})).json()
        dy = (await client.get("/api/monitor/cookie", params={"platform": "dy"})).json()

        assert xhs["present"] is True
        assert dy["present"] is False

        # The old endpoint still defaults to Xiaohongshu.
        assert (await client.get("/api/monitor/cookie")).json()["present"] is True


class TestLegacyKeyMigration:
    @pytest.mark.asyncio
    async def test_old_flat_keys_are_moved_to_the_new_namespace(self, tmp_path):
        """Existing installs must not lose their cookie on upgrade."""
        monitor_db.set_sqlite_path(tmp_path / "monitor.db")
        await monitor_db.init_db()

        async with monitor_db.get_engine().begin() as conn:
            await conn.execute(
                text(
                    "INSERT INTO monitor_setting (key, value, updated_at) "
                    "VALUES ('xhs_cookie', 'web_session=legacy', 1)"
                )
            )
            await conn.execute(
                text(
                    "INSERT INTO monitor_setting (key, value, updated_at) "
                    "VALUES ('wecom_webhook', 'https://qyapi.weixin.qq.com/x', 1)"
                )
            )

        # Re-running init performs the rename.
        await monitor_db.init_db()

        async with monitor_db.get_engine().begin() as conn:
            rows = dict(
                (await conn.execute(text("SELECT key, value FROM monitor_setting"))).all()
            )

        assert rows.get("platform.xhs.cookie") == "web_session=legacy"
        assert rows.get("system.wecom_webhook") == "https://qyapi.weixin.qq.com/x"
        assert "xhs_cookie" not in rows
        assert "wecom_webhook" not in rows

        await monitor_db.dispose_engine()

    @pytest.mark.asyncio
    async def test_migration_is_idempotent_and_keeps_the_newer_value(self, tmp_path):
        monitor_db.set_sqlite_path(tmp_path / "monitor.db")
        await monitor_db.init_db()

        async with monitor_db.get_engine().begin() as conn:
            await conn.execute(
                text(
                    "INSERT INTO monitor_setting (key, value, updated_at) VALUES "
                    "('platform.xhs.cookie', 'current', 2), ('xhs_cookie', 'stale', 1)"
                )
            )

        await monitor_db.init_db()

        async with monitor_db.get_engine().begin() as conn:
            rows = dict(
                (await conn.execute(text("SELECT key, value FROM monitor_setting"))).all()
            )

        assert rows.get("platform.xhs.cookie") == "current"
        assert "xhs_cookie" not in rows

        await monitor_db.dispose_engine()