butubb
4e60524f37
在上游 MediaCrawler 之上新增一层： - 监控层 api/monitor/ —— 多博主/多笔记的定时采集、指标快照差分、报表、 企业微信通知。每轮采集写入独立目录，差分才成立。 - WebUI 登录鉴权 api/auth.py —— PBKDF2 口令 + 服务端会话，/api 全接口防护。 WebSocket 单独加依赖：BaseHTTPMiddleware 对 ws 作用域直接放行，覆盖不到。 - 全局平台切换 + 能力矩阵 —— 如实区分「爬虫模块支持」与「监控层已接线」， 未接通的平台直接拒绝建任务，而不是静默跑空。 - 监控库改用 MySQL 5.7（可回退 SQLite 供测试）：逐表强制 utf8mb4 （服务端与库默认都是 latin1），启动校验所连 schema 以防写错库， 连接池 recycle + pre_ping 应对 MySQL 的 8 小时空闲断连。 修复上游缺陷： - xhs/core.py: 主页抓取失败会跳掉整个博主，导致一条作品都抓不到， 而那份资料只喂给一个空函数。改为尽力而为，失败不中断。 - xhs/login.py: cookie 登录只注入 web_session，冷启动签名会失败。 新增 INJECT_ALL_COOKIES 开关（默认关闭，原有行为不变）。 - requirements.txt: 补上 websockets。它在上游 pyproject.toml 里有声明、 这里漏了，导致 uvicorn 没有 WebSocket 能力，实时日志流从未工作。 改动过的上游文件清单及合并方式见 UPSTREAM.md。 测试：492 passed（另有 1 个既有的 Windows/gbk 上游测试失败，与本改动无关）
128 lines
4.1 KiB
Python
128 lines
4.1 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright (c) 2025 [email protected]
|
|
#
|
|
# This file is part of MediaCrawler project.
|
|
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/conftest.py
|
|
# GitHub: https://github.com/NanmiCoder
|
|
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
|
#
|
|
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
|
# 1. 不得用于任何商业用途。
|
|
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
|
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
|
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
|
# 5. 不得用于任何非法或不当的用途。
|
|
#
|
|
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
|
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
|
|
|
"""
|
|
Pytest configuration and shared fixtures
|
|
"""
|
|
|
|
import pytest
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# Add project root to Python path
|
|
project_root = Path(__file__).parent.parent
|
|
sys.path.insert(0, str(project_root))
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def project_root_path():
|
|
"""Return project root path"""
|
|
return project_root
|
|
|
|
|
|
@pytest.fixture
|
|
def sample_xhs_note():
|
|
"""Sample Xiaohongshu note data for testing"""
|
|
return {
|
|
"note_id": "test_note_123",
|
|
"type": "normal",
|
|
"title": "Test Title",
|
|
"desc": "This is a test description",
|
|
"video_url": "",
|
|
"time": 1700000000,
|
|
"last_update_time": 1700000000,
|
|
"user_id": "user_123",
|
|
"nickname": "Test User",
|
|
"avatar": "https://example.com/avatar.jpg",
|
|
"liked_count": 100,
|
|
"collected_count": 50,
|
|
"comment_count": 25,
|
|
"share_count": 10,
|
|
"ip_location": "Shanghai",
|
|
"image_list": "https://example.com/img1.jpg,https://example.com/img2.jpg",
|
|
"tag_list": "test,programming,Python",
|
|
"note_url": "https://www.xiaohongshu.com/explore/test_note_123",
|
|
"source_keyword": "test keyword",
|
|
"xsec_token": "test_token_123"
|
|
}
|
|
|
|
|
|
@pytest.fixture
|
|
def sample_xhs_comment():
|
|
"""Sample Xiaohongshu comment data for testing"""
|
|
return {
|
|
"comment_id": "comment_123",
|
|
"create_time": 1700000000,
|
|
"ip_location": "Beijing",
|
|
"note_id": "test_note_123",
|
|
"content": "This is a test comment",
|
|
"user_id": "user_456",
|
|
"nickname": "Comment User",
|
|
"avatar": "https://example.com/avatar2.jpg",
|
|
"sub_comment_count": 5,
|
|
"pictures": "",
|
|
"parent_comment_id": 0,
|
|
"like_count": 15
|
|
}
|
|
|
|
|
|
@pytest.fixture
|
|
def sample_xhs_creator():
|
|
"""Sample Xiaohongshu creator data for testing"""
|
|
return {
|
|
"user_id": "creator_123",
|
|
"nickname": "Creator Name",
|
|
"gender": "Female",
|
|
"avatar": "https://example.com/creator_avatar.jpg",
|
|
"desc": "This is the creator bio",
|
|
"ip_location": "Guangzhou",
|
|
"follows": 500,
|
|
"fans": 10000,
|
|
"interaction": 50000,
|
|
"tag_list": '{"profession": "Designer", "interest": "Photography"}'
|
|
}
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _bypass_auth_for_non_auth_suites(request):
|
|
"""Skip API authentication for suites that are not about authentication.
|
|
|
|
Adding auth to every /api route breaks any test that speaks HTTP, so those
|
|
suites override the dependency here. This uses FastAPI's own
|
|
``dependency_overrides`` mechanism rather than a production-visible
|
|
"test mode" switch, which could be shipped enabled by accident.
|
|
|
|
``tests/test_auth.py`` is deliberately excluded: it must exercise the real
|
|
enforcement path, including the route-enumeration guard that asserts every
|
|
other /api route really does return 401.
|
|
"""
|
|
if request.node.fspath.basename == "test_auth.py":
|
|
yield
|
|
return
|
|
|
|
from api.auth import require_auth, require_ws_auth
|
|
from api.main import app
|
|
|
|
app.dependency_overrides[require_auth] = lambda: None
|
|
app.dependency_overrides[require_ws_auth] = lambda: None
|
|
try:
|
|
yield
|
|
finally:
|
|
app.dependency_overrides.pop(require_auth, None)
|
|
app.dependency_overrides.pop(require_ws_auth, None)
|