butubb
4e60524f37
在上游 MediaCrawler 之上新增一层： - 监控层 api/monitor/ —— 多博主/多笔记的定时采集、指标快照差分、报表、 企业微信通知。每轮采集写入独立目录，差分才成立。 - WebUI 登录鉴权 api/auth.py —— PBKDF2 口令 + 服务端会话，/api 全接口防护。 WebSocket 单独加依赖：BaseHTTPMiddleware 对 ws 作用域直接放行，覆盖不到。 - 全局平台切换 + 能力矩阵 —— 如实区分「爬虫模块支持」与「监控层已接线」， 未接通的平台直接拒绝建任务，而不是静默跑空。 - 监控库改用 MySQL 5.7（可回退 SQLite 供测试）：逐表强制 utf8mb4 （服务端与库默认都是 latin1），启动校验所连 schema 以防写错库， 连接池 recycle + pre_ping 应对 MySQL 的 8 小时空闲断连。 修复上游缺陷： - xhs/core.py: 主页抓取失败会跳掉整个博主，导致一条作品都抓不到， 而那份资料只喂给一个空函数。改为尽力而为，失败不中断。 - xhs/login.py: cookie 登录只注入 web_session，冷启动签名会失败。 新增 INJECT_ALL_COOKIES 开关（默认关闭，原有行为不变）。 - requirements.txt: 补上 websockets。它在上游 pyproject.toml 里有声明、 这里漏了，导致 uvicorn 没有 WebSocket 能力，实时日志流从未工作。 改动过的上游文件清单及合并方式见 UPSTREAM.md。 测试：492 passed（另有 1 个既有的 Windows/gbk 上游测试失败，与本改动无关）
134 lines
4.5 KiB
Python
134 lines
4.5 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright (c) 2025 [email protected]
|
|
#
|
|
# This file is part of MediaCrawler project.
|
|
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/schemas/crawler.py
|
|
# GitHub: https://github.com/NanmiCoder
|
|
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
|
#
|
|
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
|
# 1. 不得用于任何商业用途。
|
|
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
|
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
|
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
|
# 5. 不得用于任何非法或不当的用途。
|
|
#
|
|
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
|
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
|
|
|
from enum import Enum
|
|
from typing import Optional, Literal
|
|
from pydantic import BaseModel, Field
|
|
|
|
|
|
MAX_API_LIMIT_COUNT = 10000
|
|
|
|
|
|
class PlatformEnum(str, Enum):
|
|
"""Supported media platforms"""
|
|
XHS = "xhs"
|
|
DOUYIN = "dy"
|
|
KUAISHOU = "ks"
|
|
BILIBILI = "bili"
|
|
WEIBO = "wb"
|
|
TIEBA = "tieba"
|
|
ZHIHU = "zhihu"
|
|
|
|
|
|
class LoginTypeEnum(str, Enum):
|
|
"""Login method"""
|
|
QRCODE = "qrcode"
|
|
PHONE = "phone"
|
|
COOKIE = "cookie"
|
|
|
|
|
|
class CrawlerTypeEnum(str, Enum):
|
|
"""Crawler type"""
|
|
SEARCH = "search"
|
|
DETAIL = "detail"
|
|
CREATOR = "creator"
|
|
|
|
|
|
class SaveDataOptionEnum(str, Enum):
|
|
"""Data save option"""
|
|
CSV = "csv"
|
|
DB = "db"
|
|
JSON = "json"
|
|
JSONL = "jsonl"
|
|
SQLITE = "sqlite"
|
|
MONGODB = "mongodb"
|
|
EXCEL = "excel"
|
|
|
|
|
|
class CrawlerStartRequest(BaseModel):
|
|
"""Crawler start request"""
|
|
platform: PlatformEnum
|
|
login_type: LoginTypeEnum = LoginTypeEnum.QRCODE
|
|
crawler_type: CrawlerTypeEnum = CrawlerTypeEnum.SEARCH
|
|
keywords: str = "" # Keywords for search mode
|
|
specified_ids: str = "" # Post/video ID list for detail mode, comma-separated
|
|
creator_ids: str = "" # Creator ID list for creator mode, comma-separated
|
|
start_page: int = 1
|
|
enable_comments: bool = True
|
|
enable_sub_comments: bool = False
|
|
enable_media: bool = False
|
|
save_option: SaveDataOptionEnum = SaveDataOptionEnum.JSONL
|
|
cookies: str = ""
|
|
headless: bool = False
|
|
max_notes_count: Optional[int] = Field(default=None, ge=1, le=MAX_API_LIMIT_COUNT)
|
|
max_comments_count: Optional[int] = Field(default=None, ge=1, le=MAX_API_LIMIT_COUNT)
|
|
|
|
# --- Options only used by scheduled monitor runs. Each defaults to None so
|
|
# the corresponding CLI flag is omitted entirely for manual Crawl-tab runs,
|
|
# which keeps their behaviour byte-identical to before.
|
|
|
|
# Isolate this run's output in its own directory. The crawler's own file
|
|
# writer names files by date only, so same-day runs would otherwise append
|
|
# into one shared file and could not be told apart.
|
|
save_data_path: Optional[str] = None
|
|
# Unattended runs must not try to attach to the user's desktop Chrome.
|
|
enable_cdp_mode: Optional[bool] = None
|
|
# XHS cookie login only injects `web_session` by default, which is not enough
|
|
# to sign API requests from a cold browser profile.
|
|
inject_all_cookies: Optional[bool] = None
|
|
save_login_state: Optional[bool] = None
|
|
# Preferred over `cookies`: a value on the command line is visible in the
|
|
# process list.
|
|
cookies_file: Optional[str] = None
|
|
max_concurrency_num: Optional[int] = Field(default=None, ge=1, le=MAX_API_LIMIT_COUNT)
|
|
|
|
# Crawl-strategy and proxy knobs surfaced on the Settings page. Like the
|
|
# fields above, each stays None unless the caller sets it, so the CLI flag is
|
|
# omitted entirely and the config-file default applies.
|
|
crawler_max_sleep_sec: Optional[int] = Field(default=None, ge=0, le=600)
|
|
enable_ip_proxy: Optional[bool] = None
|
|
ip_proxy_pool_count: Optional[int] = Field(default=None, ge=1, le=100)
|
|
ip_proxy_provider_name: Optional[str] = None
|
|
static_proxy_url: Optional[str] = None
|
|
|
|
|
|
class CrawlerStatusResponse(BaseModel):
|
|
"""Crawler status response"""
|
|
status: Literal["idle", "running", "stopping", "error"]
|
|
platform: Optional[str] = None
|
|
crawler_type: Optional[str] = None
|
|
started_at: Optional[str] = None
|
|
error_message: Optional[str] = None
|
|
|
|
|
|
class LogEntry(BaseModel):
|
|
"""Log entry"""
|
|
id: int
|
|
timestamp: str
|
|
level: Literal["info", "warning", "error", "success", "debug"]
|
|
message: str
|
|
|
|
|
|
class DataFileInfo(BaseModel):
|
|
"""Data file information"""
|
|
name: str
|
|
path: str
|
|
size: int
|
|
modified_at: str
|
|
record_count: Optional[int] = None
|