feat: 监控面板 / 登录鉴权 / 多平台切换 / MySQL
在上游 MediaCrawler 之上新增一层: - 监控层 api/monitor/ —— 多博主/多笔记的定时采集、指标快照差分、报表、 企业微信通知。每轮采集写入独立目录,差分才成立。 - WebUI 登录鉴权 api/auth.py —— PBKDF2 口令 + 服务端会话,/api 全接口防护。 WebSocket 单独加依赖:BaseHTTPMiddleware 对 ws 作用域直接放行,覆盖不到。 - 全局平台切换 + 能力矩阵 —— 如实区分「爬虫模块支持」与「监控层已接线」, 未接通的平台直接拒绝建任务,而不是静默跑空。 - 监控库改用 MySQL 5.7(可回退 SQLite 供测试):逐表强制 utf8mb4 (服务端与库默认都是 latin1),启动校验所连 schema 以防写错库, 连接池 recycle + pre_ping 应对 MySQL 的 8 小时空闲断连。 修复上游缺陷: - xhs/core.py: 主页抓取失败会跳掉整个博主,导致一条作品都抓不到, 而那份资料只喂给一个空函数。改为尽力而为,失败不中断。 - xhs/login.py: cookie 登录只注入 web_session,冷启动签名会失败。 新增 INJECT_ALL_COOKIES 开关(默认关闭,原有行为不变)。 - requirements.txt: 补上 websockets。它在上游 pyproject.toml 里有声明、 这里漏了,导致 uvicorn 没有 WebSocket 能力,实时日志流从未工作。 改动过的上游文件清单及合并方式见 UPSTREAM.md。 测试:492 passed(另有 1 个既有的 Windows/gbk 上游测试失败,与本改动无关)
This commit is contained in:
@@ -300,6 +300,14 @@ async def parse_cmd(argv: Optional[Sequence[str]] = None):
|
||||
rich_help_panel="Performance Configuration",
|
||||
),
|
||||
] = config.MAX_CONCURRENCY_NUM,
|
||||
crawler_max_sleep_sec: Annotated[
|
||||
int,
|
||||
typer.Option(
|
||||
"--crawler_max_sleep_sec",
|
||||
help="Seconds to wait between requests. Higher is slower but far less likely to trip platform rate limiting",
|
||||
rich_help_panel="Performance Configuration",
|
||||
),
|
||||
] = config.CRAWLER_MAX_SLEEP_SEC,
|
||||
save_data_path: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
@@ -308,6 +316,41 @@ async def parse_cmd(argv: Optional[Sequence[str]] = None):
|
||||
rich_help_panel="Storage Configuration",
|
||||
),
|
||||
] = config.SAVE_DATA_PATH,
|
||||
enable_cdp_mode: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
"--enable_cdp_mode",
|
||||
help="Whether to drive the user's local Chrome over CDP instead of launching a browser, supports yes/true/t/y/1 or no/false/f/n/0. Set to false for unattended/server runs",
|
||||
rich_help_panel="Runtime Configuration",
|
||||
show_default=True,
|
||||
),
|
||||
] = str(config.ENABLE_CDP_MODE),
|
||||
save_login_state: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
"--save_login_state",
|
||||
help="Whether to persist the browser profile so a previous login can be reused, supports yes/true/t/y/1 or no/false/f/n/0",
|
||||
rich_help_panel="Runtime Configuration",
|
||||
show_default=True,
|
||||
),
|
||||
] = str(config.SAVE_LOGIN_STATE),
|
||||
inject_all_cookies: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
"--inject_all_cookies",
|
||||
help="Whether to inject every cookie supplied via --cookies/--cookies_file instead of only web_session, supports yes/true/t/y/1 or no/false/f/n/0",
|
||||
rich_help_panel="Runtime Configuration",
|
||||
show_default=True,
|
||||
),
|
||||
] = str(config.INJECT_ALL_COOKIES),
|
||||
cookies_file: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
"--cookies_file",
|
||||
help="Path to a file holding the cookie string. Preferred over --cookies, whose value is visible in the process list",
|
||||
rich_help_panel="Runtime Configuration",
|
||||
),
|
||||
] = "",
|
||||
enable_ip_proxy: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
@@ -350,6 +393,18 @@ async def parse_cmd(argv: Optional[Sequence[str]] = None):
|
||||
enable_headless = _to_bool(headless)
|
||||
enable_ip_proxy_value = _to_bool(enable_ip_proxy)
|
||||
init_db_value = init_db.value if init_db else None
|
||||
enable_cdp_mode_value = _to_bool(enable_cdp_mode)
|
||||
save_login_state_value = _to_bool(save_login_state)
|
||||
inject_all_cookies_value = _to_bool(inject_all_cookies)
|
||||
|
||||
# A file is preferred over --cookies: a literal value on the command line
|
||||
# is visible to any other user on the machine via the process list.
|
||||
if cookies_file:
|
||||
try:
|
||||
with open(cookies_file, "r", encoding="utf-8") as f:
|
||||
cookies = f.read().strip()
|
||||
except OSError as e:
|
||||
raise typer.BadParameter(f"Unable to read --cookies_file: {e}")
|
||||
|
||||
# Parse specified_id and creator_id into lists
|
||||
specified_id_list = [id.strip() for id in specified_id.split(",") if id.strip()] if specified_id else []
|
||||
@@ -368,9 +423,13 @@ async def parse_cmd(argv: Optional[Sequence[str]] = None):
|
||||
config.CDP_HEADLESS = enable_headless
|
||||
config.SAVE_DATA_OPTION = save_data_option.value
|
||||
config.COOKIES = cookies
|
||||
config.ENABLE_CDP_MODE = enable_cdp_mode_value
|
||||
config.SAVE_LOGIN_STATE = save_login_state_value
|
||||
config.INJECT_ALL_COOKIES = inject_all_cookies_value
|
||||
config.CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES = max_comments_count_singlenotes
|
||||
config.CRAWLER_MAX_NOTES_COUNT = crawler_max_notes_count
|
||||
config.MAX_CONCURRENCY_NUM = max_concurrency_num
|
||||
config.CRAWLER_MAX_SLEEP_SEC = crawler_max_sleep_sec
|
||||
config.SAVE_DATA_PATH = save_data_path
|
||||
config.ENABLE_IP_PROXY = enable_ip_proxy_value
|
||||
config.IP_PROXY_POOL_COUNT = ip_proxy_pool_count
|
||||
|
||||
Reference in New Issue
Block a user