feat: 监控面板 / 登录鉴权 / 多平台切换 / MySQL
在上游 MediaCrawler 之上新增一层: - 监控层 api/monitor/ —— 多博主/多笔记的定时采集、指标快照差分、报表、 企业微信通知。每轮采集写入独立目录,差分才成立。 - WebUI 登录鉴权 api/auth.py —— PBKDF2 口令 + 服务端会话,/api 全接口防护。 WebSocket 单独加依赖:BaseHTTPMiddleware 对 ws 作用域直接放行,覆盖不到。 - 全局平台切换 + 能力矩阵 —— 如实区分「爬虫模块支持」与「监控层已接线」, 未接通的平台直接拒绝建任务,而不是静默跑空。 - 监控库改用 MySQL 5.7(可回退 SQLite 供测试):逐表强制 utf8mb4 (服务端与库默认都是 latin1),启动校验所连 schema 以防写错库, 连接池 recycle + pre_ping 应对 MySQL 的 8 小时空闲断连。 修复上游缺陷: - xhs/core.py: 主页抓取失败会跳掉整个博主,导致一条作品都抓不到, 而那份资料只喂给一个空函数。改为尽力而为,失败不中断。 - xhs/login.py: cookie 登录只注入 web_session,冷启动签名会失败。 新增 INJECT_ALL_COOKIES 开关(默认关闭,原有行为不变)。 - requirements.txt: 补上 websockets。它在上游 pyproject.toml 里有声明、 这里漏了,导致 uvicorn 没有 WebSocket 能力,实时日志流从未工作。 改动过的上游文件清单及合并方式见 UPSTREAM.md。 测试:492 passed(另有 1 个既有的 Windows/gbk 上游测试失败,与本改动无关)
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
import type { ComponentType, ReactNode, KeyboardEvent } from 'react'
|
||||
import { useState } from 'react'
|
||||
import { useEffect, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { Database, Globe, Image as ImageIcon, KeyRound, MessageSquare, Play, Square, X } from 'lucide-react'
|
||||
import { Label } from '@/components/ui/label'
|
||||
@@ -8,7 +8,9 @@ import { Input } from '@/components/ui/input'
|
||||
import { Checkbox } from '@/components/ui/checkbox'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { useCrawlerStore } from '@/store/crawlerStore'
|
||||
import { usePlatforms, useConfigOptions, useStartCrawler, useStopCrawler } from '@/hooks/useCrawler'
|
||||
import { useConfigOptions, useStartCrawler, useStopCrawler } from '@/hooks/useCrawler'
|
||||
import { useCurrentPlatform } from '@/hooks/usePlatform'
|
||||
import { useCookieStatus } from '@/hooks/useMonitor'
|
||||
import { ParsedIdList } from './ParsedIdList'
|
||||
|
||||
type SectionProps = {
|
||||
@@ -137,9 +139,19 @@ export function CrawlerConfigPanel() {
|
||||
const updateConfig = useCrawlerStore((state) => state.updateConfig)
|
||||
const status = useCrawlerStore((state) => state.status)
|
||||
|
||||
const { data: platforms } = usePlatforms()
|
||||
const { data: options } = useConfigOptions()
|
||||
const { mutate: startCrawler, isPending: isStarting } = useStartCrawler()
|
||||
|
||||
// Follow the global platform selection rather than keeping its own copy.
|
||||
const { platform: hostPlatform, capability: hostCapability } = useCurrentPlatform()
|
||||
const { data: cookieStatus } = useCookieStatus()
|
||||
const storedCookieOk = cookieStatus?.present ?? false
|
||||
|
||||
useEffect(() => {
|
||||
if (config.platform !== hostPlatform) {
|
||||
updateConfig({ platform: hostPlatform })
|
||||
}
|
||||
}, [hostPlatform, config.platform, updateConfig])
|
||||
const { mutate: stopCrawler, isPending: isStopping } = useStopCrawler()
|
||||
|
||||
const isDisabled = status === 'running' || status === 'stopping'
|
||||
@@ -165,22 +177,15 @@ export function CrawlerConfigPanel() {
|
||||
icon={Globe}
|
||||
>
|
||||
<Field label={t('field.platform')}>
|
||||
<Select
|
||||
value={config.platform}
|
||||
onValueChange={(value) => updateConfig({ platform: value })}
|
||||
disabled={isDisabled}
|
||||
>
|
||||
<SelectTrigger className="h-9 text-xs">
|
||||
<SelectValue placeholder={t('field.platformPlaceholder')} />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{platforms?.map((platform) => (
|
||||
<SelectItem key={platform.value} value={platform.value}>
|
||||
{platform.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
{/* Driven by the global switcher in the header. A second platform
|
||||
control here would be a second source of truth for the same
|
||||
decision, and the two would drift. */}
|
||||
<div className="flex items-center justify-between h-9 px-3 rounded-md border border-cyber-border-DEFAULT bg-cyber-bg-tertiary">
|
||||
<span className="font-mono text-xs text-cyber-neon-cyan">
|
||||
{hostCapability?.label ?? hostPlatform}
|
||||
</span>
|
||||
<span className="text-[10px] font-mono text-cyber-text-muted">由右上角切换</span>
|
||||
</div>
|
||||
</Field>
|
||||
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
@@ -300,14 +305,23 @@ export function CrawlerConfigPanel() {
|
||||
</Field>
|
||||
|
||||
{config.login_type === 'cookie' ? (
|
||||
<Field label={t('field.cookies')} hint={t('field.cookiesHint')}>
|
||||
<textarea
|
||||
value={config.cookies}
|
||||
onChange={(e) => updateConfig({ cookies: e.target.value })}
|
||||
disabled={isDisabled}
|
||||
placeholder={t('field.cookiesPlaceholder')}
|
||||
className="min-h-[80px] w-full rounded-md border border-cyber-border-DEFAULT bg-cyber-bg-tertiary px-3 py-2 text-xs font-mono text-cyber-text-primary placeholder:text-cyber-text-muted focus-visible:outline-none focus-visible:border-cyber-neon-cyan/50 focus-visible:shadow-cyber-soft disabled:cursor-not-allowed disabled:opacity-50 transition-all resize-none"
|
||||
/>
|
||||
// The cookie value has exactly one home now: 设置 → 登录态. A paste
|
||||
// box here was unpersisted, so it went stale between sessions and
|
||||
// could silently disagree with the cookie the monitor uses.
|
||||
<Field label={t('field.cookies')}>
|
||||
<div className="flex items-center gap-2 h-9 px-3 rounded-md border border-cyber-border-DEFAULT bg-cyber-bg-tertiary">
|
||||
<span
|
||||
className={`status-dot ${
|
||||
storedCookieOk ? 'status-dot-online' : 'status-dot-warning'
|
||||
}`}
|
||||
/>
|
||||
<span className="font-mono text-[11px] text-cyber-text-secondary">
|
||||
{storedCookieOk ? '复用「设置」中已保存的 Cookie' : '「设置」中尚未配置 Cookie'}
|
||||
</span>
|
||||
</div>
|
||||
<p className="text-[10px] font-mono text-cyber-text-muted">
|
||||
在「设置 → 登录态」里维护,此处不再单独填写
|
||||
</p>
|
||||
</Field>
|
||||
) : null}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user