feat: 监控面板 / 登录鉴权 / 多平台切换 / MySQL
在上游 MediaCrawler 之上新增一层: - 监控层 api/monitor/ —— 多博主/多笔记的定时采集、指标快照差分、报表、 企业微信通知。每轮采集写入独立目录,差分才成立。 - WebUI 登录鉴权 api/auth.py —— PBKDF2 口令 + 服务端会话,/api 全接口防护。 WebSocket 单独加依赖:BaseHTTPMiddleware 对 ws 作用域直接放行,覆盖不到。 - 全局平台切换 + 能力矩阵 —— 如实区分「爬虫模块支持」与「监控层已接线」, 未接通的平台直接拒绝建任务,而不是静默跑空。 - 监控库改用 MySQL 5.7(可回退 SQLite 供测试):逐表强制 utf8mb4 (服务端与库默认都是 latin1),启动校验所连 schema 以防写错库, 连接池 recycle + pre_ping 应对 MySQL 的 8 小时空闲断连。 修复上游缺陷: - xhs/core.py: 主页抓取失败会跳掉整个博主,导致一条作品都抓不到, 而那份资料只喂给一个空函数。改为尽力而为,失败不中断。 - xhs/login.py: cookie 登录只注入 web_session,冷启动签名会失败。 新增 INJECT_ALL_COOKIES 开关(默认关闭,原有行为不变)。 - requirements.txt: 补上 websockets。它在上游 pyproject.toml 里有声明、 这里漏了,导致 uvicorn 没有 WebSocket 能力,实时日志流从未工作。 改动过的上游文件清单及合并方式见 UPSTREAM.md。 测试:492 passed(另有 1 个既有的 Windows/gbk 上游测试失败,与本改动无关)
This commit is contained in:
+165
-5
@@ -1,4 +1,21 @@
|
||||
import axios from 'axios'
|
||||
import type {
|
||||
CommentBucket,
|
||||
CommentNoteOption,
|
||||
CookieStatus,
|
||||
MetricPoint,
|
||||
MonitorComment,
|
||||
MonitorEvent,
|
||||
MonitorNote,
|
||||
MonitorOverview,
|
||||
MonitorRun,
|
||||
MonitorTask,
|
||||
PlatformCapability,
|
||||
ReportResult,
|
||||
SettingsResponse,
|
||||
TaskCreatePayload,
|
||||
WebhookStatus,
|
||||
} from '@/types/monitor'
|
||||
|
||||
const api = axios.create({
|
||||
baseURL: '/api',
|
||||
@@ -6,8 +23,32 @@ const api = axios.create({
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
// Same-origin in both dev (Vite proxies /api) and production, so the browser
|
||||
// sends the session cookie anyway; this is explicit rather than load-bearing.
|
||||
withCredentials: true,
|
||||
})
|
||||
|
||||
/** The app registers a handler so a 401 anywhere can bounce the user to login. */
|
||||
let onUnauthorized: (() => void) | null = null
|
||||
|
||||
export function setUnauthorizedHandler(handler: (() => void) | null) {
|
||||
onUnauthorized = handler
|
||||
}
|
||||
|
||||
api.interceptors.response.use(
|
||||
(response) => response,
|
||||
(error) => {
|
||||
const status = error?.response?.status
|
||||
const url: string = error?.config?.url ?? ''
|
||||
// A failed login attempt is not an expired session -- treating it as one
|
||||
// would re-render the gate on every wrong password.
|
||||
if (status === 401 && !url.includes('/auth/login')) {
|
||||
onUnauthorized?.()
|
||||
}
|
||||
return Promise.reject(error)
|
||||
},
|
||||
)
|
||||
|
||||
// Types
|
||||
export interface CrawlerConfig {
|
||||
platform: string
|
||||
@@ -53,11 +94,7 @@ export interface FilePreviewResponse {
|
||||
columns?: string[]
|
||||
}
|
||||
|
||||
export interface Platform {
|
||||
value: string
|
||||
label: string
|
||||
icon: string
|
||||
}
|
||||
export type Platform = PlatformCapability
|
||||
|
||||
export interface ConfigOption {
|
||||
value: string
|
||||
@@ -102,4 +139,127 @@ export const envApi = {
|
||||
check: () => api.get<EnvCheckResult>('/env/check'),
|
||||
}
|
||||
|
||||
export const settingsApi = {
|
||||
getAll: (platform?: string) =>
|
||||
api.get<SettingsResponse>('/settings', { params: { platform } }),
|
||||
/** Partial update: only the keys present are written. */
|
||||
update: (values: Record<string, unknown>, platform?: string) =>
|
||||
api.put<{ changed: string[] }>('/settings', values, { params: { platform } }),
|
||||
}
|
||||
|
||||
export interface AuthStatus {
|
||||
authenticated: boolean
|
||||
expires_at: number
|
||||
}
|
||||
|
||||
export const authApi = {
|
||||
me: () => api.get<AuthStatus>('/auth/me'),
|
||||
login: (password: string) => api.post<{ expires_at: number }>('/auth/login', { password }),
|
||||
logout: () => api.post('/auth/logout'),
|
||||
changePassword: (current: string, next: string) =>
|
||||
api.post('/auth/password', { current, new: next }),
|
||||
}
|
||||
|
||||
// Every monitor read is scoped by platform, so the panel only ever shows the
|
||||
// workspace the operator selected.
|
||||
export const monitorApi = {
|
||||
getOverview: (platform?: string) =>
|
||||
api.get<MonitorOverview>('/monitor/overview', { params: { platform } }),
|
||||
|
||||
getTasks: (platform?: string) =>
|
||||
api.get<{ tasks: MonitorTask[] }>('/monitor/tasks', { params: { platform } }),
|
||||
createTask: (payload: TaskCreatePayload) => api.post('/monitor/tasks', payload),
|
||||
updateTask: (id: number, payload: Partial<TaskCreatePayload>) =>
|
||||
api.patch(`/monitor/tasks/${id}`, payload),
|
||||
deleteTask: (id: number) => api.delete(`/monitor/tasks/${id}`),
|
||||
runTaskNow: (id: number) => api.post(`/monitor/tasks/${id}/run`),
|
||||
getRuns: (id: number, limit = 50) =>
|
||||
api.get<{ runs: MonitorRun[] }>(`/monitor/tasks/${id}/runs`, { params: { limit } }),
|
||||
|
||||
getNotes: (taskId?: number, onlyNew = false, limit = 200, platform?: string) =>
|
||||
api.get<{ notes: MonitorNote[] }>('/monitor/notes', {
|
||||
params: { task_id: taskId, only_new: onlyNew, limit, platform },
|
||||
}),
|
||||
// Metric time series for one note; the chart reads this.
|
||||
getNoteSeries: (noteId: string, taskId?: number) =>
|
||||
api.get<{ series: MetricPoint[] }>(`/monitor/notes/${noteId}/series`, {
|
||||
params: { task_id: taskId },
|
||||
}),
|
||||
|
||||
getComments: (taskId?: number, noteId?: string, limit = 200, platform?: string) =>
|
||||
api.get<{ comments: MonitorComment[]; total: number }>('/monitor/comments', {
|
||||
params: { task_id: taskId, note_id: noteId, limit, platform },
|
||||
}),
|
||||
|
||||
/** Same endpoint, bucketed per work -- easier to read than a flat stream. */
|
||||
getCommentsGrouped: (taskId?: number, noteId?: string, limit = 200, platform?: string) =>
|
||||
api.get<{ groups: CommentBucket[]; total: number }>('/monitor/comments', {
|
||||
params: { task_id: taskId, note_id: noteId, group_by: 'note', limit, platform },
|
||||
}),
|
||||
|
||||
/** Works that have comments, with counts -- feeds the filter dropdown. */
|
||||
getCommentNotes: (taskId?: number, platform?: string) =>
|
||||
api.get<{ notes: CommentNoteOption[] }>('/monitor/comment-notes', {
|
||||
params: { task_id: taskId, platform },
|
||||
}),
|
||||
|
||||
/**
|
||||
* Export URL for `window.open`. Downloads are navigations and cannot carry an
|
||||
* Authorization header, which is why the session lives in a cookie.
|
||||
*/
|
||||
getExportUrl: (params: {
|
||||
kind: 'notes' | 'comments' | 'report'
|
||||
taskId?: number
|
||||
taskIds?: number[]
|
||||
noteId?: string
|
||||
format?: 'csv' | 'xlsx'
|
||||
startDate?: string
|
||||
endDate?: string
|
||||
platform?: string
|
||||
}) => {
|
||||
const search = new URLSearchParams()
|
||||
search.set('kind', params.kind)
|
||||
search.set('format', params.format ?? 'csv')
|
||||
if (params.platform) search.set('platform', params.platform)
|
||||
if (params.taskId !== undefined) search.set('task_id', String(params.taskId))
|
||||
params.taskIds?.forEach((id) => search.append('task_id', String(id)))
|
||||
if (params.noteId) search.set('note_id', params.noteId)
|
||||
// Explicit dates rather than `days`, so the file matches the range on screen.
|
||||
if (params.startDate) search.set('start_date', params.startDate)
|
||||
if (params.endDate) search.set('end_date', params.endDate)
|
||||
return `/api/monitor/export?${search.toString()}`
|
||||
},
|
||||
|
||||
getEvents: (taskId?: number, limit = 200, platform?: string) =>
|
||||
api.get<{ events: MonitorEvent[]; latest_id: number | null }>('/monitor/events', {
|
||||
params: { task_id: taskId, limit, platform },
|
||||
}),
|
||||
markEventsRead: (taskId?: number) =>
|
||||
api.post('/monitor/events/read', null, { params: { task_id: taskId } }),
|
||||
|
||||
getCookieStatus: (platform?: string) =>
|
||||
api.get<CookieStatus>('/monitor/cookie', { params: { platform } }),
|
||||
setCookie: (cookie: string, platform?: string) =>
|
||||
api.post('/monitor/cookie', { cookie }, { params: { platform } }),
|
||||
clearCookie: (platform?: string) =>
|
||||
api.delete('/monitor/cookie', { params: { platform } }),
|
||||
|
||||
// task_id is repeated (?task_id=1&task_id=2) rather than comma-joined, so the
|
||||
// query string is built by hand instead of relying on axios array encoding,
|
||||
// which would emit "task_id[]=" that FastAPI does not match.
|
||||
getReport: (taskIds: number[], startDate: string, endDate: string, platform?: string) => {
|
||||
const params = new URLSearchParams()
|
||||
taskIds.forEach((id) => params.append('task_id', String(id)))
|
||||
params.append('start_date', startDate)
|
||||
params.append('end_date', endDate)
|
||||
if (platform) params.append('platform', platform)
|
||||
return api.get<ReportResult>(`/monitor/report?${params.toString()}`)
|
||||
},
|
||||
|
||||
getWebhook: () => api.get<WebhookStatus>('/monitor/webhook'),
|
||||
setWebhook: (url: string) => api.post('/monitor/webhook', { url }),
|
||||
clearWebhook: () => api.delete('/monitor/webhook'),
|
||||
testWebhook: (url?: string) => api.post('/monitor/webhook/test', { url: url ?? null }),
|
||||
}
|
||||
|
||||
export default api
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
/** Formatting helpers for the monitoring dashboard. */
|
||||
|
||||
/** Compact count for display: mirrors how the platform itself abbreviates. */
|
||||
export function formatCount(value: number | null | undefined): string {
|
||||
if (value === null || value === undefined) return '—'
|
||||
if (value >= 100_000_000) return `${(value / 100_000_000).toFixed(1)}亿`
|
||||
if (value >= 10_000) return `${(value / 10_000).toFixed(1)}万`
|
||||
return String(value)
|
||||
}
|
||||
|
||||
/** Signed delta, or null when the change is unknown (either side unparseable). */
|
||||
export function formatDelta(value: number | null | undefined): string | null {
|
||||
if (value === null || value === undefined || value === 0) return null
|
||||
return `${value > 0 ? '+' : ''}${formatCount(value)}`
|
||||
}
|
||||
|
||||
/** "3分钟前" / "2小时后" / "—". */
|
||||
export function formatRelative(ms: number | null | undefined): string {
|
||||
if (!ms) return '—'
|
||||
|
||||
const diff = ms - Date.now()
|
||||
const abs = Math.abs(diff)
|
||||
const future = diff > 0
|
||||
const suffix = (n: number, unit: string) => (future ? `${n}${unit}后` : `${n}${unit}前`)
|
||||
|
||||
if (abs < 60_000) return future ? '即将' : '刚刚'
|
||||
if (abs < 3_600_000) return suffix(Math.floor(abs / 60_000), '分钟')
|
||||
if (abs < 86_400_000) return suffix(Math.floor(abs / 3_600_000), '小时')
|
||||
return suffix(Math.floor(abs / 86_400_000), '天')
|
||||
}
|
||||
|
||||
export function formatDateTime(ms: number | null | undefined): string {
|
||||
if (!ms) return '—'
|
||||
const d = new Date(ms)
|
||||
const pad = (n: number) => String(n).padStart(2, '0')
|
||||
return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}`
|
||||
}
|
||||
|
||||
/** Human interval label for task cards. */
|
||||
export function formatInterval(minutes: number): string {
|
||||
if (minutes % 1440 === 0) return `${minutes / 1440} 天`
|
||||
if (minutes % 60 === 0) return `${minutes / 60} 小时`
|
||||
return `${minutes} 分钟`
|
||||
}
|
||||
Reference in New Issue
Block a user