butubb
61808444ad
**评论能采了。** 之前 comment/list 一直回 200 + 空 body，被读成「这条没评论」—— 而它其实只是被网关挡了。缺的就是 a_bogus 签名，仓库里本来就有 （libs/douyin.js + execjs）。签上之后实测 200 / 9960 字节真评论。 只给评论接口签：作品、详情、博主资料三个不带签名也照常返回，而给它们加签名是 没验证过的改动。签名按需 import —— 那个模块 import 时就把 JS 喂给 execjs， 不该拖进监控层热路径。 **作品导出那几列一直是空的。** 列名写的是裸键 liked_count，而作品行的指标嵌在 metrics / deltas 里，row.get() 永远取到 None —— 导出来的表有「点赞/评论/收藏/ 分享」四列，每一格都没有数。原来的测试只断言了「作品ID」，所以没发现。 顺手补上：导出带上 博主备注/昵称、作品备注、发布时间，时间戳格式化成人能读的 形态（原来是一串 13 位毫秒，Excel 里没法看也没法排序）。
724 lines
28 KiB
Python
724 lines
28 KiB
Python
# -*- coding: utf-8 -*-
|
||
# Copyright (c) 2025 [email protected]
|
||
#
|
||
# This file is part of MediaCrawler project.
|
||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/routers/monitor.py
|
||
# GitHub: https://github.com/NanmiCoder
|
||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||
#
|
||
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
||
# 1. 不得用于任何商业用途。
|
||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
||
# 5. 不得用于任何非法或不当的用途。
|
||
#
|
||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||
|
||
"""HTTP API for scheduled monitoring tasks."""
|
||
|
||
from datetime import date, datetime, timedelta
|
||
from typing import Any, Dict, List, Optional
|
||
|
||
from fastapi import APIRouter, HTTPException, Query, Response
|
||
from fastapi.responses import FileResponse
|
||
|
||
from ..monitor import covers, notify, qrlogin, report, service, upstream
|
||
from ..monitor.db import get_session
|
||
from ..monitor.platforms import PLATFORM_XHS
|
||
from ..monitor.settings import (
|
||
cookie_key,
|
||
delete_setting,
|
||
get_cookie_status,
|
||
get_setting,
|
||
set_cookie,
|
||
set_setting,
|
||
)
|
||
from ..monitor.models import SETTING_WECOM_WEBHOOK, MonitorTask
|
||
from ..schemas.monitor import (
|
||
CookiePayload,
|
||
CreatorAliasPayload,
|
||
NoteAliasPayload,
|
||
MonitorTaskCreate,
|
||
MonitorTaskUpdate,
|
||
WebhookPayload,
|
||
WebhookTestPayload,
|
||
)
|
||
|
||
router = APIRouter(prefix="/monitor", tags=["monitor"])
|
||
|
||
|
||
@router.get("/overview")
|
||
async def get_overview(platform: Optional[str] = None):
|
||
"""Headline numbers for the dashboard tiles, scoped to one platform."""
|
||
async with get_session() as session:
|
||
return await service.overview(session, platform)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Tasks
|
||
# ---------------------------------------------------------------------------
|
||
|
||
@router.get("/tasks")
|
||
async def list_tasks(platform: Optional[str] = None):
|
||
async with get_session() as session:
|
||
return {"tasks": await service.list_tasks(session, platform)}
|
||
|
||
|
||
@router.post("/tasks", status_code=201)
|
||
async def create_task(payload: MonitorTaskCreate):
|
||
async with get_session() as session:
|
||
try:
|
||
task = await service.create_task(session, payload.model_dump())
|
||
except service.TargetParseError as exc:
|
||
raise HTTPException(status_code=400, detail=str(exc))
|
||
except ValueError as exc:
|
||
raise HTTPException(status_code=400, detail=str(exc))
|
||
return {"id": task.id, "message": "Monitoring task created"}
|
||
|
||
|
||
@router.patch("/tasks/{task_id}")
|
||
async def update_task(task_id: int, payload: MonitorTaskUpdate):
|
||
async with get_session() as session:
|
||
try:
|
||
await service.update_task(session, task_id, payload.model_dump(exclude_unset=True))
|
||
except service.TargetParseError as exc:
|
||
raise HTTPException(status_code=400, detail=str(exc))
|
||
except ValueError as exc:
|
||
raise HTTPException(status_code=404, detail=str(exc))
|
||
return {"message": "Monitoring task updated"}
|
||
|
||
|
||
@router.delete("/tasks/{task_id}")
|
||
async def delete_task(task_id: int):
|
||
async with get_session() as session:
|
||
try:
|
||
await service.delete_task(session, task_id)
|
||
except ValueError as exc:
|
||
raise HTTPException(status_code=404, detail=str(exc))
|
||
return {"message": "Monitoring task deleted"}
|
||
|
||
|
||
@router.post("/tasks/{task_id}/run")
|
||
async def run_task_now(task_id: int):
|
||
"""Queue a run immediately and return; the crawl itself takes minutes."""
|
||
async with get_session() as session:
|
||
task = await session.get(MonitorTask, task_id)
|
||
if task is None:
|
||
raise HTTPException(status_code=404, detail=f"Task {task_id} not found")
|
||
if not any(target.enabled for target in task.targets):
|
||
raise HTTPException(status_code=400, detail="Task has no enabled targets")
|
||
|
||
service.trigger_manual_run(task_id)
|
||
return {"message": "Run queued"}
|
||
|
||
|
||
@router.get("/tasks/{task_id}/runs")
|
||
async def list_runs(task_id: int, limit: int = Query(default=50, ge=1, le=500)):
|
||
async with get_session() as session:
|
||
return {"runs": await service.list_runs(session, task_id, limit=limit)}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Collected data
|
||
# ---------------------------------------------------------------------------
|
||
|
||
@router.get("/notes")
|
||
async def list_notes(
|
||
task_id: Optional[int] = None,
|
||
only_new: bool = False,
|
||
limit: int = Query(default=200, ge=1, le=2000),
|
||
platform: Optional[str] = None,
|
||
):
|
||
async with get_session() as session:
|
||
return {
|
||
"notes": await service.list_notes(session, task_id, only_new, limit, platform),
|
||
# 博主**单独给一份**，而不是让前端从作品里推。作品推不出「一条作品都没有的
|
||
# 博主」—— 那正是最该显示的一类（还在涨粉，只是最近没发）。
|
||
"creators": await service.list_creators(session, task_id, platform),
|
||
}
|
||
|
||
|
||
@router.get("/notes/{note_id}/series")
|
||
async def note_series(note_id: str, task_id: Optional[int] = None):
|
||
"""Metric time series for a single note."""
|
||
async with get_session() as session:
|
||
return {"series": await service.note_series(session, note_id, task_id)}
|
||
|
||
|
||
@router.get("/comments")
|
||
async def list_comments(
|
||
task_id: Optional[int] = None,
|
||
note_id: Optional[str] = None,
|
||
group_by: Optional[str] = Query(
|
||
default=None, description="传 note 则按作品分组返回，便于阅读"
|
||
),
|
||
limit: int = Query(default=200, ge=1, le=2000),
|
||
platform: Optional[str] = None,
|
||
):
|
||
"""Comments, each carrying the work it belongs to.
|
||
|
||
``note_id`` filters to one work; ``group_by=note`` returns them bucketed per
|
||
work instead of as a flat stream.
|
||
"""
|
||
async with get_session() as session:
|
||
comments = await service.list_comments(session, task_id, note_id, limit, platform)
|
||
|
||
if group_by != "note":
|
||
return {"comments": comments, "total": len(comments)}
|
||
|
||
buckets: Dict[str, Dict[str, Any]] = {}
|
||
for comment in comments:
|
||
bucket = buckets.setdefault(
|
||
comment["note_id"],
|
||
{
|
||
"note_id": comment["note_id"],
|
||
"note_title": comment["note_title"],
|
||
"note_cover": comment["note_cover"],
|
||
"note_url": comment["note_url"],
|
||
# 作品所属的创作者。评论流按 博主 → 作品 → 评论 三级展开时，最外层
|
||
# 就是按这两个字段分组的 —— 少了它们，前端拿到的是 undefined，
|
||
# 于是所有博主塌成同一个分组、标签回退成「未知博主」。
|
||
# 同一个桶里的评论必然同属一个作品，所以取哪一条都一样。
|
||
"creator_hash": comment["note_creator_hash"],
|
||
"creator_name": comment["note_creator_name"],
|
||
# 作品的发布时间。评论流按 博主 → 作品 → 评论 展开时，作品那一层
|
||
# 光有标题不够 —— 同名作品不少，日期能帮着认。
|
||
"published_at": comment["note_published_at"],
|
||
"comments": [],
|
||
},
|
||
)
|
||
bucket["comments"].append(comment)
|
||
|
||
ordered = sorted(
|
||
buckets.values(),
|
||
key=lambda group: group["comments"][0]["first_seen_at"],
|
||
reverse=True,
|
||
)
|
||
return {"groups": ordered, "total": len(comments)}
|
||
|
||
|
||
@router.get("/comment-notes")
|
||
async def list_comment_notes(task_id: Optional[int] = None, platform: Optional[str] = None):
|
||
"""Works that have comments, newest first, with counts.
|
||
|
||
Feeds the comment filter dropdown so the operator can pick by title.
|
||
"""
|
||
async with get_session() as session:
|
||
return {"notes": await service.comment_note_groups(session, task_id, platform)}
|
||
|
||
|
||
@router.get("/events")
|
||
async def list_events(
|
||
task_id: Optional[int] = None,
|
||
type: Optional[str] = None,
|
||
since_id: Optional[int] = None,
|
||
limit: int = Query(default=200, ge=1, le=2000),
|
||
platform: Optional[str] = None,
|
||
):
|
||
async with get_session() as session:
|
||
events = await service.list_events(session, task_id, type, since_id, limit, platform)
|
||
return {"events": events, "latest_id": events[0]["id"] if events else since_id}
|
||
|
||
|
||
@router.post("/events/read")
|
||
async def mark_events_read(task_id: Optional[int] = None):
|
||
async with get_session() as session:
|
||
count = await service.mark_events_read(session, task_id)
|
||
return {"marked": count}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Cookie / login health
|
||
# ---------------------------------------------------------------------------
|
||
|
||
@router.get("/cookie")
|
||
async def get_cookie_endpoint(platform: str = Query(default=PLATFORM_XHS)):
|
||
"""Cookie health only -- deliberately never returns the cookie value.
|
||
|
||
``platform`` defaults to Xiaohongshu so existing callers keep working; the
|
||
key it reads is the namespaced one.
|
||
"""
|
||
async with get_session() as session:
|
||
return await get_cookie_status(session, platform)
|
||
|
||
|
||
@router.post("/cookie")
|
||
async def set_cookie_endpoint(payload: CookiePayload, platform: str = Query(default=PLATFORM_XHS)):
|
||
async with get_session() as session:
|
||
await set_cookie(session, payload.cookie.strip(), platform)
|
||
return {"message": "Cookie saved"}
|
||
|
||
|
||
@router.delete("/cookie")
|
||
async def clear_cookie_endpoint(platform: str = Query(default=PLATFORM_XHS)):
|
||
async with get_session() as session:
|
||
await delete_setting(session, cookie_key(platform))
|
||
return {"message": "Cookie cleared"}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 博主备注
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@router.put("/creators/{creator_hash}")
|
||
async def set_creator_alias_endpoint(
|
||
creator_hash: str,
|
||
payload: CreatorAliasPayload,
|
||
platform: str = Query(default=PLATFORM_XHS),
|
||
):
|
||
"""给博主起个备注（界面上的「备注」）。
|
||
|
||
作品栏按 creator_hash 归组，可那是个哈希、昵称又常常认不出是谁 —— 备注是人自己起的
|
||
名字。空串表示清掉这条备注。
|
||
"""
|
||
async with get_session() as session:
|
||
await service.set_creator_alias(session, platform, creator_hash, payload.alias)
|
||
return {"creator_hash": creator_hash, "alias": payload.alias.strip()}
|
||
|
||
|
||
@router.put("/notes/{note_id}")
|
||
async def set_note_alias_endpoint(
|
||
note_id: str,
|
||
payload: NoteAliasPayload,
|
||
platform: str = Query(default=PLATFORM_XHS),
|
||
):
|
||
"""给**作品**起个备注。
|
||
|
||
和上面那条博主备注是一对：博主备注回答「这个账号是谁」，这条回答「这条作品我要盯着」。
|
||
两者不能合并 —— 一个博主底下常常只有一两件值得盯的作品。
|
||
"""
|
||
async with get_session() as session:
|
||
await service.set_note_alias(session, platform, note_id, payload.alias)
|
||
return {"note_id": note_id, "alias": payload.alias.strip()}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# QR login
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# These drive the browser already listening on the CDP debug port, which is the
|
||
# same browser -- and therefore the same profile -- that monitor runs attach to.
|
||
# Scanning once is what makes later unattended runs logged in.
|
||
|
||
|
||
@router.get("/covers/{note_id}")
|
||
async def get_cover(note_id: str):
|
||
"""作品封面，从本地缓存读。
|
||
|
||
**为什么不让前端直连图床**：图床地址是带签名、会过期的 —— 实测隔天即 403，
|
||
而且带不带 Referer 都一样，所以那是过期而不是防盗链。本地那份与签名无关。
|
||
|
||
这个路由是带鉴权的（整条 monitor 路由都挂了 require_auth），所以封面不会被
|
||
匿名读走；前端用同源的 <img> 请求会自动带上会话 cookie。
|
||
"""
|
||
path = covers.find_cached(note_id)
|
||
if path is None:
|
||
raise HTTPException(status_code=404, detail="封面未缓存")
|
||
|
||
media_types = {
|
||
".jpg": "image/jpeg",
|
||
".png": "image/png",
|
||
".webp": "image/webp",
|
||
".gif": "image/gif",
|
||
".heic": "image/heic",
|
||
}
|
||
return FileResponse(
|
||
path,
|
||
media_type=media_types.get(path.suffix.lower(), "application/octet-stream"),
|
||
# 本地文件不会变（note_id 唯一），让浏览器自己缓存，省掉重复请求。
|
||
headers={"Cache-Control": "private, max-age=86400"},
|
||
)
|
||
|
||
|
||
@router.post("/login/qr")
|
||
async def start_qr_login(platform: str = Query(default=PLATFORM_XHS)):
|
||
"""Open the login page in the CDP browser and return its QR code.
|
||
|
||
A server deployment has no display (Chrome sits under Xvfb), so the code is
|
||
surfaced here for the operator to scan instead of in a desktop window that
|
||
does not exist.
|
||
"""
|
||
try:
|
||
return await qrlogin.start(platform)
|
||
except ValueError as exc:
|
||
raise HTTPException(status_code=400, detail=str(exc))
|
||
except RuntimeError as exc:
|
||
raise HTTPException(status_code=502, detail=str(exc))
|
||
|
||
|
||
@router.get("/login/qr")
|
||
async def get_qr_login():
|
||
"""Poll the live session: waiting -> success / expired / error.
|
||
|
||
扫码成功时**把 cookie 一并存进库**。扫码本来只写浏览器 profile，那只够 CDP 模式用；
|
||
存一份之后，CDP 关掉、任务改用 --cookies_file 注入也照样能跑 —— 两种机制同时填上，
|
||
开关怎么切都不会断。
|
||
"""
|
||
snapshot = await qrlogin.status()
|
||
|
||
if snapshot["status"] == qrlogin.STATUS_SUCCESS:
|
||
cookie = await qrlogin.take_cookie()
|
||
if cookie:
|
||
async with get_session() as session:
|
||
await set_cookie(session, cookie)
|
||
snapshot["cookie_saved"] = True
|
||
snapshot["message"] = f"{snapshot['message']}；登录态已同时存入 Cookie"
|
||
|
||
return snapshot
|
||
|
||
|
||
@router.delete("/login/qr")
|
||
async def cancel_qr_login():
|
||
"""Drop our tab and stop polling."""
|
||
return await qrlogin.cancel()
|
||
|
||
|
||
@router.get("/login/state")
|
||
async def get_login_state(force: bool = Query(default=False)):
|
||
"""Ask the browser itself whether it is signed in.
|
||
|
||
Deliberately separate from the QR session above. That session is in-memory and
|
||
dies with the process -- a redeploy is enough -- so "am I logged in?" must not
|
||
hinge on it, or a successful scan looks like nothing happened.
|
||
|
||
``force`` reloads the page first, for when the login may have lapsed somewhere
|
||
else and the page's copy of the state is stale.
|
||
"""
|
||
return await qrlogin.check_login_state(force=force)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Report
|
||
# ---------------------------------------------------------------------------
|
||
|
||
async def _resolve_scope(
|
||
session, task_ids: Optional[List[int]], platform: Optional[str]
|
||
) -> Optional[List[int]]:
|
||
"""Combine an explicit task selection with an optional platform filter.
|
||
|
||
``None`` means "no restriction"; an explicit list is intersected with the
|
||
platform's tasks so a stale selection cannot leak another platform's data
|
||
into a scoped report.
|
||
"""
|
||
if platform is None:
|
||
return task_ids
|
||
|
||
platform_ids = set(await service.platform_task_ids(session, platform))
|
||
if task_ids is None:
|
||
return list(platform_ids)
|
||
return [task for task in task_ids if task in platform_ids]
|
||
|
||
|
||
@router.get("/export")
|
||
async def export_data(
|
||
kind: str = Query(..., description="notes | comments | report"),
|
||
task_id: Optional[List[int]] = Query(default=None),
|
||
note_id: Optional[str] = None,
|
||
start_date: Optional[str] = Query(default=None, description="YYYY-MM-DD，report 用"),
|
||
end_date: Optional[str] = Query(default=None, description="YYYY-MM-DD，report 用"),
|
||
days: int = Query(default=7, ge=1, le=365),
|
||
file_format: str = Query(default="csv", alias="format", description="csv | xlsx"),
|
||
platform: Optional[str] = None,
|
||
):
|
||
"""Download collected data as CSV or Excel.
|
||
|
||
Reached by the browser as a navigation (``window.open``), which cannot carry
|
||
an Authorization header -- this is one of the reasons the session lives in a
|
||
cookie.
|
||
"""
|
||
if kind not in ("notes", "comments", "report"):
|
||
raise HTTPException(status_code=400, detail="kind 必须是 notes / comments / report")
|
||
if file_format not in ("csv", "xlsx"):
|
||
raise HTTPException(status_code=400, detail="format 必须是 csv 或 xlsx")
|
||
|
||
async with get_session() as session:
|
||
scoped = await _resolve_scope(session, task_id, platform)
|
||
|
||
if kind == "notes":
|
||
single_task = scoped[0] if scoped and len(scoped) == 1 else None
|
||
rows = await service.list_notes(session, single_task, False, 5000, platform)
|
||
elif kind == "comments":
|
||
single_task = scoped[0] if scoped and len(scoped) == 1 else None
|
||
rows = await service.list_comments(session, single_task, note_id, 5000, platform)
|
||
else:
|
||
try:
|
||
end_day = date.fromisoformat(end_date) if end_date else date.today()
|
||
start_day = (
|
||
date.fromisoformat(start_date)
|
||
if start_date
|
||
else end_day - timedelta(days=days - 1)
|
||
)
|
||
except ValueError:
|
||
raise HTTPException(status_code=400, detail="日期格式应为 YYYY-MM-DD")
|
||
# Not `report = ...`: that would make `report` a local name for the
|
||
# whole function and shadow the module import on this very line.
|
||
report_data = await report.build_report(session, scoped, start_day, end_day)
|
||
rows = report_data["rows"]
|
||
|
||
if not rows:
|
||
raise HTTPException(status_code=404, detail="该范围内没有数据可导出")
|
||
|
||
columns = _export_columns(kind)
|
||
stamp = date.today().isoformat()
|
||
# ASCII on purpose: a non-ASCII filename needs RFC 5987 encoding in
|
||
# Content-Disposition, and the plain `filename="..."` form used below would
|
||
# mangle it.
|
||
filename = f"export_{kind}_{stamp}.{file_format}"
|
||
|
||
if file_format == "xlsx":
|
||
payload = _to_xlsx(rows, columns, kind)
|
||
media_type = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||
else:
|
||
payload = _to_csv(rows, columns)
|
||
media_type = "text/csv; charset=utf-8"
|
||
|
||
return Response(
|
||
content=payload,
|
||
media_type=media_type,
|
||
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
||
)
|
||
|
||
|
||
def _export_columns(kind: str) -> List[tuple[str, str]]:
|
||
"""(key, header) pairs per export kind."""
|
||
if kind == "notes":
|
||
return [
|
||
# 先放「这是谁」：导出来是拿去比对和汇报的，一行只有作品 ID 没法用。
|
||
# 备注优先 —— 昵称常常认不出是谁（见 notes 表那一层的说明）。
|
||
("creator_alias", "博主备注"),
|
||
("creator_name", "博主昵称"),
|
||
("note_alias", "作品备注"),
|
||
("title", "标题"),
|
||
("note_id", "作品ID"),
|
||
("note_url", "链接"),
|
||
("published_at", "发布时间"),
|
||
# 指标嵌在 row["metrics"] 里，所以这里必须写成路径 —— 写成裸键名的话这几列
|
||
# 全空（见 _lookup）。
|
||
("metrics.liked_count", "点赞"),
|
||
("metrics.comment_count", "评论"),
|
||
("metrics.collected_count", "收藏"),
|
||
("metrics.share_count", "分享"),
|
||
("deltas.liked_count", "点赞增量"),
|
||
("deltas.comment_count", "评论增量"),
|
||
("first_seen_at", "首次发现"),
|
||
("last_seen_at", "最近采集"),
|
||
]
|
||
if kind == "comments":
|
||
return [
|
||
("note_creator_name", "博主昵称"),
|
||
("note_title", "所属作品"),
|
||
("note_id", "作品ID"),
|
||
("comment_id", "评论ID"),
|
||
("content", "内容"),
|
||
("nickname", "昵称"),
|
||
("like_count", "点赞"),
|
||
("sub_comment_count", "子评论数"),
|
||
("create_time", "发布时间"),
|
||
("first_seen_at", "首次发现"),
|
||
]
|
||
return [
|
||
("date", "日期"),
|
||
("new_notes", "新增作品"),
|
||
("new_comments", "新增评论"),
|
||
("liked_count_delta", "点赞增量"),
|
||
("comment_count_delta", "评论增量"),
|
||
("collected_count_delta", "收藏增量"),
|
||
("share_count_delta", "分享增量"),
|
||
]
|
||
|
||
|
||
# 表里存的是毫秒时间戳。直接倒进 CSV 就是一串 13 位数字 —— 打开 Excel 的人没法看，
|
||
# 也没法排序。这几个键统一格式化成人能读的形态。
|
||
_TIME_KEYS = {"published_at", "first_seen_at", "last_seen_at", "create_time"}
|
||
|
||
|
||
def _fmt_time(value: Any) -> str:
|
||
"""毫秒 → ``YYYY-MM-DD HH:MM``（服务器本地时区）。"""
|
||
try:
|
||
return datetime.fromtimestamp(int(value) / 1000).strftime("%Y-%m-%d %H:%M")
|
||
except (TypeError, ValueError, OSError, OverflowError):
|
||
return ""
|
||
|
||
|
||
def _cell_for(row: Dict[str, Any], key: str) -> Any:
|
||
"""一列的值：时间键格式化成人能读的，其余照原样（None 变空串）。"""
|
||
value = _lookup(row, key)
|
||
if key.rsplit(".", 1)[-1] in _TIME_KEYS:
|
||
return _fmt_time(value)
|
||
return _cell(value)
|
||
|
||
|
||
def _lookup(row: Dict[str, Any], key: str) -> Any:
|
||
"""取一列的值。键可以是 ``metrics.liked_count`` 这种路径。
|
||
|
||
作品行的指标是**嵌在** ``metrics`` / ``deltas`` 里的，而 ``_export_columns`` 里写的
|
||
是 ``liked_count`` —— 照顶层键直接 ``row.get()`` 的话，点赞/评论/收藏/分享四列连带
|
||
两个增量列**永远是空的**，导出来的表看着有这几列，其实一格都没有。
|
||
"""
|
||
value: Any = row
|
||
for part in key.split("."):
|
||
if not isinstance(value, dict):
|
||
return None
|
||
value = value.get(part)
|
||
return value
|
||
|
||
|
||
def _cell(value: Any) -> Any:
|
||
if value is None:
|
||
return ""
|
||
if isinstance(value, (list, dict)):
|
||
return ", ".join(str(v) for v in value) if isinstance(value, list) else str(value)
|
||
return value
|
||
|
||
|
||
def _to_csv(rows: List[Dict[str, Any]], columns: List[tuple[str, str]]) -> bytes:
|
||
import csv
|
||
import io
|
||
|
||
buffer = io.StringIO()
|
||
writer = csv.writer(buffer)
|
||
writer.writerow([header for _, header in columns])
|
||
for row in rows:
|
||
writer.writerow([_cell_for(row, key) for key, _ in columns])
|
||
|
||
# utf-8-sig: without the BOM Excel opens Chinese CSV as mojibake, which is
|
||
# the single most common complaint about CSV exports here.
|
||
return buffer.getvalue().encode("utf-8-sig")
|
||
|
||
|
||
def _to_xlsx(rows: List[Dict[str, Any]], columns: List[tuple[str, str]], sheet: str) -> bytes:
|
||
import io
|
||
|
||
from openpyxl import Workbook
|
||
|
||
workbook = Workbook()
|
||
worksheet = workbook.active
|
||
worksheet.title = {"notes": "作品", "comments": "评论"}.get(sheet, "报表")
|
||
worksheet.append([header for _, header in columns])
|
||
for row in rows:
|
||
worksheet.append([_cell_for(row, key) for key, _ in columns])
|
||
|
||
output = io.BytesIO()
|
||
workbook.save(output)
|
||
return output.getvalue()
|
||
|
||
|
||
@router.get("/report")
|
||
async def get_report(
|
||
task_id: Optional[List[int]] = Query(
|
||
default=None, description="Repeat to include several tasks; omit for all"
|
||
),
|
||
start_date: Optional[str] = Query(default=None, description="YYYY-MM-DD"),
|
||
end_date: Optional[str] = Query(default=None, description="YYYY-MM-DD"),
|
||
days: int = Query(default=7, ge=1, le=365, description="Window used when dates are omitted"),
|
||
platform: Optional[str] = None,
|
||
):
|
||
"""Daily new-content counts and interaction deltas for the selected tasks."""
|
||
try:
|
||
end_day = date.fromisoformat(end_date) if end_date else date.today()
|
||
start_day = date.fromisoformat(start_date) if start_date else end_day - timedelta(days=days - 1)
|
||
except ValueError:
|
||
raise HTTPException(status_code=400, detail="日期格式应为 YYYY-MM-DD")
|
||
|
||
if start_day > end_day:
|
||
raise HTTPException(status_code=400, detail="开始日期不能晚于结束日期")
|
||
|
||
async with get_session() as session:
|
||
scoped = await _resolve_scope(session, task_id, platform)
|
||
return await report.build_report(session, scoped, start_day, end_day)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# WeCom webhook
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _mask_webhook(url: str) -> str:
|
||
"""Show enough of the URL to recognise it, without exposing the robot key."""
|
||
if not url:
|
||
return ""
|
||
key_marker = "key="
|
||
index = url.find(key_marker)
|
||
if index == -1:
|
||
return url[:12] + "..." if len(url) > 12 else url
|
||
prefix = url[: index + len(key_marker)]
|
||
key = url[index + len(key_marker) :]
|
||
if len(key) <= 8:
|
||
return prefix + "*" * len(key)
|
||
return f"{prefix}{key[:4]}...{key[-4:]}"
|
||
|
||
|
||
@router.get("/webhook")
|
||
async def get_webhook():
|
||
async with get_session() as session:
|
||
url = (await get_setting(session, SETTING_WECOM_WEBHOOK)) or ""
|
||
return {"configured": bool(url), "masked": _mask_webhook(url)}
|
||
|
||
|
||
@router.post("/webhook")
|
||
async def set_webhook(payload: WebhookPayload):
|
||
url = payload.url.strip()
|
||
if url and "qyapi.weixin.qq.com" not in url:
|
||
# Catches the common mistake of pasting a group-chat invite or the app
|
||
# URL instead of the robot webhook.
|
||
raise HTTPException(
|
||
status_code=400,
|
||
detail="这不像企业微信机器人 Webhook 地址（应包含 qyapi.weixin.qq.com）",
|
||
)
|
||
|
||
async with get_session() as session:
|
||
await set_setting(session, SETTING_WECOM_WEBHOOK, url)
|
||
return {"message": "Webhook 已保存" if url else "Webhook 已清空", "configured": bool(url)}
|
||
|
||
|
||
@router.delete("/webhook")
|
||
async def clear_webhook():
|
||
async with get_session() as session:
|
||
await delete_setting(session, SETTING_WECOM_WEBHOOK)
|
||
return {"message": "Webhook 已删除"}
|
||
|
||
|
||
@router.post("/webhook/test")
|
||
async def test_webhook(payload: WebhookTestPayload):
|
||
"""Send a test message so the user can verify the robot works before relying on it."""
|
||
async with get_session() as session:
|
||
url = payload.url.strip() if payload.url else await notify.get_webhook_url(session)
|
||
|
||
ok, detail = await notify.send_wecom(
|
||
url, "**综合采集平台 通知测试**\n> 如果你看到这条消息，说明 Webhook 配置成功。"
|
||
)
|
||
if not ok:
|
||
raise HTTPException(status_code=400, detail=detail)
|
||
return {"message": detail}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 上游更新
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@router.get("/upstream")
|
||
async def get_upstream_status():
|
||
"""最近一次上游检查的结果。
|
||
|
||
只读缓存，不触发检查：fetch 要走网络、最长两分钟，不该由一个 GET 顺手发起。
|
||
没有查过时返回空对象，前端据此显示「尚未检查」。
|
||
"""
|
||
async with get_session() as session:
|
||
return await upstream.load_state(session)
|
||
|
||
|
||
@router.post("/upstream/check")
|
||
async def run_upstream_check():
|
||
"""立刻检查一次上游仓库，并把结果写回缓存。
|
||
|
||
即使「定期检查」开关是关的也照查 —— 手动点这一次的意义正在于此。这里会一直
|
||
等到 fetch 结束（前端给这条请求单独放长了超时），因为结果就是要给人看的。
|
||
|
||
``notify_when_new=False``：点这个按钮的人正看着结果，没必要再给自己推一条群消息。
|
||
没有推过的那批提交会留给下一次「定时检查」推 —— 推送状态记的是 tip，不是「推过没」。
|
||
"""
|
||
return await upstream.run_check(notify_when_new=False)
|