butubb
2112c1a870
移植自 mac-agent-os 的 mediacrawler_adapter：不起子进程、不开页面，用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin，所以 ingest 那条链路一个字不用改。
**目前能用的（真环境实测，非推断）**：
profile/other : 200, 7075 字节 —— 博主主页指标（粉丝/获赞/作品数/昵称）
aweme/detail : 200, 45425 字节 —— 单条作品详情（含点赞/评论/收藏/分享）
**目前不能用的：作品列表 `aweme/post`。** 两个互相独立的原因：
1. 这个接口被抖音单独升级成了真校验：不带 x-tt-argus 回 403「Uifid Not Found」，
带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过，「真校验」过不了。
同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的，
挑中的恰好是「批量拉作品列表」这个最敏感的动作。
2. 改走页面截获也不行：CDP 浏览器打开博主主页会落到「验证码中间页」（当天大量探测的
代价，过几小时要重测）。
所以现在的边界是：**已知作品的指标刷新能做，自动发现新作品做不了**。
**排查中控住变量后得到的两条事实**（都写进注释了）：
· `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
UA+client hints+Cookie 时是 200 但仅 121 字节的空壳，补上这三个头变 7074 字节。
（我先前猜的 sec-ch-ua 不是关键。）
· 因此 UA 与 client hints 必须**成套地取自同一个浏览器**，所以 BrowserIdentity 一次
从 CDP 取齐 cookie + UA + hints，而不是各自写死。
「200 + 空 body 必须当场报错」也是刻意写死的：放过去它会在下游变成「这个博主没作品」，
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的，还被翻译成「账号被封」。
测试 +11：cookie 解析、请求头成套性（含 uifid 缺失/回退）、产物键名与 store 对齐、
以及 _get 的三条失败路径（空 body / 403 带网关原话 / 正常返回）。
712 lines
27 KiB
Python
712 lines
27 KiB
Python
"""
|
||
routes/scrape.py — 抓取系统 API 路由 v1
|
||
|
||
端点：
|
||
POST /api/scrape/run — 执行抓取
|
||
POST /api/scrape/resolve — 批量解析 URL
|
||
GET /api/scrape/result — 异步任务结果查询
|
||
GET /api/scrape/tasks — 任务列表
|
||
GET /api/scrape/items — 内容列表
|
||
GET /api/scrape/items/{id} — 单条详情
|
||
GET /api/scrape/stats — 抓取统计
|
||
POST /api/scrape/sources — 创建抓取源
|
||
GET /api/scrape/sources — 抓取源列表
|
||
DELETE /api/scrape/sources/{id} — 删除抓取源
|
||
"""
|
||
import asyncio, json, logging
|
||
from fastapi import APIRouter, HTTPException
|
||
from services.scrape_engine import ScrapeEngine
|
||
|
||
logger = logging.getLogger("dashboard.routes.scrape")
|
||
router = APIRouter(prefix="/api/scrape", tags=["scrape"])
|
||
|
||
_engine = None
|
||
|
||
def _get_engine():
|
||
global _engine
|
||
if _engine is None:
|
||
_engine = ScrapeEngine()
|
||
return _engine
|
||
|
||
|
||
@router.post("/douyin-stats")
|
||
async def api_douyin_stats(data: dict = {}):
|
||
"""获取抖音视频详细统计数据（隔离于养号系统）"""
|
||
from services.douyin_stats import get_video_data
|
||
url = (data.get("url") or "").strip()
|
||
if not url:
|
||
return {"status": "error", "message": "url 必填"}
|
||
result = await get_video_data(url)
|
||
return {"status": "ok" if "error" not in result else "error", **result}
|
||
|
||
|
||
@router.post("/run")
|
||
async def api_scrape_run(data: dict = {}):
|
||
"""执行抓取任务"""
|
||
try:
|
||
targets = data.get("targets", data.get("target", []))
|
||
if isinstance(targets, str):
|
||
targets = [targets]
|
||
|
||
request = {
|
||
"targets": targets,
|
||
"platform": data.get("platform", "auto"),
|
||
"depth": data.get("depth", "light"),
|
||
"tool_level": data.get("tool_level", 2),
|
||
"machine": data.get("machine", ""),
|
||
"multi_machine": data.get("multi_machine", False),
|
||
"async_mode": data.get("async_mode", False),
|
||
}
|
||
engine = _get_engine()
|
||
result = await engine.run(request)
|
||
return {"status": "ok", **result}
|
||
except Exception as e:
|
||
logger.exception("抓取执行失败")
|
||
return {"status": "error", "message": str(e)}
|
||
|
||
|
||
@router.post("/resolve")
|
||
async def api_scrape_resolve(data: dict = {}):
|
||
"""批量解析 URL，不执行抓取"""
|
||
try:
|
||
urls = data.get("urls", data.get("targets", []))
|
||
if isinstance(urls, str):
|
||
urls = [urls]
|
||
engine = _get_engine()
|
||
results = engine.resolve_urls(urls)
|
||
# 解析短链
|
||
for r in results:
|
||
if r["type"] == "shortlink":
|
||
aweme_id = engine._resolve_shortlink(r["input"])
|
||
if aweme_id:
|
||
r["type"] = "video"
|
||
r["target_id"] = aweme_id
|
||
r["platform"] = "douyin"
|
||
r["status"] = "resolved"
|
||
else:
|
||
r["status"] = "unresolved"
|
||
return {"status": "ok", "data": results}
|
||
except Exception as e:
|
||
return {"status": "error", "message": str(e)}
|
||
|
||
|
||
@router.get("/title")
|
||
async def api_scrape_title(url: str = ""):
|
||
"""获取网页标题（扫码级轻量请求，供互动页导入用）"""
|
||
if not url:
|
||
return {"status": "ok", "title": ""}
|
||
if not url.startswith("http"):
|
||
url = "https://" + url
|
||
try:
|
||
import urllib.request, re
|
||
req = urllib.request.Request(url, headers={
|
||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
|
||
"Accept": "text/html",
|
||
})
|
||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||
html = resp.read(65536).decode("utf-8", errors="replace")
|
||
m = re.search(r'<title[^>]*>([^<]+)</title>', html, re.IGNORECASE)
|
||
title = m.group(1).strip() if m else ""
|
||
# 抖音特殊处理：去掉 " - 抖音" 后缀
|
||
for suffix in [" - 抖音", " - 抖音视频", " - 快手", " - 小红书"]:
|
||
if title.endswith(suffix):
|
||
title = title[:-len(suffix)]
|
||
break
|
||
return {"status": "ok", "title": title}
|
||
except Exception as e:
|
||
return {"status": "error", "message": str(e), "title": ""}
|
||
|
||
|
||
@router.get("/result")
|
||
async def api_scrape_result(run_id: str = ""):
|
||
"""查询异步任务结果"""
|
||
if not run_id:
|
||
return {"status": "error", "message": "run_id 必填"}
|
||
engine = _get_engine()
|
||
result = engine.get_async_result(run_id)
|
||
if result is None:
|
||
return {"status": "error", "message": f"任务 {run_id} 不存在"}
|
||
return {"status": "ok", "data": result}
|
||
|
||
|
||
@router.get("/tasks")
|
||
async def api_scrape_tasks(status: str = "", platform: str = "",
|
||
limit: int = 50):
|
||
"""任务列表"""
|
||
engine = _get_engine()
|
||
tasks = engine.db.list_tasks(
|
||
status=status or None,
|
||
platform=platform or None,
|
||
limit=min(limit, 200)
|
||
)
|
||
return {"status": "ok", "data": tasks}
|
||
|
||
|
||
@router.get("/items")
|
||
async def api_scrape_items(platform: str = "", author_id: str = "",
|
||
limit: int = 100):
|
||
"""内容列表"""
|
||
engine = _get_engine()
|
||
items = engine.db.list_items(
|
||
platform=platform or None,
|
||
author_id=author_id or None,
|
||
limit=min(limit, 200)
|
||
)
|
||
return {"status": "ok", "data": items}
|
||
|
||
|
||
@router.get("/items/{item_id}")
|
||
async def api_scrape_item_detail(item_id: int):
|
||
"""单条内容详情（含评论区）"""
|
||
engine = _get_engine()
|
||
item = engine.db.get_item(item_id)
|
||
if not item:
|
||
raise HTTPException(status_code=404, detail="内容不存在")
|
||
comments = engine.db.get_comments(item_id)
|
||
item["comments"] = comments
|
||
return {"status": "ok", "data": item}
|
||
|
||
|
||
@router.get("/stats")
|
||
async def api_scrape_stats():
|
||
"""抓取统计"""
|
||
engine = _get_engine()
|
||
stats = engine.get_stats()
|
||
return {"status": "ok", **stats}
|
||
|
||
|
||
@router.post("/sources")
|
||
async def api_scrape_create_source(data: dict = {}):
|
||
"""创建抓取源"""
|
||
platform = data.get("platform", "")
|
||
source_type = data.get("source_type", "")
|
||
target = data.get("target", "")
|
||
if not platform or not target:
|
||
return {"status": "error", "message": "platform 和 target 必填"}
|
||
engine = _get_engine()
|
||
engine.db.upsert_source(
|
||
platform=platform,
|
||
source_type=source_type,
|
||
target=target,
|
||
display_name=data.get("display_name", ""),
|
||
category=data.get("category", ""),
|
||
notes=data.get("notes", ""),
|
||
schedule=data.get("schedule", ""),
|
||
depth=data.get("depth", "light"),
|
||
tool_level=data.get("tool_level", 2),
|
||
)
|
||
return {"status": "ok", "message": "抓取源已创建"}
|
||
|
||
|
||
@router.patch("/sources/{source_id}")
|
||
async def api_scrape_update_source(source_id: int, data: dict = {}):
|
||
"""编辑抓取源（名称/类别/说明/类型/目标/状态等）"""
|
||
engine = _get_engine()
|
||
ok = engine.db.update_source(source_id, **data)
|
||
if not ok:
|
||
return {"status": "error", "message": "没有可更新的字段或源不存在"}
|
||
return {"status": "ok", "message": "抓取源已更新"}
|
||
|
||
|
||
@router.get("/sources")
|
||
async def api_scrape_list_sources():
|
||
"""抓取源列表"""
|
||
engine = _get_engine()
|
||
sources = engine.db.list_sources()
|
||
return {"status": "ok", "data": sources}
|
||
|
||
|
||
@router.delete("/sources/{source_id}")
|
||
async def api_scrape_delete_source(source_id: int):
|
||
"""删除抓取源"""
|
||
engine = _get_engine()
|
||
engine.db.delete_source(source_id)
|
||
return {"status": "ok", "message": "抓取源已删除"}
|
||
|
||
# ── 抖音追踪系统（隔离于养号） ──
|
||
|
||
import json as _js, os as _os, time as _time, subprocess as _sp
|
||
from pathlib import Path as _Path
|
||
|
||
_TRACKER_DB = _Path(_os.environ.get("AGENT_LOCAL",
|
||
str(_Path.home() / "workbuddy-agent-os" / "agent-local"))) / "data" / "douyin_tracker.json"
|
||
|
||
|
||
def _load_tracker() -> list:
|
||
if _TRACKER_DB.exists():
|
||
try:
|
||
return _js.loads(_TRACKER_DB.read_text())
|
||
except:
|
||
pass
|
||
return []
|
||
|
||
|
||
def _save_tracker(items: list):
|
||
_TRACKER_DB.parent.mkdir(parents=True, exist_ok=True)
|
||
_TRACKER_DB.write_text(_js.dumps(items, ensure_ascii=False, indent=2))
|
||
|
||
|
||
def _clean_douyin_title(raw: str) -> str:
|
||
"""清洗抖音标题：去掉『2.07 复制打开抖音，看看【xxx的作品】』前缀"""
|
||
if not raw:
|
||
return ""
|
||
t = str(raw).strip()
|
||
# 去掉开头的数字+空格（如 "2.07 "）
|
||
import re as _re
|
||
t = _re.sub(r'^[\d.]+[\s]*', '', t)
|
||
# 去掉 "复制打开抖音" 及其变体
|
||
t = _re.sub(r'^复制打开抖音[，,。.]*\s*', '', t)
|
||
t = _re.sub(r'^打开抖音[，,。.]*\s*', '', t)
|
||
t = _re.sub(r'^看看[，,。.]*\s*', '', t)
|
||
# 去掉【xxx的作品】前缀
|
||
t = _re.sub(r'^【[^】]+的作品】', '', t)
|
||
# 再次去掉可能残留的 "复制打开抖音"
|
||
t = _re.sub(r'复制打开抖音', '', t)
|
||
return t.strip() or raw[:60]
|
||
|
||
|
||
@router.post("/import-topics")
|
||
async def api_import_topics(data: dict = {}):
|
||
"""从 tyhtak API 导入视频列表（轻量化，不调用 ops）"""
|
||
import urllib.request as _urq
|
||
api_url = (data.get("api_url") or "").strip()
|
||
if not api_url:
|
||
return {"status": "error", "message": "api_url 必填"}
|
||
page = int(data.get("page", 1))
|
||
page_size = int(data.get("page_size", 100))
|
||
try:
|
||
# api_url 可能已带查询参数（如 ?page=1&pageSize=100）→ 用 & 追加，避免 ?page= 重复
|
||
sep = "&" if "?" in api_url else "?"
|
||
full_url = f"{api_url}{sep}page={page}&pageSize={page_size}"
|
||
resp = _urq.urlopen(full_url, timeout=15)
|
||
body = _js.loads(resp.read().decode())
|
||
except Exception as e:
|
||
return {"status": "error", "message": f"请求失败: {e}"}
|
||
raw_items = body.get("data", {}).get("items", [])
|
||
items = []
|
||
for item in raw_items:
|
||
url = (item.get("share_url") or "").strip()
|
||
if not url:
|
||
continue
|
||
vid = str(item.get("id", ""))
|
||
items.append({
|
||
"id": vid,
|
||
"url": url,
|
||
"title": _clean_douyin_title(item.get("share_text", "")),
|
||
"author": item.get("dyname", "") or item.get("nickname", "") or item.get("name", ""),
|
||
# 结构化字段（归一化，任何 API 按常见键名映射；未采集也能展示/筛选）
|
||
"dyid": item.get("dyid", ""),
|
||
"likes": item.get("like_count", 0),
|
||
"comments": item.get("comment_count", 0),
|
||
"collects": item.get("favorite_count", 0),
|
||
"plays": item.get("play_count", 0),
|
||
"account_level": item.get("account_level", ""),
|
||
"task_id": item.get("task_id", ""),
|
||
"share_check_status": item.get("share_check_status", ""),
|
||
"submit_time": item.get("submit_time", ""),
|
||
"created_at": item.get("created_at", ""),
|
||
"raw": item, # 完整原始对象（列表展开查看）
|
||
})
|
||
return {"status": "ok", "items": items, "total": len(items)}
|
||
|
||
|
||
@router.post("/track-video")
|
||
async def api_track_video(data: dict = {}):
|
||
"""采集单条视频并加入跟踪"""
|
||
from services.douyin_stats import get_video_data
|
||
url = (data.get("url") or "").strip()
|
||
if not url:
|
||
return {"status": "error", "message": "url 必填"}
|
||
stats = await get_video_data(url)
|
||
if "error" in stats:
|
||
return {"status": "error", "message": stats["error"]}
|
||
# 生成唯一 ID
|
||
item_id = f"dy_{stats.get('aweme_id','')}"
|
||
record = {
|
||
"id": item_id,
|
||
"url": url,
|
||
"title": stats.get("title", ""),
|
||
"author": stats.get("author", ""),
|
||
"author_uid": stats.get("author_uid", ""), # 博主数字ID（用于判断是否已跟踪）
|
||
"stats": {
|
||
"likes": stats.get("likes", 0),
|
||
"comments": stats.get("comments", 0),
|
||
"collects": stats.get("collects", 0),
|
||
"shares": stats.get("shares", 0),
|
||
},
|
||
"comment_texts": stats.get("comment_texts", []),
|
||
"collected_at": _time.strftime("%Y-%m-%d %H:%M:%S"),
|
||
}
|
||
# 保存
|
||
items = _load_tracker()
|
||
# 去重（同 ID 替换）
|
||
# ⚠️ 修复：不能整个删掉重建（会丢失 prev_stats 对比链）。
|
||
# 视频已存在时 → 保留旧 stats 作为 prev_stats，再更新新 stats。
|
||
for i, it in enumerate(items):
|
||
if it["id"] == item_id:
|
||
# 已存在：保留历史对比数据
|
||
if "stats" in it:
|
||
record["prev_stats"] = it.get("stats")
|
||
record["prev_collected_at"] = it.get("collected_at", "")
|
||
# 保留旧的 video 级字段（author_uid 等）
|
||
record["author_uid"] = record.get("author_uid") or it.get("author_uid", "")
|
||
items.pop(i)
|
||
break
|
||
items.insert(0, record)
|
||
_save_tracker(items)
|
||
return {"status": "ok", "item": record}
|
||
|
||
|
||
@router.get("/tracked-videos")
|
||
async def api_tracked_videos():
|
||
"""获取已跟踪的视频列表"""
|
||
return {"status": "ok", "items": _load_tracker()}
|
||
|
||
|
||
@router.post("/delete-tracked/{item_id}")
|
||
async def api_delete_tracked(item_id: str):
|
||
"""删除单条跟踪视频"""
|
||
items = _load_tracker()
|
||
before = len(items)
|
||
items = [it for it in items if it["id"] != item_id]
|
||
if len(items) < before:
|
||
_save_tracker(items)
|
||
return {"status": "ok"}
|
||
return {"status": "error", "message": "未找到"}
|
||
|
||
|
||
@router.post("/refresh-video/{item_id}")
|
||
async def api_refresh_video(item_id: str):
|
||
"""刷新单条跟踪视频的数据"""
|
||
from services.douyin_stats import get_video_data
|
||
items = _load_tracker()
|
||
found = None
|
||
for it in items:
|
||
if it["id"] == item_id:
|
||
found = it
|
||
break
|
||
if not found:
|
||
return {"status": "error", "message": "未找到该视频"}
|
||
stats = await get_video_data(found["url"])
|
||
if "error" in stats:
|
||
resp = {"status": "error", "message": stats["error"]}
|
||
if stats.get("login_expired"):
|
||
resp["login_expired"] = True
|
||
resp["message"] = "⛔ 抖音登录已过期，请重新登录。点击顶部「📱 打开抖音登录」按钮"
|
||
return resp
|
||
# 保存旧数据（用于对比）
|
||
if "stats" in found:
|
||
found["prev_stats"] = found.get("stats")
|
||
found["prev_collected_at"] = found.get("collected_at", "")
|
||
found["stats"] = {
|
||
"likes": stats.get("likes", 0),
|
||
"comments": stats.get("comments", 0),
|
||
"collects": stats.get("collects", 0),
|
||
"shares": stats.get("shares", 0),
|
||
}
|
||
found["comment_texts"] = stats.get("comment_texts", [])
|
||
found["collected_at"] = _time.strftime("%Y-%m-%d %H:%M:%S")
|
||
_save_tracker(items)
|
||
return {"status": "ok", "item": found}
|
||
|
||
|
||
# ── 抖音博主监控（跟踪博主 + 每日快照 + 新视频发现） ──
|
||
# 数据流：
|
||
# 视频URL/博主主页URL → 拿 uid（数字ID）→ 存 douyin_authors.json
|
||
# → profile/other 打基线（粉丝/获赞/作品数）
|
||
# → aweme/post 拉最新视频 → 发现新视频自动加入视频跟踪
|
||
# 存储：agent-local/data/douyin_authors.json
|
||
# [{author_id, nickname, unique_id, uid, added_at, last_collected_at,
|
||
# snapshots: [{date, fans, total_favorited, works, new_videos: [id]}],
|
||
# video_ids: [已见视频ID集合，用于新视频发现]}]
|
||
|
||
_AUTHORS_DB = _Path(_os.environ.get("AGENT_LOCAL",
|
||
str(_Path.home() / "workbuddy-agent-os" / "agent-local"))) / "data" / "douyin_authors.json"
|
||
|
||
|
||
def _load_authors() -> list:
|
||
if _AUTHORS_DB.exists():
|
||
try:
|
||
return _js.loads(_AUTHORS_DB.read_text())
|
||
except:
|
||
pass
|
||
return []
|
||
|
||
|
||
def _save_authors(items: list):
|
||
_AUTHORS_DB.parent.mkdir(parents=True, exist_ok=True)
|
||
_AUTHORS_DB.write_text(_js.dumps(items, ensure_ascii=False, indent=2))
|
||
|
||
|
||
def _author_id_from_url(url: str) -> str:
|
||
"""从 URL 提取博主 ID：支持 /user/{sec_uid} 或 /video/{id}"""
|
||
import re as _re
|
||
m = _re.search(r'douyin\.com/user/([^/?#]+)', url)
|
||
if m:
|
||
return m.group(1)
|
||
return ""
|
||
|
||
|
||
@router.post("/track-author")
|
||
async def api_track_author(data: dict = {}):
|
||
"""加入博主跟踪（传视频URL或博主主页URL，自动解析 uid 并打基线）"""
|
||
from services.mediacrawler_adapter import get_video_data, get_author_profile, get_author_videos
|
||
url = (data.get("url") or data.get("video_url") or "").strip()
|
||
if not url:
|
||
return {"status": "error", "message": "url 必填（视频链接或博主主页链接）"}
|
||
|
||
# 情况1：博主主页链接 douyin.com/user/xxx → xxx 可能是 sec_uid
|
||
author_id = _author_id_from_url(url)
|
||
if author_id:
|
||
# 主页链接里的 ID 是 sec_uid，需要解析成数字 uid
|
||
# 方式：先当作视频无解，直接尝试用 sec_uid 查 profile（会失败），
|
||
# 因此这里先提示用视频链接，或尝试用 sec_uid 反查
|
||
return await _add_author_by_sec_uid(author_id, url)
|
||
|
||
# 情况2：视频链接 → 先采视频详情拿 uid
|
||
stats = await get_video_data(url)
|
||
if "error" in stats:
|
||
return {"status": "error", "message": stats["error"]}
|
||
uid = str(stats.get("author_uid", ""))
|
||
nickname = stats.get("author", "")
|
||
if not uid:
|
||
return {"status": "error", "message": "未能从视频解析博主ID"}
|
||
return await _add_author_by_uid(uid, nickname, url)
|
||
|
||
|
||
async def _add_author_by_sec_uid(sec_uid: str, source_url: str):
|
||
"""通过 sec_uid 添加博主（先采集一次打基线）"""
|
||
# sec_uid 不能直接查 profile/other（会报 UserId不合法）
|
||
# 尝试通过视频详情反查：没有视频 URL 时先用 sec_uid 查 aweme/post（同样要数字uid）
|
||
# 目前无法直接转换 → 提示用户使用视频链接
|
||
return {"status": "error", "message": "博主主页链接暂不支持直接添加，请改用视频链接（主页 ID 是加密格式）"}
|
||
|
||
|
||
async def _add_author_by_uid(uid: str, nickname: str, source_url: str):
|
||
"""通过数字 uid 添加博主并打基线"""
|
||
from services.mediacrawler_adapter import get_author_profile, get_author_videos
|
||
authors = _load_authors()
|
||
for a in authors:
|
||
if a.get("uid") == uid:
|
||
return {"status": "ok", "message": f"博主 {nickname} 已在跟踪中", "item": a, "already": True}
|
||
|
||
profile = await get_author_profile(uid)
|
||
if "error" in profile:
|
||
return {"status": "error", "message": profile["error"]}
|
||
|
||
# 拉最新视频（用于新视频发现基线）
|
||
videos_res = await get_author_videos(uid, count=20)
|
||
video_ids = [v["aweme_id"] for v in videos_res.get("videos", [])]
|
||
|
||
author = {
|
||
"author_id": f"author_{uid}",
|
||
"uid": uid,
|
||
"nickname": profile.get("nickname", nickname),
|
||
"unique_id": profile.get("unique_id", ""),
|
||
"sec_uid": profile.get("sec_uid", ""),
|
||
"added_at": _time.strftime("%Y-%m-%d %H:%M:%S"),
|
||
"source_url": source_url,
|
||
"last_collected_at": _time.strftime("%Y-%m-%d %H:%M:%S"),
|
||
"latest_videos": videos_res.get("videos", [])[:3], # 最新3条视频（标题+数据）
|
||
"snapshots": [{
|
||
"date": _time.strftime("%Y-%m-%d"),
|
||
"fans": profile.get("fans", 0),
|
||
"total_favorited": profile.get("total_favorited", 0),
|
||
"works": profile.get("works", 0),
|
||
"new_videos": video_ids,
|
||
}],
|
||
"video_ids": video_ids,
|
||
}
|
||
authors.insert(0, author)
|
||
_save_authors(authors)
|
||
return {"status": "ok", "message": f"已跟踪博主 {author['nickname']}", "item": author}
|
||
|
||
|
||
@router.get("/tracked-authors")
|
||
async def api_tracked_authors():
|
||
"""博主跟踪列表（含最新快照、今日涨粉）"""
|
||
authors = _load_authors()
|
||
today = _time.strftime("%Y-%m-%d")
|
||
result = []
|
||
for a in authors:
|
||
snaps = a.get("snapshots", [])
|
||
latest = snaps[-1] if snaps else {}
|
||
prev = snaps[-2] if len(snaps) > 1 else {}
|
||
result.append({
|
||
"author_id": a.get("author_id"),
|
||
"uid": a.get("uid"),
|
||
"nickname": a.get("nickname"),
|
||
"unique_id": a.get("unique_id"),
|
||
"added_at": a.get("added_at"),
|
||
"last_collected_at": a.get("last_collected_at"),
|
||
"fans": latest.get("fans", 0),
|
||
"fans_delta": (latest.get("fans", 0) - prev.get("fans", 0)) if prev else 0,
|
||
"total_favorited": latest.get("total_favorited", 0),
|
||
"works": latest.get("works", 0),
|
||
"snapshot_count": len(snaps),
|
||
"last_snapshot_date": latest.get("date", ""),
|
||
"today_collected": latest.get("date") == today,
|
||
"latest_videos": a.get("latest_videos", []), # 最新3条视频（标题+数据）
|
||
})
|
||
return {"status": "ok", "items": result}
|
||
|
||
|
||
@router.post("/refresh-author/{author_id}")
|
||
async def api_refresh_author(author_id: str):
|
||
"""手动刷新单个博主：采主页快照 + 发现新视频（自动加入视频跟踪）"""
|
||
from services.mediacrawler_adapter import get_author_profile, get_author_videos
|
||
authors = _load_authors()
|
||
found = next((a for a in authors if a.get("author_id") == author_id), None)
|
||
if not found:
|
||
return {"status": "error", "message": "未找到该博主"}
|
||
|
||
profile = await get_author_profile(found["uid"])
|
||
if "error" in profile:
|
||
resp = {"status": "error", "message": profile["error"]}
|
||
if profile.get("login_expired"):
|
||
resp["login_expired"] = True
|
||
resp["message"] = "⛔ 抖音登录已过期，请重新登录"
|
||
return resp
|
||
|
||
# 拉最新视频，发现新视频
|
||
videos_res = await get_author_videos(found["uid"], count=20)
|
||
new_videos = []
|
||
known = set(found.get("video_ids", []))
|
||
for v in videos_res.get("videos", []):
|
||
if v["aweme_id"] not in known:
|
||
new_videos.append(v["aweme_id"])
|
||
# 自动加入视频跟踪
|
||
await api_track_video({"url": v["url"]})
|
||
found["video_ids"] = list(known) + new_videos
|
||
# 更新最新3条视频（标题+数据）
|
||
found["latest_videos"] = videos_res.get("videos", [])[:3]
|
||
|
||
# 存快照
|
||
today = _time.strftime("%Y-%m-%d")
|
||
found.setdefault("snapshots", []).append({
|
||
"date": today,
|
||
"fans": profile.get("fans", 0),
|
||
"total_favorited": profile.get("total_favorited", 0),
|
||
"works": profile.get("works", 0),
|
||
"new_videos": new_videos,
|
||
})
|
||
# 同一天重复刷新 → 替换今天的快照（不重复堆积）
|
||
snaps = found["snapshots"]
|
||
same_day = [i for i, s in enumerate(snaps) if s.get("date") == today]
|
||
if len(same_day) > 1:
|
||
# 保留第一次，删除后续同日快照
|
||
for i in sorted(same_day[1:], reverse=True):
|
||
snaps.pop(i)
|
||
found["last_collected_at"] = _time.strftime("%Y-%m-%d %H:%M:%S")
|
||
_save_authors(authors)
|
||
|
||
return {"status": "ok", "item": {
|
||
"author_id": found["author_id"],
|
||
"nickname": found["nickname"],
|
||
"fans": profile.get("fans", 0),
|
||
"total_favorited": profile.get("total_favorited", 0),
|
||
"works": profile.get("works", 0),
|
||
"new_videos": new_videos,
|
||
"new_video_count": len(new_videos),
|
||
}}
|
||
|
||
|
||
@router.post("/refresh-all-authors")
|
||
async def api_refresh_all_authors():
|
||
"""刷新全部博主（限速 3 秒/个）"""
|
||
from services.mediacrawler_adapter import get_author_profile
|
||
authors = _load_authors()
|
||
results = []
|
||
for a in authors:
|
||
try:
|
||
r = await api_refresh_author(a["author_id"])
|
||
results.append({"author_id": a["author_id"], "nickname": a.get("nickname"), "status": r.get("status"), "new_videos": r.get("item", {}).get("new_video_count", 0)})
|
||
except Exception as e:
|
||
results.append({"author_id": a["author_id"], "nickname": a.get("nickname"), "status": "error", "error": str(e)[:100]})
|
||
await asyncio.sleep(3)
|
||
return {"status": "ok", "results": results, "total": len(authors)}
|
||
|
||
|
||
@router.get("/author-history/{author_id}")
|
||
async def api_author_history(author_id: str):
|
||
"""博主历史趋势（每日序列）"""
|
||
authors = _load_authors()
|
||
found = next((a for a in authors if a.get("author_id") == author_id), None)
|
||
if not found:
|
||
return {"status": "error", "message": "未找到该博主"}
|
||
return {"status": "ok", "item": {
|
||
"author_id": found["author_id"],
|
||
"nickname": found["nickname"],
|
||
"uid": found["uid"],
|
||
"snapshots": found.get("snapshots", []),
|
||
}}
|
||
|
||
|
||
@router.get("/author-videos/{author_id}")
|
||
async def api_author_videos(author_id: str):
|
||
"""博主最新视频列表（实时拉取）"""
|
||
from services.mediacrawler_adapter import get_author_videos
|
||
authors = _load_authors()
|
||
found = next((a for a in authors if a.get("author_id") == author_id), None)
|
||
if not found:
|
||
return {"status": "error", "message": "未找到该博主"}
|
||
res = await get_author_videos(found["uid"], count=30)
|
||
return {"status": "ok", "videos": res.get("videos", []), "has_more": res.get("has_more", False)}
|
||
|
||
|
||
@router.delete("/tracked-authors/{author_id}")
|
||
async def api_delete_tracked_author(author_id: str):
|
||
"""取消博主跟踪"""
|
||
authors = _load_authors()
|
||
before = len(authors)
|
||
authors = [a for a in authors if a.get("author_id") != author_id]
|
||
if len(authors) < before:
|
||
_save_authors(authors)
|
||
return {"status": "ok"}
|
||
return {"status": "error", "message": "未找到"}
|
||
|
||
|
||
@router.get("/author-new-videos")
|
||
async def api_author_new_videos():
|
||
"""全部博主的新视频汇总（最近一次刷新发现的新视频）"""
|
||
authors = _load_authors()
|
||
items = []
|
||
for a in authors:
|
||
snaps = a.get("snapshots", [])
|
||
if snaps:
|
||
latest = snaps[-1]
|
||
for vid in latest.get("new_videos", []):
|
||
items.append({"author_id": a.get("author_id"), "nickname": a.get("nickname"), "video_id": vid})
|
||
return {"status": "ok", "items": items}
|
||
|
||
|
||
@router.get("/chrome-status")
|
||
async def api_chrome_status():
|
||
"""检查采集 Chrome 是否可用（只检查 profile 目录，不检查 CDP 端口）"""
|
||
CHROME = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
|
||
profile_ok = _os.path.exists("/tmp/chrome-douyin-profile/Default/Cookies")
|
||
chrome_installed = _os.path.exists(CHROME)
|
||
return {"status": "ok", "online": True, "chrome_installed": chrome_installed, "profile_ok": profile_ok}
|
||
|
||
|
||
@router.get("/check-login")
|
||
async def api_check_login():
|
||
"""检查抖音登录状态"""
|
||
from services.mediacrawler_adapter import check_login_status
|
||
try:
|
||
result = await check_login_status()
|
||
return {"status": "ok", **result}
|
||
except Exception as e:
|
||
return {"status": "error", "message": str(e)}
|
||
|
||
|
||
@router.post("/open-login")
|
||
async def api_open_login():
|
||
"""在 Chrome 中打开抖音登录页"""
|
||
from services.mediacrawler_adapter import open_login_page
|
||
try:
|
||
result = await open_login_page()
|
||
return result
|
||
except Exception as e:
|
||
return {"status": "error", "message": str(e)}
|