""" routes/scrape.py — 抓取系统 API 路由 v1 端点: POST /api/scrape/run — 执行抓取 POST /api/scrape/resolve — 批量解析 URL GET /api/scrape/result — 异步任务结果查询 GET /api/scrape/tasks — 任务列表 GET /api/scrape/items — 内容列表 GET /api/scrape/items/{id} — 单条详情 GET /api/scrape/stats — 抓取统计 POST /api/scrape/sources — 创建抓取源 GET /api/scrape/sources — 抓取源列表 DELETE /api/scrape/sources/{id} — 删除抓取源 """ import asyncio, json, logging from fastapi import APIRouter, HTTPException from services.scrape_engine import ScrapeEngine logger = logging.getLogger("dashboard.routes.scrape") router = APIRouter(prefix="/api/scrape", tags=["scrape"]) _engine = None def _get_engine(): global _engine if _engine is None: _engine = ScrapeEngine() return _engine @router.post("/douyin-stats") async def api_douyin_stats(data: dict = {}): """获取抖音视频详细统计数据(隔离于养号系统)""" from services.douyin_stats import get_video_data url = (data.get("url") or "").strip() if not url: return {"status": "error", "message": "url 必填"} result = await get_video_data(url) return {"status": "ok" if "error" not in result else "error", **result} @router.post("/run") async def api_scrape_run(data: dict = {}): """执行抓取任务""" try: targets = data.get("targets", data.get("target", [])) if isinstance(targets, str): targets = [targets] request = { "targets": targets, "platform": data.get("platform", "auto"), "depth": data.get("depth", "light"), "tool_level": data.get("tool_level", 2), "machine": data.get("machine", ""), "multi_machine": data.get("multi_machine", False), "async_mode": data.get("async_mode", False), } engine = _get_engine() result = await engine.run(request) return {"status": "ok", **result} except Exception as e: logger.exception("抓取执行失败") return {"status": "error", "message": str(e)} @router.post("/resolve") async def api_scrape_resolve(data: dict = {}): """批量解析 URL,不执行抓取""" try: urls = data.get("urls", data.get("targets", [])) if isinstance(urls, str): urls = [urls] engine = _get_engine() results = engine.resolve_urls(urls) # 解析短链 for r in results: if r["type"] == "shortlink": aweme_id = engine._resolve_shortlink(r["input"]) if aweme_id: r["type"] = "video" r["target_id"] = aweme_id r["platform"] = "douyin" r["status"] = "resolved" else: r["status"] = "unresolved" return {"status": "ok", "data": results} except Exception as e: return {"status": "error", "message": str(e)} @router.get("/title") async def api_scrape_title(url: str = ""): """获取网页标题(扫码级轻量请求,供互动页导入用)""" if not url: return {"status": "ok", "title": ""} if not url.startswith("http"): url = "https://" + url try: import urllib.request, re req = urllib.request.Request(url, headers={ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36", "Accept": "text/html", }) with urllib.request.urlopen(req, timeout=10) as resp: html = resp.read(65536).decode("utf-8", errors="replace") m = re.search(r']*>([^<]+)', html, re.IGNORECASE) title = m.group(1).strip() if m else "" # 抖音特殊处理:去掉 " - 抖音" 后缀 for suffix in [" - 抖音", " - 抖音视频", " - 快手", " - 小红书"]: if title.endswith(suffix): title = title[:-len(suffix)] break return {"status": "ok", "title": title} except Exception as e: return {"status": "error", "message": str(e), "title": ""} @router.get("/result") async def api_scrape_result(run_id: str = ""): """查询异步任务结果""" if not run_id: return {"status": "error", "message": "run_id 必填"} engine = _get_engine() result = engine.get_async_result(run_id) if result is None: return {"status": "error", "message": f"任务 {run_id} 不存在"} return {"status": "ok", "data": result} @router.get("/tasks") async def api_scrape_tasks(status: str = "", platform: str = "", limit: int = 50): """任务列表""" engine = _get_engine() tasks = engine.db.list_tasks( status=status or None, platform=platform or None, limit=min(limit, 200) ) return {"status": "ok", "data": tasks} @router.get("/items") async def api_scrape_items(platform: str = "", author_id: str = "", limit: int = 100): """内容列表""" engine = _get_engine() items = engine.db.list_items( platform=platform or None, author_id=author_id or None, limit=min(limit, 200) ) return {"status": "ok", "data": items} @router.get("/items/{item_id}") async def api_scrape_item_detail(item_id: int): """单条内容详情(含评论区)""" engine = _get_engine() item = engine.db.get_item(item_id) if not item: raise HTTPException(status_code=404, detail="内容不存在") comments = engine.db.get_comments(item_id) item["comments"] = comments return {"status": "ok", "data": item} @router.get("/stats") async def api_scrape_stats(): """抓取统计""" engine = _get_engine() stats = engine.get_stats() return {"status": "ok", **stats} @router.post("/sources") async def api_scrape_create_source(data: dict = {}): """创建抓取源""" platform = data.get("platform", "") source_type = data.get("source_type", "") target = data.get("target", "") if not platform or not target: return {"status": "error", "message": "platform 和 target 必填"} engine = _get_engine() engine.db.upsert_source( platform=platform, source_type=source_type, target=target, display_name=data.get("display_name", ""), category=data.get("category", ""), notes=data.get("notes", ""), schedule=data.get("schedule", ""), depth=data.get("depth", "light"), tool_level=data.get("tool_level", 2), ) return {"status": "ok", "message": "抓取源已创建"} @router.patch("/sources/{source_id}") async def api_scrape_update_source(source_id: int, data: dict = {}): """编辑抓取源(名称/类别/说明/类型/目标/状态等)""" engine = _get_engine() ok = engine.db.update_source(source_id, **data) if not ok: return {"status": "error", "message": "没有可更新的字段或源不存在"} return {"status": "ok", "message": "抓取源已更新"} @router.get("/sources") async def api_scrape_list_sources(): """抓取源列表""" engine = _get_engine() sources = engine.db.list_sources() return {"status": "ok", "data": sources} @router.delete("/sources/{source_id}") async def api_scrape_delete_source(source_id: int): """删除抓取源""" engine = _get_engine() engine.db.delete_source(source_id) return {"status": "ok", "message": "抓取源已删除"} # ── 抖音追踪系统(隔离于养号) ── import json as _js, os as _os, time as _time, subprocess as _sp from pathlib import Path as _Path _TRACKER_DB = _Path(_os.environ.get("AGENT_LOCAL", str(_Path.home() / "workbuddy-agent-os" / "agent-local"))) / "data" / "douyin_tracker.json" def _load_tracker() -> list: if _TRACKER_DB.exists(): try: return _js.loads(_TRACKER_DB.read_text()) except: pass return [] def _save_tracker(items: list): _TRACKER_DB.parent.mkdir(parents=True, exist_ok=True) _TRACKER_DB.write_text(_js.dumps(items, ensure_ascii=False, indent=2)) def _clean_douyin_title(raw: str) -> str: """清洗抖音标题:去掉『2.07 复制打开抖音,看看【xxx的作品】』前缀""" if not raw: return "" t = str(raw).strip() # 去掉开头的数字+空格(如 "2.07 ") import re as _re t = _re.sub(r'^[\d.]+[\s]*', '', t) # 去掉 "复制打开抖音" 及其变体 t = _re.sub(r'^复制打开抖音[,,。.]*\s*', '', t) t = _re.sub(r'^打开抖音[,,。.]*\s*', '', t) t = _re.sub(r'^看看[,,。.]*\s*', '', t) # 去掉【xxx的作品】前缀 t = _re.sub(r'^【[^】]+的作品】', '', t) # 再次去掉可能残留的 "复制打开抖音" t = _re.sub(r'复制打开抖音', '', t) return t.strip() or raw[:60] @router.post("/import-topics") async def api_import_topics(data: dict = {}): """从 tyhtak API 导入视频列表(轻量化,不调用 ops)""" import urllib.request as _urq api_url = (data.get("api_url") or "").strip() if not api_url: return {"status": "error", "message": "api_url 必填"} page = int(data.get("page", 1)) page_size = int(data.get("page_size", 100)) try: # api_url 可能已带查询参数(如 ?page=1&pageSize=100)→ 用 & 追加,避免 ?page= 重复 sep = "&" if "?" in api_url else "?" full_url = f"{api_url}{sep}page={page}&pageSize={page_size}" resp = _urq.urlopen(full_url, timeout=15) body = _js.loads(resp.read().decode()) except Exception as e: return {"status": "error", "message": f"请求失败: {e}"} raw_items = body.get("data", {}).get("items", []) items = [] for item in raw_items: url = (item.get("share_url") or "").strip() if not url: continue vid = str(item.get("id", "")) items.append({ "id": vid, "url": url, "title": _clean_douyin_title(item.get("share_text", "")), "author": item.get("dyname", "") or item.get("nickname", "") or item.get("name", ""), # 结构化字段(归一化,任何 API 按常见键名映射;未采集也能展示/筛选) "dyid": item.get("dyid", ""), "likes": item.get("like_count", 0), "comments": item.get("comment_count", 0), "collects": item.get("favorite_count", 0), "plays": item.get("play_count", 0), "account_level": item.get("account_level", ""), "task_id": item.get("task_id", ""), "share_check_status": item.get("share_check_status", ""), "submit_time": item.get("submit_time", ""), "created_at": item.get("created_at", ""), "raw": item, # 完整原始对象(列表展开查看) }) return {"status": "ok", "items": items, "total": len(items)} @router.post("/track-video") async def api_track_video(data: dict = {}): """采集单条视频并加入跟踪""" from services.douyin_stats import get_video_data url = (data.get("url") or "").strip() if not url: return {"status": "error", "message": "url 必填"} stats = await get_video_data(url) if "error" in stats: return {"status": "error", "message": stats["error"]} # 生成唯一 ID item_id = f"dy_{stats.get('aweme_id','')}" record = { "id": item_id, "url": url, "title": stats.get("title", ""), "author": stats.get("author", ""), "author_uid": stats.get("author_uid", ""), # 博主数字ID(用于判断是否已跟踪) "stats": { "likes": stats.get("likes", 0), "comments": stats.get("comments", 0), "collects": stats.get("collects", 0), "shares": stats.get("shares", 0), }, "comment_texts": stats.get("comment_texts", []), "collected_at": _time.strftime("%Y-%m-%d %H:%M:%S"), } # 保存 items = _load_tracker() # 去重(同 ID 替换) # ⚠️ 修复:不能整个删掉重建(会丢失 prev_stats 对比链)。 # 视频已存在时 → 保留旧 stats 作为 prev_stats,再更新新 stats。 for i, it in enumerate(items): if it["id"] == item_id: # 已存在:保留历史对比数据 if "stats" in it: record["prev_stats"] = it.get("stats") record["prev_collected_at"] = it.get("collected_at", "") # 保留旧的 video 级字段(author_uid 等) record["author_uid"] = record.get("author_uid") or it.get("author_uid", "") items.pop(i) break items.insert(0, record) _save_tracker(items) return {"status": "ok", "item": record} @router.get("/tracked-videos") async def api_tracked_videos(): """获取已跟踪的视频列表""" return {"status": "ok", "items": _load_tracker()} @router.post("/delete-tracked/{item_id}") async def api_delete_tracked(item_id: str): """删除单条跟踪视频""" items = _load_tracker() before = len(items) items = [it for it in items if it["id"] != item_id] if len(items) < before: _save_tracker(items) return {"status": "ok"} return {"status": "error", "message": "未找到"} @router.post("/refresh-video/{item_id}") async def api_refresh_video(item_id: str): """刷新单条跟踪视频的数据""" from services.douyin_stats import get_video_data items = _load_tracker() found = None for it in items: if it["id"] == item_id: found = it break if not found: return {"status": "error", "message": "未找到该视频"} stats = await get_video_data(found["url"]) if "error" in stats: resp = {"status": "error", "message": stats["error"]} if stats.get("login_expired"): resp["login_expired"] = True resp["message"] = "⛔ 抖音登录已过期,请重新登录。点击顶部「📱 打开抖音登录」按钮" return resp # 保存旧数据(用于对比) if "stats" in found: found["prev_stats"] = found.get("stats") found["prev_collected_at"] = found.get("collected_at", "") found["stats"] = { "likes": stats.get("likes", 0), "comments": stats.get("comments", 0), "collects": stats.get("collects", 0), "shares": stats.get("shares", 0), } found["comment_texts"] = stats.get("comment_texts", []) found["collected_at"] = _time.strftime("%Y-%m-%d %H:%M:%S") _save_tracker(items) return {"status": "ok", "item": found} # ── 抖音博主监控(跟踪博主 + 每日快照 + 新视频发现) ── # 数据流: # 视频URL/博主主页URL → 拿 uid(数字ID)→ 存 douyin_authors.json # → profile/other 打基线(粉丝/获赞/作品数) # → aweme/post 拉最新视频 → 发现新视频自动加入视频跟踪 # 存储:agent-local/data/douyin_authors.json # [{author_id, nickname, unique_id, uid, added_at, last_collected_at, # snapshots: [{date, fans, total_favorited, works, new_videos: [id]}], # video_ids: [已见视频ID集合,用于新视频发现]}] _AUTHORS_DB = _Path(_os.environ.get("AGENT_LOCAL", str(_Path.home() / "workbuddy-agent-os" / "agent-local"))) / "data" / "douyin_authors.json" def _load_authors() -> list: if _AUTHORS_DB.exists(): try: return _js.loads(_AUTHORS_DB.read_text()) except: pass return [] def _save_authors(items: list): _AUTHORS_DB.parent.mkdir(parents=True, exist_ok=True) _AUTHORS_DB.write_text(_js.dumps(items, ensure_ascii=False, indent=2)) def _author_id_from_url(url: str) -> str: """从 URL 提取博主 ID:支持 /user/{sec_uid} 或 /video/{id}""" import re as _re m = _re.search(r'douyin\.com/user/([^/?#]+)', url) if m: return m.group(1) return "" @router.post("/track-author") async def api_track_author(data: dict = {}): """加入博主跟踪(传视频URL或博主主页URL,自动解析 uid 并打基线)""" from services.mediacrawler_adapter import get_video_data, get_author_profile, get_author_videos url = (data.get("url") or data.get("video_url") or "").strip() if not url: return {"status": "error", "message": "url 必填(视频链接或博主主页链接)"} # 情况1:博主主页链接 douyin.com/user/xxx → xxx 可能是 sec_uid author_id = _author_id_from_url(url) if author_id: # 主页链接里的 ID 是 sec_uid,需要解析成数字 uid # 方式:先当作视频无解,直接尝试用 sec_uid 查 profile(会失败), # 因此这里先提示用视频链接,或尝试用 sec_uid 反查 return await _add_author_by_sec_uid(author_id, url) # 情况2:视频链接 → 先采视频详情拿 uid stats = await get_video_data(url) if "error" in stats: return {"status": "error", "message": stats["error"]} uid = str(stats.get("author_uid", "")) nickname = stats.get("author", "") if not uid: return {"status": "error", "message": "未能从视频解析博主ID"} return await _add_author_by_uid(uid, nickname, url) async def _add_author_by_sec_uid(sec_uid: str, source_url: str): """通过 sec_uid 添加博主(先采集一次打基线)""" # sec_uid 不能直接查 profile/other(会报 UserId不合法) # 尝试通过视频详情反查:没有视频 URL 时先用 sec_uid 查 aweme/post(同样要数字uid) # 目前无法直接转换 → 提示用户使用视频链接 return {"status": "error", "message": "博主主页链接暂不支持直接添加,请改用视频链接(主页 ID 是加密格式)"} async def _add_author_by_uid(uid: str, nickname: str, source_url: str): """通过数字 uid 添加博主并打基线""" from services.mediacrawler_adapter import get_author_profile, get_author_videos authors = _load_authors() for a in authors: if a.get("uid") == uid: return {"status": "ok", "message": f"博主 {nickname} 已在跟踪中", "item": a, "already": True} profile = await get_author_profile(uid) if "error" in profile: return {"status": "error", "message": profile["error"]} # 拉最新视频(用于新视频发现基线) videos_res = await get_author_videos(uid, count=20) video_ids = [v["aweme_id"] for v in videos_res.get("videos", [])] author = { "author_id": f"author_{uid}", "uid": uid, "nickname": profile.get("nickname", nickname), "unique_id": profile.get("unique_id", ""), "sec_uid": profile.get("sec_uid", ""), "added_at": _time.strftime("%Y-%m-%d %H:%M:%S"), "source_url": source_url, "last_collected_at": _time.strftime("%Y-%m-%d %H:%M:%S"), "latest_videos": videos_res.get("videos", [])[:3], # 最新3条视频(标题+数据) "snapshots": [{ "date": _time.strftime("%Y-%m-%d"), "fans": profile.get("fans", 0), "total_favorited": profile.get("total_favorited", 0), "works": profile.get("works", 0), "new_videos": video_ids, }], "video_ids": video_ids, } authors.insert(0, author) _save_authors(authors) return {"status": "ok", "message": f"已跟踪博主 {author['nickname']}", "item": author} @router.get("/tracked-authors") async def api_tracked_authors(): """博主跟踪列表(含最新快照、今日涨粉)""" authors = _load_authors() today = _time.strftime("%Y-%m-%d") result = [] for a in authors: snaps = a.get("snapshots", []) latest = snaps[-1] if snaps else {} prev = snaps[-2] if len(snaps) > 1 else {} result.append({ "author_id": a.get("author_id"), "uid": a.get("uid"), "nickname": a.get("nickname"), "unique_id": a.get("unique_id"), "added_at": a.get("added_at"), "last_collected_at": a.get("last_collected_at"), "fans": latest.get("fans", 0), "fans_delta": (latest.get("fans", 0) - prev.get("fans", 0)) if prev else 0, "total_favorited": latest.get("total_favorited", 0), "works": latest.get("works", 0), "snapshot_count": len(snaps), "last_snapshot_date": latest.get("date", ""), "today_collected": latest.get("date") == today, "latest_videos": a.get("latest_videos", []), # 最新3条视频(标题+数据) }) return {"status": "ok", "items": result} @router.post("/refresh-author/{author_id}") async def api_refresh_author(author_id: str): """手动刷新单个博主:采主页快照 + 发现新视频(自动加入视频跟踪)""" from services.mediacrawler_adapter import get_author_profile, get_author_videos authors = _load_authors() found = next((a for a in authors if a.get("author_id") == author_id), None) if not found: return {"status": "error", "message": "未找到该博主"} profile = await get_author_profile(found["uid"]) if "error" in profile: resp = {"status": "error", "message": profile["error"]} if profile.get("login_expired"): resp["login_expired"] = True resp["message"] = "⛔ 抖音登录已过期,请重新登录" return resp # 拉最新视频,发现新视频 videos_res = await get_author_videos(found["uid"], count=20) new_videos = [] known = set(found.get("video_ids", [])) for v in videos_res.get("videos", []): if v["aweme_id"] not in known: new_videos.append(v["aweme_id"]) # 自动加入视频跟踪 await api_track_video({"url": v["url"]}) found["video_ids"] = list(known) + new_videos # 更新最新3条视频(标题+数据) found["latest_videos"] = videos_res.get("videos", [])[:3] # 存快照 today = _time.strftime("%Y-%m-%d") found.setdefault("snapshots", []).append({ "date": today, "fans": profile.get("fans", 0), "total_favorited": profile.get("total_favorited", 0), "works": profile.get("works", 0), "new_videos": new_videos, }) # 同一天重复刷新 → 替换今天的快照(不重复堆积) snaps = found["snapshots"] same_day = [i for i, s in enumerate(snaps) if s.get("date") == today] if len(same_day) > 1: # 保留第一次,删除后续同日快照 for i in sorted(same_day[1:], reverse=True): snaps.pop(i) found["last_collected_at"] = _time.strftime("%Y-%m-%d %H:%M:%S") _save_authors(authors) return {"status": "ok", "item": { "author_id": found["author_id"], "nickname": found["nickname"], "fans": profile.get("fans", 0), "total_favorited": profile.get("total_favorited", 0), "works": profile.get("works", 0), "new_videos": new_videos, "new_video_count": len(new_videos), }} @router.post("/refresh-all-authors") async def api_refresh_all_authors(): """刷新全部博主(限速 3 秒/个)""" from services.mediacrawler_adapter import get_author_profile authors = _load_authors() results = [] for a in authors: try: r = await api_refresh_author(a["author_id"]) results.append({"author_id": a["author_id"], "nickname": a.get("nickname"), "status": r.get("status"), "new_videos": r.get("item", {}).get("new_video_count", 0)}) except Exception as e: results.append({"author_id": a["author_id"], "nickname": a.get("nickname"), "status": "error", "error": str(e)[:100]}) await asyncio.sleep(3) return {"status": "ok", "results": results, "total": len(authors)} @router.get("/author-history/{author_id}") async def api_author_history(author_id: str): """博主历史趋势(每日序列)""" authors = _load_authors() found = next((a for a in authors if a.get("author_id") == author_id), None) if not found: return {"status": "error", "message": "未找到该博主"} return {"status": "ok", "item": { "author_id": found["author_id"], "nickname": found["nickname"], "uid": found["uid"], "snapshots": found.get("snapshots", []), }} @router.get("/author-videos/{author_id}") async def api_author_videos(author_id: str): """博主最新视频列表(实时拉取)""" from services.mediacrawler_adapter import get_author_videos authors = _load_authors() found = next((a for a in authors if a.get("author_id") == author_id), None) if not found: return {"status": "error", "message": "未找到该博主"} res = await get_author_videos(found["uid"], count=30) return {"status": "ok", "videos": res.get("videos", []), "has_more": res.get("has_more", False)} @router.delete("/tracked-authors/{author_id}") async def api_delete_tracked_author(author_id: str): """取消博主跟踪""" authors = _load_authors() before = len(authors) authors = [a for a in authors if a.get("author_id") != author_id] if len(authors) < before: _save_authors(authors) return {"status": "ok"} return {"status": "error", "message": "未找到"} @router.get("/author-new-videos") async def api_author_new_videos(): """全部博主的新视频汇总(最近一次刷新发现的新视频)""" authors = _load_authors() items = [] for a in authors: snaps = a.get("snapshots", []) if snaps: latest = snaps[-1] for vid in latest.get("new_videos", []): items.append({"author_id": a.get("author_id"), "nickname": a.get("nickname"), "video_id": vid}) return {"status": "ok", "items": items} @router.get("/chrome-status") async def api_chrome_status(): """检查采集 Chrome 是否可用(只检查 profile 目录,不检查 CDP 端口)""" CHROME = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" profile_ok = _os.path.exists("/tmp/chrome-douyin-profile/Default/Cookies") chrome_installed = _os.path.exists(CHROME) return {"status": "ok", "online": True, "chrome_installed": chrome_installed, "profile_ok": profile_ok} @router.get("/check-login") async def api_check_login(): """检查抖音登录状态""" from services.mediacrawler_adapter import check_login_status try: result = await check_login_status() return {"status": "ok", **result} except Exception as e: return {"status": "error", "message": str(e)} @router.post("/open-login") async def api_open_login(): """在 Chrome 中打开抖音登录页""" from services.mediacrawler_adapter import open_login_page try: result = await open_login_page() return result except Exception as e: return {"status": "error", "message": str(e)}