feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程,直接发 HTTP
Deploy VitePress site to Pages / build (push) Canceled after 0s
Deploy VitePress site to Pages / Deploy (push) Canceled after 0s

移植自 mac-agent-os 的 mediacrawler_adapter:不起子进程、不开页面,用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin,所以 ingest 那条链路一个字不用改。

**目前能用的(真环境实测,非推断)**:

    profile/other : 200, 7075 字节  —— 博主主页指标(粉丝/获赞/作品数/昵称)
    aweme/detail  : 200, 45425 字节 —— 单条作品详情(含点赞/评论/收藏/分享)

**目前不能用的:作品列表 `aweme/post`。** 两个互相独立的原因:
  1. 这个接口被抖音单独升级成了真校验:不带 x-tt-argus 回 403「Uifid Not Found」,
     带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过,「真校验」过不了。
     同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的,
     挑中的恰好是「批量拉作品列表」这个最敏感的动作。
  2. 改走页面截获也不行:CDP 浏览器打开博主主页会落到「验证码中间页」(当天大量探测的
     代价,过几小时要重测)。

  所以现在的边界是:**已知作品的指标刷新能做,自动发现新作品做不了**。

**排查中控住变量后得到的两条事实**(都写进注释了):
  · `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
    UA+client hints+Cookie 时是 200 但仅 121 字节的空壳,补上这三个头变 7074 字节。
    (我先前猜的 sec-ch-ua 不是关键。)
  · 因此 UA 与 client hints 必须**成套地取自同一个浏览器**,所以 BrowserIdentity 一次
    从 CDP 取齐 cookie + UA + hints,而不是各自写死。

「200 + 空 body 必须当场报错」也是刻意写死的:放过去它会在下游变成「这个博主没作品」,
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的,还被翻译成「账号被封」。

测试 +11:cookie 解析、请求头成套性(含 uifid 缺失/回退)、产物键名与 store 对齐、
以及 _get 的三条失败路径(空 body / 403 带网关原话 / 正常返回)。
This commit is contained in:
2026-10-10 17:12:25 +08:00
parent 21ff01b894
commit 2112c1a870
1601 changed files with 367425 additions and 0 deletions
@@ -0,0 +1,654 @@
"""
routes/ops.py — 统一操作执行路由 v5
⚠️ 本文件是 Dashboard 通往执行层的唯一入口。
所有操作(养号/采集/登录/评论/点赞)都经过这里。
调用规范:
前端统一发 POST /api/ops/run {type, accounts, params}
→ CommandBus.dispatch() → CMD_REGISTRY 模板映射 → mc 引擎
禁止:
- 在前端直接调用 routes/matrix.py 的 POST 路由
- 在 routes/matrix.py 中新增 POST 写操作
- 绕过 CommandBus 直接 subprocess
操作类型(定义在 CMD_REGISTRY 中):
- nurture: 养号执行,走 nurture_runner.sh 包装器
- collect: 主页采集,auto_blueprint 自动按账号平台选蓝图
- login: 智能登录,走 mc smart-login
- logout: 登出
- comment: 定向评论
- like: 点赞
修复记录:
2026-06-28: /api/ops/queue 改为 ThreadPoolExecutor 并行查询,避免远程超时堵塞
"""
import logging
from fastapi import APIRouter, HTTPException
from services.command_bus import CommandBus, MachineSession
logger = logging.getLogger("dashboard.ops")
router = APIRouter(prefix="/api/ops", tags=["ops"])
@router.post("/run")
def api_ops_run(data: dict = {}):
"""
统一操作执行入口
请求体:
type: str — 操作类型 (nurture/collect/login/logout/comment/like/smart_comment)
accounts: [str] — 账号ID列表
params: dict — 操作特定参数(可选)
blueprint: str — 蓝图名(nurture 用,不传则自动检测)
rounds: int — 轮数(nurture 用,默认 10)
phone: str — 手机号(collect 用)
url: str — 视频链接(comment/like 用)
urls: [str] — 视频链接列表(smart_comment 用)
direction: str — 评论方向(praise/question/empathy)
machine: str — 强制指定目标机器
dry_run: bool — 仅预览不执行
"""
try:
op_type = data.get("type", "")
accounts = data.get("accounts", [])
params = data.get("params", {})
if not op_type:
return {"status": "error", "message": "type 必填 (nurture/collect/login/logout/comment/like/smart_comment)"}
if not accounts:
return {"status": "error", "message": "accounts 必填"}
wait = data.get("wait", False)
if data.get("dry_run") or params.get("dry_run"):
params["dry_run"] = True
result = CommandBus.dispatch(op_type, accounts, params, wait=wait)
# wait=True 时响应可能包含执行结果摘要
if wait and result.get("status") == "completed":
summary = result.get("per_machine", {})
s = f"完成: "
for m, info in summary.items():
s += f"{m}: {info.get('success',0)}成功/{info.get('failed',0)}失败 "
logger.info(f"批量执行完毕 — {s}")
return result
except Exception as e:
logger.exception("ops/run 失败")
raise HTTPException(500, detail=str(e))
@router.get("/status")
def api_ops_status(machine: str = None, account: str = None):
"""查询所有命令状态"""
return {"commands": CommandBus.get_status(machine=machine, account=account)}
@router.get("/history")
def api_ops_history(limit: int = 20):
"""查询命令执行历史(含每台机器的执行结果)"""
return {"commands": CommandBus.get_status()[:limit]}
@router.get("/machines")
def api_ops_machines():
"""查询所有机器聚合状态"""
return CommandBus.get_all_machines_status()
@router.get("/running")
def api_ops_running():
"""聚合所有机器的运行中任务(从 guardd 获取)"""
from services.command_bus import _guardd_api, HOSTNAME, _get_machine_info
machines = []
# 本机
local_tasks = _guardd_api("GET", "/scheduler/tasks") or {}
local_active = local_tasks.get("active") or local_tasks
if local_active:
machines.append({"machine": HOSTNAME, "tasks": [local_active] if isinstance(local_active, dict) else local_active})
# 远程机器
info = _get_machine_info("dummy")
for name, minfo in getattr(_get_machine_info, "_oracle_machines", {}).items():
if name == HOSTNAME:
continue
ip = minfo.get("tailscale_ip", "")
if ip:
remote_tasks = _guardd_api("GET", "/scheduler/tasks", machine=name) or {}
if remote_tasks:
machines.append({"machine": name, "tasks": remote_tasks})
return {"machines": machines}
@router.post("/test-atom")
def api_ops_test_atom(data: dict = {}):
"""
单步原子操作测试
动态生成单步骤蓝图 → 通过 command_bus 执行
请求体:
op: str — 原子操作名称 (goto_home/scroll_feed/like 等)
account: str — 账号ID
platform: str — 平台 (douyin/xiaohongshu),选填,自动检测
"""
import json, tempfile, os, uuid
from pathlib import Path
op = data.get("op", "")
account = data.get("account", "")
platform = data.get("platform", "douyin")
if not op:
return {"status": "error", "message": "op 必填"}
if not account:
return {"status": "error", "message": "account 必填"}
try:
# 动态生成单步骤蓝图
bp = {
"id": f"test_{op}_{uuid.uuid4().hex[:8]}",
"name": f"测试-{op}",
"description": f"单步测试: {op}",
"version": "1.0",
"platform": platform,
"steps": [{"step_id": 1, "op": op, "args": {}}]
}
# 写入临时蓝图文件
bp_dir = Path(__file__).parent.parent / ".." / "07_matrix" / "blueprints"
bp_file = bp_dir / f"_test_{op}.json"
bp_file.parent.mkdir(parents=True, exist_ok=True)
bp_file.write_text(json.dumps(bp, ensure_ascii=False, indent=2))
# 执行 (1轮,不等待 → 后台异步执行)
result = CommandBus.dispatch("nurture", [account], {
"blueprint": f"_test_{op}".replace(".json",""),
"rounds": 1,
"single_op": op,
}, wait=False)
# 清理临时蓝图
if bp_file.exists():
bp_file.unlink()
return {
"status": result.get("status", "submitted"),
"op": op,
"account": account,
"details": result.get("message", result.get("per_machine", "已提交")),
}
except Exception as e:
logger.exception("test-atom 失败")
raise HTTPException(500, detail=str(e))
@router.get("/policy")
def api_ops_policy():
"""获取当前执行策略配置(供看板前端展示约束条)"""
import sys, os
scripts_dir = os.path.join(os.path.dirname(__file__), "..", "..", "07_matrix", "scripts")
sys.path.insert(0, scripts_dir)
try:
from mc.execution_policy import get_policy, preflight as _pf
policy = get_policy()
status = _pf()
policy["current_status"] = status
return policy
except Exception as e:
return {"version": "unknown", "error": str(e), "max_concurrent": 3}
@router.post("/cancel/{run_id}")
def api_ops_cancel(run_id: str):
"""取消命令"""
return CommandBus.cancel(run_id)
@router.post("/cleanup-stale")
def api_ops_cleanup_stale():
"""清理僵尸命令:进程已死但状态为 running 的标记为 CRASHED,并释放队列"""
from services.command_bus import cleanup_stale_commands
count = cleanup_stale_commands()
return {"status": "ok", "cleaned": count}
@router.get("/log/{run_id}")
def api_ops_log(run_id: str):
"""查看命令日志"""
from pathlib import Path
log_dir = Path.home() / "workbuddy-agent-os" / "agent-local" / "runtime" / "commands"
for f in sorted(log_dir.glob(f"{run_id}*"), key=lambda x: x.stat().st_mtime, reverse=True):
if f.exists():
tail = f.read_text(encoding="utf-8", errors="replace")[-5000:]
return {"run_id": run_id, "log": tail, "path": str(f)}
return {"run_id": run_id, "log": "", "path": ""}
@router.get("/queue")
def api_ops_queue(machine: str = None):
"""查看各机任务队列详情(含活跃任务、排队、槽位)
使用 ThreadPoolExecutor 并行查询所有机器,避免单机超时堵塞整体。
个别机器超时/离线不影响其他机器结果。
"""
from services.command_bus import _guardd_api, ORACLE_PATH
machines = [machine] if machine else []
if not machines:
try:
import yaml
oracle = yaml.safe_load(ORACLE_PATH.read_text())
machines = list(oracle.get("machines", {}).keys())
except Exception:
machines = [__import__("utils.identity", fromlist=["resolve_hostname"]).resolve_hostname()]
results = {}
def query_machine(m):
"""查询单台机器,返回 (machine_name, data)"""
try:
data = _guardd_api("GET", "/scheduler/tasks", machine=m)
if data:
return (m, data)
else:
from services.command_bus import CommandBus
ms = CommandBus.get_machine_status(m)
return (m, {"active": ms.get("active_task"), "error": "guardd scheduler 未响应"})
except Exception as e:
return (m, {"error": str(e)})
# 并行查询所有机器(本机快,远程可能超时,但不阻塞彼此)
from concurrent.futures import ThreadPoolExecutor, as_completed
with ThreadPoolExecutor(max_workers=len(machines)) as pool:
futures = {pool.submit(query_machine, m): m for m in machines}
for future in as_completed(futures):
try:
m, data = future.result()
results[m] = data
except Exception as e:
m = futures[future]
results[m] = {"error": str(e)}
return {"machines": results}
@router.post("/task/cancel")
def api_ops_task_cancel(data: dict = {}):
"""取消任务 — 发送到对应机器的 guardd"""
task_id = data.get("task_id", "")
machine = data.get("machine", "")
if not task_id:
return {"status": "error", "message": "task_id 必填"}
from services.command_bus import _guardd_api
result = _guardd_api("POST", f"/task/{task_id}/stop", {}, machine=machine)
return {"status": "ok", "task_id": task_id, "result": result}
@router.post("/task/submit")
def api_ops_task_submit(data: dict = {}):
"""提交任务到调度器 — 发到目标机器的 guardd"""
task = data.get("task", {})
machine = data.get("machine", "")
if not task.get("task_id"):
return {"status": "error", "message": "task.task_id 必填"}
from services.command_bus import _guardd_api
result = _guardd_api("POST", "/scheduler/submit", task, machine=machine)
return {"status": "accepted", "result": result}
@router.post("/reset")
def api_ops_reset(data: dict = {}):
"""重置机器:清空任务队列 + 杀浏览器
Body:
machine: str — 指定机器名,不传则重置所有机器
"""
from services.command_bus import _guardd_api, ORACLE_PATH, HOSTNAME
import yaml
requested = data.get("machine", "")
if requested:
machines = [requested]
else:
# 重置所有机器
try:
oracle = yaml.safe_load(ORACLE_PATH.read_text())
machines = list(oracle.get("machines", {}).keys())
except:
machines = [HOSTNAME]
results = {}
for m in machines:
try:
# 调用 guardd scheduler/reset(杀活跃+清队列+杀浏览器)
r = _guardd_api("POST", "/scheduler/reset", {}, machine=m)
results[m] = "ok" if r else "no response"
except Exception as e:
results[m] = str(e)
return {"status": "ok", "machines": results}
import time as _time
import re as _re
def _extract_title(share_text: str) -> str:
"""从抖音分享文案中提取视频标题"""
if not share_text:
return ""
# 格式: "2.38 复制打开抖音,看看【「作者」的作品】标题文字#话题 https://..."
m = _re.search(r'】(.+?)(?:#|https?://)', share_text)
if m:
return m.group(1).strip()
# 兜底: 取 "看看【..." 之后到 # 或 URL 之前的内容
m2 = _re.search(r'看看【.*?】(.+?)(?:#|https?://)', share_text)
if m2:
return m2.group(1).strip()
return share_text[:80].strip()
@router.post("/import-topics")
def api_import_topics(data: dict = {}):
"""从外部 API 导入今日热门视频列表"""
import urllib.request as _urq
import json as _js
api_url = (data.get("api_url") or "").strip()
if not api_url:
return {"status": "error", "message": "api_url 必填"}
page = int(data.get("page", 1))
page_size = int(data.get("page_size", 100))
# 查询已成功评论的 ID 列表(去重用)
commented_ids = set()
try:
from services.command_bus import _load_batch_logs
for log in _load_batch_logs():
for vid in (log.get("video_ids") or []):
commented_ids.add(str(vid))
except Exception:
pass
try:
full_url = f"{api_url}?page={page}&pageSize={page_size}"
resp = _urq.urlopen(full_url, timeout=15)
body = _js.loads(resp.read().decode())
except Exception as e:
return {"status": "error", "message": f"请求外部 API 失败: {e}"}
raw_items = body.get("data", {}).get("items", [])
items = []
for item in raw_items:
url = (item.get("share_url") or "").strip()
if not url:
continue
vid = str(item.get("id", ""))
title = _extract_title(item.get("share_text", ""))
# 从 share_text 提取话题标签
share_text = item.get("share_text", "")
tags = _re.findall(r'#([^#\s]+)', share_text)
items.append({
"id": vid,
"url": url,
"title": title,
"author": item.get("nickname", "") or item.get("name", ""),
"tags": tags[:5], # 最多5个标签
"created_at": item.get("created_at", ""),
"completed": item.get("completion_status", 0) == 1,
"already_commented": vid in commented_ids,
})
return {
"status": "ok",
"items": items,
"total": len(items),
"page": page,
"page_size": page_size,
}
_BATCH_LOG_DIR = None
def _get_batch_log_dir():
global _BATCH_LOG_DIR
if _BATCH_LOG_DIR is None:
from pathlib import Path
_BATCH_LOG_DIR = Path.home() / "workbuddy-agent-os" / "agent-local" / "runtime" / "commands" / "batches"
_BATCH_LOG_DIR.mkdir(parents=True, exist_ok=True)
return _BATCH_LOG_DIR
@router.post("/clear-all")
def api_ops_clear_all(data: dict = {}):
"""清空所有机器的所有任务(杀死运行中 + 清空队列 + 重置状态)
改进:增加超时、逐台返回结果、整体 status 反映实际结果
"""
from services.command_bus import _guardd_api, ORACLE_PATH, HOSTNAME
import yaml
try:
oracle = yaml.safe_load(ORACLE_PATH.read_text())
machines = list(oracle.get("machines", {}).keys())
except Exception:
machines = [HOSTNAME]
results = {}
all_ok = True
for m in machines:
try:
r = _guardd_api("POST", "/scheduler/clear-all", machine=m, timeout=10)
r_status = r.get("status", "error")
results[m] = r_status
if r_status != "ok":
all_ok = False
except Exception as e:
results[m] = str(e)[:60]
all_ok = False
return {"status": "ok" if all_ok else "partial", "machines": results}
@router.post("/collect-p2")
def api_ops_collect_p2(data: dict = {}):
"""提交 P2 级别采集任务(填空闲 slot 用)"""
accounts = data.get("accounts", [])
if not accounts:
return {"status": "error", "message": "accounts 必填"}
from services.command_bus import CommandBus
try:
result = CommandBus.dispatch("collect", accounts, {
"priority": 2,
"blueprint": data.get("blueprint", "douyin_read_profile"),
"rounds": 1,
})
return {
"status": "ok",
"total": len(accounts),
"detail": result,
}
except Exception as e:
return {"status": "error", "message": str(e)}
@router.post("/batch-log")
def api_batch_log_save(data: dict = {}):
"""保存一批任务的执行记录"""
import json, threading, time
log_dir = _get_batch_log_dir()
batch_id = data.get("batch_id", f"batch_{int(time.time())}")
log_entry = {
"batch_id": batch_id,
"time": data.get("time", time.strftime("%Y-%m-%d %H:%M:%S")),
"type": data.get("type", "smart_comment"),
"total": data.get("total", 0),
"video_count": data.get("video_count", 0),
"account_count": data.get("account_count", 0),
"urls": data.get("urls", []),
"video_ids": data.get("video_ids", []),
"direction": data.get("direction", ""),
"interval": data.get("interval", ""),
"machine_counts": data.get("machine_counts", {}),
}
today = time.strftime("%Y-%m-%d")
log_file = log_dir / f"{today}.jsonl"
with threading.Lock():
with open(log_file, "a", encoding="utf-8") as f:
f.write(json.dumps(log_entry, ensure_ascii=False) + "\n")
return {"status": "ok", "batch_id": batch_id}
@router.get("/batch-logs")
def api_batch_logs_list(limit: int = 50):
"""获取执行记录列表(最新 N 条)"""
import json
log_dir = _get_batch_log_dir()
if not log_dir.exists():
return {"logs": []}
entries = []
for f in sorted(log_dir.glob("*.jsonl"), reverse=True):
lines = f.read_text(encoding="utf-8").strip().split("\n")
for line in reversed(lines):
if line.strip():
try:
entries.append(json.loads(line))
except json.JSONDecodeError:
pass
if len(entries) >= limit:
break
if len(entries) >= limit:
break
return {"logs": entries}
# ── 任务状态推送(guardd → Dashboard)───────────────
_TASK_EVENTS = [] # 内存环形缓冲,保存最近 200 条
_MAX_EVENTS = 200
supported_plugins = {}
@router.post("/push/task-event")
def api_push_task_event(data: dict = {}):
"""接收 guardd 推送的任务状态变化事件"""
global _TASK_EVENTS
event = {
"event": data.get("event", "unknown"),
"machine": data.get("machine", ""),
"task_id": data.get("task_id", ""),
"cmd_type": data.get("cmd_type", ""),
"accounts": data.get("accounts", []),
"slot_id": data.get("slot_id"),
"status": data.get("status", ""),
"time": data.get("time", ""),
"received_at": _time.strftime("%Y-%m-%d %H:%M:%S"),
}
_TASK_EVENTS.append(event)
if len(_TASK_EVENTS) > _MAX_EVENTS:
_TASK_EVENTS = _TASK_EVENTS[-_MAX_EVENTS:]
return {"status": "ok"}
@router.get("/task-events")
def api_task_events(limit: int = 50):
"""获取最近的任务事件列表(供前端轮询/实时展示)"""
return {"events": _TASK_EVENTS[-limit:]}
# ── AI 消耗统计(看板首页展示)───────────────
@router.get("/ai/usage")
def api_ai_usage(days: int = 7):
"""AI 消耗统计(读 agent-local/runtime/ai_usage.jsonl)
Returns:
today: 今日聚合(calls/tokens/缓存命中/费用估算)
total: 总计聚合
recent: 最近 20 条明细
"""
import json as _json
from datetime import datetime as _dt
from pathlib import Path as _Path
_log_path = _Path.home() / "workbuddy-agent-os" / "agent-local" / "runtime" / "ai_usage.jsonl"
entries = []
if _log_path.exists():
for line in _log_path.read_text(encoding="utf-8", errors="replace").splitlines():
line = line.strip()
if not line:
continue
try:
entries.append(_json.loads(line))
except Exception:
pass
today_str = _dt.now().strftime("%Y-%m-%d")
today_entries = [e for e in entries if str(e.get("ts", "")).startswith(today_str)]
# flash 官方空闲价(元/百万 token):缓存命中 0.05 / 未命中 1.5 / 输出 4.5
def _agg(lst: list) -> dict:
prompt = sum(e.get("prompt_tokens", 0) for e in lst)
completion = sum(e.get("completion_tokens", 0) for e in lst)
hit = sum(e.get("cache_hit", 0) for e in lst)
miss = sum(e.get("cache_miss", 0) for e in lst)
reasoning = sum(e.get("reasoning_tokens", 0) for e in lst)
cost = (miss * 1.5 + hit * 0.05 + completion * 4.5) / 1e6
return {
"calls": len(lst),
"prompt_tokens": prompt,
"completion_tokens": completion,
"total_tokens": prompt + completion,
"cache_hit": hit,
"cache_miss": miss,
"cache_hit_rate": round(hit / (hit + miss), 3) if (hit + miss) else 0,
"reasoning_tokens": reasoning,
"cost_estimate": round(cost, 4), # 元(按 flash 空闲价估算)
}
return {"today": _agg(today_entries), "total": _agg(entries), "recent": entries[-20:]}
@router.post("/fetch-titles")
def api_fetch_titles(data: dict = {}):
"""从视频 URL 列表中提取页面标题
Body: {"urls": ["https://www.douyin.com/video/...", ...]}
Returns: {"titles": [{"url": "...", "title": "...", "error": "..."}]}
"""
import urllib.request as _urq
import re as _re
urls = data.get("urls", [])
if not urls or not isinstance(urls, list):
return {"status": "error", "message": "urls 必填"}
results = []
for url in urls:
url = url.strip()
if not url:
continue
title = ""
error = ""
try:
req = _urq.Request(url, headers={
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
"Accept-Language": "zh-CN,zh;q=0.9",
})
resp = _urq.urlopen(req, timeout=10)
html = resp.read().decode("utf-8", errors="replace")
# 提取 <title> 标签
m = _re.search(r'<title>([^<]+)</title>', html)
if m:
title = m.group(1).strip()
# 去掉平台后缀
title = _re.sub(r'\s*[-–—|]\s*抖音$', '', title).strip()
# 如果标题太长(含大量SEO关键词),截取到有用部分
if len(title) > 60:
title = title[:60]
except Exception as e:
error = str(e)[:60]
results.append({"url": url, "title": title, "error": error})
logger.info(" 📋 提取 %d 个URL标题: %d 成功", len(results), sum(1 for r in results if r["title"]))
return {"status": "ok", "titles": results}