feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程,直接发 HTTP
Deploy VitePress site to Pages / build (push) Canceled after 0s
Deploy VitePress site to Pages / Deploy (push) Canceled after 0s

移植自 mac-agent-os 的 mediacrawler_adapter:不起子进程、不开页面,用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin,所以 ingest 那条链路一个字不用改。

**目前能用的(真环境实测,非推断)**:

    profile/other : 200, 7075 字节  —— 博主主页指标(粉丝/获赞/作品数/昵称)
    aweme/detail  : 200, 45425 字节 —— 单条作品详情(含点赞/评论/收藏/分享)

**目前不能用的:作品列表 `aweme/post`。** 两个互相独立的原因:
  1. 这个接口被抖音单独升级成了真校验:不带 x-tt-argus 回 403「Uifid Not Found」,
     带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过,「真校验」过不了。
     同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的,
     挑中的恰好是「批量拉作品列表」这个最敏感的动作。
  2. 改走页面截获也不行:CDP 浏览器打开博主主页会落到「验证码中间页」(当天大量探测的
     代价,过几小时要重测)。

  所以现在的边界是:**已知作品的指标刷新能做,自动发现新作品做不了**。

**排查中控住变量后得到的两条事实**(都写进注释了):
  · `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
    UA+client hints+Cookie 时是 200 但仅 121 字节的空壳,补上这三个头变 7074 字节。
    (我先前猜的 sec-ch-ua 不是关键。)
  · 因此 UA 与 client hints 必须**成套地取自同一个浏览器**,所以 BrowserIdentity 一次
    从 CDP 取齐 cookie + UA + hints,而不是各自写死。

「200 + 空 body 必须当场报错」也是刻意写死的:放过去它会在下游变成「这个博主没作品」,
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的,还被翻译成「账号被封」。

测试 +11:cookie 解析、请求头成套性(含 uifid 缺失/回退)、产物键名与 store 对齐、
以及 _get 的三条失败路径(空 body / 403 带网关原话 / 正常返回)。
This commit is contained in:
2026-10-10 17:12:25 +08:00
parent 21ff01b894
commit 2112c1a870
1601 changed files with 367425 additions and 0 deletions
@@ -0,0 +1,456 @@
"""
command_chain.py — 接力执行链系统
将一个或多个命令按顺序串联,上一步成功自动触发下一步。
支持失败重试、超时控制、状态查询、取消。
使用:
from services.command_chain import CommandChain
chain = CommandChain.create("nurture-then-collect", tasks)
status = CommandChain.get_status(chain.chain_id)
"""
import json
import logging
import os
import threading
import time
from datetime import datetime, timezone
from pathlib import Path
from typing import Optional
from uuid import uuid4
logger = logging.getLogger("dashboard.command_chain")
_THIS_DIR = Path(__file__).resolve().parent
AGENT_LOCAL = Path(os.environ.get("AGENT_LOCAL", str(Path.home() / "workbuddy-agent-os" / "agent-local")))
CHAINS_DIR = AGENT_LOCAL / "runtime" / "chains"
CHAINS_DIR.mkdir(parents=True, exist_ok=True)
class ChainTask:
"""接力链中的单个任务定义"""
def __init__(self, task_id: str, cmd_type: str, accounts: list,
params: dict = None, depends_on: str = None,
max_retries: int = 2, timeout: int = 600,
on_failure: str = "abort"):
self.task_id = task_id
self.cmd_type = cmd_type # nurture/collect/login/comment
self.accounts = accounts
self.params = params or {}
self.depends_on = depends_on # 前置 task_id, None=第一个
self.max_retries = max_retries
self.timeout = timeout
self.on_failure = on_failure # "abort" | "skip" | "retry"
def to_dict(self):
return {
"task_id": self.task_id,
"type": self.cmd_type,
"accounts": self.accounts,
"params": self.params,
"depends_on": self.depends_on,
"max_retries": self.max_retries,
"timeout": self.timeout,
"on_failure": self.on_failure,
}
@classmethod
def from_dict(cls, d):
return cls(
task_id=d["task_id"],
cmd_type=d["type"],
accounts=d["accounts"],
params=d.get("params", {}),
depends_on=d.get("depends_on"),
max_retries=d.get("max_retries", 2),
timeout=d.get("timeout", 600),
on_failure=d.get("on_failure", "abort"),
)
class ChainRun:
"""一次接力链的执行实例"""
def __init__(self, chain_id: str, name: str, tasks: list):
self.chain_id = chain_id
self.name = name
self.tasks = tasks # list[ChainTask]
self.status = "pending" # pending/running/completed/failed
self.current_index = -1 # 当前执行到的 task 索引
self.task_results = {} # {task_id: ChainTaskResult}
self.error = ""
self.created_at = datetime.now(timezone.utc).isoformat()
self.completed_at = None
self._stop_requested = False
def to_dict(self):
return {
"chain_id": self.chain_id,
"name": self.name,
"status": self.status,
"current_index": self.current_index,
"total_tasks": len(self.tasks),
"tasks": [t.to_dict() for t in self.tasks],
"task_results": {k: v.to_dict() for k, v in self.task_results.items()},
"error": self.error,
"created_at": self.created_at,
"completed_at": self.completed_at,
}
class ChainTaskResult:
"""单个任务的执行结果"""
def __init__(self, task_id: str):
self.task_id = task_id
self.status = "pending" # pending/running/completed/failed/skipped
self.attempts = 0
self.command_result = None
self.error = ""
self.started_at = None
self.completed_at = None
def to_dict(self):
return {
"task_id": self.task_id,
"status": self.status,
"attempts": self.attempts,
"command_result": self.command_result,
"error": self.error,
"started_at": self.started_at,
"completed_at": self.completed_at,
}
class CommandChain:
"""接力执行链调度器"""
_locks = {}
_running = {}
@classmethod
def create(cls, name: str, tasks: list) -> ChainRun:
"""创建一条接力链
Args:
name: 链名称
tasks: list[ChainTask] 或 list[dict]
Returns:
ChainRun 实例
"""
# 转换 dict 为 ChainTask
task_objects = []
for t in tasks:
if isinstance(t, dict):
task_objects.append(ChainTask.from_dict(t))
else:
task_objects.append(t)
chain_id = str(uuid4())[:12]
run = ChainRun(chain_id, name, task_objects)
# 持久化
cls._save(run)
# 异步启动执行
thread = threading.Thread(target=cls._execute, args=(chain_id,), daemon=True)
cls._running[chain_id] = thread
thread.start()
return run
@classmethod
def get_status(cls, chain_id: str) -> Optional[dict]:
"""查询接力链状态"""
run = cls._load(chain_id)
if run:
return run.to_dict()
return None
@classmethod
def cancel(cls, chain_id: str) -> dict:
"""取消接力链"""
run = cls._load(chain_id)
if not run:
return {"error": f"接力链不存在: {chain_id}"}
run._stop_requested = True
run.status = "cancelled"
run.completed_at = datetime.now(timezone.utc).isoformat()
cls._save(run)
return {"ok": True, "chain_id": chain_id, "status": "cancelled"}
@classmethod
def list_chains(cls, limit: int = 20) -> list:
"""列出最近创建的接力链"""
chains = []
if CHAINS_DIR.exists():
for f in sorted(CHAINS_DIR.iterdir(), reverse=True)[:limit]:
if f.suffix == ".json":
try:
data = json.loads(f.read_text())
chains.append({
"chain_id": data.get("chain_id", f.stem),
"name": data.get("name", ""),
"status": data.get("status", ""),
"total_tasks": len(data.get("tasks", [])),
"current_index": data.get("current_index", -1),
"created_at": data.get("created_at", ""),
"completed_at": data.get("completed_at", ""),
})
except Exception:
pass
return chains
# ── 内部方法 ──
# ── 并行执行引擎 ──
@classmethod
def _build_levels(cls, run: ChainRun):
"""将任务按依赖关系分组为层级
Level 0: 无 depends_on,并行启动
Level 1: 依赖 Level 0 任务
Level N: 依赖 Level N-1 任务
"""
task_map = {t.task_id: t for t in run.tasks}
levels = []
remaining = set(task_map.keys())
while remaining:
# 找到所有前置任务都已完成的剩余任务
ready = set()
for tid in remaining:
task = task_map[tid]
if not task.depends_on or task.depends_on not in remaining:
ready.add(tid)
if not ready:
# 死锁:剩余的都有未满足的依赖
break
levels.append(list(ready))
remaining -= ready
return levels
@classmethod
def _execute_task(cls, run: ChainRun, task: ChainTask):
"""执行单个任务(线程安全)"""
result = ChainTaskResult(task.task_id)
result.status = "running"
result.started_at = datetime.now(timezone.utc).isoformat()
# 检查前置任务状态
if task.depends_on:
dep_result = run.task_results.get(task.depends_on)
if dep_result and dep_result.status != "completed":
result.status = "skipped"
result.error = f"前置任务 {task.depends_on} 未成功完成"
run.task_results[task.task_id] = result
cls._save(run)
return
# 跳过空账号
if not task.accounts:
result.status = "skipped"
result.error = "账号列表为空"
result.completed_at = datetime.now(timezone.utc).isoformat()
run.task_results[task.task_id] = result
cls._save(run)
return
# 执行并重试
success = False
for attempt in range(1, task.max_retries + 1):
if run._stop_requested:
break
result.attempts = attempt
run.task_results[task.task_id] = result
cls._save(run)
try:
from services.command_bus import CommandBus
cmd_result = CommandBus.dispatch(
task.cmd_type, task.accounts,
task.params, wait=True
)
result.command_result = cmd_result
# 检查各个机器的子任务状态
sub_statuses = cmd_result.get("per_machine", {})
machine_statuses = {m: s.get("status", "unknown") for m, s in sub_statuses.items()}
all_sub_ok = all(s == "completed" or s == "accepted" or s == "started"
for s in machine_statuses.values())
if cmd_result.get("status") in ("completed", "accepted") or all_sub_ok:
success = True
break
else:
result.error = cmd_result.get("message", "") or f"机器状态: {machine_statuses}"
logger.warning(f" [{run.chain_id}] {task.task_id} 第{attempt}次失败: {result.error}")
if attempt < task.max_retries:
time.sleep(5)
except Exception as e:
result.error = str(e)
logger.warning(f" [{run.chain_id}] {task.task_id} 异常: {e}")
if attempt < task.max_retries:
time.sleep(5)
# 结果
if success:
result.status = "completed"
else:
result.status = "failed"
result.completed_at = datetime.now(timezone.utc).isoformat()
run.task_results[task.task_id] = result
cls._save(run)
@classmethod
def _execute(cls, chain_id: str):
"""执行接力链(按依赖层级并行执行)"""
run = cls._load(chain_id)
if not run:
return
run.status = "running"
cls._save(run)
# 构建依赖层级
task_map = {t.task_id: t for t in run.tasks}
levels = cls._build_levels(run)
for level_idx, level_tasks in enumerate(levels):
if run._stop_requested:
break
run.current_index = level_idx
logger.info(f" [{chain_id}] 执行层级 {level_idx+1}/{len(levels)}: {level_tasks}")
cls._save(run)
# 同一层级并行执行
threads = []
for tid in level_tasks:
task = task_map[tid]
t = threading.Thread(
target=cls._execute_task,
args=(run, task),
daemon=True
)
threads.append(t)
t.start()
# 等待本层级所有任务完成
for t in threads:
t.join()
# 检查是否有任务失败需要终止
if not run._stop_requested:
for tid in level_tasks:
tr = run.task_results.get(tid)
if tr and tr.status == "failed":
task = task_map[tid]
if task.on_failure == "abort":
run.status = "failed"
run.error = f"任务 {tid} 失败后终止"
run.completed_at = datetime.now(timezone.utc).isoformat()
cls._save(run)
cls._running.pop(chain_id, None)
return
# 层间冷却
if level_idx < len(levels) - 1:
time.sleep(3)
# 最终状态
if not run._stop_requested:
all_ok = all(
run.task_results.get(t.task_id, ChainTaskResult(t.task_id)).status == "completed"
for t in run.tasks if t.accounts # 空账号skipped不计入失败
)
run.status = "completed" if all_ok else "failed"
if not run.error:
failed = [t.task_id for t in run.tasks
if run.task_results.get(t.task_id, ChainTaskResult(t.task_id)).status not in ("completed", "skipped")]
if failed:
run.status = "failed"
run.error = f"以下任务失败: {', '.join(failed)}"
else:
run.status = "cancelled"
run.completed_at = datetime.now(timezone.utc).isoformat()
cls._save(run)
cls._running.pop(chain_id, None)
# 链完成后自动触发联邦采集器拉取最新数据
if run.status == "completed":
try:
from services.fleet_collector import collect_all
logger.info(f" [{chain_id}] 链完成,自动触发联邦采集...")
collect_all()
except Exception as e:
logger.warning(f" [{chain_id}] 联邦采集自动触发失败: {e}")
_save_locks: dict = {}
@classmethod
def _get_save_lock(cls, chain_id: str) -> threading.Lock:
"""按 chain_id 获取锁,确保同一链的写入串行化"""
if chain_id not in cls._save_locks:
cls._save_locks[chain_id] = threading.Lock()
return cls._save_locks[chain_id]
@classmethod
def _save(cls, run: ChainRun):
"""持久化接力链状态(按chain_id加锁,原子写入)"""
with cls._get_save_lock(run.chain_id):
path = CHAINS_DIR / f"{run.chain_id}.json"
tmp = path.with_suffix(".tmp")
tmp.write_text(json.dumps(run.to_dict(), indent=2, ensure_ascii=False))
tmp.replace(path)
@classmethod
def _load(cls, chain_id: str) -> Optional[ChainRun]:
"""加载接力链"""
path = CHAINS_DIR / f"{chain_id}.json"
if not path.exists():
return None
try:
data = json.loads(path.read_text())
run = ChainRun(data["chain_id"], data["name"],
[ChainTask.from_dict(t) for t in data["tasks"]])
run.status = data.get("status", "pending")
run.current_index = data.get("current_index", -1)
run.task_results = {}
for tid, rd in data.get("task_results", {}).items():
r = ChainTaskResult(tid)
r.status = rd.get("status", "pending")
r.attempts = rd.get("attempts", 0)
r.command_result = rd.get("command_result")
r.error = rd.get("error", "")
r.started_at = rd.get("started_at")
r.completed_at = rd.get("completed_at")
run.task_results[tid] = r
run.error = data.get("error", "")
run.created_at = data.get("created_at", "")
run.completed_at = data.get("completed_at")
return run
except Exception as e:
logger.warning(f" ⚠️ 接力链加载失败 [{chain_id}]: {e}")
# 尝试修复:删除最后一行非JSON内容
try:
path = CHAINS_DIR / f"{chain_id}.json"
raw = path.read_text()
# 找到第一个完整的 JSON 对象
import re as _re
m = _re.search(r'^(\{[^{}]*(\{[^{}]*\})*[^{}]*\})', raw, _re.DOTALL)
if m:
data = json.loads(m.group(1))
path.write_text(m.group(1))
logger.info(f" ✅ 接力链文件已修复 [{chain_id}]")
return cls._load(chain_id)
except Exception:
pass
return None