feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程,直接发 HTTP
Deploy VitePress site to Pages / build (push) Canceled after 0s
Deploy VitePress site to Pages / Deploy (push) Canceled after 0s

移植自 mac-agent-os 的 mediacrawler_adapter:不起子进程、不开页面,用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin,所以 ingest 那条链路一个字不用改。

**目前能用的(真环境实测,非推断)**:

    profile/other : 200, 7075 字节  —— 博主主页指标(粉丝/获赞/作品数/昵称)
    aweme/detail  : 200, 45425 字节 —— 单条作品详情(含点赞/评论/收藏/分享)

**目前不能用的:作品列表 `aweme/post`。** 两个互相独立的原因:
  1. 这个接口被抖音单独升级成了真校验:不带 x-tt-argus 回 403「Uifid Not Found」,
     带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过,「真校验」过不了。
     同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的,
     挑中的恰好是「批量拉作品列表」这个最敏感的动作。
  2. 改走页面截获也不行:CDP 浏览器打开博主主页会落到「验证码中间页」(当天大量探测的
     代价,过几小时要重测)。

  所以现在的边界是:**已知作品的指标刷新能做,自动发现新作品做不了**。

**排查中控住变量后得到的两条事实**(都写进注释了):
  · `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
    UA+client hints+Cookie 时是 200 但仅 121 字节的空壳,补上这三个头变 7074 字节。
    (我先前猜的 sec-ch-ua 不是关键。)
  · 因此 UA 与 client hints 必须**成套地取自同一个浏览器**,所以 BrowserIdentity 一次
    从 CDP 取齐 cookie + UA + hints,而不是各自写死。

「200 + 空 body 必须当场报错」也是刻意写死的:放过去它会在下游变成「这个博主没作品」,
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的,还被翻译成「账号被封」。

测试 +11:cookie 解析、请求头成套性(含 uifid 缺失/回退)、产物键名与 store 对齐、
以及 _get 的三条失败路径(空 body / 403 带网关原话 / 正常返回)。
This commit is contained in:
2026-10-10 17:12:25 +08:00
parent 21ff01b894
commit 2112c1a870
1601 changed files with 367425 additions and 0 deletions
@@ -0,0 +1,169 @@
"""
heartbeat.py — 增强心跳上报 (guardd 模块)
职责:
- 每15秒采集系统状态 + 任务状态 + 槽位状态
- 上报到 Dashboard (POST /api/push/heartbeat)
- 本地写入 status/{hostname}/heartbeat.json
"""
import json
import logging
import os
import subprocess
import time
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
from modules.task_store import TaskStore
from modules.slot_manager import BrowserSlotManager
from modules.scheduler import Scheduler
from modules.account_monitor import AccountMonitor
logger = logging.getLogger("guardd.heartbeat")
class HeartbeatReporter:
"""增强心跳上报器"""
def __init__(self, task_store: TaskStore, slot_manager: BrowserSlotManager,
scheduler: Scheduler, hostname: str, machine_uid: str,
dashboard_url: str = None,
account_monitor: AccountMonitor = None):
self.task_store = task_store
self.slot_manager = slot_manager
self.scheduler = scheduler
self.hostname = hostname
self.machine_uid = machine_uid
self.dashboard_url = dashboard_url or "http://127.0.0.1:9988"
self.account_monitor = account_monitor
def collect(self) -> dict:
"""采集完整状态(v2: 含 per-slot 详细进度)"""
slot_usage = self.slot_manager.get_usage() if self.slot_manager else {"max": 0, "used": 0, "slots": []}
# 收集任务状态(v2: active_tasks 字典,支持多 slot 并行)
active_tasks = list(self.scheduler.active_tasks.values()) if hasattr(self.scheduler, 'active_tasks') and self.scheduler.active_tasks else []
active_list = []
for t in active_tasks:
at = self._task_to_heartbeat(t)
if at:
active_list.append(at)
queued = self.scheduler.get_all_queued() if hasattr(self.scheduler, 'get_all_queued') else []
task_counts = self.task_store.count()
# 增强 slot 信息:从 slot_manager 获取每个浏览器的当前步骤/时间
slot_details = []
for s in slot_usage.get("slots", []):
if s.get("account_id"):
slot_details.append({
"slot_id": s.get("slot_id"),
"account_id": s.get("account_id"),
"nickname": s.get("nickname", ""),
"platform": s.get("platform", ""),
"current_step": s.get("current_step", ""),
"step_index": s.get("step_index", 0),
"total_steps": s.get("total_steps", 0),
"elapsed_sec": s.get("elapsed_sec", 0),
"health": s.get("health", "healthy"),
"pid": s.get("pid"),
})
# 系统状态
cpu_load = -1
try:
cpu_load = round(os.getloadavg()[0], 2)
except OSError:
pass
# 采集每个账号的登录状态
account_status = {}
if self.account_monitor:
try:
account_status = self.account_monitor.collect_status()
except Exception as e:
logger.debug(f"账号状态采集失败: {e}")
heartbeat = {
"hostname": self.hostname,
"machine_uid": self.machine_uid,
"status": "online",
"last_seen": datetime.now(timezone.utc).isoformat(),
"version": "4.3.0",
"account_status": account_status,
"slots": {
"max": slot_usage.get("max", 3),
"used": slot_usage.get("used", 0),
"list": slot_details,
},
"tasks": {
"active": active_list,
"queued": [
{"task_id": q["task_id"], "priority": q.get("priority", 1),
"estimated_at": q.get("scheduled_at", 0)}
for q in queued[:20]
],
"counts": task_counts,
},
"system": {
"cpu_load_1m": cpu_load,
"browsers_open": slot_usage.get("used", 0),
},
}
return heartbeat
def _task_to_heartbeat(self, task: dict) -> dict:
"""把 task dict 转成心跳用的紧凑格式"""
if not task:
return None
return {
"task_id": task.get("task_id", ""),
"cmd_type": task.get("cmd_type", ""),
"type": task.get("task_type", task.get("cmd_type", "")),
"account": (task.get("accounts") or [""])[0],
"blueprint": task.get("blueprint", ""),
"status": task.get("status", ""),
"started_at": task.get("started_at", 0),
"elapsed_sec": int(time.time() - task.get("started_at", time.time())),
"progress": {
"current_step": task.get("current_step", ""),
"step_index": task.get("step_index", 0),
"total_steps": task.get("total_steps", 0),
},
}
def send_to_dashboard(self, heartbeat: dict = None):
"""发送心跳到 Dashboard"""
if heartbeat is None:
heartbeat = self.collect()
try:
payload = json.dumps(heartbeat, ensure_ascii=False).encode("utf-8")
req = urllib.request.Request(
f"{self.dashboard_url}/api/push/heartbeat",
data=payload,
headers={"Content-Type": "application/json"},
method="POST",
)
urllib.request.urlopen(req, timeout=5)
except Exception as e:
logger.debug(f"心跳推送失败: {e}")
def write_local(self, heartbeat: dict = None):
"""写入本地 heartbeat.json"""
if heartbeat is None:
heartbeat = self.collect()
try:
home = Path.home()
status_dir = home / "workbuddy-agent-os" / "agent-local" / "runtime" / "guardd" / self.hostname
status_dir.mkdir(parents=True, exist_ok=True)
path = status_dir / "heartbeat.json"
path.write_text(json.dumps(heartbeat, indent=2, ensure_ascii=False), encoding="utf-8")
# also write live file
live_dir = status_dir.parent / "live"
live_dir.mkdir(parents=True, exist_ok=True)
(live_dir / f"{self.machine_uid}.json").write_text(
json.dumps(heartbeat, indent=2, ensure_ascii=False), encoding="utf-8"
)
except Exception as e:
logger.error(f"本地心跳写入失败: {e}")