feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程,直接发 HTTP
移植自 mac-agent-os 的 mediacrawler_adapter:不起子进程、不开页面,用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin,所以 ingest 那条链路一个字不用改。
**目前能用的(真环境实测,非推断)**:
profile/other : 200, 7075 字节 —— 博主主页指标(粉丝/获赞/作品数/昵称)
aweme/detail : 200, 45425 字节 —— 单条作品详情(含点赞/评论/收藏/分享)
**目前不能用的:作品列表 `aweme/post`。** 两个互相独立的原因:
1. 这个接口被抖音单独升级成了真校验:不带 x-tt-argus 回 403「Uifid Not Found」,
带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过,「真校验」过不了。
同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的,
挑中的恰好是「批量拉作品列表」这个最敏感的动作。
2. 改走页面截获也不行:CDP 浏览器打开博主主页会落到「验证码中间页」(当天大量探测的
代价,过几小时要重测)。
所以现在的边界是:**已知作品的指标刷新能做,自动发现新作品做不了**。
**排查中控住变量后得到的两条事实**(都写进注释了):
· `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
UA+client hints+Cookie 时是 200 但仅 121 字节的空壳,补上这三个头变 7074 字节。
(我先前猜的 sec-ch-ua 不是关键。)
· 因此 UA 与 client hints 必须**成套地取自同一个浏览器**,所以 BrowserIdentity 一次
从 CDP 取齐 cookie + UA + hints,而不是各自写死。
「200 + 空 body 必须当场报错」也是刻意写死的:放过去它会在下游变成「这个博主没作品」,
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的,还被翻译成「账号被封」。
测试 +11:cookie 解析、请求头成套性(含 uifid 缺失/回退)、产物键名与 store 对齐、
以及 _get 的三条失败路径(空 body / 403 带网关原话 / 正常返回)。
This commit is contained in:
@@ -0,0 +1,169 @@
|
||||
"""
|
||||
heartbeat.py — 增强心跳上报 (guardd 模块)
|
||||
|
||||
职责:
|
||||
- 每15秒采集系统状态 + 任务状态 + 槽位状态
|
||||
- 上报到 Dashboard (POST /api/push/heartbeat)
|
||||
- 本地写入 status/{hostname}/heartbeat.json
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from modules.task_store import TaskStore
|
||||
from modules.slot_manager import BrowserSlotManager
|
||||
from modules.scheduler import Scheduler
|
||||
from modules.account_monitor import AccountMonitor
|
||||
|
||||
logger = logging.getLogger("guardd.heartbeat")
|
||||
|
||||
|
||||
class HeartbeatReporter:
|
||||
"""增强心跳上报器"""
|
||||
|
||||
def __init__(self, task_store: TaskStore, slot_manager: BrowserSlotManager,
|
||||
scheduler: Scheduler, hostname: str, machine_uid: str,
|
||||
dashboard_url: str = None,
|
||||
account_monitor: AccountMonitor = None):
|
||||
self.task_store = task_store
|
||||
self.slot_manager = slot_manager
|
||||
self.scheduler = scheduler
|
||||
self.hostname = hostname
|
||||
self.machine_uid = machine_uid
|
||||
self.dashboard_url = dashboard_url or "http://127.0.0.1:9988"
|
||||
self.account_monitor = account_monitor
|
||||
|
||||
def collect(self) -> dict:
|
||||
"""采集完整状态(v2: 含 per-slot 详细进度)"""
|
||||
slot_usage = self.slot_manager.get_usage() if self.slot_manager else {"max": 0, "used": 0, "slots": []}
|
||||
|
||||
# 收集任务状态(v2: active_tasks 字典,支持多 slot 并行)
|
||||
active_tasks = list(self.scheduler.active_tasks.values()) if hasattr(self.scheduler, 'active_tasks') and self.scheduler.active_tasks else []
|
||||
active_list = []
|
||||
for t in active_tasks:
|
||||
at = self._task_to_heartbeat(t)
|
||||
if at:
|
||||
active_list.append(at)
|
||||
queued = self.scheduler.get_all_queued() if hasattr(self.scheduler, 'get_all_queued') else []
|
||||
task_counts = self.task_store.count()
|
||||
|
||||
# 增强 slot 信息:从 slot_manager 获取每个浏览器的当前步骤/时间
|
||||
slot_details = []
|
||||
for s in slot_usage.get("slots", []):
|
||||
if s.get("account_id"):
|
||||
slot_details.append({
|
||||
"slot_id": s.get("slot_id"),
|
||||
"account_id": s.get("account_id"),
|
||||
"nickname": s.get("nickname", ""),
|
||||
"platform": s.get("platform", ""),
|
||||
"current_step": s.get("current_step", ""),
|
||||
"step_index": s.get("step_index", 0),
|
||||
"total_steps": s.get("total_steps", 0),
|
||||
"elapsed_sec": s.get("elapsed_sec", 0),
|
||||
"health": s.get("health", "healthy"),
|
||||
"pid": s.get("pid"),
|
||||
})
|
||||
|
||||
# 系统状态
|
||||
cpu_load = -1
|
||||
try:
|
||||
cpu_load = round(os.getloadavg()[0], 2)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# 采集每个账号的登录状态
|
||||
account_status = {}
|
||||
if self.account_monitor:
|
||||
try:
|
||||
account_status = self.account_monitor.collect_status()
|
||||
except Exception as e:
|
||||
logger.debug(f"账号状态采集失败: {e}")
|
||||
|
||||
heartbeat = {
|
||||
"hostname": self.hostname,
|
||||
"machine_uid": self.machine_uid,
|
||||
"status": "online",
|
||||
"last_seen": datetime.now(timezone.utc).isoformat(),
|
||||
"version": "4.3.0",
|
||||
"account_status": account_status,
|
||||
"slots": {
|
||||
"max": slot_usage.get("max", 3),
|
||||
"used": slot_usage.get("used", 0),
|
||||
"list": slot_details,
|
||||
},
|
||||
"tasks": {
|
||||
"active": active_list,
|
||||
"queued": [
|
||||
{"task_id": q["task_id"], "priority": q.get("priority", 1),
|
||||
"estimated_at": q.get("scheduled_at", 0)}
|
||||
for q in queued[:20]
|
||||
],
|
||||
"counts": task_counts,
|
||||
},
|
||||
"system": {
|
||||
"cpu_load_1m": cpu_load,
|
||||
"browsers_open": slot_usage.get("used", 0),
|
||||
},
|
||||
}
|
||||
return heartbeat
|
||||
|
||||
def _task_to_heartbeat(self, task: dict) -> dict:
|
||||
"""把 task dict 转成心跳用的紧凑格式"""
|
||||
if not task:
|
||||
return None
|
||||
return {
|
||||
"task_id": task.get("task_id", ""),
|
||||
"cmd_type": task.get("cmd_type", ""),
|
||||
"type": task.get("task_type", task.get("cmd_type", "")),
|
||||
"account": (task.get("accounts") or [""])[0],
|
||||
"blueprint": task.get("blueprint", ""),
|
||||
"status": task.get("status", ""),
|
||||
"started_at": task.get("started_at", 0),
|
||||
"elapsed_sec": int(time.time() - task.get("started_at", time.time())),
|
||||
"progress": {
|
||||
"current_step": task.get("current_step", ""),
|
||||
"step_index": task.get("step_index", 0),
|
||||
"total_steps": task.get("total_steps", 0),
|
||||
},
|
||||
}
|
||||
|
||||
def send_to_dashboard(self, heartbeat: dict = None):
|
||||
"""发送心跳到 Dashboard"""
|
||||
if heartbeat is None:
|
||||
heartbeat = self.collect()
|
||||
|
||||
try:
|
||||
payload = json.dumps(heartbeat, ensure_ascii=False).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
f"{self.dashboard_url}/api/push/heartbeat",
|
||||
data=payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
urllib.request.urlopen(req, timeout=5)
|
||||
except Exception as e:
|
||||
logger.debug(f"心跳推送失败: {e}")
|
||||
|
||||
def write_local(self, heartbeat: dict = None):
|
||||
"""写入本地 heartbeat.json"""
|
||||
if heartbeat is None:
|
||||
heartbeat = self.collect()
|
||||
try:
|
||||
home = Path.home()
|
||||
status_dir = home / "workbuddy-agent-os" / "agent-local" / "runtime" / "guardd" / self.hostname
|
||||
status_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = status_dir / "heartbeat.json"
|
||||
path.write_text(json.dumps(heartbeat, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
# also write live file
|
||||
live_dir = status_dir.parent / "live"
|
||||
live_dir.mkdir(parents=True, exist_ok=True)
|
||||
(live_dir / f"{self.machine_uid}.json").write_text(
|
||||
json.dumps(heartbeat, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"本地心跳写入失败: {e}")
|
||||
Reference in New Issue
Block a user