feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程,直接发 HTTP
Deploy VitePress site to Pages / build (push) Canceled after 0s
Deploy VitePress site to Pages / Deploy (push) Canceled after 0s

移植自 mac-agent-os 的 mediacrawler_adapter:不起子进程、不开页面,用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin,所以 ingest 那条链路一个字不用改。

**目前能用的(真环境实测,非推断)**:

    profile/other : 200, 7075 字节  —— 博主主页指标(粉丝/获赞/作品数/昵称)
    aweme/detail  : 200, 45425 字节 —— 单条作品详情(含点赞/评论/收藏/分享)

**目前不能用的:作品列表 `aweme/post`。** 两个互相独立的原因:
  1. 这个接口被抖音单独升级成了真校验:不带 x-tt-argus 回 403「Uifid Not Found」,
     带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过,「真校验」过不了。
     同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的,
     挑中的恰好是「批量拉作品列表」这个最敏感的动作。
  2. 改走页面截获也不行:CDP 浏览器打开博主主页会落到「验证码中间页」(当天大量探测的
     代价,过几小时要重测)。

  所以现在的边界是:**已知作品的指标刷新能做,自动发现新作品做不了**。

**排查中控住变量后得到的两条事实**(都写进注释了):
  · `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
    UA+client hints+Cookie 时是 200 但仅 121 字节的空壳,补上这三个头变 7074 字节。
    (我先前猜的 sec-ch-ua 不是关键。)
  · 因此 UA 与 client hints 必须**成套地取自同一个浏览器**,所以 BrowserIdentity 一次
    从 CDP 取齐 cookie + UA + hints,而不是各自写死。

「200 + 空 body 必须当场报错」也是刻意写死的:放过去它会在下游变成「这个博主没作品」,
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的,还被翻译成「账号被封」。

测试 +11:cookie 解析、请求头成套性(含 uifid 缺失/回退)、产物键名与 store 对齐、
以及 _get 的三条失败路径(空 body / 403 带网关原话 / 正常返回)。
This commit is contained in:
2026-10-10 17:12:25 +08:00
parent 21ff01b894
commit 2112c1a870
1601 changed files with 367425 additions and 0 deletions
@@ -0,0 +1,454 @@
"""
account_service.py — 统一账号数据聚合层 (Dashboard)
职责:
- 联邦账号中心的数据后端
- ORACLE.yaml → 账号归属映射(只读)
- 所有机器统一走 guardd HTTP API (port 9090) 查询登录状态
- 合并 profile 数据 (profiles.json + fleet_collector)
设计原则:
- ORACLE.yaml 是权威来源,账号归属从 ORACLE 读
- 所有机器对等,本机不特殊对待
- 远程状态有 30 秒 TTL 缓存,避免频繁 HTTP 调用
- 离线机器显示 last_known 状态 + offline 标记
"""
import json
import logging
import os
import sys
import time
from datetime import datetime, timezone
from typing import Optional
from services.command_bus import _guardd_api
from pathlib import Path
logger = logging.getLogger("dashboard.account_service")
_THIS_DIR = Path(__file__).resolve().parent.parent
AGENT_SYNC = Path(os.environ.get("AGENT_SYNC",
str(Path.home() / "workbuddy-agent-os" / "agent-sync")))
AGENT_LOCAL = Path(os.environ.get("AGENT_LOCAL",
str(Path.home() / "workbuddy-agent-os" / "agent-local")))
GUARDD_PORT = 9090
# ── 本机身份 ──
try:
sys.path.insert(0, str(AGENT_SYNC / "05_tools" / "10_dashboard"))
from utils.identity import resolve_hostname
HOSTNAME = resolve_hostname()
except Exception:
HOSTNAME = os.uname().nodename
# ── Registry 标签读取 ──
_REGISTRY_PATH = AGENT_SYNC / "05_tools" / "07_matrix" / "accounts_registry.yaml"
def _load_registry_tags(account_id: str) -> list:
"""从 accounts_registry.yaml 读取账号的 tags"""
if not _REGISTRY_PATH.exists():
return []
try:
import yaml
data = yaml.safe_load(_REGISTRY_PATH.read_text()) or {}
for acct in data.get("accounts", []):
if acct.get("id") == account_id:
return acct.get("tags", [])
except Exception:
pass
return []
def _load_tags_cache(account_id: str) -> list:
"""从本地缓存文件读取远程账号的 tags(由 sync_tags.sh 同步)"""
_CACHE_PATH = AGENT_LOCAL / "data" / "account_tags_cache.json"
if not _CACHE_PATH.exists():
return []
try:
data = json.loads(_CACHE_PATH.read_text())
return data.get(account_id, [])
except Exception:
return []
class AccountService:
"""账户统一聚合服务"""
def __init__(self):
self._oracle = None
self._oracle_loaded = 0
# 远程状态缓存 {machine_name: {"accounts": {...}, "cached_at": ts}}
self._remote_cache = {}
# 远程 profile 缓存 {machine_name: {"profiles": {...}, "cached_at": ts}}
self._profile_cache = {}
self._cache_ttl = 300 # 秒(5 分钟)
# ═══════════════════════════════════════════════════════════
# 公开 API
# ═══════════════════════════════════════════════════════════
def get_all_accounts(self) -> list[dict]:
"""获取所有机器上所有账号的统一视图
Returns:
list[dict]: 每个 dict 包含完整账号信息
"""
oracle = self._load_oracle()
machines = oracle.get("machines", {})
oracle_accounts = oracle.get("accounts", [])
# 1. 构建 ORACLE 账号→机器映射
# {account_id: {phone, machine, platforms, identity_dir}}
oracle_map = {}
for entry in oracle_accounts:
identity = entry.get("identity", "")
phone = entry.get("phone", "")
machine = entry.get("machine") or entry.get("assigned_machine", "")
platforms = entry.get("platforms", {})
# 旧格式兼容:platforms 是 {platform: account_id}
for plat, aid in platforms.items():
oracle_map[aid] = {
"phone": phone,
"identity": identity,
"machine": machine,
"platform": plat,
}
# 2. 从本机 MatrixManager 获取本地账号的完整信息(含 profiles)
local_accounts = self._get_local_accounts_full()
local_map = {a["id"]: a for a in local_accounts}
# 3. 并行查询所有机器的 guardd 状态(含本机)
all_machines = set()
for entry in oracle_accounts:
machine = entry.get("machine") or entry.get("assigned_machine", "")
if machine:
all_machines.add(machine)
# 确保本机也在列表中(用实际 hostname 而非空字符串)
local_machine = HOSTNAME or "chengzigedeAir"
all_machines.add(local_machine)
self._prefetch_remote_statuses(list(all_machines))
# 4. 聚合所有账号
seen = set()
results = []
# 先处理 ORACLE 中定义的所有账号(权威列表)
for aid, entry in oracle_map.items():
if aid in seen:
continue
seen.add(aid)
machine = entry["machine"]
is_local = (machine == HOSTNAME or not machine)
acct = {
"id": aid,
"phone": entry["phone"],
"platform": entry["platform"],
"identity_dir": f"identities/{entry['identity']}" if entry.get("identity") else "",
"owner_machine": machine or HOSTNAME,
"is_local": is_local,
}
# 登录状态(所有机器对等:统一走 guardd HTTP API)
remote_status, status_detail = self._get_remote_account_status(machine, aid)
acct["login_status"] = remote_status
if status_detail:
acct["status_detail"] = status_detail
# 补充 profile 数据
if is_local:
local = local_map.get(aid, {})
acct["nickname"] = local.get("nickname", "")
acct["fans"] = local.get("fans", "")
acct["following"] = local.get("following", "")
acct["likes"] = local.get("likes", "")
acct["posts"] = local.get("posts", "")
acct["avatar"] = local.get("avatar", "")
acct["_banned"] = local.get("_banned", False)
acct["_identity_dir_exists"] = local.get("_identity_dir_exists", False)
acct["notes"] = local.get("notes", "")
acct["tags"] = local.get("tags", [])
# 从 registry 补充 tags
if not acct.get("tags"):
_reg_tags = _load_registry_tags(aid)
if _reg_tags:
acct["tags"] = _reg_tags
# 远程账号:通过 guardd HTTP 获取 profile(昵称/粉丝/采集信息),再补 tags 缓存
profile = self._get_profile_for_account(aid, machine)
acct.update(profile)
if not acct.get("tags"):
_cache_tags = _load_tags_cache(aid)
if _cache_tags:
acct["tags"] = _cache_tags
results.append(acct)
# 5. 补全 ORACLE 中没有但本机有的账号(兼容过渡期)
for local_acct in local_accounts:
aid = local_acct["id"]
if aid not in seen:
seen.add(aid)
local_acct["is_local"] = True
local_acct["login_status"] = local_acct.get("_status", "unknown")
local_acct["_identity_dir_exists"] = local_acct.get("_identity_dir_exists", False)
results.append(local_acct)
# 排序:按机器→平台→账号ID
machine_order = ["chengzigedeAir", "5kechengdeAir", "7kecheng"]
def sort_key(a):
m = a.get("owner_machine", "")
try:
mi = machine_order.index(m)
except ValueError:
mi = 99
plat = 0 if a.get("platform") == "douyin" else 1
return (mi, plat, a["id"])
results.sort(key=sort_key)
return results
def get_account_detail(self, account_id: str) -> dict | None:
"""获取单个账号完整详情(含任务历史、代理信息等)"""
results = self.get_all_accounts()
for a in results:
if a["id"] == account_id:
return a
return None
def batch_refresh_status(self, account_ids: list[str] = None) -> dict:
"""强制刷新指定账号(或全部账号)的状态
Args:
account_ids: 如果为 None,刷新所有账号
Returns:
{account_id: status, ...}
"""
oracle = self._load_oracle()
machines = oracle.get("machines", {})
# 按机器分组
by_machine = {}
for aid in (account_ids or []):
machine = self._get_account_machine(aid)
if machine not in by_machine:
by_machine[machine] = []
by_machine[machine].append(aid)
results = {}
for machine, aids in by_machine.items():
if machine == HOSTNAME or not machine:
# 本机:用 MatrixManager
try:
sys.path.insert(0, str(AGENT_SYNC / "05_tools" / "07_matrix" / "scripts"))
from matrix_mgmt import MatrixManager
mgr = MatrixManager()
for aid in aids:
acct = mgr.get_account(aid)
results[aid] = acct.get("_status", "unknown") if acct else "unknown"
except Exception as e:
logger.warning(f"本机刷新状态失败: {e}")
for aid in aids:
results[aid] = "unknown"
else:
# 远程:清缓存 + 重新查询
self._remote_cache.pop(machine, None)
statuses = self._query_remote_statuses(machine)
for aid in aids:
results[aid] = statuses.get(aid, "offline")
return results
# ═══════════════════════════════════════════════════════════
# 内部方法
# ═══════════════════════════════════════════════════════════
def _load_oracle(self) -> dict:
"""读取 ORACLE.yaml(只读)"""
now = time.time()
if self._oracle and now - self._oracle_loaded < 60:
return self._oracle
path = AGENT_SYNC / "ORACLE.yaml"
if not path.exists():
logger.warning("ORACLE.yaml 不存在")
self._oracle = {"machines": {}, "accounts": []}
self._oracle_loaded = now
return self._oracle
try:
import yaml
self._oracle = yaml.safe_load(path.read_text()) or {}
self._oracle_loaded = now
return self._oracle
except Exception as e:
logger.warning(f"ORACLE.yaml 解析失败: {e}")
return self._oracle or {"machines": {}, "accounts": []}
def _get_account_machine(self, account_id: str) -> str:
"""从 ORACLE 查账号归属机器"""
oracle = self._load_oracle()
for entry in oracle.get("accounts", []):
platforms = entry.get("platforms", {})
for plat, aid in platforms.items():
if aid == account_id:
return entry.get("machine") or entry.get("assigned_machine", "")
# 也检查 identity 字段
if entry.get("identity", "").replace("phone_", "") == account_id.replace("douyin_", "").replace("xhs_", ""):
return entry.get("machine") or entry.get("assigned_machine", "")
return HOSTNAME
def _get_local_accounts_full(self) -> list[dict]:
"""获取本机账号完整信息(从 MatrixManager)"""
try:
sys.path.insert(0, str(AGENT_SYNC / "05_tools" / "07_matrix" / "scripts"))
from matrix_mgmt import MatrixManager
mgr = MatrixManager()
accounts = mgr.list_accounts()
# 合并 profiles.json(通过 guardd API,对等原则)
profiles = {}
try:
data = _guardd_api("GET", "/accounts/profiles")
if isinstance(data, dict):
profiles = data.get("profiles", {})
except Exception:
pass
for acct in accounts:
aid = acct.get("id", "")
pid = acct.get("identity_dir", "").replace("identities/", "")
profile = profiles.get(aid) or profiles.get(pid) or {}
if profile:
acct.setdefault("nickname", profile.get("nickname", ""))
acct.setdefault("fans", profile.get("fans", ""))
acct.setdefault("avatar", profile.get("avatar", ""))
acct.setdefault("following", profile.get("following", ""))
acct.setdefault("likes", profile.get("likes", ""))
acct.setdefault("posts", profile.get("posts", ""))
if profile.get("status") == "banned":
acct["_banned"] = True
return accounts
except Exception as e:
logger.warning(f"获取本机账号失败: {e}")
return []
def _get_remote_account_status(self, machine: str, account_id: str):
"""获取单个远程账号的登录状态 + 详情
Returns: (status_str, detail_str)
"""
statuses, details = self._get_all_remote_statuses(machine)
detail = details.get(account_id, "") if details else ""
return (statuses.get(account_id, "unknown"), detail)
def _prefetch_remote_statuses(self, machines: list[str]):
"""并行预取远程机器状态
使用 ThreadPoolExecutor 并行查询所有远程机器,
避免逐个查询导致的总延迟累加。
"""
if not machines:
return
now = time.time()
# 只查缓存过期或未缓存的机器
to_fetch = [m for m in machines
if m not in self._remote_cache
or now - self._remote_cache[m].get("cached_at", 0) >= self._cache_ttl]
if not to_fetch:
return
logger.info(f"并行查询远程状态: {to_fetch}")
from concurrent.futures import ThreadPoolExecutor, as_completed
with ThreadPoolExecutor(max_workers=len(to_fetch)) as pool:
future_map = {pool.submit(self._query_remote_statuses, m): m for m in to_fetch}
for future in as_completed(future_map):
machine = future_map[future]
try:
statuses, details = future.result()
self._remote_cache[machine] = {
"accounts": statuses,
"details": details,
"cached_at": now,
}
logger.debug(f"远程状态已缓存: {machine} ({len(statuses)} 个账号)")
except Exception as e:
logger.debug(f"远程状态查询失败 {machine}: {e}")
self._remote_cache[machine] = {"accounts": {}, "cached_at": now}
def _get_all_remote_statuses(self, machine: str):
"""获取一台远程机器上所有账号的状态 + 详情
使用 30 秒 TTL 缓存
Returns: (accounts_dict, details_dict)
"""
now = time.time()
cached = self._remote_cache.get(machine)
if cached and now - cached.get("cached_at", 0) < self._cache_ttl:
return (cached.get("accounts", {}), cached.get("details", {}))
statuses, details = self._query_remote_statuses(machine)
self._remote_cache[machine] = {
"accounts": statuses,
"details": details,
"cached_at": now,
}
return (statuses, details)
def _query_remote_statuses(self, machine: str):
"""通过 guardd HTTP API 查询远程机器账号状态 + 详情
Returns: (accounts_dict, details_dict)
"""
if not machine:
return ({}, {})
from services.command_bus import _guardd_api
# Tailscale 网络抖动容错:加大 timeout + 重试一次(默认 5s 对跨机大响应不够)
data = {}
for _ in range(2):
try:
data = _guardd_api("GET", "/accounts/status", machine=machine, timeout=15)
if isinstance(data, dict) and data.get("accounts"):
break
except Exception:
pass
if isinstance(data, dict):
return (data.get("accounts", {}), data.get("details", {}))
return ({}, {})
def _get_profile_for_account(self, account_id: str, machine: str) -> dict:
"""通过 guardd HTTP API 获取账号 profile 数据(带机器级缓存)"""
try:
# 检查缓存
now = time.time()
cached = self._profile_cache.get(machine)
if not cached or now - cached.get("cached_at", 0) >= self._cache_ttl:
# Tailscale 网络抖动容错:profiles 响应较大(~12KB,多次 recv),
# 默认 5s timeout 易超时 → 加大到 15s + 重试一次
data = {}
for _ in range(2):
try:
data = _guardd_api("GET", "/accounts/profiles", machine=machine, timeout=15)
if isinstance(data, dict) and data.get("profiles"):
break
except Exception:
pass
profiles = data.get("profiles", {}) if isinstance(data, dict) else {}
if profiles:
self._profile_cache[machine] = {"profiles": profiles, "cached_at": now}
elif machine in self._profile_cache:
# 查询失败:沿用上次成功的缓存(不覆盖为空,避免整片昵称消失)
profiles = self._profile_cache[machine].get("profiles", {})
logger.warning(" ⚠️ %s profiles 查询失败,沿用缓存(%d 项)", machine, len(profiles))
else:
# 首次即失败:不写缓存(留待下次请求重试),本次返回空
logger.warning(" ⚠️ %s profiles 查询失败(无缓存可沿用)", machine)
else:
profiles = cached.get("profiles", {})
p = profiles.get(account_id, {})
if p and p.get("nickname"):
return {
"nickname": p.get("nickname", ""),
"fans": str(p.get("fans", "")),
"avatar": p.get("avatar", ""),
"following": str(p.get("following", "")),
"likes": str(p.get("likes", "")),
"posts": str(p.get("posts", "")),
}
except Exception:
pass
return {"nickname": "", "fans": "", "following": "", "likes": "", "posts": "", "avatar": ""}