feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程,直接发 HTTP
移植自 mac-agent-os 的 mediacrawler_adapter:不起子进程、不开页面,用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin,所以 ingest 那条链路一个字不用改。
**目前能用的(真环境实测,非推断)**:
profile/other : 200, 7075 字节 —— 博主主页指标(粉丝/获赞/作品数/昵称)
aweme/detail : 200, 45425 字节 —— 单条作品详情(含点赞/评论/收藏/分享)
**目前不能用的:作品列表 `aweme/post`。** 两个互相独立的原因:
1. 这个接口被抖音单独升级成了真校验:不带 x-tt-argus 回 403「Uifid Not Found」,
带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过,「真校验」过不了。
同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的,
挑中的恰好是「批量拉作品列表」这个最敏感的动作。
2. 改走页面截获也不行:CDP 浏览器打开博主主页会落到「验证码中间页」(当天大量探测的
代价,过几小时要重测)。
所以现在的边界是:**已知作品的指标刷新能做,自动发现新作品做不了**。
**排查中控住变量后得到的两条事实**(都写进注释了):
· `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
UA+client hints+Cookie 时是 200 但仅 121 字节的空壳,补上这三个头变 7074 字节。
(我先前猜的 sec-ch-ua 不是关键。)
· 因此 UA 与 client hints 必须**成套地取自同一个浏览器**,所以 BrowserIdentity 一次
从 CDP 取齐 cookie + UA + hints,而不是各自写死。
「200 + 空 body 必须当场报错」也是刻意写死的:放过去它会在下游变成「这个博主没作品」,
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的,还被翻译成「账号被封」。
测试 +11:cookie 解析、请求头成套性(含 uifid 缺失/回退)、产物键名与 store 对齐、
以及 _get 的三条失败路径(空 body / 403 带网关原话 / 正常返回)。
This commit is contained in:
@@ -0,0 +1,454 @@
|
||||
"""
|
||||
account_service.py — 统一账号数据聚合层 (Dashboard)
|
||||
|
||||
职责:
|
||||
- 联邦账号中心的数据后端
|
||||
- ORACLE.yaml → 账号归属映射(只读)
|
||||
- 所有机器统一走 guardd HTTP API (port 9090) 查询登录状态
|
||||
- 合并 profile 数据 (profiles.json + fleet_collector)
|
||||
|
||||
设计原则:
|
||||
- ORACLE.yaml 是权威来源,账号归属从 ORACLE 读
|
||||
- 所有机器对等,本机不特殊对待
|
||||
- 远程状态有 30 秒 TTL 缓存,避免频繁 HTTP 调用
|
||||
- 离线机器显示 last_known 状态 + offline 标记
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
from services.command_bus import _guardd_api
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger("dashboard.account_service")
|
||||
|
||||
_THIS_DIR = Path(__file__).resolve().parent.parent
|
||||
AGENT_SYNC = Path(os.environ.get("AGENT_SYNC",
|
||||
str(Path.home() / "workbuddy-agent-os" / "agent-sync")))
|
||||
AGENT_LOCAL = Path(os.environ.get("AGENT_LOCAL",
|
||||
str(Path.home() / "workbuddy-agent-os" / "agent-local")))
|
||||
GUARDD_PORT = 9090
|
||||
|
||||
# ── 本机身份 ──
|
||||
try:
|
||||
sys.path.insert(0, str(AGENT_SYNC / "05_tools" / "10_dashboard"))
|
||||
from utils.identity import resolve_hostname
|
||||
HOSTNAME = resolve_hostname()
|
||||
except Exception:
|
||||
HOSTNAME = os.uname().nodename
|
||||
|
||||
# ── Registry 标签读取 ──
|
||||
_REGISTRY_PATH = AGENT_SYNC / "05_tools" / "07_matrix" / "accounts_registry.yaml"
|
||||
def _load_registry_tags(account_id: str) -> list:
|
||||
"""从 accounts_registry.yaml 读取账号的 tags"""
|
||||
if not _REGISTRY_PATH.exists():
|
||||
return []
|
||||
try:
|
||||
import yaml
|
||||
data = yaml.safe_load(_REGISTRY_PATH.read_text()) or {}
|
||||
for acct in data.get("accounts", []):
|
||||
if acct.get("id") == account_id:
|
||||
return acct.get("tags", [])
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
|
||||
def _load_tags_cache(account_id: str) -> list:
|
||||
"""从本地缓存文件读取远程账号的 tags(由 sync_tags.sh 同步)"""
|
||||
_CACHE_PATH = AGENT_LOCAL / "data" / "account_tags_cache.json"
|
||||
if not _CACHE_PATH.exists():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(_CACHE_PATH.read_text())
|
||||
return data.get(account_id, [])
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
class AccountService:
|
||||
"""账户统一聚合服务"""
|
||||
|
||||
def __init__(self):
|
||||
self._oracle = None
|
||||
self._oracle_loaded = 0
|
||||
# 远程状态缓存 {machine_name: {"accounts": {...}, "cached_at": ts}}
|
||||
self._remote_cache = {}
|
||||
# 远程 profile 缓存 {machine_name: {"profiles": {...}, "cached_at": ts}}
|
||||
self._profile_cache = {}
|
||||
self._cache_ttl = 300 # 秒(5 分钟)
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# 公开 API
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
def get_all_accounts(self) -> list[dict]:
|
||||
"""获取所有机器上所有账号的统一视图
|
||||
|
||||
Returns:
|
||||
list[dict]: 每个 dict 包含完整账号信息
|
||||
"""
|
||||
oracle = self._load_oracle()
|
||||
machines = oracle.get("machines", {})
|
||||
oracle_accounts = oracle.get("accounts", [])
|
||||
|
||||
# 1. 构建 ORACLE 账号→机器映射
|
||||
# {account_id: {phone, machine, platforms, identity_dir}}
|
||||
oracle_map = {}
|
||||
for entry in oracle_accounts:
|
||||
identity = entry.get("identity", "")
|
||||
phone = entry.get("phone", "")
|
||||
machine = entry.get("machine") or entry.get("assigned_machine", "")
|
||||
platforms = entry.get("platforms", {})
|
||||
# 旧格式兼容:platforms 是 {platform: account_id}
|
||||
for plat, aid in platforms.items():
|
||||
oracle_map[aid] = {
|
||||
"phone": phone,
|
||||
"identity": identity,
|
||||
"machine": machine,
|
||||
"platform": plat,
|
||||
}
|
||||
|
||||
# 2. 从本机 MatrixManager 获取本地账号的完整信息(含 profiles)
|
||||
local_accounts = self._get_local_accounts_full()
|
||||
local_map = {a["id"]: a for a in local_accounts}
|
||||
|
||||
# 3. 并行查询所有机器的 guardd 状态(含本机)
|
||||
all_machines = set()
|
||||
for entry in oracle_accounts:
|
||||
machine = entry.get("machine") or entry.get("assigned_machine", "")
|
||||
if machine:
|
||||
all_machines.add(machine)
|
||||
# 确保本机也在列表中(用实际 hostname 而非空字符串)
|
||||
local_machine = HOSTNAME or "chengzigedeAir"
|
||||
all_machines.add(local_machine)
|
||||
self._prefetch_remote_statuses(list(all_machines))
|
||||
|
||||
# 4. 聚合所有账号
|
||||
seen = set()
|
||||
results = []
|
||||
|
||||
# 先处理 ORACLE 中定义的所有账号(权威列表)
|
||||
for aid, entry in oracle_map.items():
|
||||
if aid in seen:
|
||||
continue
|
||||
seen.add(aid)
|
||||
machine = entry["machine"]
|
||||
is_local = (machine == HOSTNAME or not machine)
|
||||
|
||||
acct = {
|
||||
"id": aid,
|
||||
"phone": entry["phone"],
|
||||
"platform": entry["platform"],
|
||||
"identity_dir": f"identities/{entry['identity']}" if entry.get("identity") else "",
|
||||
"owner_machine": machine or HOSTNAME,
|
||||
"is_local": is_local,
|
||||
}
|
||||
|
||||
# 登录状态(所有机器对等:统一走 guardd HTTP API)
|
||||
remote_status, status_detail = self._get_remote_account_status(machine, aid)
|
||||
acct["login_status"] = remote_status
|
||||
if status_detail:
|
||||
acct["status_detail"] = status_detail
|
||||
# 补充 profile 数据
|
||||
if is_local:
|
||||
local = local_map.get(aid, {})
|
||||
acct["nickname"] = local.get("nickname", "")
|
||||
acct["fans"] = local.get("fans", "")
|
||||
acct["following"] = local.get("following", "")
|
||||
acct["likes"] = local.get("likes", "")
|
||||
acct["posts"] = local.get("posts", "")
|
||||
acct["avatar"] = local.get("avatar", "")
|
||||
acct["_banned"] = local.get("_banned", False)
|
||||
acct["_identity_dir_exists"] = local.get("_identity_dir_exists", False)
|
||||
acct["notes"] = local.get("notes", "")
|
||||
acct["tags"] = local.get("tags", [])
|
||||
# 从 registry 补充 tags
|
||||
if not acct.get("tags"):
|
||||
_reg_tags = _load_registry_tags(aid)
|
||||
if _reg_tags:
|
||||
acct["tags"] = _reg_tags
|
||||
# 远程账号:通过 guardd HTTP 获取 profile(昵称/粉丝/采集信息),再补 tags 缓存
|
||||
profile = self._get_profile_for_account(aid, machine)
|
||||
acct.update(profile)
|
||||
if not acct.get("tags"):
|
||||
_cache_tags = _load_tags_cache(aid)
|
||||
if _cache_tags:
|
||||
acct["tags"] = _cache_tags
|
||||
|
||||
results.append(acct)
|
||||
|
||||
# 5. 补全 ORACLE 中没有但本机有的账号(兼容过渡期)
|
||||
for local_acct in local_accounts:
|
||||
aid = local_acct["id"]
|
||||
if aid not in seen:
|
||||
seen.add(aid)
|
||||
local_acct["is_local"] = True
|
||||
local_acct["login_status"] = local_acct.get("_status", "unknown")
|
||||
local_acct["_identity_dir_exists"] = local_acct.get("_identity_dir_exists", False)
|
||||
results.append(local_acct)
|
||||
|
||||
# 排序:按机器→平台→账号ID
|
||||
machine_order = ["chengzigedeAir", "5kechengdeAir", "7kecheng"]
|
||||
def sort_key(a):
|
||||
m = a.get("owner_machine", "")
|
||||
try:
|
||||
mi = machine_order.index(m)
|
||||
except ValueError:
|
||||
mi = 99
|
||||
plat = 0 if a.get("platform") == "douyin" else 1
|
||||
return (mi, plat, a["id"])
|
||||
results.sort(key=sort_key)
|
||||
|
||||
return results
|
||||
|
||||
def get_account_detail(self, account_id: str) -> dict | None:
|
||||
"""获取单个账号完整详情(含任务历史、代理信息等)"""
|
||||
results = self.get_all_accounts()
|
||||
for a in results:
|
||||
if a["id"] == account_id:
|
||||
return a
|
||||
return None
|
||||
|
||||
def batch_refresh_status(self, account_ids: list[str] = None) -> dict:
|
||||
"""强制刷新指定账号(或全部账号)的状态
|
||||
|
||||
Args:
|
||||
account_ids: 如果为 None,刷新所有账号
|
||||
|
||||
Returns:
|
||||
{account_id: status, ...}
|
||||
"""
|
||||
oracle = self._load_oracle()
|
||||
machines = oracle.get("machines", {})
|
||||
|
||||
# 按机器分组
|
||||
by_machine = {}
|
||||
for aid in (account_ids or []):
|
||||
machine = self._get_account_machine(aid)
|
||||
if machine not in by_machine:
|
||||
by_machine[machine] = []
|
||||
by_machine[machine].append(aid)
|
||||
|
||||
results = {}
|
||||
for machine, aids in by_machine.items():
|
||||
if machine == HOSTNAME or not machine:
|
||||
# 本机:用 MatrixManager
|
||||
try:
|
||||
sys.path.insert(0, str(AGENT_SYNC / "05_tools" / "07_matrix" / "scripts"))
|
||||
from matrix_mgmt import MatrixManager
|
||||
mgr = MatrixManager()
|
||||
for aid in aids:
|
||||
acct = mgr.get_account(aid)
|
||||
results[aid] = acct.get("_status", "unknown") if acct else "unknown"
|
||||
except Exception as e:
|
||||
logger.warning(f"本机刷新状态失败: {e}")
|
||||
for aid in aids:
|
||||
results[aid] = "unknown"
|
||||
else:
|
||||
# 远程:清缓存 + 重新查询
|
||||
self._remote_cache.pop(machine, None)
|
||||
statuses = self._query_remote_statuses(machine)
|
||||
for aid in aids:
|
||||
results[aid] = statuses.get(aid, "offline")
|
||||
|
||||
return results
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# 内部方法
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
def _load_oracle(self) -> dict:
|
||||
"""读取 ORACLE.yaml(只读)"""
|
||||
now = time.time()
|
||||
if self._oracle and now - self._oracle_loaded < 60:
|
||||
return self._oracle
|
||||
path = AGENT_SYNC / "ORACLE.yaml"
|
||||
if not path.exists():
|
||||
logger.warning("ORACLE.yaml 不存在")
|
||||
self._oracle = {"machines": {}, "accounts": []}
|
||||
self._oracle_loaded = now
|
||||
return self._oracle
|
||||
try:
|
||||
import yaml
|
||||
self._oracle = yaml.safe_load(path.read_text()) or {}
|
||||
self._oracle_loaded = now
|
||||
return self._oracle
|
||||
except Exception as e:
|
||||
logger.warning(f"ORACLE.yaml 解析失败: {e}")
|
||||
return self._oracle or {"machines": {}, "accounts": []}
|
||||
|
||||
def _get_account_machine(self, account_id: str) -> str:
|
||||
"""从 ORACLE 查账号归属机器"""
|
||||
oracle = self._load_oracle()
|
||||
for entry in oracle.get("accounts", []):
|
||||
platforms = entry.get("platforms", {})
|
||||
for plat, aid in platforms.items():
|
||||
if aid == account_id:
|
||||
return entry.get("machine") or entry.get("assigned_machine", "")
|
||||
# 也检查 identity 字段
|
||||
if entry.get("identity", "").replace("phone_", "") == account_id.replace("douyin_", "").replace("xhs_", ""):
|
||||
return entry.get("machine") or entry.get("assigned_machine", "")
|
||||
return HOSTNAME
|
||||
|
||||
def _get_local_accounts_full(self) -> list[dict]:
|
||||
"""获取本机账号完整信息(从 MatrixManager)"""
|
||||
try:
|
||||
sys.path.insert(0, str(AGENT_SYNC / "05_tools" / "07_matrix" / "scripts"))
|
||||
from matrix_mgmt import MatrixManager
|
||||
mgr = MatrixManager()
|
||||
accounts = mgr.list_accounts()
|
||||
|
||||
# 合并 profiles.json(通过 guardd API,对等原则)
|
||||
profiles = {}
|
||||
try:
|
||||
data = _guardd_api("GET", "/accounts/profiles")
|
||||
if isinstance(data, dict):
|
||||
profiles = data.get("profiles", {})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for acct in accounts:
|
||||
aid = acct.get("id", "")
|
||||
pid = acct.get("identity_dir", "").replace("identities/", "")
|
||||
profile = profiles.get(aid) or profiles.get(pid) or {}
|
||||
if profile:
|
||||
acct.setdefault("nickname", profile.get("nickname", ""))
|
||||
acct.setdefault("fans", profile.get("fans", ""))
|
||||
acct.setdefault("avatar", profile.get("avatar", ""))
|
||||
acct.setdefault("following", profile.get("following", ""))
|
||||
acct.setdefault("likes", profile.get("likes", ""))
|
||||
acct.setdefault("posts", profile.get("posts", ""))
|
||||
if profile.get("status") == "banned":
|
||||
acct["_banned"] = True
|
||||
return accounts
|
||||
except Exception as e:
|
||||
logger.warning(f"获取本机账号失败: {e}")
|
||||
return []
|
||||
|
||||
def _get_remote_account_status(self, machine: str, account_id: str):
|
||||
"""获取单个远程账号的登录状态 + 详情
|
||||
Returns: (status_str, detail_str)
|
||||
"""
|
||||
statuses, details = self._get_all_remote_statuses(machine)
|
||||
detail = details.get(account_id, "") if details else ""
|
||||
return (statuses.get(account_id, "unknown"), detail)
|
||||
|
||||
def _prefetch_remote_statuses(self, machines: list[str]):
|
||||
"""并行预取远程机器状态
|
||||
|
||||
使用 ThreadPoolExecutor 并行查询所有远程机器,
|
||||
避免逐个查询导致的总延迟累加。
|
||||
"""
|
||||
if not machines:
|
||||
return
|
||||
now = time.time()
|
||||
# 只查缓存过期或未缓存的机器
|
||||
to_fetch = [m for m in machines
|
||||
if m not in self._remote_cache
|
||||
or now - self._remote_cache[m].get("cached_at", 0) >= self._cache_ttl]
|
||||
if not to_fetch:
|
||||
return
|
||||
|
||||
logger.info(f"并行查询远程状态: {to_fetch}")
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
with ThreadPoolExecutor(max_workers=len(to_fetch)) as pool:
|
||||
future_map = {pool.submit(self._query_remote_statuses, m): m for m in to_fetch}
|
||||
for future in as_completed(future_map):
|
||||
machine = future_map[future]
|
||||
try:
|
||||
statuses, details = future.result()
|
||||
self._remote_cache[machine] = {
|
||||
"accounts": statuses,
|
||||
"details": details,
|
||||
"cached_at": now,
|
||||
}
|
||||
logger.debug(f"远程状态已缓存: {machine} ({len(statuses)} 个账号)")
|
||||
except Exception as e:
|
||||
logger.debug(f"远程状态查询失败 {machine}: {e}")
|
||||
self._remote_cache[machine] = {"accounts": {}, "cached_at": now}
|
||||
|
||||
def _get_all_remote_statuses(self, machine: str):
|
||||
"""获取一台远程机器上所有账号的状态 + 详情
|
||||
使用 30 秒 TTL 缓存
|
||||
Returns: (accounts_dict, details_dict)
|
||||
"""
|
||||
now = time.time()
|
||||
cached = self._remote_cache.get(machine)
|
||||
if cached and now - cached.get("cached_at", 0) < self._cache_ttl:
|
||||
return (cached.get("accounts", {}), cached.get("details", {}))
|
||||
|
||||
statuses, details = self._query_remote_statuses(machine)
|
||||
self._remote_cache[machine] = {
|
||||
"accounts": statuses,
|
||||
"details": details,
|
||||
"cached_at": now,
|
||||
}
|
||||
return (statuses, details)
|
||||
|
||||
def _query_remote_statuses(self, machine: str):
|
||||
"""通过 guardd HTTP API 查询远程机器账号状态 + 详情
|
||||
Returns: (accounts_dict, details_dict)
|
||||
"""
|
||||
if not machine:
|
||||
return ({}, {})
|
||||
|
||||
from services.command_bus import _guardd_api
|
||||
# Tailscale 网络抖动容错:加大 timeout + 重试一次(默认 5s 对跨机大响应不够)
|
||||
data = {}
|
||||
for _ in range(2):
|
||||
try:
|
||||
data = _guardd_api("GET", "/accounts/status", machine=machine, timeout=15)
|
||||
if isinstance(data, dict) and data.get("accounts"):
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
if isinstance(data, dict):
|
||||
return (data.get("accounts", {}), data.get("details", {}))
|
||||
return ({}, {})
|
||||
|
||||
def _get_profile_for_account(self, account_id: str, machine: str) -> dict:
|
||||
"""通过 guardd HTTP API 获取账号 profile 数据(带机器级缓存)"""
|
||||
try:
|
||||
# 检查缓存
|
||||
now = time.time()
|
||||
cached = self._profile_cache.get(machine)
|
||||
if not cached or now - cached.get("cached_at", 0) >= self._cache_ttl:
|
||||
# Tailscale 网络抖动容错:profiles 响应较大(~12KB,多次 recv),
|
||||
# 默认 5s timeout 易超时 → 加大到 15s + 重试一次
|
||||
data = {}
|
||||
for _ in range(2):
|
||||
try:
|
||||
data = _guardd_api("GET", "/accounts/profiles", machine=machine, timeout=15)
|
||||
if isinstance(data, dict) and data.get("profiles"):
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
profiles = data.get("profiles", {}) if isinstance(data, dict) else {}
|
||||
if profiles:
|
||||
self._profile_cache[machine] = {"profiles": profiles, "cached_at": now}
|
||||
elif machine in self._profile_cache:
|
||||
# 查询失败:沿用上次成功的缓存(不覆盖为空,避免整片昵称消失)
|
||||
profiles = self._profile_cache[machine].get("profiles", {})
|
||||
logger.warning(" ⚠️ %s profiles 查询失败,沿用缓存(%d 项)", machine, len(profiles))
|
||||
else:
|
||||
# 首次即失败:不写缓存(留待下次请求重试),本次返回空
|
||||
logger.warning(" ⚠️ %s profiles 查询失败(无缓存可沿用)", machine)
|
||||
else:
|
||||
profiles = cached.get("profiles", {})
|
||||
p = profiles.get(account_id, {})
|
||||
if p and p.get("nickname"):
|
||||
return {
|
||||
"nickname": p.get("nickname", ""),
|
||||
"fans": str(p.get("fans", "")),
|
||||
"avatar": p.get("avatar", ""),
|
||||
"following": str(p.get("following", "")),
|
||||
"likes": str(p.get("likes", "")),
|
||||
"posts": str(p.get("posts", "")),
|
||||
}
|
||||
except Exception:
|
||||
pass
|
||||
return {"nickname": "", "fans": "", "following": "", "likes": "", "posts": "", "avatar": ""}
|
||||
Reference in New Issue
Block a user