butubb
2112c1a870
移植自 mac-agent-os 的 mediacrawler_adapter：不起子进程、不开页面，用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin，所以 ingest 那条链路一个字不用改。
**目前能用的（真环境实测，非推断）**：
profile/other : 200, 7075 字节 —— 博主主页指标（粉丝/获赞/作品数/昵称）
aweme/detail : 200, 45425 字节 —— 单条作品详情（含点赞/评论/收藏/分享）
**目前不能用的：作品列表 `aweme/post`。** 两个互相独立的原因：
1. 这个接口被抖音单独升级成了真校验：不带 x-tt-argus 回 403「Uifid Not Found」，
带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过，「真校验」过不了。
同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的，
挑中的恰好是「批量拉作品列表」这个最敏感的动作。
2. 改走页面截获也不行：CDP 浏览器打开博主主页会落到「验证码中间页」（当天大量探测的
代价，过几小时要重测）。
所以现在的边界是：**已知作品的指标刷新能做，自动发现新作品做不了**。
**排查中控住变量后得到的两条事实**（都写进注释了）：
· `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
UA+client hints+Cookie 时是 200 但仅 121 字节的空壳，补上这三个头变 7074 字节。
（我先前猜的 sec-ch-ua 不是关键。）
· 因此 UA 与 client hints 必须**成套地取自同一个浏览器**，所以 BrowserIdentity 一次
从 CDP 取齐 cookie + UA + hints，而不是各自写死。
「200 + 空 body 必须当场报错」也是刻意写死的：放过去它会在下游变成「这个博主没作品」，
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的，还被翻译成「账号被封」。
测试 +11：cookie 解析、请求头成套性（含 uifid 缺失/回退）、产物键名与 store 对齐、
以及 _get 的三条失败路径（空 body / 403 带网关原话 / 正常返回）。
455 lines
19 KiB
Python
455 lines
19 KiB
Python
"""
|
||
account_service.py — 统一账号数据聚合层 (Dashboard)
|
||
|
||
职责:
|
||
- 联邦账号中心的数据后端
|
||
- ORACLE.yaml → 账号归属映射（只读）
|
||
- 所有机器统一走 guardd HTTP API (port 9090) 查询登录状态
|
||
- 合并 profile 数据 (profiles.json + fleet_collector)
|
||
|
||
设计原则:
|
||
- ORACLE.yaml 是权威来源，账号归属从 ORACLE 读
|
||
- 所有机器对等，本机不特殊对待
|
||
- 远程状态有 30 秒 TTL 缓存，避免频繁 HTTP 调用
|
||
- 离线机器显示 last_known 状态 + offline 标记
|
||
"""
|
||
import json
|
||
import logging
|
||
import os
|
||
import sys
|
||
import time
|
||
from datetime import datetime, timezone
|
||
from typing import Optional
|
||
from services.command_bus import _guardd_api
|
||
from pathlib import Path
|
||
|
||
logger = logging.getLogger("dashboard.account_service")
|
||
|
||
_THIS_DIR = Path(__file__).resolve().parent.parent
|
||
AGENT_SYNC = Path(os.environ.get("AGENT_SYNC",
|
||
str(Path.home() / "workbuddy-agent-os" / "agent-sync")))
|
||
AGENT_LOCAL = Path(os.environ.get("AGENT_LOCAL",
|
||
str(Path.home() / "workbuddy-agent-os" / "agent-local")))
|
||
GUARDD_PORT = 9090
|
||
|
||
# ── 本机身份 ──
|
||
try:
|
||
sys.path.insert(0, str(AGENT_SYNC / "05_tools" / "10_dashboard"))
|
||
from utils.identity import resolve_hostname
|
||
HOSTNAME = resolve_hostname()
|
||
except Exception:
|
||
HOSTNAME = os.uname().nodename
|
||
|
||
# ── Registry 标签读取 ──
|
||
_REGISTRY_PATH = AGENT_SYNC / "05_tools" / "07_matrix" / "accounts_registry.yaml"
|
||
def _load_registry_tags(account_id: str) -> list:
|
||
"""从 accounts_registry.yaml 读取账号的 tags"""
|
||
if not _REGISTRY_PATH.exists():
|
||
return []
|
||
try:
|
||
import yaml
|
||
data = yaml.safe_load(_REGISTRY_PATH.read_text()) or {}
|
||
for acct in data.get("accounts", []):
|
||
if acct.get("id") == account_id:
|
||
return acct.get("tags", [])
|
||
except Exception:
|
||
pass
|
||
return []
|
||
|
||
|
||
def _load_tags_cache(account_id: str) -> list:
|
||
"""从本地缓存文件读取远程账号的 tags（由 sync_tags.sh 同步）"""
|
||
_CACHE_PATH = AGENT_LOCAL / "data" / "account_tags_cache.json"
|
||
if not _CACHE_PATH.exists():
|
||
return []
|
||
try:
|
||
data = json.loads(_CACHE_PATH.read_text())
|
||
return data.get(account_id, [])
|
||
except Exception:
|
||
return []
|
||
|
||
|
||
class AccountService:
|
||
"""账户统一聚合服务"""
|
||
|
||
def __init__(self):
|
||
self._oracle = None
|
||
self._oracle_loaded = 0
|
||
# 远程状态缓存 {machine_name: {"accounts": {...}, "cached_at": ts}}
|
||
self._remote_cache = {}
|
||
# 远程 profile 缓存 {machine_name: {"profiles": {...}, "cached_at": ts}}
|
||
self._profile_cache = {}
|
||
self._cache_ttl = 300 # 秒（5 分钟）
|
||
|
||
# ═══════════════════════════════════════════════════════════
|
||
# 公开 API
|
||
# ═══════════════════════════════════════════════════════════
|
||
|
||
def get_all_accounts(self) -> list[dict]:
|
||
"""获取所有机器上所有账号的统一视图
|
||
|
||
Returns:
|
||
list[dict]: 每个 dict 包含完整账号信息
|
||
"""
|
||
oracle = self._load_oracle()
|
||
machines = oracle.get("machines", {})
|
||
oracle_accounts = oracle.get("accounts", [])
|
||
|
||
# 1. 构建 ORACLE 账号→机器映射
|
||
# {account_id: {phone, machine, platforms, identity_dir}}
|
||
oracle_map = {}
|
||
for entry in oracle_accounts:
|
||
identity = entry.get("identity", "")
|
||
phone = entry.get("phone", "")
|
||
machine = entry.get("machine") or entry.get("assigned_machine", "")
|
||
platforms = entry.get("platforms", {})
|
||
# 旧格式兼容：platforms 是 {platform: account_id}
|
||
for plat, aid in platforms.items():
|
||
oracle_map[aid] = {
|
||
"phone": phone,
|
||
"identity": identity,
|
||
"machine": machine,
|
||
"platform": plat,
|
||
}
|
||
|
||
# 2. 从本机 MatrixManager 获取本地账号的完整信息（含 profiles）
|
||
local_accounts = self._get_local_accounts_full()
|
||
local_map = {a["id"]: a for a in local_accounts}
|
||
|
||
# 3. 并行查询所有机器的 guardd 状态（含本机）
|
||
all_machines = set()
|
||
for entry in oracle_accounts:
|
||
machine = entry.get("machine") or entry.get("assigned_machine", "")
|
||
if machine:
|
||
all_machines.add(machine)
|
||
# 确保本机也在列表中（用实际 hostname 而非空字符串）
|
||
local_machine = HOSTNAME or "chengzigedeAir"
|
||
all_machines.add(local_machine)
|
||
self._prefetch_remote_statuses(list(all_machines))
|
||
|
||
# 4. 聚合所有账号
|
||
seen = set()
|
||
results = []
|
||
|
||
# 先处理 ORACLE 中定义的所有账号（权威列表）
|
||
for aid, entry in oracle_map.items():
|
||
if aid in seen:
|
||
continue
|
||
seen.add(aid)
|
||
machine = entry["machine"]
|
||
is_local = (machine == HOSTNAME or not machine)
|
||
|
||
acct = {
|
||
"id": aid,
|
||
"phone": entry["phone"],
|
||
"platform": entry["platform"],
|
||
"identity_dir": f"identities/{entry['identity']}" if entry.get("identity") else "",
|
||
"owner_machine": machine or HOSTNAME,
|
||
"is_local": is_local,
|
||
}
|
||
|
||
# 登录状态（所有机器对等：统一走 guardd HTTP API）
|
||
remote_status, status_detail = self._get_remote_account_status(machine, aid)
|
||
acct["login_status"] = remote_status
|
||
if status_detail:
|
||
acct["status_detail"] = status_detail
|
||
# 补充 profile 数据
|
||
if is_local:
|
||
local = local_map.get(aid, {})
|
||
acct["nickname"] = local.get("nickname", "")
|
||
acct["fans"] = local.get("fans", "")
|
||
acct["following"] = local.get("following", "")
|
||
acct["likes"] = local.get("likes", "")
|
||
acct["posts"] = local.get("posts", "")
|
||
acct["avatar"] = local.get("avatar", "")
|
||
acct["_banned"] = local.get("_banned", False)
|
||
acct["_identity_dir_exists"] = local.get("_identity_dir_exists", False)
|
||
acct["notes"] = local.get("notes", "")
|
||
acct["tags"] = local.get("tags", [])
|
||
# 从 registry 补充 tags
|
||
if not acct.get("tags"):
|
||
_reg_tags = _load_registry_tags(aid)
|
||
if _reg_tags:
|
||
acct["tags"] = _reg_tags
|
||
# 远程账号：通过 guardd HTTP 获取 profile（昵称/粉丝/采集信息），再补 tags 缓存
|
||
profile = self._get_profile_for_account(aid, machine)
|
||
acct.update(profile)
|
||
if not acct.get("tags"):
|
||
_cache_tags = _load_tags_cache(aid)
|
||
if _cache_tags:
|
||
acct["tags"] = _cache_tags
|
||
|
||
results.append(acct)
|
||
|
||
# 5. 补全 ORACLE 中没有但本机有的账号（兼容过渡期）
|
||
for local_acct in local_accounts:
|
||
aid = local_acct["id"]
|
||
if aid not in seen:
|
||
seen.add(aid)
|
||
local_acct["is_local"] = True
|
||
local_acct["login_status"] = local_acct.get("_status", "unknown")
|
||
local_acct["_identity_dir_exists"] = local_acct.get("_identity_dir_exists", False)
|
||
results.append(local_acct)
|
||
|
||
# 排序：按机器→平台→账号ID
|
||
machine_order = ["chengzigedeAir", "5kechengdeAir", "7kecheng"]
|
||
def sort_key(a):
|
||
m = a.get("owner_machine", "")
|
||
try:
|
||
mi = machine_order.index(m)
|
||
except ValueError:
|
||
mi = 99
|
||
plat = 0 if a.get("platform") == "douyin" else 1
|
||
return (mi, plat, a["id"])
|
||
results.sort(key=sort_key)
|
||
|
||
return results
|
||
|
||
def get_account_detail(self, account_id: str) -> dict | None:
|
||
"""获取单个账号完整详情（含任务历史、代理信息等）"""
|
||
results = self.get_all_accounts()
|
||
for a in results:
|
||
if a["id"] == account_id:
|
||
return a
|
||
return None
|
||
|
||
def batch_refresh_status(self, account_ids: list[str] = None) -> dict:
|
||
"""强制刷新指定账号（或全部账号）的状态
|
||
|
||
Args:
|
||
account_ids: 如果为 None，刷新所有账号
|
||
|
||
Returns:
|
||
{account_id: status, ...}
|
||
"""
|
||
oracle = self._load_oracle()
|
||
machines = oracle.get("machines", {})
|
||
|
||
# 按机器分组
|
||
by_machine = {}
|
||
for aid in (account_ids or []):
|
||
machine = self._get_account_machine(aid)
|
||
if machine not in by_machine:
|
||
by_machine[machine] = []
|
||
by_machine[machine].append(aid)
|
||
|
||
results = {}
|
||
for machine, aids in by_machine.items():
|
||
if machine == HOSTNAME or not machine:
|
||
# 本机：用 MatrixManager
|
||
try:
|
||
sys.path.insert(0, str(AGENT_SYNC / "05_tools" / "07_matrix" / "scripts"))
|
||
from matrix_mgmt import MatrixManager
|
||
mgr = MatrixManager()
|
||
for aid in aids:
|
||
acct = mgr.get_account(aid)
|
||
results[aid] = acct.get("_status", "unknown") if acct else "unknown"
|
||
except Exception as e:
|
||
logger.warning(f"本机刷新状态失败: {e}")
|
||
for aid in aids:
|
||
results[aid] = "unknown"
|
||
else:
|
||
# 远程：清缓存 + 重新查询
|
||
self._remote_cache.pop(machine, None)
|
||
statuses = self._query_remote_statuses(machine)
|
||
for aid in aids:
|
||
results[aid] = statuses.get(aid, "offline")
|
||
|
||
return results
|
||
|
||
# ═══════════════════════════════════════════════════════════
|
||
# 内部方法
|
||
# ═══════════════════════════════════════════════════════════
|
||
|
||
def _load_oracle(self) -> dict:
|
||
"""读取 ORACLE.yaml（只读）"""
|
||
now = time.time()
|
||
if self._oracle and now - self._oracle_loaded < 60:
|
||
return self._oracle
|
||
path = AGENT_SYNC / "ORACLE.yaml"
|
||
if not path.exists():
|
||
logger.warning("ORACLE.yaml 不存在")
|
||
self._oracle = {"machines": {}, "accounts": []}
|
||
self._oracle_loaded = now
|
||
return self._oracle
|
||
try:
|
||
import yaml
|
||
self._oracle = yaml.safe_load(path.read_text()) or {}
|
||
self._oracle_loaded = now
|
||
return self._oracle
|
||
except Exception as e:
|
||
logger.warning(f"ORACLE.yaml 解析失败: {e}")
|
||
return self._oracle or {"machines": {}, "accounts": []}
|
||
|
||
def _get_account_machine(self, account_id: str) -> str:
|
||
"""从 ORACLE 查账号归属机器"""
|
||
oracle = self._load_oracle()
|
||
for entry in oracle.get("accounts", []):
|
||
platforms = entry.get("platforms", {})
|
||
for plat, aid in platforms.items():
|
||
if aid == account_id:
|
||
return entry.get("machine") or entry.get("assigned_machine", "")
|
||
# 也检查 identity 字段
|
||
if entry.get("identity", "").replace("phone_", "") == account_id.replace("douyin_", "").replace("xhs_", ""):
|
||
return entry.get("machine") or entry.get("assigned_machine", "")
|
||
return HOSTNAME
|
||
|
||
def _get_local_accounts_full(self) -> list[dict]:
|
||
"""获取本机账号完整信息（从 MatrixManager）"""
|
||
try:
|
||
sys.path.insert(0, str(AGENT_SYNC / "05_tools" / "07_matrix" / "scripts"))
|
||
from matrix_mgmt import MatrixManager
|
||
mgr = MatrixManager()
|
||
accounts = mgr.list_accounts()
|
||
|
||
# 合并 profiles.json（通过 guardd API，对等原则）
|
||
profiles = {}
|
||
try:
|
||
data = _guardd_api("GET", "/accounts/profiles")
|
||
if isinstance(data, dict):
|
||
profiles = data.get("profiles", {})
|
||
except Exception:
|
||
pass
|
||
|
||
for acct in accounts:
|
||
aid = acct.get("id", "")
|
||
pid = acct.get("identity_dir", "").replace("identities/", "")
|
||
profile = profiles.get(aid) or profiles.get(pid) or {}
|
||
if profile:
|
||
acct.setdefault("nickname", profile.get("nickname", ""))
|
||
acct.setdefault("fans", profile.get("fans", ""))
|
||
acct.setdefault("avatar", profile.get("avatar", ""))
|
||
acct.setdefault("following", profile.get("following", ""))
|
||
acct.setdefault("likes", profile.get("likes", ""))
|
||
acct.setdefault("posts", profile.get("posts", ""))
|
||
if profile.get("status") == "banned":
|
||
acct["_banned"] = True
|
||
return accounts
|
||
except Exception as e:
|
||
logger.warning(f"获取本机账号失败: {e}")
|
||
return []
|
||
|
||
def _get_remote_account_status(self, machine: str, account_id: str):
|
||
"""获取单个远程账号的登录状态 + 详情
|
||
Returns: (status_str, detail_str)
|
||
"""
|
||
statuses, details = self._get_all_remote_statuses(machine)
|
||
detail = details.get(account_id, "") if details else ""
|
||
return (statuses.get(account_id, "unknown"), detail)
|
||
|
||
def _prefetch_remote_statuses(self, machines: list[str]):
|
||
"""并行预取远程机器状态
|
||
|
||
使用 ThreadPoolExecutor 并行查询所有远程机器，
|
||
避免逐个查询导致的总延迟累加。
|
||
"""
|
||
if not machines:
|
||
return
|
||
now = time.time()
|
||
# 只查缓存过期或未缓存的机器
|
||
to_fetch = [m for m in machines
|
||
if m not in self._remote_cache
|
||
or now - self._remote_cache[m].get("cached_at", 0) >= self._cache_ttl]
|
||
if not to_fetch:
|
||
return
|
||
|
||
logger.info(f"并行查询远程状态: {to_fetch}")
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
with ThreadPoolExecutor(max_workers=len(to_fetch)) as pool:
|
||
future_map = {pool.submit(self._query_remote_statuses, m): m for m in to_fetch}
|
||
for future in as_completed(future_map):
|
||
machine = future_map[future]
|
||
try:
|
||
statuses, details = future.result()
|
||
self._remote_cache[machine] = {
|
||
"accounts": statuses,
|
||
"details": details,
|
||
"cached_at": now,
|
||
}
|
||
logger.debug(f"远程状态已缓存: {machine} ({len(statuses)} 个账号)")
|
||
except Exception as e:
|
||
logger.debug(f"远程状态查询失败 {machine}: {e}")
|
||
self._remote_cache[machine] = {"accounts": {}, "cached_at": now}
|
||
|
||
def _get_all_remote_statuses(self, machine: str):
|
||
"""获取一台远程机器上所有账号的状态 + 详情
|
||
使用 30 秒 TTL 缓存
|
||
Returns: (accounts_dict, details_dict)
|
||
"""
|
||
now = time.time()
|
||
cached = self._remote_cache.get(machine)
|
||
if cached and now - cached.get("cached_at", 0) < self._cache_ttl:
|
||
return (cached.get("accounts", {}), cached.get("details", {}))
|
||
|
||
statuses, details = self._query_remote_statuses(machine)
|
||
self._remote_cache[machine] = {
|
||
"accounts": statuses,
|
||
"details": details,
|
||
"cached_at": now,
|
||
}
|
||
return (statuses, details)
|
||
|
||
def _query_remote_statuses(self, machine: str):
|
||
"""通过 guardd HTTP API 查询远程机器账号状态 + 详情
|
||
Returns: (accounts_dict, details_dict)
|
||
"""
|
||
if not machine:
|
||
return ({}, {})
|
||
|
||
from services.command_bus import _guardd_api
|
||
# Tailscale 网络抖动容错：加大 timeout + 重试一次（默认 5s 对跨机大响应不够）
|
||
data = {}
|
||
for _ in range(2):
|
||
try:
|
||
data = _guardd_api("GET", "/accounts/status", machine=machine, timeout=15)
|
||
if isinstance(data, dict) and data.get("accounts"):
|
||
break
|
||
except Exception:
|
||
pass
|
||
if isinstance(data, dict):
|
||
return (data.get("accounts", {}), data.get("details", {}))
|
||
return ({}, {})
|
||
|
||
def _get_profile_for_account(self, account_id: str, machine: str) -> dict:
|
||
"""通过 guardd HTTP API 获取账号 profile 数据（带机器级缓存）"""
|
||
try:
|
||
# 检查缓存
|
||
now = time.time()
|
||
cached = self._profile_cache.get(machine)
|
||
if not cached or now - cached.get("cached_at", 0) >= self._cache_ttl:
|
||
# Tailscale 网络抖动容错：profiles 响应较大（~12KB，多次 recv），
|
||
# 默认 5s timeout 易超时 → 加大到 15s + 重试一次
|
||
data = {}
|
||
for _ in range(2):
|
||
try:
|
||
data = _guardd_api("GET", "/accounts/profiles", machine=machine, timeout=15)
|
||
if isinstance(data, dict) and data.get("profiles"):
|
||
break
|
||
except Exception:
|
||
pass
|
||
profiles = data.get("profiles", {}) if isinstance(data, dict) else {}
|
||
if profiles:
|
||
self._profile_cache[machine] = {"profiles": profiles, "cached_at": now}
|
||
elif machine in self._profile_cache:
|
||
# 查询失败：沿用上次成功的缓存（不覆盖为空，避免整片昵称消失）
|
||
profiles = self._profile_cache[machine].get("profiles", {})
|
||
logger.warning(" ⚠️ %s profiles 查询失败，沿用缓存（%d 项）", machine, len(profiles))
|
||
else:
|
||
# 首次即失败：不写缓存（留待下次请求重试），本次返回空
|
||
logger.warning(" ⚠️ %s profiles 查询失败（无缓存可沿用）", machine)
|
||
else:
|
||
profiles = cached.get("profiles", {})
|
||
p = profiles.get(account_id, {})
|
||
if p and p.get("nickname"):
|
||
return {
|
||
"nickname": p.get("nickname", ""),
|
||
"fans": str(p.get("fans", "")),
|
||
"avatar": p.get("avatar", ""),
|
||
"following": str(p.get("following", "")),
|
||
"likes": str(p.get("likes", "")),
|
||
"posts": str(p.get("posts", "")),
|
||
}
|
||
except Exception:
|
||
pass
|
||
return {"nickname": "", "fans": "", "following": "", "likes": "", "posts": "", "avatar": ""}
|