feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程,直接发 HTTP
Deploy VitePress site to Pages / build (push) Canceled after 0s
Deploy VitePress site to Pages / Deploy (push) Canceled after 0s

移植自 mac-agent-os 的 mediacrawler_adapter:不起子进程、不开页面,用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin,所以 ingest 那条链路一个字不用改。

**目前能用的(真环境实测,非推断)**:

    profile/other : 200, 7075 字节  —— 博主主页指标(粉丝/获赞/作品数/昵称)
    aweme/detail  : 200, 45425 字节 —— 单条作品详情(含点赞/评论/收藏/分享)

**目前不能用的:作品列表 `aweme/post`。** 两个互相独立的原因:
  1. 这个接口被抖音单独升级成了真校验:不带 x-tt-argus 回 403「Uifid Not Found」,
     带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过,「真校验」过不了。
     同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的,
     挑中的恰好是「批量拉作品列表」这个最敏感的动作。
  2. 改走页面截获也不行:CDP 浏览器打开博主主页会落到「验证码中间页」(当天大量探测的
     代价,过几小时要重测)。

  所以现在的边界是:**已知作品的指标刷新能做,自动发现新作品做不了**。

**排查中控住变量后得到的两条事实**(都写进注释了):
  · `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
    UA+client hints+Cookie 时是 200 但仅 121 字节的空壳,补上这三个头变 7074 字节。
    (我先前猜的 sec-ch-ua 不是关键。)
  · 因此 UA 与 client hints 必须**成套地取自同一个浏览器**,所以 BrowserIdentity 一次
    从 CDP 取齐 cookie + UA + hints,而不是各自写死。

「200 + 空 body 必须当场报错」也是刻意写死的:放过去它会在下游变成「这个博主没作品」,
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的,还被翻译成「账号被封」。

测试 +11:cookie 解析、请求头成套性(含 uifid 缺失/回退)、产物键名与 store 对齐、
以及 _get 的三条失败路径(空 body / 403 带网关原话 / 正常返回)。
This commit is contained in:
2026-10-10 17:12:25 +08:00
parent 21ff01b894
commit 2112c1a870
1601 changed files with 367425 additions and 0 deletions
@@ -0,0 +1,231 @@
#!/usr/bin/env python3
"""
capability_doctor.py — 能力体检(一条命令验证「清单说的 ready 是不是真能跑」)
背景:capabilities.yaml 是能力唯一真相源,但清单可能与实际脱节
(实测曾发现:frame_chain 代码早已实现却报 missing;
lipsync 实际用百炼却登记成 fal)。
本工具做静态体检,把「名不副实」的能力揪出来。
用法:
python capability_doctor.py # 全部能力体检
python capability_doctor.py --capability lipsync
python capability_doctor.py --json # 机器可读(供 Dashboard)
python capability_doctor.py --only-issues # 只看有问题的
检查项(全部零成本,不调用任何付费接口):
1. impl 文件是否存在(清单 impl 字段指向的代码)
2. provider 的 api_key 是否已配置(api_key_ref → config.yaml + local.yaml)
3. 状态一致性:status=ready 但没有任何可用 provider → ✗ 名不副实
4. 结构完整性:是否声明了 providers / status
退出码:0 = 无问题;1 = 存在 ⚠️ / ✗
"""
import argparse
import json
import re
import sys
from pathlib import Path
# ── 定位 ──
_SCRIPTS = Path(__file__).resolve().parent
_AVE = _SCRIPTS.parent # 05_tools/09_ave
if str(_SCRIPTS) not in sys.path:
sys.path.insert(0, str(_SCRIPTS))
try:
import yaml
except ImportError:
print("需要 pyyaml:pip install pyyaml")
sys.exit(2)
CAP_FILE = _AVE / "config" / "capabilities.yaml"
PLACEHOLDERS = {"", "sk-xxx", "xxx", "your_key", "your_openai_api_key", "omlx", "changeme", "todo"}
def load_caps() -> list:
d = yaml.safe_load(CAP_FILE.read_text(encoding="utf-8")) or {}
return d.get("capabilities") or []
def load_config_merged() -> dict:
"""合并 config.yaml(模板)与 local.yaml(本机真实密钥,优先级更高)"""
merged: dict = {}
def _merge(src: dict):
for k, v in (src or {}).items():
if isinstance(v, dict) and isinstance(merged.get(k), dict):
merged[k].update(v)
else:
merged[k] = v
tmpl = _AVE / "config.yaml"
if tmpl.exists():
try:
_merge(yaml.safe_load(tmpl.read_text(encoding="utf-8")) or {})
except Exception:
pass
local = Path.home() / "workbuddy-agent-os/agent-local/tools/ave/config/local.yaml"
if local.exists():
try:
_merge(yaml.safe_load(local.read_text(encoding="utf-8")) or {})
except Exception:
pass
return merged
def _get_by_path(cfg: dict, dotted: str):
cur = cfg
for part in (dotted or "").split("."):
if not isinstance(cur, dict) or part not in cur:
return None
cur = cur[part]
return cur
def _key_ok(val) -> bool:
if val is None:
return False
s = str(val).strip()
return bool(s) and s.lower() not in PLACEHOLDERS and not s.startswith("${")
def _impl_targets(impl: str) -> list:
"""从 impl 描述提取可核对的路径(文件或目录都可)
· 'composer/lipsync.py(lipsync_videoretalk)' → ['composer/lipsync.py']
· 'music_selector/' → ['music_selector']
· 'assets_manager/ + character_generator/' → ['assets_manager', 'character_generator']
"""
if not impl:
return []
files = re.findall(r"([\w./-]+\.py)", impl)
if files:
return files
dirs = re.findall(r"([\w][\w./-]*)/", impl)
seen, out = set(), []
for d in dirs:
d = d.rstrip('/')
if d and d not in seen:
seen.add(d)
out.append(d)
return out or [impl.strip()]
def check_capability(cap: dict, cfg: dict) -> dict:
"""单个能力的静态体检结果"""
cid = cap.get("id", "?")
status = (cap.get("status") or "").strip() or "(未声明)"
providers = cap.get("providers") or []
issues: list[str] = []
warns: list[str] = []
# ① impl 实现(文件或目录)
impl = cap.get("impl") or ""
targets = _impl_targets(impl)
impl_ok = None
if targets:
found = []
missing = []
for rel in targets:
cands = [_SCRIPTS / rel, _SCRIPTS / "capabilities" / rel, _AVE / rel]
(found if any(c.exists() for c in cands) else missing).append(rel)
impl_ok = bool(found)
# planned / missing 能力允许尚无实现;ready/partial 缺实现才是问题
if missing and status in ("ready", "partial"):
warns.append(f"impl 未找到: {', '.join(missing[:3])}")
elif status in ("ready", "partial"):
warns.append("未声明 impl(无法核对实现)")
# ② provider 的 key 配置情况
prov_ok: list[str] = []
prov_missing: list[str] = []
for p in providers:
ref = p.get("api_key_ref") or ""
pid = p.get("id") or "?"
if not ref:
prov_ok.append(f"{pid}(无需key)") # 本地实现(ffmpeg 等)
continue
val = _get_by_path(cfg, ref)
if _key_ok(val):
prov_ok.append(pid)
else:
prov_missing.append(f"{pid}({ref})")
# ③ 状态一致性
if status == "ready":
if providers and not prov_ok:
issues.append(f"status=ready 但没有任何可用 provider(缺: {', '.join(prov_missing)})")
if impl_ok is False:
issues.append("status=ready 但 impl 文件不存在")
elif status == "partial":
if providers and not prov_ok:
warns.append(f"partial 且全部 provider 不可用(缺: {', '.join(prov_missing)})")
elif status in ("planned", "missing", ""):
pass
# ④ 结构完整性
if not providers:
warns.append("未声明 providers")
verdict = "✗" if issues else ("⚠️" if warns else "✓")
return {
"id": cid,
"name": cap.get("name", ""),
"status": status,
"impl": impl,
"impl_ok": impl_ok,
"providers_ok": prov_ok,
"providers_missing": prov_missing,
"issues": issues,
"warns": warns,
"verdict": verdict,
}
def main():
ap = argparse.ArgumentParser(description="能力体检(静态,零成本)")
ap.add_argument("--capability", help="只查某个能力 id")
ap.add_argument("--json", action="store_true", help="输出 JSON")
ap.add_argument("--only-issues", action="store_true", help="只显示 ⚠️/✗")
args = ap.parse_args()
caps = load_caps()
cfg = load_config_merged()
if args.capability:
caps = [c for c in caps if c.get("id") == args.capability]
if not caps:
print(f"未找到能力: {args.capability}")
return 2
results = [check_capability(c, cfg) for c in caps]
if args.only_issues:
results = [r for r in results if r["verdict"] != "✓"]
if args.json:
print(json.dumps({"total": len(results), "results": results}, ensure_ascii=False, indent=1))
else:
bad = [r for r in results if r["verdict"] == "✗"]
warn = [r for r in results if r["verdict"] == "⚠️"]
good = [r for r in results if r["verdict"] == "✓"]
print(f"\n能力体检(共 {len(results)} 项) ✓ {len(good)} · ⚠️ {len(warn)} · ✗ {len(bad)}\n")
for r in results:
if args.only_issues and r["verdict"] == "✓":
continue
line = f" {r['verdict']} {r['id']:26s} [{r['status']:8s}]"
if r["providers_ok"]:
line += f" ✓{','.join(r['providers_ok'])[:36]}"
if r["providers_missing"]:
line += f" ✗{','.join(r['providers_missing'])[:36]}"
print(line)
for m in r["issues"]:
print(f" ✗ {m}")
for m in r["warns"]:
print(f" · {m}")
print()
return 1 if any(r["verdict"] == "✗" for r in results) else 0
if __name__ == "__main__":
sys.exit(main())