Files
MediaCrawler/mac-agent-os-main/05_tools/09_ave/scripts/capability_doctor.py
T
butubb 2112c1a870
Deploy VitePress site to Pages / build (push) Canceled after 0s
Deploy VitePress site to Pages / Deploy (push) Canceled after 0s
feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程,直接发 HTTP
移植自 mac-agent-os 的 mediacrawler_adapter:不起子进程、不开页面,用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin,所以 ingest 那条链路一个字不用改。

**目前能用的(真环境实测,非推断)**:

    profile/other : 200, 7075 字节  —— 博主主页指标(粉丝/获赞/作品数/昵称)
    aweme/detail  : 200, 45425 字节 —— 单条作品详情(含点赞/评论/收藏/分享)

**目前不能用的:作品列表 `aweme/post`。** 两个互相独立的原因:
  1. 这个接口被抖音单独升级成了真校验:不带 x-tt-argus 回 403「Uifid Not Found」,
     带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过,「真校验」过不了。
     同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的,
     挑中的恰好是「批量拉作品列表」这个最敏感的动作。
  2. 改走页面截获也不行:CDP 浏览器打开博主主页会落到「验证码中间页」(当天大量探测的
     代价,过几小时要重测)。

  所以现在的边界是:**已知作品的指标刷新能做,自动发现新作品做不了**。

**排查中控住变量后得到的两条事实**(都写进注释了):
  · `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
    UA+client hints+Cookie 时是 200 但仅 121 字节的空壳,补上这三个头变 7074 字节。
    (我先前猜的 sec-ch-ua 不是关键。)
  · 因此 UA 与 client hints 必须**成套地取自同一个浏览器**,所以 BrowserIdentity 一次
    从 CDP 取齐 cookie + UA + hints,而不是各自写死。

「200 + 空 body 必须当场报错」也是刻意写死的:放过去它会在下游变成「这个博主没作品」,
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的,还被翻译成「账号被封」。

测试 +11:cookie 解析、请求头成套性(含 uifid 缺失/回退)、产物键名与 store 对齐、
以及 _get 的三条失败路径(空 body / 403 带网关原话 / 正常返回)。
2026-10-10 17:12:25 +08:00

232 lines
7.9 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""
capability_doctor.py — 能力体检(一条命令验证「清单说的 ready 是不是真能跑」)
背景:capabilities.yaml 是能力唯一真相源,但清单可能与实际脱节
(实测曾发现:frame_chain 代码早已实现却报 missing;
lipsync 实际用百炼却登记成 fal)。
本工具做静态体检,把「名不副实」的能力揪出来。
用法:
python capability_doctor.py # 全部能力体检
python capability_doctor.py --capability lipsync
python capability_doctor.py --json # 机器可读(供 Dashboard)
python capability_doctor.py --only-issues # 只看有问题的
检查项(全部零成本,不调用任何付费接口):
1. impl 文件是否存在(清单 impl 字段指向的代码)
2. provider 的 api_key 是否已配置(api_key_ref → config.yaml + local.yaml)
3. 状态一致性:status=ready 但没有任何可用 provider → ✗ 名不副实
4. 结构完整性:是否声明了 providers / status
退出码:0 = 无问题;1 = 存在 ⚠️ / ✗
"""
import argparse
import json
import re
import sys
from pathlib import Path
# ── 定位 ──
_SCRIPTS = Path(__file__).resolve().parent
_AVE = _SCRIPTS.parent # 05_tools/09_ave
if str(_SCRIPTS) not in sys.path:
sys.path.insert(0, str(_SCRIPTS))
try:
import yaml
except ImportError:
print("需要 pyyaml:pip install pyyaml")
sys.exit(2)
CAP_FILE = _AVE / "config" / "capabilities.yaml"
PLACEHOLDERS = {"", "sk-xxx", "xxx", "your_key", "your_openai_api_key", "omlx", "changeme", "todo"}
def load_caps() -> list:
d = yaml.safe_load(CAP_FILE.read_text(encoding="utf-8")) or {}
return d.get("capabilities") or []
def load_config_merged() -> dict:
"""合并 config.yaml(模板)与 local.yaml(本机真实密钥,优先级更高)"""
merged: dict = {}
def _merge(src: dict):
for k, v in (src or {}).items():
if isinstance(v, dict) and isinstance(merged.get(k), dict):
merged[k].update(v)
else:
merged[k] = v
tmpl = _AVE / "config.yaml"
if tmpl.exists():
try:
_merge(yaml.safe_load(tmpl.read_text(encoding="utf-8")) or {})
except Exception:
pass
local = Path.home() / "workbuddy-agent-os/agent-local/tools/ave/config/local.yaml"
if local.exists():
try:
_merge(yaml.safe_load(local.read_text(encoding="utf-8")) or {})
except Exception:
pass
return merged
def _get_by_path(cfg: dict, dotted: str):
cur = cfg
for part in (dotted or "").split("."):
if not isinstance(cur, dict) or part not in cur:
return None
cur = cur[part]
return cur
def _key_ok(val) -> bool:
if val is None:
return False
s = str(val).strip()
return bool(s) and s.lower() not in PLACEHOLDERS and not s.startswith("${")
def _impl_targets(impl: str) -> list:
"""从 impl 描述提取可核对的路径(文件或目录都可)
· 'composer/lipsync.py(lipsync_videoretalk)' → ['composer/lipsync.py']
· 'music_selector/' → ['music_selector']
· 'assets_manager/ + character_generator/' → ['assets_manager', 'character_generator']
"""
if not impl:
return []
files = re.findall(r"([\w./-]+\.py)", impl)
if files:
return files
dirs = re.findall(r"([\w][\w./-]*)/", impl)
seen, out = set(), []
for d in dirs:
d = d.rstrip('/')
if d and d not in seen:
seen.add(d)
out.append(d)
return out or [impl.strip()]
def check_capability(cap: dict, cfg: dict) -> dict:
"""单个能力的静态体检结果"""
cid = cap.get("id", "?")
status = (cap.get("status") or "").strip() or "(未声明)"
providers = cap.get("providers") or []
issues: list[str] = []
warns: list[str] = []
# ① impl 实现(文件或目录)
impl = cap.get("impl") or ""
targets = _impl_targets(impl)
impl_ok = None
if targets:
found = []
missing = []
for rel in targets:
cands = [_SCRIPTS / rel, _SCRIPTS / "capabilities" / rel, _AVE / rel]
(found if any(c.exists() for c in cands) else missing).append(rel)
impl_ok = bool(found)
# planned / missing 能力允许尚无实现;ready/partial 缺实现才是问题
if missing and status in ("ready", "partial"):
warns.append(f"impl 未找到: {', '.join(missing[:3])}")
elif status in ("ready", "partial"):
warns.append("未声明 impl(无法核对实现)")
# ② provider 的 key 配置情况
prov_ok: list[str] = []
prov_missing: list[str] = []
for p in providers:
ref = p.get("api_key_ref") or ""
pid = p.get("id") or "?"
if not ref:
prov_ok.append(f"{pid}(无需key)") # 本地实现(ffmpeg 等)
continue
val = _get_by_path(cfg, ref)
if _key_ok(val):
prov_ok.append(pid)
else:
prov_missing.append(f"{pid}({ref})")
# ③ 状态一致性
if status == "ready":
if providers and not prov_ok:
issues.append(f"status=ready 但没有任何可用 provider(缺: {', '.join(prov_missing)})")
if impl_ok is False:
issues.append("status=ready 但 impl 文件不存在")
elif status == "partial":
if providers and not prov_ok:
warns.append(f"partial 且全部 provider 不可用(缺: {', '.join(prov_missing)})")
elif status in ("planned", "missing", ""):
pass
# ④ 结构完整性
if not providers:
warns.append("未声明 providers")
verdict = "✗" if issues else ("⚠️" if warns else "✓")
return {
"id": cid,
"name": cap.get("name", ""),
"status": status,
"impl": impl,
"impl_ok": impl_ok,
"providers_ok": prov_ok,
"providers_missing": prov_missing,
"issues": issues,
"warns": warns,
"verdict": verdict,
}
def main():
ap = argparse.ArgumentParser(description="能力体检(静态,零成本)")
ap.add_argument("--capability", help="只查某个能力 id")
ap.add_argument("--json", action="store_true", help="输出 JSON")
ap.add_argument("--only-issues", action="store_true", help="只显示 ⚠️/✗")
args = ap.parse_args()
caps = load_caps()
cfg = load_config_merged()
if args.capability:
caps = [c for c in caps if c.get("id") == args.capability]
if not caps:
print(f"未找到能力: {args.capability}")
return 2
results = [check_capability(c, cfg) for c in caps]
if args.only_issues:
results = [r for r in results if r["verdict"] != "✓"]
if args.json:
print(json.dumps({"total": len(results), "results": results}, ensure_ascii=False, indent=1))
else:
bad = [r for r in results if r["verdict"] == "✗"]
warn = [r for r in results if r["verdict"] == "⚠️"]
good = [r for r in results if r["verdict"] == "✓"]
print(f"\n能力体检(共 {len(results)} 项) ✓ {len(good)} · ⚠️ {len(warn)} · ✗ {len(bad)}\n")
for r in results:
if args.only_issues and r["verdict"] == "✓":
continue
line = f" {r['verdict']} {r['id']:26s} [{r['status']:8s}]"
if r["providers_ok"]:
line += f" ✓{','.join(r['providers_ok'])[:36]}"
if r["providers_missing"]:
line += f" ✗{','.join(r['providers_missing'])[:36]}"
print(line)
for m in r["issues"]:
print(f" ✗ {m}")
for m in r["warns"]:
print(f" · {m}")
print()
return 1 if any(r["verdict"] == "✗" for r in results) else 0
if __name__ == "__main__":
sys.exit(main())