feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程,直接发 HTTP
移植自 mac-agent-os 的 mediacrawler_adapter:不起子进程、不开页面,用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin,所以 ingest 那条链路一个字不用改。
**目前能用的(真环境实测,非推断)**:
profile/other : 200, 7075 字节 —— 博主主页指标(粉丝/获赞/作品数/昵称)
aweme/detail : 200, 45425 字节 —— 单条作品详情(含点赞/评论/收藏/分享)
**目前不能用的:作品列表 `aweme/post`。** 两个互相独立的原因:
1. 这个接口被抖音单独升级成了真校验:不带 x-tt-argus 回 403「Uifid Not Found」,
带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过,「真校验」过不了。
同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的,
挑中的恰好是「批量拉作品列表」这个最敏感的动作。
2. 改走页面截获也不行:CDP 浏览器打开博主主页会落到「验证码中间页」(当天大量探测的
代价,过几小时要重测)。
所以现在的边界是:**已知作品的指标刷新能做,自动发现新作品做不了**。
**排查中控住变量后得到的两条事实**(都写进注释了):
· `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
UA+client hints+Cookie 时是 200 但仅 121 字节的空壳,补上这三个头变 7074 字节。
(我先前猜的 sec-ch-ua 不是关键。)
· 因此 UA 与 client hints 必须**成套地取自同一个浏览器**,所以 BrowserIdentity 一次
从 CDP 取齐 cookie + UA + hints,而不是各自写死。
「200 + 空 body 必须当场报错」也是刻意写死的:放过去它会在下游变成「这个博主没作品」,
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的,还被翻译成「账号被封」。
测试 +11:cookie 解析、请求头成套性(含 uifid 缺失/回退)、产物键名与 store 对齐、
以及 _get 的三条失败路径(空 body / 403 带网关原话 / 正常返回)。
This commit is contained in:
@@ -0,0 +1,231 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
capability_doctor.py — 能力体检(一条命令验证「清单说的 ready 是不是真能跑」)
|
||||
|
||||
背景:capabilities.yaml 是能力唯一真相源,但清单可能与实际脱节
|
||||
(实测曾发现:frame_chain 代码早已实现却报 missing;
|
||||
lipsync 实际用百炼却登记成 fal)。
|
||||
本工具做静态体检,把「名不副实」的能力揪出来。
|
||||
|
||||
用法:
|
||||
python capability_doctor.py # 全部能力体检
|
||||
python capability_doctor.py --capability lipsync
|
||||
python capability_doctor.py --json # 机器可读(供 Dashboard)
|
||||
python capability_doctor.py --only-issues # 只看有问题的
|
||||
|
||||
检查项(全部零成本,不调用任何付费接口):
|
||||
1. impl 文件是否存在(清单 impl 字段指向的代码)
|
||||
2. provider 的 api_key 是否已配置(api_key_ref → config.yaml + local.yaml)
|
||||
3. 状态一致性:status=ready 但没有任何可用 provider → ✗ 名不副实
|
||||
4. 结构完整性:是否声明了 providers / status
|
||||
|
||||
退出码:0 = 无问题;1 = 存在 ⚠️ / ✗
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# ── 定位 ──
|
||||
_SCRIPTS = Path(__file__).resolve().parent
|
||||
_AVE = _SCRIPTS.parent # 05_tools/09_ave
|
||||
if str(_SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(_SCRIPTS))
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
print("需要 pyyaml:pip install pyyaml")
|
||||
sys.exit(2)
|
||||
|
||||
CAP_FILE = _AVE / "config" / "capabilities.yaml"
|
||||
PLACEHOLDERS = {"", "sk-xxx", "xxx", "your_key", "your_openai_api_key", "omlx", "changeme", "todo"}
|
||||
|
||||
|
||||
def load_caps() -> list:
|
||||
d = yaml.safe_load(CAP_FILE.read_text(encoding="utf-8")) or {}
|
||||
return d.get("capabilities") or []
|
||||
|
||||
|
||||
def load_config_merged() -> dict:
|
||||
"""合并 config.yaml(模板)与 local.yaml(本机真实密钥,优先级更高)"""
|
||||
merged: dict = {}
|
||||
|
||||
def _merge(src: dict):
|
||||
for k, v in (src or {}).items():
|
||||
if isinstance(v, dict) and isinstance(merged.get(k), dict):
|
||||
merged[k].update(v)
|
||||
else:
|
||||
merged[k] = v
|
||||
|
||||
tmpl = _AVE / "config.yaml"
|
||||
if tmpl.exists():
|
||||
try:
|
||||
_merge(yaml.safe_load(tmpl.read_text(encoding="utf-8")) or {})
|
||||
except Exception:
|
||||
pass
|
||||
local = Path.home() / "workbuddy-agent-os/agent-local/tools/ave/config/local.yaml"
|
||||
if local.exists():
|
||||
try:
|
||||
_merge(yaml.safe_load(local.read_text(encoding="utf-8")) or {})
|
||||
except Exception:
|
||||
pass
|
||||
return merged
|
||||
|
||||
|
||||
def _get_by_path(cfg: dict, dotted: str):
|
||||
cur = cfg
|
||||
for part in (dotted or "").split("."):
|
||||
if not isinstance(cur, dict) or part not in cur:
|
||||
return None
|
||||
cur = cur[part]
|
||||
return cur
|
||||
|
||||
|
||||
def _key_ok(val) -> bool:
|
||||
if val is None:
|
||||
return False
|
||||
s = str(val).strip()
|
||||
return bool(s) and s.lower() not in PLACEHOLDERS and not s.startswith("${")
|
||||
|
||||
|
||||
def _impl_targets(impl: str) -> list:
|
||||
"""从 impl 描述提取可核对的路径(文件或目录都可)
|
||||
|
||||
· 'composer/lipsync.py(lipsync_videoretalk)' → ['composer/lipsync.py']
|
||||
· 'music_selector/' → ['music_selector']
|
||||
· 'assets_manager/ + character_generator/' → ['assets_manager', 'character_generator']
|
||||
"""
|
||||
if not impl:
|
||||
return []
|
||||
files = re.findall(r"([\w./-]+\.py)", impl)
|
||||
if files:
|
||||
return files
|
||||
dirs = re.findall(r"([\w][\w./-]*)/", impl)
|
||||
seen, out = set(), []
|
||||
for d in dirs:
|
||||
d = d.rstrip('/')
|
||||
if d and d not in seen:
|
||||
seen.add(d)
|
||||
out.append(d)
|
||||
return out or [impl.strip()]
|
||||
|
||||
|
||||
def check_capability(cap: dict, cfg: dict) -> dict:
|
||||
"""单个能力的静态体检结果"""
|
||||
cid = cap.get("id", "?")
|
||||
status = (cap.get("status") or "").strip() or "(未声明)"
|
||||
providers = cap.get("providers") or []
|
||||
issues: list[str] = []
|
||||
warns: list[str] = []
|
||||
|
||||
# ① impl 实现(文件或目录)
|
||||
impl = cap.get("impl") or ""
|
||||
targets = _impl_targets(impl)
|
||||
impl_ok = None
|
||||
if targets:
|
||||
found = []
|
||||
missing = []
|
||||
for rel in targets:
|
||||
cands = [_SCRIPTS / rel, _SCRIPTS / "capabilities" / rel, _AVE / rel]
|
||||
(found if any(c.exists() for c in cands) else missing).append(rel)
|
||||
impl_ok = bool(found)
|
||||
# planned / missing 能力允许尚无实现;ready/partial 缺实现才是问题
|
||||
if missing and status in ("ready", "partial"):
|
||||
warns.append(f"impl 未找到: {', '.join(missing[:3])}")
|
||||
elif status in ("ready", "partial"):
|
||||
warns.append("未声明 impl(无法核对实现)")
|
||||
|
||||
# ② provider 的 key 配置情况
|
||||
prov_ok: list[str] = []
|
||||
prov_missing: list[str] = []
|
||||
for p in providers:
|
||||
ref = p.get("api_key_ref") or ""
|
||||
pid = p.get("id") or "?"
|
||||
if not ref:
|
||||
prov_ok.append(f"{pid}(无需key)") # 本地实现(ffmpeg 等)
|
||||
continue
|
||||
val = _get_by_path(cfg, ref)
|
||||
if _key_ok(val):
|
||||
prov_ok.append(pid)
|
||||
else:
|
||||
prov_missing.append(f"{pid}({ref})")
|
||||
|
||||
# ③ 状态一致性
|
||||
if status == "ready":
|
||||
if providers and not prov_ok:
|
||||
issues.append(f"status=ready 但没有任何可用 provider(缺: {', '.join(prov_missing)})")
|
||||
if impl_ok is False:
|
||||
issues.append("status=ready 但 impl 文件不存在")
|
||||
elif status == "partial":
|
||||
if providers and not prov_ok:
|
||||
warns.append(f"partial 且全部 provider 不可用(缺: {', '.join(prov_missing)})")
|
||||
elif status in ("planned", "missing", ""):
|
||||
pass
|
||||
|
||||
# ④ 结构完整性
|
||||
if not providers:
|
||||
warns.append("未声明 providers")
|
||||
|
||||
verdict = "✗" if issues else ("⚠️" if warns else "✓")
|
||||
return {
|
||||
"id": cid,
|
||||
"name": cap.get("name", ""),
|
||||
"status": status,
|
||||
"impl": impl,
|
||||
"impl_ok": impl_ok,
|
||||
"providers_ok": prov_ok,
|
||||
"providers_missing": prov_missing,
|
||||
"issues": issues,
|
||||
"warns": warns,
|
||||
"verdict": verdict,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="能力体检(静态,零成本)")
|
||||
ap.add_argument("--capability", help="只查某个能力 id")
|
||||
ap.add_argument("--json", action="store_true", help="输出 JSON")
|
||||
ap.add_argument("--only-issues", action="store_true", help="只显示 ⚠️/✗")
|
||||
args = ap.parse_args()
|
||||
|
||||
caps = load_caps()
|
||||
cfg = load_config_merged()
|
||||
if args.capability:
|
||||
caps = [c for c in caps if c.get("id") == args.capability]
|
||||
if not caps:
|
||||
print(f"未找到能力: {args.capability}")
|
||||
return 2
|
||||
|
||||
results = [check_capability(c, cfg) for c in caps]
|
||||
if args.only_issues:
|
||||
results = [r for r in results if r["verdict"] != "✓"]
|
||||
|
||||
if args.json:
|
||||
print(json.dumps({"total": len(results), "results": results}, ensure_ascii=False, indent=1))
|
||||
else:
|
||||
bad = [r for r in results if r["verdict"] == "✗"]
|
||||
warn = [r for r in results if r["verdict"] == "⚠️"]
|
||||
good = [r for r in results if r["verdict"] == "✓"]
|
||||
print(f"\n能力体检(共 {len(results)} 项) ✓ {len(good)} · ⚠️ {len(warn)} · ✗ {len(bad)}\n")
|
||||
for r in results:
|
||||
if args.only_issues and r["verdict"] == "✓":
|
||||
continue
|
||||
line = f" {r['verdict']} {r['id']:26s} [{r['status']:8s}]"
|
||||
if r["providers_ok"]:
|
||||
line += f" ✓{','.join(r['providers_ok'])[:36]}"
|
||||
if r["providers_missing"]:
|
||||
line += f" ✗{','.join(r['providers_missing'])[:36]}"
|
||||
print(line)
|
||||
for m in r["issues"]:
|
||||
print(f" ✗ {m}")
|
||||
for m in r["warns"]:
|
||||
print(f" · {m}")
|
||||
print()
|
||||
|
||||
return 1 if any(r["verdict"] == "✗" for r in results) else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user