feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程,直接发 HTTP
移植自 mac-agent-os 的 mediacrawler_adapter:不起子进程、不开页面,用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin,所以 ingest 那条链路一个字不用改。
**目前能用的(真环境实测,非推断)**:
profile/other : 200, 7075 字节 —— 博主主页指标(粉丝/获赞/作品数/昵称)
aweme/detail : 200, 45425 字节 —— 单条作品详情(含点赞/评论/收藏/分享)
**目前不能用的:作品列表 `aweme/post`。** 两个互相独立的原因:
1. 这个接口被抖音单独升级成了真校验:不带 x-tt-argus 回 403「Uifid Not Found」,
带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过,「真校验」过不了。
同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的,
挑中的恰好是「批量拉作品列表」这个最敏感的动作。
2. 改走页面截获也不行:CDP 浏览器打开博主主页会落到「验证码中间页」(当天大量探测的
代价,过几小时要重测)。
所以现在的边界是:**已知作品的指标刷新能做,自动发现新作品做不了**。
**排查中控住变量后得到的两条事实**(都写进注释了):
· `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
UA+client hints+Cookie 时是 200 但仅 121 字节的空壳,补上这三个头变 7074 字节。
(我先前猜的 sec-ch-ua 不是关键。)
· 因此 UA 与 client hints 必须**成套地取自同一个浏览器**,所以 BrowserIdentity 一次
从 CDP 取齐 cookie + UA + hints,而不是各自写死。
「200 + 空 body 必须当场报错」也是刻意写死的:放过去它会在下游变成「这个博主没作品」,
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的,还被翻译成「账号被封」。
测试 +11:cookie 解析、请求头成套性(含 uifid 缺失/回退)、产物键名与 store 对齐、
以及 _get 的三条失败路径(空 body / 403 带网关原话 / 正常返回)。
This commit is contained in:
@@ -0,0 +1,228 @@
|
||||
"""
|
||||
agentos check — 全系统健康检查
|
||||
"""
|
||||
|
||||
import sys
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from .utils import (
|
||||
get_sync_root, get_local_root, get_python, get_npx,
|
||||
info, ok, warn, err, run, banner
|
||||
)
|
||||
|
||||
|
||||
def check_item(name: str, status: str, detail: str = ""):
|
||||
"""输出检查项"""
|
||||
icon = {"ok": "✅", "warn": "⚠️ ", "fail": "❌"}
|
||||
label = {"ok": "通过", "warn": "警告", "fail": "失败"}
|
||||
print(f" {icon.get(status, '❓')} [{label.get(status, '未知')}] {name}")
|
||||
if detail:
|
||||
print(f" {detail}")
|
||||
|
||||
|
||||
def cmd_check(quick: bool = False):
|
||||
"""执行全系统健康检查"""
|
||||
banner()
|
||||
sync_root = get_sync_root()
|
||||
local_root = get_local_root()
|
||||
|
||||
print("=" * 60)
|
||||
print(" 全系统健康检查")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
# 1. 目录结构
|
||||
print("📁 目录结构:")
|
||||
check_item("agent-sync 根目录", "ok" if sync_root.exists() else "fail", str(sync_root))
|
||||
check_item("agent-local 根目录", "ok" if local_root.exists() else "fail", str(local_root))
|
||||
|
||||
sync_skills = sync_root / "02_skills"
|
||||
sync_skills_count = len([d for d in sync_skills.iterdir() if d.is_dir() and (d / "SKILL.md").exists()]) if sync_skills.exists() else 0
|
||||
check_item(f"技能目录 ({sync_skills_count} 个)", "ok" if sync_skills_count > 0 else "warn")
|
||||
|
||||
local_materials = local_root / "materials" if local_root.exists() else None
|
||||
local_raw = local_root / "memory/raw" if local_root.exists() else None
|
||||
materials_exist = local_materials and local_materials.exists()
|
||||
raw_exist = local_raw and local_raw.exists()
|
||||
check_item("materials/", "ok" if materials_exist else "warn", "采集素材目录")
|
||||
check_item("memory/raw/", "ok" if raw_exist else "warn", "L3 记忆原文目录")
|
||||
|
||||
# 2. 软链接
|
||||
print("\n🔗 软链接:")
|
||||
symlinks_to_check = [
|
||||
("04_memory/long_term/raw", local_root / "memory/raw"),
|
||||
("04_memory/vector_db", local_root / "memory/vector_db"),
|
||||
("06_runtime/cache", local_root / "runtime/cache"),
|
||||
]
|
||||
for link_rel, target in symlinks_to_check:
|
||||
link_path = sync_root / link_rel
|
||||
if link_path.is_symlink():
|
||||
actual = link_path.resolve()
|
||||
expected = target.resolve()
|
||||
if actual == expected:
|
||||
check_item(f"{link_rel} → OK", "ok")
|
||||
else:
|
||||
check_item(f"{link_rel} → 指向错误", "warn", f"实际: {actual}, 期望: {expected}")
|
||||
elif link_path.exists():
|
||||
check_item(f"{link_rel} → 真实目录", "ok")
|
||||
else:
|
||||
check_item(f"{link_rel} → 不存在", "fail")
|
||||
|
||||
# 3. Python 环境
|
||||
print("\n🐍 Python 环境:")
|
||||
py = get_python()
|
||||
if Path(py).exists():
|
||||
result = run([py, "--version"])
|
||||
check_item("Python 版本", "ok" if result.returncode == 0 else "fail", result.stdout.strip() if result.returncode == 0 else "")
|
||||
else:
|
||||
check_item("Python 可执行文件", "fail", py)
|
||||
|
||||
# 关键依赖检查
|
||||
deps = ["chromadb", "trafilatura", "feedparser", "sqlite_utils"]
|
||||
for dep in deps:
|
||||
result = run([py, "-c", f"import {dep}"])
|
||||
check_item(f"依赖: {dep}", "ok" if result.returncode == 0 else "fail")
|
||||
|
||||
if quick:
|
||||
print("\n" + "=" * 60)
|
||||
ok("快速检查完成")
|
||||
return
|
||||
|
||||
# 4. oMLX
|
||||
print("\n🤖 本地 LLM 引擎:")
|
||||
try:
|
||||
import json, urllib.request
|
||||
data = json.dumps({"model": "Qwen3-Embedding-0.6B-4bit-DWQ", "input": ["test"]}).encode()
|
||||
req = urllib.request.Request(
|
||||
"http://localhost:8000/v1/embeddings",
|
||||
data=data,
|
||||
headers={"Authorization": "Bearer omlx", "Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
resp = urllib.request.urlopen(req, timeout=5)
|
||||
check_item("oMLX Embedding API", "ok" if resp.status == 200 else "fail", f"状态码 {resp.status}")
|
||||
except Exception as e:
|
||||
check_item("oMLX Embedding API", "fail", str(e))
|
||||
|
||||
# 音频转文字能力检查
|
||||
print("\n🎙️ 音频转文字:")
|
||||
ffmpeg = run(["which", "ffmpeg"])
|
||||
if ffmpeg.returncode == 0:
|
||||
check_item("ffmpeg(系统工具)", "ok", "已安装")
|
||||
else:
|
||||
check_item("ffmpeg(系统工具)", "warn", "未安装,音频转文字不可用。运行: brew install ffmpeg")
|
||||
|
||||
whisper_check = run([py, "-m", "pip", "show", "openai-whisper"])
|
||||
if whisper_check.returncode == 0:
|
||||
import re
|
||||
ver_match = re.search(r"Version:\s*(\S+)", whisper_check.stdout)
|
||||
ver = ver_match.group(1) if ver_match else "?"
|
||||
check_item("openai-whisper(Python包)", "ok", f"v{ver}")
|
||||
# 检查模型是否已下载
|
||||
import os as _os
|
||||
whisper_cache = _os.path.expanduser("~/.cache/whisper")
|
||||
if _os.path.exists(whisper_cache):
|
||||
models = [f for f in _os.listdir(whisper_cache) if f.endswith(('.pt', '.bin'))]
|
||||
if models:
|
||||
model_sizes = [_os.path.getsize(_os.path.join(whisper_cache, m)) / 1024 / 1024 for m in models]
|
||||
detail = ", ".join([f"{m} ({s:.0f}MB)" for m, s in zip(models, model_sizes)])
|
||||
check_item("Whisper 模型权重", "ok", f"已缓存: {detail}")
|
||||
else:
|
||||
check_item("Whisper 模型权重", "warn", "首次使用时会自动下载(~1.5GB)")
|
||||
else:
|
||||
check_item("Whisper 模型权重", "warn", "未下载,首次运行自动下载(~1.5GB)")
|
||||
else:
|
||||
check_item("openai-whisper(Python包)", "fail", "未安装,运行: pip install openai-whisper")
|
||||
|
||||
# 5. facts.db
|
||||
print("\n🗄️ 记忆层:")
|
||||
facts_db = sync_root / "04_memory" / "long_term" / "facts.db"
|
||||
if facts_db.exists():
|
||||
size_kb = facts_db.stat().st_size / 1024
|
||||
result = run([py, "-c", f"""
|
||||
import sqlite3
|
||||
conn = sqlite3.connect('{facts_db}')
|
||||
c = conn.cursor()
|
||||
c.execute("SELECT COUNT(*) FROM facts")
|
||||
count = c.fetchone()[0]
|
||||
c.execute("SELECT COUNT(DISTINCT subject) FROM facts")
|
||||
subjects = c.fetchone()[0]
|
||||
conn.close()
|
||||
print(f"条目: {{count}}, 主题: {{subjects}}")
|
||||
"""])
|
||||
check_item(f"facts.db ({size_kb:.1f} KB)", "ok", result.stdout.strip() if result.returncode == 0 else "读取异常")
|
||||
else:
|
||||
check_item("facts.db", "warn", "不存在(首次初始化正常)")
|
||||
|
||||
# 6. 向量库
|
||||
vector_db = local_root / "vector_db" / "global" / "chroma"
|
||||
if vector_db.exists():
|
||||
chroma_files = list(vector_db.glob("*.sqlite3"))
|
||||
check_item("global 向量库", "ok" if chroma_files else "warn", f"目录存在但无 .sqlite3 文件" if not chroma_files else "")
|
||||
else:
|
||||
check_item("global 向量库", "warn", "未建,运行 agentos init 或 vector_db_rebuild.py 重建")
|
||||
|
||||
# 7. WorkBuddy 技能注册
|
||||
print("\n📦 技能注册状态:")
|
||||
wb_skills = Path.home() / ".workbuddy" / "skills"
|
||||
sync_skills_dir = sync_root / "02_skills"
|
||||
if wb_skills.exists() and sync_skills_dir.exists():
|
||||
wb_names = {d.name for d in wb_skills.iterdir() if d.is_dir()}
|
||||
src_names = {d.name for d in sync_skills_dir.iterdir() if d.is_dir() and (d / "SKILL.md").exists()}
|
||||
missing = src_names - wb_names
|
||||
extra = wb_names - src_names
|
||||
if missing:
|
||||
check_item(f"未注册到 WorkBuddy ({len(missing)} 个)", "warn", ", ".join(sorted(missing)))
|
||||
if extra:
|
||||
check_item(f"WB 中多余的 ({len(extra)} 个)", "warn", ", ".join(sorted(extra)))
|
||||
if not missing and not extra:
|
||||
check_item(f"技能一致性 ({len(src_names)} 个)", "ok", "源目录与 WB 注册一致")
|
||||
|
||||
# 8. 磁盘
|
||||
print("\n💾 磁盘空间:")
|
||||
import shutil
|
||||
usage = shutil.disk_usage(str(Path.home()))
|
||||
free_gb = usage.free / (1024 ** 3)
|
||||
total_gb = usage.total / (1024 ** 3)
|
||||
pct = usage.free / usage.total * 100
|
||||
status = "ok" if pct > 10 else "warn"
|
||||
check_item(f"磁盘剩余", status, f"{free_gb:.0f} GB / {total_gb:.0f} GB ({pct:.0f}%)")
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
ok("健康检查完成")
|
||||
print()
|
||||
|
||||
|
||||
def install_deps(dry_run: bool = False):
|
||||
"""安装 Python 依赖(被 init 调用)"""
|
||||
py = get_python()
|
||||
req_file = get_sync_root() / "requirements.txt"
|
||||
if not req_file.exists():
|
||||
warn(f"requirements.txt 不存在: {req_file}")
|
||||
return
|
||||
|
||||
if dry_run:
|
||||
info(f" [DRY-RUN] pip install -r {req_file}")
|
||||
return
|
||||
|
||||
info("安装 Python 依赖...")
|
||||
result = run([py, "-m", "pip", "install", "-r", str(req_file), "--quiet"])
|
||||
if result.returncode == 0:
|
||||
ok("Python 依赖安装完成")
|
||||
else:
|
||||
warn(f"部分依赖安装失败:\n{result.stderr[:200]}")
|
||||
|
||||
# 检查并提示安装 ffmpeg(音频转文字需要)
|
||||
ffmpeg = run(["which", "ffmpeg"])
|
||||
if ffmpeg.returncode != 0:
|
||||
print()
|
||||
warn("音频转文字功能需要 ffmpeg 系统工具")
|
||||
info(" 运行以下命令安装:")
|
||||
info(" brew install ffmpeg")
|
||||
print()
|
||||
|
||||
|
||||
def cmd_run(args):
|
||||
cmd_check(quick=args.quick)
|
||||
Reference in New Issue
Block a user