feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程,直接发 HTTP
Deploy VitePress site to Pages / build (push) Canceled after 0s
Deploy VitePress site to Pages / Deploy (push) Canceled after 0s

移植自 mac-agent-os 的 mediacrawler_adapter:不起子进程、不开页面,用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin,所以 ingest 那条链路一个字不用改。

**目前能用的(真环境实测,非推断)**:

    profile/other : 200, 7075 字节  —— 博主主页指标(粉丝/获赞/作品数/昵称)
    aweme/detail  : 200, 45425 字节 —— 单条作品详情(含点赞/评论/收藏/分享)

**目前不能用的:作品列表 `aweme/post`。** 两个互相独立的原因:
  1. 这个接口被抖音单独升级成了真校验:不带 x-tt-argus 回 403「Uifid Not Found」,
     带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过,「真校验」过不了。
     同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的,
     挑中的恰好是「批量拉作品列表」这个最敏感的动作。
  2. 改走页面截获也不行:CDP 浏览器打开博主主页会落到「验证码中间页」(当天大量探测的
     代价,过几小时要重测)。

  所以现在的边界是:**已知作品的指标刷新能做,自动发现新作品做不了**。

**排查中控住变量后得到的两条事实**(都写进注释了):
  · `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
    UA+client hints+Cookie 时是 200 但仅 121 字节的空壳,补上这三个头变 7074 字节。
    (我先前猜的 sec-ch-ua 不是关键。)
  · 因此 UA 与 client hints 必须**成套地取自同一个浏览器**,所以 BrowserIdentity 一次
    从 CDP 取齐 cookie + UA + hints,而不是各自写死。

「200 + 空 body 必须当场报错」也是刻意写死的:放过去它会在下游变成「这个博主没作品」,
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的,还被翻译成「账号被封」。

测试 +11:cookie 解析、请求头成套性(含 uifid 缺失/回退)、产物键名与 store 对齐、
以及 _get 的三条失败路径(空 body / 403 带网关原话 / 正常返回)。
This commit is contained in:
2026-10-10 17:12:25 +08:00
parent 21ff01b894
commit 2112c1a870
1601 changed files with 367425 additions and 0 deletions
@@ -0,0 +1,424 @@
"""
agentos config — 配置管理模块 (v1.0.0)
管理 01_core/ ←→ ~/.workbuddy/ 的配置同步。
三类配置:
A: 自动覆盖(内核身份文件)
B: 对比选择(服务配置,diff 后用户选择)
C: 仅报告(本机个性化,不同步)
子命令:
agentos config status 查看配置状态
agentos config diff 查看差异详情
agentos config apply 应用配置更新
agentos config rollback 回滚到备份
最后更新: 2026-05-02
"""
import json
import os
import shutil
import subprocess
import sys
from datetime import datetime
from pathlib import Path
from typing import Optional
from .const import __agent_sync_root__, __agent_local_root__, __version__
SYNC_ROOT = Path(__agent_sync_root__)
MANIFEST_PATH = SYNC_ROOT / "01_core" / "CONFIG_MANIFEST.yaml"
# ══════════════════════════════════════════════════════════════
# 配置清单解析
# ══════════════════════════════════════════════════════════════
def load_manifest() -> list:
"""解析 CONFIG_MANIFEST.yaml,返回配置项列表"""
import yaml
if not MANIFEST_PATH.exists():
print(f"❌ 未找到配置清单: {MANIFEST_PATH}")
return []
with open(MANIFEST_PATH) as f:
data = yaml.safe_load(f)
return data.get("configs", [])
def resolve_path(path_str: str) -> Optional[Path]:
"""将 ~/ 开头的路径解析为绝对路径"""
if not path_str:
return None
path_str = path_str.replace("~", str(Path.home()))
p = Path(path_str)
return p if p.exists() else p # 即使不存在也返回路径对象
# ══════════════════════════════════════════════════════════════
# 状态检查
# ══════════════════════════════════════════════════════════════
def check_status() -> list:
"""检查所有配置项状态,返回状态列表"""
items = load_manifest()
if not items:
return []
results = []
for item in items:
cat = item["category"]
name = item["file"]
source = resolve_path(item.get("source"))
target = resolve_path(item.get("target"))
status = {
"name": name,
"category": cat,
"source": str(source) if source else "—",
"target": str(target) if target else "—",
"exists_source": source and source.exists(),
"exists_target": target and target.exists(),
"identical": False,
"diff_lines": 0,
}
# 比较内容
if status["exists_source"] and status["exists_target"]:
try:
s_content = source.read_bytes()
t_content = target.read_bytes()
status["identical"] = s_content == t_content
if not status["identical"]:
# 粗略统计差异行数
s_lines = s_content.decode().splitlines()
t_lines = t_content.decode().splitlines()
status["diff_lines"] = abs(len(s_lines) - len(t_lines))
except Exception:
status["identical"] = False
results.append(status)
return results
# ══════════════════════════════════════════════════════════════
# 差异显示
# ══════════════════════════════════════════════════════════════
def show_diff(item: dict):
"""显示指定配置项的 diff"""
source = resolve_path(item.get("source"))
target = resolve_path(item.get("target"))
if not source or not source.exists():
print(f" ⚠️ 源文件不存在: {source}")
return
if not target or not target.exists():
print(f" ⚠️ 目标文件不存在: {target}")
print(f" → 将创建: {target}")
return
try:
result = subprocess.run(
["diff", "-u", str(source), str(target)],
capture_output=True, text=True, timeout=10
)
if result.returncode == 0:
print(f" ✅ 文件一致,无差异")
else:
diff_out = result.stdout
lines = diff_out.splitlines()
# 只显示前 30 行
display = lines[:30]
if len(lines) > 30:
display.append(f" ... (还有 {len(lines)-30} 行差异)")
for line in display:
marker = " "
if line.startswith("+"):
marker = "+"
elif line.startswith("-"):
marker = "-"
elif line.startswith("@@"):
marker = "@"
print(f" {marker} {line}")
except Exception as e:
print(f" ❌ diff 失败: {e}")
# ══════════════════════════════════════════════════════════════
# 备份
# ══════════════════════════════════════════════════════════════
def backup_targets(items: list) -> Optional[Path]:
"""备份所有目标文件到时间戳目录"""
backup_dir = Path.home() / ".workbuddy" / f"config_backup_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
count = 0
for item in items:
if item["category"] == "C":
continue
target = resolve_path(item.get("target"))
if target and target.exists():
backup_dir.mkdir(parents=True, exist_ok=True)
shutil.copy2(str(target), str(backup_dir / target.name))
count += 1
if count > 0:
print(f" 💾 已备份 {count} 个文件到: {backup_dir}")
return backup_dir
return None
# ══════════════════════════════════════════════════════════════
# 应用配置
# ══════════════════════════════════════════════════════════════
def apply_config(item: dict) -> bool:
"""应用单个配置项"""
cat = item["category"]
name = item["file"]
source = resolve_path(item.get("source"))
target = resolve_path(item.get("target"))
if not target:
print(f" ⚠️ {name}: 目标路径为空,跳过")
return False
if cat == "A":
# A类: 自动覆盖
if not source or not source.exists():
print(f" ⚠️ {name}: 源文件不存在 ({source}),跳过")
return False
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(str(source), str(target))
print(f" ✅ {name}: 已部署(A类自动覆盖)")
return True
elif cat == "B":
# B类: 对比后选择
if not source or not source.exists():
print(f" ⚠️ {name}: 源文件不存在 ({source}),跳过")
return False
target.parent.mkdir(parents=True, exist_ok=True)
if target.exists():
# 检查是否一致
try:
s_content = source.read_bytes()
t_content = target.read_bytes()
if s_content == t_content:
print(f" ✅ {name}: 一致,无需更新")
return True
except Exception:
pass
# 有差异,尝试交互(非交互环境自动跳过)
print(f"\n 📋 {name}: 源与目标存在差异")
print(f" 源: {source}")
print(f" 目标: {target}")
show_diff(item)
print()
try:
choice = input(f" 选择 [K]保留本地 [N]用新版 [D]查看差异 [S]跳过: ").strip().upper()
except (EOFError, OSError):
print(f" ⏭️ {name}: 非交互环境,跳过(运行 agentos config diff {name} 手动处理)")
return False
while True:
if choice == "K":
print(f" ⏭️ {name}: 保留本地版本")
return False
elif choice == "N":
shutil.copy2(str(source), str(target))
print(f" ✅ {name}: 已更新为新版")
return True
elif choice == "D":
show_diff(item)
break
elif choice == "S":
print(f" ⏭️ {name}: 跳过")
return False
else:
break
else:
# 目标不存在,直接部署
shutil.copy2(str(source), str(target))
print(f" ✅ {name}: 已创建(首次部署)")
return True
elif cat == "C":
print(f" ⏭️ {name}: C类本机配置,不同步")
return False
return False
# ══════════════════════════════════════════════════════════════
# CLI 命令
# ══════════════════════════════════════════════════════════════
def cmd_status(args):
"""agentos config status — 查看配置状态"""
results = check_status()
if not results:
print("❌ 无法读取配置清单")
return
print(f"\n{'='*60}")
print(f" 📋 AgentOS 配置状态")
print(f"{'='*60}\n")
for r in results:
cat = r["category"]
name = r["name"]
tag = {"A": "🔵 A-自动", "B": "🟡 B-选择", "C": "⚪ C-本地"}.get(cat, "?")
if cat == "C":
exists = r["exists_target"]
print(f" {tag} {name:20s} {'✅ 存在' if exists else '⏹️ 不存在'}")
elif r["identical"]:
print(f" {tag} {name:20s} ✅ 一致")
elif r["exists_source"] and r["exists_target"]:
print(f" {tag} {name:20s} ⚠️ 有差异(~{r['diff_lines']}行不同)")
elif r["exists_source"] and not r["exists_target"]:
print(f" {tag} {name:20s} 📋 待部署(目标不存在)")
elif not r["exists_source"]:
print(f" {tag} {name:20s} ❌ 源文件不存在")
else:
print(f" {tag} {name:20s} ❓ 未知")
print(f"\n{'─'*60}")
a_count = sum(1 for r in results if r["category"] == "A" and not r["identical"])
b_count = sum(1 for r in results if r["category"] == "B" and not r["identical"])
if a_count + b_count > 0:
print(f" ⚠️ {a_count} 个 A类 + {b_count} 个 B类 待处理")
print(f" 运行: agentos config apply")
else:
print(f" ✅ 所有配置一致")
print()
def cmd_diff(args):
"""agentos config diff — 查看所有差异"""
results = check_status()
if not results:
return
items = load_manifest()
for i, r in enumerate(results):
if r["identical"] or r["category"] == "C":
continue
if r["exists_source"] and r["exists_target"]:
print(f"\n{'='*60}")
print(f" 📄 {r['name']}")
print(f"{'='*60}")
show_diff(items[i])
print()
def cmd_apply(args):
"""agentos config apply — 应用配置更新"""
items = load_manifest()
if not items:
return
print(f"\n{'='*60}")
print(f" 🔄 AgentOS 配置更新")
print(f"{'='*60}\n")
# 先备份
print(f"▶ 备份现有配置...")
backup_dir = backup_targets(items)
if backup_dir:
print(f" 回滚命令: agentos config rollback {backup_dir.name}")
print()
# 逐项应用
changed = 0
for item in items:
if item["category"] == "C":
continue
ok = apply_config(item)
if ok:
changed += 1
print(f"\n{'─'*60}")
if changed > 0:
print(f" ✅ 已更新 {changed} 个配置项")
print(f" 提示: 如果 WorkBuddy 正在运行,可能需要重启生效")
else:
print(f" ⏭️ 无变更")
print()
def cmd_rollback(args):
"""agentos config rollback — 回滚配置"""
backup_name = getattr(args, "backup_name", None)
backup_base = Path.home() / ".workbuddy"
if backup_name:
backup_dir = backup_base / backup_name
else:
# 找最近的备份
backups = sorted(backup_base.glob("config_backup_*"))
if not backups:
print("❌ 未找到备份")
return
backup_dir = backups[-1]
print(f" 使用最近备份: {backup_dir.name}")
if not backup_dir.exists():
print(f"❌ 备份目录不存在: {backup_dir}")
return
files = list(backup_dir.iterdir())
if not files:
print(f"❌ 备份目录为空: {backup_dir}")
return
print(f"\n{'='*60}")
print(f" ↩️ 回滚配置")
print(f"{'='*60}\n")
print(f" 从: {backup_dir}")
print()
for f in files:
target = Path.home() / ".workbuddy" / f.name
shutil.copy2(str(f), str(target))
print(f" ✅ {f.name}: 已恢复")
print(f"\n 共恢复 {len(files)} 个文件")
print()
# ══════════════════════════════════════════════════════════════
# 子命令注册
# ══════════════════════════════════════════════════════════════
def setup_parser(subparsers):
"""向主 CLI 注册 config 子命令"""
p = subparsers.add_parser("config", help="配置管理:状态查看 / 差异对比 / 部署 / 回滚")
config_sub = p.add_subparsers(dest="config_cmd", help="配置子命令")
config_sub.add_parser("status", help="查看配置状态(哪些文件有差异)")
config_sub.add_parser("diff", help="查看配置差异详情")
config_sub.add_parser("apply", help="应用配置更新(A类自动 / B类选择)")
p_rollback = config_sub.add_parser("rollback", help="回滚到上次备份")
p_rollback.add_argument("backup_name", nargs="?", default=None,
help="备份目录名(可选,默认使用最近备份)")
def run(args):
cmd = args.config_cmd
if not cmd:
print("agentos config 子命令: status | diff | apply | rollback")
return
if cmd == "status":
cmd_status(args)
elif cmd == "diff":
cmd_diff(args)
elif cmd == "apply":
cmd_apply(args)
elif cmd == "rollback":
cmd_rollback(args)