Files
MediaCrawler/mac-agent-os-main/05_tools/00_setup/agentos/sync.py
T
butubb 2112c1a870
Deploy VitePress site to Pages / build (push) Canceled after 0s
Deploy VitePress site to Pages / Deploy (push) Canceled after 0s
feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程,直接发 HTTP
移植自 mac-agent-os 的 mediacrawler_adapter:不起子进程、不开页面,用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin,所以 ingest 那条链路一个字不用改。

**目前能用的(真环境实测,非推断)**:

    profile/other : 200, 7075 字节  —— 博主主页指标(粉丝/获赞/作品数/昵称)
    aweme/detail  : 200, 45425 字节 —— 单条作品详情(含点赞/评论/收藏/分享)

**目前不能用的:作品列表 `aweme/post`。** 两个互相独立的原因:
  1. 这个接口被抖音单独升级成了真校验:不带 x-tt-argus 回 403「Uifid Not Found」,
     带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过,「真校验」过不了。
     同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的,
     挑中的恰好是「批量拉作品列表」这个最敏感的动作。
  2. 改走页面截获也不行:CDP 浏览器打开博主主页会落到「验证码中间页」(当天大量探测的
     代价,过几小时要重测)。

  所以现在的边界是:**已知作品的指标刷新能做,自动发现新作品做不了**。

**排查中控住变量后得到的两条事实**(都写进注释了):
  · `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
    UA+client hints+Cookie 时是 200 但仅 121 字节的空壳,补上这三个头变 7074 字节。
    (我先前猜的 sec-ch-ua 不是关键。)
  · 因此 UA 与 client hints 必须**成套地取自同一个浏览器**,所以 BrowserIdentity 一次
    从 CDP 取齐 cookie + UA + hints,而不是各自写死。

「200 + 空 body 必须当场报错」也是刻意写死的:放过去它会在下游变成「这个博主没作品」,
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的,还被翻译成「账号被封」。

测试 +11:cookie 解析、请求头成套性(含 uifid 缺失/回退)、产物键名与 store 对齐、
以及 _get 的三条失败路径(空 body / 403 带网关原话 / 正常返回)。
2026-10-10 17:12:25 +08:00

200 lines
6.2 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
agentos sync — 双机增量同步
检查 agent-sync/ 与本地 WorkBuddy 的差异并增量更新:
1. 技能差异检测(对比版本号和文件变化)
2. MCP 配置差异检测
3. 自动化任务差异检测
4. 增量修复差异项
"""
import sys
import json
import shutil
import hashlib
from pathlib import Path
from .utils import (
get_sync_root, get_local_root, get_python,
info, ok, warn, err, run, banner
)
from . import skill_mgr
def file_hash(path: Path) -> str:
"""计算文件 SHA256"""
h = hashlib.sha256()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(65536), b""):
h.update(chunk)
return h.hexdigest()[:16]
def diff_skills(dry_run: bool) -> list:
"""检测技能差异"""
sync_root = get_sync_root()
wb_skills = Path.home() / ".workbuddy" / "skills"
source_dir = sync_root / "02_skills"
changes = []
source_dirs = {d.name: d for d in skill_mgr.get_skill_dirs(source_dir)}
wb_dirs = {d.name: d for d in skill_mgr.get_skill_dirs(wb_skills)} if wb_skills.exists() else {}
for name, src_dir in source_dirs.items():
if name not in wb_dirs:
changes.append({"type": "missing", "name": name, "action": "install"})
continue
src_card = skill_mgr.read_skill_card(src_dir)
wb_card = skill_mgr.read_skill_card(wb_dirs[name])
if src_card["version"] != wb_card["version"]:
changes.append({
"type": "version_diff",
"name": name,
"action": "update",
"src_version": src_card["version"],
"wb_version": wb_card["version"],
})
for name in wb_dirs:
if name not in source_dirs:
changes.append({"type": "extra", "name": name, "action": "uninstall"})
return changes
def diff_mcp(dry_run: bool) -> list:
"""检测 MCP 配置差异"""
sync_root = get_sync_root()
src = sync_root / "01_core" / "mcp.json"
dst = Path.home() / ".workbuddy" / "mcp.json"
changes = []
if not src.exists():
return changes
if not dst.exists():
changes.append({"type": "missing", "name": "mcp.json", "action": "deploy"})
else:
src_hash = file_hash(src)
dst_hash = file_hash(dst)
if src_hash != dst_hash:
changes.append({"type": "content_diff", "name": "mcp.json", "action": "update"})
return changes
def diff_automations(dry_run: bool) -> list:
"""检测自动化任务差异"""
sync_root = get_sync_root()
automations_file = sync_root / "01_core" / "automations.yaml"
if not automations_file.exists():
return []
return [{"type": "info", "name": "automations.yaml", "action": "需手动在 WorkBuddy 中比对"}]
def apply_changes(changes: list, dry_run: bool, force: bool):
"""执行差异修复"""
if dry_run:
return
for ch in changes:
action = ch["action"]
if action == "install":
info(f" 安装技能: {ch['name']}")
src_dir = get_sync_root() / "02_skills" / ch["name"]
skill_mgr.install_all(get_sync_root() / "02_skills")
elif action == "update":
info(f" 更新技能: {ch['name']} ({ch.get('src_version', '?')})")
skill_mgr.install_all(get_sync_root() / "02_skills")
elif action == "uninstall":
warn(f" agent-sync 中已移除: {ch['name']}(建议手动清理 WorkBuddy)")
elif action == "deploy":
info(f" 部署 MCP 配置")
src = get_sync_root() / "01_core" / "mcp.json"
dst = Path.home() / ".workbuddy" / "mcp.json"
if dst.exists():
dst.rename(dst.with_suffix(".json.syncbak"))
shutil.copy2(str(src), str(dst))
ok(" MCP 配置已更新")
elif action == "update" and ch["name"] == "mcp.json":
info(f" 更新 MCP 配置")
src = get_sync_root() / "01_core" / "mcp.json"
dst = Path.home() / ".workbuddy" / "mcp.json"
dst.rename(dst.with_suffix(".json.syncbak"))
shutil.copy2(str(src), str(dst))
ok(" MCP 配置已更新")
def run(args):
dry_run = args.dry_run
force = args.force
banner()
info("双机增量同步" + (" [DRY-RUN]" if dry_run else ""))
print()
# 1. 检测技能差异
print("📦 技能差异:")
skill_changes = diff_skills(dry_run)
if not skill_changes:
ok(" 无差异")
else:
for ch in skill_changes:
action_icon = {"install": "➕", "update": "🔄", "uninstall": "🗑️"}.get(ch["action"], "?")
if ch["type"] == "missing":
warn(f" {action_icon} 新增技能: {ch['name']}")
elif ch["type"] == "version_diff":
info(f" {action_icon} 版本变更: {ch['name']} ({ch['wb_version']} → {ch['src_version']})")
elif ch["type"] == "extra":
warn(f" {action_icon} 多余的: {ch['name']}(源目录已移除)")
print()
# 2. MCP 差异
print("🔧 MCP 配置差异:")
mcp_changes = diff_mcp(dry_run)
if not mcp_changes:
ok(" 无差异")
else:
for ch in mcp_changes:
if ch["action"] == "deploy":
warn(f" 缺失: {ch['name']}(待部署)")
elif ch["action"] == "update":
info(f" 内容变更: {ch['name']}(待更新)")
print()
# 3. 自动化差异
print("⏰ 自动化任务差异:")
auto_changes = diff_automations(dry_run)
if not auto_changes:
ok(" 无自动化配置文件(无差异检测)")
else:
for ch in auto_changes:
info(f" {ch['name']}: {ch['action']}")
print()
# 汇总
all_changes = skill_changes + mcp_changes + [c for c in auto_changes if c["action"] != "需手动在 WorkBuddy 中比对"]
if not all_changes:
ok("完全一致,无需同步")
return
print(f"共 {len(all_changes)} 项差异")
if dry_run:
info("添加 --force 参数执行同步")
else:
# 执行同步
info("执行同步...")
apply_changes(all_changes, dry_run, force)
ok("同步完成,请重启 WorkBuddy 使更改生效")