Files
MediaCrawler/mac-agent-os-main/00_bootstrap/deploy.sh
T
butubb 2112c1a870
Deploy VitePress site to Pages / build (push) Canceled after 0s
Deploy VitePress site to Pages / Deploy (push) Canceled after 0s
feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程,直接发 HTTP
移植自 mac-agent-os 的 mediacrawler_adapter:不起子进程、不开页面,用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin,所以 ingest 那条链路一个字不用改。

**目前能用的(真环境实测,非推断)**:

    profile/other : 200, 7075 字节  —— 博主主页指标(粉丝/获赞/作品数/昵称)
    aweme/detail  : 200, 45425 字节 —— 单条作品详情(含点赞/评论/收藏/分享)

**目前不能用的:作品列表 `aweme/post`。** 两个互相独立的原因:
  1. 这个接口被抖音单独升级成了真校验:不带 x-tt-argus 回 403「Uifid Not Found」,
     带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过,「真校验」过不了。
     同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的,
     挑中的恰好是「批量拉作品列表」这个最敏感的动作。
  2. 改走页面截获也不行:CDP 浏览器打开博主主页会落到「验证码中间页」(当天大量探测的
     代价,过几小时要重测)。

  所以现在的边界是:**已知作品的指标刷新能做,自动发现新作品做不了**。

**排查中控住变量后得到的两条事实**(都写进注释了):
  · `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
    UA+client hints+Cookie 时是 200 但仅 121 字节的空壳,补上这三个头变 7074 字节。
    (我先前猜的 sec-ch-ua 不是关键。)
  · 因此 UA 与 client hints 必须**成套地取自同一个浏览器**,所以 BrowserIdentity 一次
    从 CDP 取齐 cookie + UA + hints,而不是各自写死。

「200 + 空 body 必须当场报错」也是刻意写死的:放过去它会在下游变成「这个博主没作品」,
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的,还被翻译成「账号被封」。

测试 +11:cookie 解析、请求头成套性(含 uifid 缺失/回退)、产物键名与 store 对齐、
以及 _get 的三条失败路径(空 body / 403 带网关原话 / 正常返回)。
2026-10-10 17:12:25 +08:00

232 lines
7.6 KiB
Bash
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/bin/bash
# deploy.sh — AgentOS 一键部署脚本
# 其他机器在 git pull 后运行此脚本,即可同步本机所有变更
# 用法: bash 00_bootstrap/deploy.sh
# 目标: 在其他机器上运行一次,环境变量 → 本地配置 → 依赖检查 → 全部就绪
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
AGENT_SYNC="$(cd "$SCRIPT_DIR/.." && pwd)"
info() { echo -e "\033[36m[INFO]\033[0m $1"; }
ok() { echo -e "\033[32m[OK]\033[0m $1"; }
warn() { echo -e "\033[33m[WARN]\033[0m $1"; }
err() { echo -e "\033[31m[ERROR]\033[0m $1"; }
echo "============================================"
echo " AgentOS 一键部署脚本"
echo " 仓库: $AGENT_SYNC"
echo " 时间: $(date '+%Y-%m-%d %H:%M')"
echo "============================================"
# ── 步骤1: 设置环境变量 AGENT_SYNC / AGENT_LOCAL ──
echo ""
info "步骤1/5: 设置环境变量..."
if [ -z "$AGENT_SYNC" ]; then
export AGENT_SYNC="$AGENT_SYNC"
fi
if [ -z "$AGENT_LOCAL" ]; then
export AGENT_LOCAL="$HOME/workbuddy-agent-os/agent-local"
fi
# 写入 .zshrc
for RC_FILE in "$HOME/.zshrc" "$HOME/.bash_profile"; do
if [ -f "$RC_FILE" ]; then
if ! grep -q "export AGENT_SYNC" "$RC_FILE" 2>/dev/null; then
echo "" >> "$RC_FILE"
echo "# AgentOS 路径变量(由 deploy.sh 自动添加)" >> "$RC_FILE"
echo "export AGENT_SYNC=\"$AGENT_SYNC\"" >> "$RC_FILE"
echo "export AGENT_LOCAL=\"$AGENT_LOCAL\"" >> "$RC_FILE"
ok "已写入 $RC_FILE"
else
ok "$RC_FILE 已有 AGENT_SYNC 设置"
fi
break
fi
done
# ── 步骤2: 创建 agent-local 目录结构 ──
echo ""
info "步骤2/5: 创建本机目录结构..."
mkdir -p "$AGENT_LOCAL/identity"
mkdir -p "$AGENT_LOCAL/memory/raw"
mkdir -p "$AGENT_LOCAL/memory/vector_db"
mkdir -p "$AGENT_LOCAL/memory/daily"
mkdir -p "$AGENT_LOCAL/runtime/cache"
mkdir -p "$AGENT_LOCAL/tools/matrix/config"
mkdir -p "$AGENT_LOCAL/tools/matrix/identities"
mkdir -p "$AGENT_LOCAL/tools/matrix/data"
ok "目录结构已创建"
# ── 步骤3: 创建本地配置(如不存在)──
echo ""
info "步骤3/5: 配置本地参数..."
LOCAL_CONFIG="$AGENT_LOCAL/config.yaml"
if [ ! -f "$LOCAL_CONFIG" ]; then
HOST="$(hostname)"
cat > "$LOCAL_CONFIG" << CONFEOF
# agent-local/config.yaml — 本机唯一配置入口
# 此文件每台机器独立,不同步到 Git。覆盖 ORACLE.yaml 中的默认值。
hostname: "$HOST"
machine_uid: ""
# 代理设置(各机器不同)
proxy:
socks5: "socks5://127.0.0.1:10800"
# 端口分配
ports:
dashboard: 9988
camouflage_base: 9200
CONFEOF
ok "已创建: $LOCAL_CONFIG"
else
ok "本地配置已存在: $LOCAL_CONFIG"
fi
# ── 步骤4: 检查 Python 和依赖 ──
echo ""
info "步骤4/5: 检查运行环境..."
PYTHON_OK=false
for py in "$AGENT_LOCAL/../.workbuddy/binaries/python/envs/agent-os/bin/python3" \
"$HOME/.workbuddy/binaries/python/envs/agent-os/bin/python3" \
"/Users/chengzige/.workbuddy/binaries/python/envs/agent-os/bin/python3" \
"/usr/local/bin/python3" "/usr/bin/python3"; do
if [ -x "$py" ]; then
VER=$("$py" --version 2>&1)
info "Python: $VER ($py)"
PYTHON_OK=true
break
fi
done
if [ "$PYTHON_OK" = false ]; then
warn "未找到可用的 Python 3。请先安装 WorkBuddy 或 Python 3.13+"
fi
# 检查关键依赖
if [ "$PYTHON_OK" = true ]; then
# 找可用的 pip
for py in "$HOME/.workbuddy/binaries/python/envs/agent-os/bin/pip3" \
"$HOME/.workbuddy/binaries/python/versions/3.13.12/bin/pip3" \
"pip3"; do
if command -v "$py" &>/dev/null; then
PIP="$py"
break
fi
done
fi
# ── 步骤5: 检查 Tailscale(远程连接)──
echo ""
info "步骤5/6: 检查远程连接..."
if command -v tailscale &>/dev/null; then
TS_STATUS=$(tailscale status 2>/dev/null | head -3)
if [ -n "$TS_STATUS" ]; then
ok "Tailscale 已连接"
echo "$TS_STATUS" | while read line; do echo " $line"; done
else
warn "Tailscale 已安装但未登录,请运行: tailscale up"
fi
else
warn "Tailscale 未安装。远程连接需要 Tailscale 实现跨局域网通信。"
warn "安装方法: brew install --cask tailscale 或从 https://tailscale.com/download 下载"
echo ""
fi
# ── 步骤6: 检查 Python 代码签名(新增:解决 macOS 代码签名问题)──
echo ""
info "步骤6/7: 检查 Python 代码签名..."
# 找到 WorkBuddy 管理的 Python
for PY_BIN in "$HOME/.workbuddy/binaries/python/versions/3.13.12/bin/python3" \
"$HOME/.workbuddy/binaries/python/versions/"*/bin/python3; do
if [ -x "$PY_BIN" ]; then
break
fi
done
if [ -x "$PY_BIN" ]; then
# 检查是否有 Team ID(腾讯签名会导致 orjson 无法加载)
TEAM_ID=$(codesign -dvv "$PY_BIN" 2>&1 | grep "TeamIdentifier" | awk '{print $2}')
if [ -n "$TEAM_ID" ] && [ "$TEAM_ID" != "not set" ]; then
warn "Python 带有 Team ID 签名: $TEAM_ID → 可能导致 orjson 加载失败"
info "正在修复: codesign -f -s - ..."
codesign -f -s - "$PY_BIN" 2>&1 && ok "Python 代码签名已修复" || warn "修复失败,需手动执行"
else
ok "Python 代码签名正常(adhoc)"
fi
# 验证 orjson
OK=$("$PY_BIN" -c "import orjson; print(orjson.__version__)" 2>/dev/null)
if [ -n "$OK" ]; then
ok "orjson $OK ✅"
else
warn "orjson 不可用,尝试安装依赖..."
"$PY_BIN" -m pip install -r "$SCRIPT_DIR/requirements.txt" 2>&1 | tail -2
fi
# 验证 Camoufox
C_OK=$("$PY_BIN" -c "from camoufox import Camoufox; print(1)" 2>/dev/null)
if [ "$C_OK" = "1" ]; then
ok "Camoufox ✅"
else
warn "Camoufox 不可用,安装中..."
"$PY_BIN" -m pip install camoufox 2>&1 | tail -2
fi
fi
# ── 步骤7: 同步 ORACLE 和授权 ──
echo ""
info "步骤7/7: 同步配置..."
# 确保 ORACLE.yaml 可读
if [ -f "$AGENT_SYNC/ORACLE.yaml" ]; then
ok "ORACLE.yaml 已就绪"
else
warn "ORACLE.yaml 不存在(可能尚未创建)"
fi
# ── 安装防泄露 pre-commit hook ──
echo ""
info "安装防泄露 hook..."
HOOK_SRC="$AGENT_SYNC/00_bootstrap/hooks/pre-commit"
HOOK_DST="$AGENT_SYNC/.git/hooks/pre-commit"
if [ -f "$HOOK_SRC" ]; then
cp "$HOOK_SRC" "$HOOK_DST" && chmod +x "$HOOK_DST"
ok "pre-commit hook 已安装(提交前自动拦截 API Key)"
else
warn "未找到 pre-commit hook 源文件,跳过"
fi
# 提醒用户
echo ""
echo "============================================"
ok "部署完成!"
echo ""
echo " ✅ 环境变量: AGENT_SYNC/AGENT_LOCAL 已写入 shell 配置文件"
echo " ✅ 目录结构: $AGENT_LOCAL 已创建"
echo " ✅ 本地配置: $AGENT_LOCAL/config.yaml"
echo " ✅ ORACLE 宪法: $AGENT_SYNC/ORACLE.yaml"
echo " ✅ 防泄露 hook: 已安装(API Key 提交将被拦截)"
echo ""
echo " 后续手动步骤:"
echo " 1. 使环境变量生效: source ~/.zshrc"
echo " 2. 配置你自己的 API 密钥(重要!):"
echo " 打开 Dashboard → 服务 → 🔑 API 配置"
echo " 或用环境变量: export DEEPSEEK_API_KEY=sk-xxx / DASHSCOPE_API_KEY=sk-xxx"
echo " (密钥只存本机 agent-local,不会提交到仓库)"
echo " 3. (如未安装 Tailscale)安装并登录:"
echo " brew install --cask tailscale"
echo " 或从 https://tailscale.com/download 下载"
echo " 4. 开启远程登录(每台机器一次):"
echo " sudo systemsetup -setremotelogin on"
echo "============================================"