移植自 mac-agent-os 的 mediacrawler_adapter:不起子进程、不开页面,用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin,所以 ingest 那条链路一个字不用改。
**目前能用的(真环境实测,非推断)**:
profile/other : 200, 7075 字节 —— 博主主页指标(粉丝/获赞/作品数/昵称)
aweme/detail : 200, 45425 字节 —— 单条作品详情(含点赞/评论/收藏/分享)
**目前不能用的:作品列表 `aweme/post`。** 两个互相独立的原因:
1. 这个接口被抖音单独升级成了真校验:不带 x-tt-argus 回 403「Uifid Not Found」,
带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过,「真校验」过不了。
同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的,
挑中的恰好是「批量拉作品列表」这个最敏感的动作。
2. 改走页面截获也不行:CDP 浏览器打开博主主页会落到「验证码中间页」(当天大量探测的
代价,过几小时要重测)。
所以现在的边界是:**已知作品的指标刷新能做,自动发现新作品做不了**。
**排查中控住变量后得到的两条事实**(都写进注释了):
· `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
UA+client hints+Cookie 时是 200 但仅 121 字节的空壳,补上这三个头变 7074 字节。
(我先前猜的 sec-ch-ua 不是关键。)
· 因此 UA 与 client hints 必须**成套地取自同一个浏览器**,所以 BrowserIdentity 一次
从 CDP 取齐 cookie + UA + hints,而不是各自写死。
「200 + 空 body 必须当场报错」也是刻意写死的:放过去它会在下游变成「这个博主没作品」,
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的,还被翻译成「账号被封」。
测试 +11:cookie 解析、请求头成套性(含 uifid 缺失/回退)、产物键名与 store 对齐、
以及 _get 的三条失败路径(空 body / 403 带网关原话 / 正常返回)。
185 lines
7.4 KiB
Python
185 lines
7.4 KiB
Python
"""
|
|
远程执行引擎
|
|
|
|
⚠️ 已废弃 — 请使用 services/command_bus.py 替代。
|
|
本文件保留以供 app.py/routes 引用过渡使用,新代码禁止导入。
|
|
command_bus.py 的 MachineSession._send_remote() 已实现相同功能。
|
|
|
|
通过 SSH 在远程机器执行命令,返回执行结果。
|
|
机器信息从 ORACLE.yaml 读取。
|
|
"""
|
|
|
|
import logging, subprocess, sys
|
|
from pathlib import Path
|
|
|
|
logger = logging.getLogger("dashboard.remote_exec")
|
|
|
|
_THIS_DIR = Path(__file__).resolve().parent.parent
|
|
if str(_THIS_DIR) not in sys.path:
|
|
sys.path.insert(0, str(_THIS_DIR))
|
|
|
|
_HOSTNAME_FILE = Path.home() / "workbuddy-agent-os" / "agent-local" / "identity" / "cached_hostname"
|
|
_LOCAL_HOSTNAME = _HOSTNAME_FILE.read_text().strip() if _HOSTNAME_FILE.exists() else subprocess.run(["hostname"], capture_output=True, text=True).stdout.strip()
|
|
|
|
ORACLE_PATH = _THIS_DIR.parent.parent / "ORACLE.yaml"
|
|
_MACHINE_CACHE = None
|
|
|
|
|
|
def _get_machine_info(machine_name: str) -> dict:
|
|
"""从 ORACLE 查找机器信息"""
|
|
import yaml, os
|
|
global _MACHINE_CACHE
|
|
|
|
if not ORACLE_PATH.exists():
|
|
return {}
|
|
|
|
if _MACHINE_CACHE is None:
|
|
with open(ORACLE_PATH) as f:
|
|
_MACHINE_CACHE = yaml.safe_load(f)
|
|
|
|
for name, info in _MACHINE_CACHE.get("machines", {}).items():
|
|
if name == machine_name or info.get("hostname") == machine_name:
|
|
return {
|
|
"name": name,
|
|
"hostname": info.get("hostname", name),
|
|
"ip": info.get("tailscale_ip", ""),
|
|
"user": info.get("ssh_user", ""),
|
|
}
|
|
return {}
|
|
|
|
|
|
def exec_remote(machine: str, command: str, timeout: int = 60, fire_and_forget: bool = False) -> dict:
|
|
"""在远程机器执行命令
|
|
|
|
Args:
|
|
machine: 机器名(对应 ORACLE 中的定义)
|
|
command: 要执行的命令
|
|
timeout: 超时秒数
|
|
fire_and_forget: 是否立即返回(适合浏览器交互类命令)
|
|
|
|
Returns:
|
|
{"status": "ok"/"error", "stdout": "...", "stderr": "...", "returncode": 0}
|
|
"""
|
|
info = _get_machine_info(machine)
|
|
if not info or not info.get("ip"):
|
|
return {"status": "error", "message": f"未知机器: {machine}"}
|
|
|
|
hostname = info.get("hostname", "")
|
|
user = info.get("user", "")
|
|
ip = info["ip"]
|
|
|
|
# ── 本机:直接 subprocess ──
|
|
is_local = (machine == _LOCAL_HOSTNAME or hostname == _LOCAL_HOSTNAME)
|
|
if is_local:
|
|
try:
|
|
r = subprocess.run(
|
|
command, shell=True, capture_output=True, text=True, timeout=timeout
|
|
)
|
|
return {
|
|
"status": "ok",
|
|
"returncode": r.returncode,
|
|
"output": r.stdout[:5000],
|
|
"stderr": r.stderr[:500],
|
|
}
|
|
except subprocess.TimeoutExpired:
|
|
return {"status": "error", "message": f"命令超时({timeout}s)"}
|
|
except Exception as e:
|
|
return {"status": "error", "message": str(e)}
|
|
|
|
# ── 远程:SSH ──
|
|
ssh_target = f"{user}@{ip}" if user else ip
|
|
|
|
# 自动设置环境变量 + PYTHON 路径发现
|
|
py_discover = 'PY=$(ls $HOME/.workbuddy/binaries/python/envs/agent-os/bin/python3 2>/dev/null || which python3); '
|
|
env_setup = "export AGENT_SYNC=\"$HOME/workbuddy-agent-os/agent-sync\"; "
|
|
env_setup += "export AGENT_LOCAL=\"$HOME/workbuddy-agent-os/agent-local\"; "
|
|
env_setup += "export MC_PYTHON=\"$PY\"; "
|
|
|
|
if fire_and_forget:
|
|
# 立即返回,不等待远程命令完成(适用浏览器登录等交互命令)
|
|
full_cmd = f"{py_discover} {env_setup} nohup {command} > /tmp/remote_exec.log 2>&1 &"
|
|
try:
|
|
subprocess.run(
|
|
["ssh", "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no",
|
|
ssh_target, full_cmd],
|
|
capture_output=True, text=True, timeout=15
|
|
)
|
|
return {"status": "ok", "message": "命令已发送", "fire_and_forget": True}
|
|
except Exception as e:
|
|
return {"status": "error", "message": str(e)}
|
|
|
|
full_cmd = f"{py_discover} {env_setup} {command}"
|
|
try:
|
|
r = subprocess.run(
|
|
["ssh", "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no",
|
|
ssh_target, full_cmd],
|
|
capture_output=True, text=True, timeout=timeout
|
|
)
|
|
return {
|
|
"status": "ok" if r.returncode == 0 else "error",
|
|
"returncode": r.returncode,
|
|
"stdout": r.stdout[:5000],
|
|
"stderr": r.stderr[:500],
|
|
}
|
|
except subprocess.TimeoutExpired:
|
|
return {"status": "error", "message": f"执行超时 ({timeout}s)"}
|
|
except FileNotFoundError:
|
|
return {"status": "error", "message": "ssh 命令未找到"}
|
|
except Exception as e:
|
|
return {"status": "error", "message": str(e)}
|
|
|
|
|
|
def exec_nurture(machine: str, accounts: list, blueprints: list, rounds: int = 3) -> dict:
|
|
"""在远程机器启动养号"""
|
|
accts = ",".join(accounts)
|
|
bps = ",".join(blueprints)
|
|
cmd = f"cd $AGENT_SYNC/05_tools/07_matrix/scripts && $MC_PYTHON -m mc run --accounts={accts} --blueprints={bps} --rounds={rounds} --mix --interval=45-90"
|
|
return exec_remote(machine, cmd, timeout=30)
|
|
|
|
|
|
def exec_collect(machine: str, phone: str = "") -> dict:
|
|
"""在远程机器启动采集"""
|
|
cmd = f"cd $AGENT_SYNC/05_tools/07_matrix/scripts && $MC_PYTHON -m mc collect"
|
|
if phone:
|
|
cmd += f" --phone {phone}"
|
|
else:
|
|
cmd += " --all"
|
|
return exec_remote(machine, cmd, timeout=30)
|
|
|
|
|
|
def exec_login(machine: str, account_id: str) -> dict:
|
|
"""在远程机器智能登录(全自动,立即返回)"""
|
|
cmd = f"cd $AGENT_SYNC/05_tools/07_matrix/scripts && $MC_PYTHON -m mc smart-login {account_id}"
|
|
return exec_remote(machine, cmd, timeout=120, fire_and_forget=True)
|
|
|
|
|
|
def exec_logout(machine: str, account_id: str) -> dict:
|
|
"""在远程机器清除账号登录状态"""
|
|
cmd = f"cd $AGENT_SYNC/05_tools/07_matrix/scripts && $MC_PYTHON -m mc account logout {account_id}"
|
|
return exec_remote(machine, cmd, timeout=30)
|
|
|
|
|
|
def exec_comment(machine: str, account_id: str, url: str, direction: str = "") -> dict:
|
|
"""在远程机器执行定向评论"""
|
|
dir_flag = f" --direction {direction}" if direction else ""
|
|
cmd = f"cd $AGENT_SYNC/05_tools/07_matrix/scripts && $MC_PYTHON -m mc task comment --account={account_id} --url={url}{dir_flag}"
|
|
return exec_remote(machine, cmd, timeout=120)
|
|
|
|
|
|
def exec_record(machine: str, account_id: str, platform: str = "douyin") -> dict:
|
|
"""在远程机器启动交互式录制"""
|
|
cmd = f"cd $AGENT_SYNC/05_tools/07_matrix/scripts/mc && $MC_PYTHON recorder.py {account_id} {platform}"
|
|
return exec_remote(machine, cmd, timeout=30, fire_and_forget=True)
|
|
|
|
|
|
def exec_nurture_stop(machine: str) -> dict:
|
|
"""停止远程机器的养号任务"""
|
|
cmd = "pkill -f 'mc run' 2>/dev/null; pkill -f camoufox 2>/dev/null; echo 'stopped'"
|
|
return exec_remote(machine, cmd, timeout=15)
|
|
|
|
|
|
def exec_status(machine: str) -> dict:
|
|
"""获取远程机器的完整状态"""
|
|
cmd = "echo '---guardd---'; pgrep -f guardd && echo 'running' || echo 'stopped'; echo '---camoufox---'; pgrep -f camoufox && echo 'running' || echo 'stopped'; echo '---disk---'; df -h / | tail -1 | awk '{print $3 \"/\" $2 \" (\" $5 \")\"}'; echo '---load---'; uptime | awk -F'load averages:' '{print $2}'"
|
|
return exec_remote(machine, cmd, timeout=10)
|