feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程,直接发 HTTP
移植自 mac-agent-os 的 mediacrawler_adapter:不起子进程、不开页面,用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin,所以 ingest 那条链路一个字不用改。
**目前能用的(真环境实测,非推断)**:
profile/other : 200, 7075 字节 —— 博主主页指标(粉丝/获赞/作品数/昵称)
aweme/detail : 200, 45425 字节 —— 单条作品详情(含点赞/评论/收藏/分享)
**目前不能用的:作品列表 `aweme/post`。** 两个互相独立的原因:
1. 这个接口被抖音单独升级成了真校验:不带 x-tt-argus 回 403「Uifid Not Found」,
带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过,「真校验」过不了。
同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的,
挑中的恰好是「批量拉作品列表」这个最敏感的动作。
2. 改走页面截获也不行:CDP 浏览器打开博主主页会落到「验证码中间页」(当天大量探测的
代价,过几小时要重测)。
所以现在的边界是:**已知作品的指标刷新能做,自动发现新作品做不了**。
**排查中控住变量后得到的两条事实**(都写进注释了):
· `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
UA+client hints+Cookie 时是 200 但仅 121 字节的空壳,补上这三个头变 7074 字节。
(我先前猜的 sec-ch-ua 不是关键。)
· 因此 UA 与 client hints 必须**成套地取自同一个浏览器**,所以 BrowserIdentity 一次
从 CDP 取齐 cookie + UA + hints,而不是各自写死。
「200 + 空 body 必须当场报错」也是刻意写死的:放过去它会在下游变成「这个博主没作品」,
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的,还被翻译成「账号被封」。
测试 +11:cookie 解析、请求头成套性(含 uifid 缺失/回退)、产物键名与 store 对齐、
以及 _get 的三条失败路径(空 body / 403 带网关原话 / 正常返回)。
This commit is contained in:
@@ -0,0 +1,119 @@
|
||||
"""
|
||||
account_monitor.py — 本机账号登录状态采集 (guardd 模块)
|
||||
|
||||
职责:
|
||||
- 读取本机 identities/ 目录下所有账号的 cookies.sqlite
|
||||
- 返回 {account_id: status} 映射
|
||||
- 每次调用实时检测(不缓存)
|
||||
|
||||
状态定义:
|
||||
logged_in — cookite 文件存在且含 session cookie
|
||||
no_cookie — 身份目录存在但无 cookies.sqlite 文件
|
||||
empty_cookie — cookies.sqlite 文件小于 100 字节
|
||||
no_session — cookies.sqlite 存在但无 session 类型 cookie
|
||||
no_identity — 身份目录不存在
|
||||
error — 读取异常
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger("guardd.account_monitor")
|
||||
|
||||
# ── 路径 ──────────────────────────────────────────────
|
||||
AGENT_LOCAL = Path(os.environ.get(
|
||||
"AGENT_LOCAL",
|
||||
str(Path.home() / "workbuddy-agent-os" / "agent-local")
|
||||
))
|
||||
MATRIX_IDENTITIES = AGENT_LOCAL / "tools" / "matrix" / "identities"
|
||||
OVERRIDE_PATH = AGENT_LOCAL / "tools" / "matrix" / "config" / "accounts.override.yaml"
|
||||
|
||||
|
||||
class AccountMonitor:
|
||||
"""本机账号登录状态实时采集器(60 秒缓存)"""
|
||||
|
||||
def __init__(self):
|
||||
self._identities_root = MATRIX_IDENTITIES
|
||||
self._override_path = OVERRIDE_PATH
|
||||
self._cache = {} # {account_id: status}
|
||||
self._cache_time = 0 # 上次缓存时间
|
||||
self._cache_ttl = 60 # 秒
|
||||
|
||||
def collect_status(self) -> dict:
|
||||
"""采集本机所有账号的登录状态(带 60 秒缓存)"""
|
||||
import time as _t
|
||||
now = _t.time()
|
||||
if self._cache and now - self._cache_time < self._cache_ttl:
|
||||
return dict(self._cache)
|
||||
accounts = self._get_local_accounts()
|
||||
result = {}
|
||||
for account_id, identity_hint in accounts.items():
|
||||
status = self._check_login_status(identity_hint)
|
||||
result[account_id] = status
|
||||
self._cache = result
|
||||
self._cache_time = now
|
||||
return result
|
||||
|
||||
def get_cached_status(self, account_id: str) -> str:
|
||||
"""快速查某个账号的登录状态(只走缓存,不读盘)"""
|
||||
if self._cache and account_id in self._cache:
|
||||
return self._cache[account_id]
|
||||
return "unknown"
|
||||
|
||||
def _get_local_accounts(self) -> dict:
|
||||
"""获取本机账号列表 {account_id: identity_hint}"""
|
||||
# 优先级 1: override.yaml
|
||||
if self._override_path.exists():
|
||||
try:
|
||||
import yaml
|
||||
data = yaml.safe_load(self._override_path.read_text()) or {}
|
||||
accts = {}
|
||||
for a in data.get("accounts", []):
|
||||
aid = a.get("id", "")
|
||||
if not aid:
|
||||
continue
|
||||
hint = a.get("identity_dir") or a.get("identity_hint") or aid
|
||||
hint = hint.replace("identities/", "")
|
||||
accts[aid] = hint
|
||||
if accts:
|
||||
return accts
|
||||
except Exception as e:
|
||||
logger.warning(f"读取 override.yaml 失败: {e}")
|
||||
|
||||
# 优先级 2: 扫描 identities/ 目录(降级)
|
||||
if self._identities_root.exists():
|
||||
accts = {}
|
||||
for entry in sorted(self._identities_root.iterdir()):
|
||||
if entry.is_dir() and not entry.name.startswith("."):
|
||||
accts[entry.name] = entry.name
|
||||
return accts
|
||||
|
||||
return {}
|
||||
|
||||
def _check_login_status(self, identity_hint: str) -> str:
|
||||
"""检查单个身份目录的登录状态(逻辑同 matrix_mgmt._check_login_status_by_hint)"""
|
||||
if not identity_hint:
|
||||
return "no_identity"
|
||||
identity_path = self._identities_root / identity_hint
|
||||
if not identity_path.exists():
|
||||
return "no_identity"
|
||||
cookie_path = identity_path / "user_data" / "cookies.sqlite"
|
||||
if not cookie_path.exists():
|
||||
return "no_cookie"
|
||||
if cookie_path.stat().st_size < 100:
|
||||
return "empty_cookie"
|
||||
try:
|
||||
conn = sqlite3.connect(str(cookie_path), timeout=2)
|
||||
cur = conn.cursor()
|
||||
total = cur.execute("SELECT count(*) FROM moz_cookies").fetchone()[0]
|
||||
session = cur.execute(
|
||||
"SELECT count(*) FROM moz_cookies WHERE name LIKE '%session%'"
|
||||
).fetchone()[0]
|
||||
conn.close()
|
||||
return "logged_in" if session > 0 else "no_session"
|
||||
except Exception as e:
|
||||
logger.debug(f"cookie 检查异常 {identity_hint}: {e}")
|
||||
return "error"
|
||||
@@ -0,0 +1,268 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
executor.py — 任务执行器 (guardd 模块 同步版)
|
||||
|
||||
职责:
|
||||
- 调 mc run / mc interact 等命令
|
||||
- 实时解析 stdout 提取进度
|
||||
- 超时控制(含 60 秒无输出检测)
|
||||
- 执行结果写入 task_store
|
||||
- shell=False(不再通过 /bin/sh 启动,避免僵尸 shell PID)
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
import logging
|
||||
from modules.task_store import TaskStore, STATUS_RUNNING, STATUS_COMPLETED, STATUS_FAILED
|
||||
|
||||
logger = logging.getLogger("guardd.executor")
|
||||
|
||||
HOME = Path.home()
|
||||
SCRIPTS_DIR = HOME / "workbuddy-agent-os" / "agent-sync" / "05_tools" / "07_matrix" / "scripts"
|
||||
PYTHON_PATH = HOME / ".workbuddy" / "binaries" / "python" / "envs" / "agent-os" / "bin" / "python3"
|
||||
|
||||
|
||||
class Executor:
|
||||
"""任务执行器(同步版本,shell=False)"""
|
||||
|
||||
def __init__(self, task_store: TaskStore, slot_manager=None):
|
||||
self.task_store = task_store
|
||||
self.slot_manager = slot_manager
|
||||
self._running_procs = {} # task_id → proc
|
||||
|
||||
def execute(self, task: dict) -> dict:
|
||||
"""执行一个任务,实时解析输出,返回结果"""
|
||||
task_id = task["task_id"]
|
||||
account_id = task.get("accounts", [""])[0] if task.get("accounts") else ""
|
||||
identity_dir = task.get("identity_dir", account_id)
|
||||
|
||||
cmd_info = self._build_cmd(task)
|
||||
if not cmd_info:
|
||||
task["status"] = STATUS_FAILED
|
||||
task["error"] = "无法构建执行命令"
|
||||
self.task_store.save(task)
|
||||
return {"success": False, "error": "无法构建执行命令"}
|
||||
|
||||
logger.info(f" ▶️ executor: task_id={task.get('task_id','')} accounts={task.get('accounts',[])} cmd_type={task.get('cmd_type','')}")
|
||||
logger.info(f" ▶️ cmd: {' '.join(cmd_info['args'])[:200] if cmd_info.get('args') else 'NONE'}")
|
||||
|
||||
task["status"] = STATUS_RUNNING
|
||||
task["started_at"] = time.time()
|
||||
self.task_store.save(task)
|
||||
|
||||
# 获取槽位
|
||||
slot = None
|
||||
if self.slot_manager:
|
||||
try:
|
||||
nickname = task.get("nickname", "")
|
||||
platform = task.get("platform", "")
|
||||
if not platform and account_id:
|
||||
platform = "xiaohongshu" if account_id.startswith("xhs_") else "douyin"
|
||||
if not nickname:
|
||||
try:
|
||||
import json
|
||||
pf = HOME / "workbuddy-agent-os" / "agent-local" / "tools" / "matrix" / "data" / "profiles.json"
|
||||
if pf.exists():
|
||||
profiles = json.loads(pf.read_text())
|
||||
if account_id in profiles:
|
||||
nickname = profiles[account_id].get("nickname", "")
|
||||
except Exception:
|
||||
pass
|
||||
slot = self.slot_manager.acquire(account_id, identity_dir, nickname, platform)
|
||||
except Exception as e:
|
||||
task["status"] = STATUS_FAILED
|
||||
task["error"] = str(e)
|
||||
self.task_store.save(task)
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
log_lines = []
|
||||
# 任务日志文件
|
||||
log_path = HOME / "workbuddy-agent-os" / "agent-local" / "runtime" / "guardd" / "tasks" / f"{task_id}.log"
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
log_fh = open(log_path, "w", encoding="utf-8")
|
||||
try:
|
||||
# shell=False:直接启动 python3 -m ...,不经过 /bin/sh
|
||||
proc = subprocess.Popen(
|
||||
cmd_info["args"],
|
||||
cwd=cmd_info.get("cwd"),
|
||||
env=cmd_info.get("env"),
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
preexec_fn=os.setsid
|
||||
)
|
||||
if slot:
|
||||
self.slot_manager.set_pid(slot["browser_id"], proc.pid)
|
||||
self._running_procs[task_id] = proc
|
||||
|
||||
max_execution_sec = task.get("max_execution_sec", 7200)
|
||||
cmd_type = task.get("cmd_type", "")
|
||||
# 仅登录任务给 5 分钟静默等待(首次登录需手动操作),其余保持 60 秒
|
||||
idle_timeout = 300 if cmd_type in ("login", "smart-login") else 60
|
||||
start_time = time.time()
|
||||
last_output_time = time.time()
|
||||
|
||||
import select as _select
|
||||
|
||||
# 实时读取输出(非阻塞版)
|
||||
while True:
|
||||
now = time.time()
|
||||
elapsed = now - start_time
|
||||
|
||||
# 浏览器启动超时:登录任务 5 分钟无输出,其他任务 60 秒无输出
|
||||
if elapsed > idle_timeout and (now - last_output_time) > idle_timeout:
|
||||
try:
|
||||
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
||||
except Exception:
|
||||
proc.kill()
|
||||
task["status"] = STATUS_FAILED
|
||||
task["error"] = "浏览器启动超时(60秒无输出)"
|
||||
task["completed_at"] = time.time()
|
||||
self.task_store.save(task)
|
||||
logger.warning(f" ⏰ [{task_id}] 浏览器启动超时(60秒无输出)")
|
||||
return {"success": False, "error": "浏览器启动超时"}
|
||||
|
||||
if elapsed > max_execution_sec:
|
||||
try:
|
||||
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
||||
except Exception:
|
||||
proc.kill()
|
||||
task["status"] = STATUS_FAILED
|
||||
task["error"] = f"超时 ({max_execution_sec:.0f}s)"
|
||||
task["completed_at"] = time.time()
|
||||
self.task_store.save(task)
|
||||
logger.warning(f" ⏰ [{task_id}] 超时 ({elapsed:.0f}s > {max_execution_sec}s)")
|
||||
return {"success": False, "error": task["error"]}
|
||||
|
||||
r, _, _ = _select.select([proc.stdout], [], [], 2.0)
|
||||
if r:
|
||||
line = proc.stdout.readline()
|
||||
if not line:
|
||||
if proc.poll() is not None:
|
||||
break
|
||||
continue
|
||||
text = line.decode("utf-8", errors="replace").strip()
|
||||
log_lines.append(text)
|
||||
log_fh.write(text + "\n")
|
||||
log_fh.flush()
|
||||
last_output_time = time.time()
|
||||
self._parse_and_update(task_id, account_id, identity_dir, text)
|
||||
else:
|
||||
if proc.poll() is not None:
|
||||
break
|
||||
|
||||
exit_code = proc.wait()
|
||||
if exit_code != 0 and task.get("cmd_type") in ("collect", "nurture"):
|
||||
try:
|
||||
import json
|
||||
pf = HOME / "workbuddy-agent-os" / "agent-local" / "tools" / "matrix" / "data" / "profiles.json"
|
||||
if pf.exists():
|
||||
mtime = pf.stat().st_mtime
|
||||
started = task.get("started_at", 0)
|
||||
if mtime >= started - 5:
|
||||
logger.info(f" ✅ [{task_id}] profiles.json 已更新 (mtime={mtime:.0f} >= start={started:.0f})")
|
||||
exit_code = 0
|
||||
except Exception:
|
||||
pass
|
||||
success = exit_code == 0
|
||||
task["status"] = STATUS_COMPLETED if success else STATUS_FAILED
|
||||
task["completed_at"] = time.time()
|
||||
task["exit_code"] = exit_code
|
||||
self.task_store.save(task)
|
||||
return {"success": success, "exit_code": exit_code, "task_id": task_id}
|
||||
|
||||
except Exception as e:
|
||||
task["status"] = STATUS_FAILED
|
||||
task["error"] = str(e)
|
||||
task["completed_at"] = time.time()
|
||||
self.task_store.save(task)
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
finally:
|
||||
if slot:
|
||||
self.slot_manager.release(slot["browser_id"])
|
||||
self._running_procs.pop(task_id, None)
|
||||
try:
|
||||
log_fh.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _build_cmd(self, task: dict) -> Optional[dict]:
|
||||
"""构建执行命令(返回 args/cwd/env 字典,shell=False)"""
|
||||
scripts_dir = str(SCRIPTS_DIR)
|
||||
python_path = str(PYTHON_PATH)
|
||||
env = os.environ.copy()
|
||||
env["PYTHONPATH"] = f"{scripts_dir}:{env.get('PYTHONPATH', '')}"
|
||||
|
||||
cmd_line = task.get("command_line", "")
|
||||
if cmd_line:
|
||||
# command_line 形如: mc task comment --account=X --url=Y -y
|
||||
# 也有旧格式: mc run --accounts=X --blueprints=Y ...
|
||||
args = [python_path, "-m"] + shlex.split(cmd_line)
|
||||
return {"args": args, "cwd": scripts_dir, "env": env}
|
||||
|
||||
cmd_type = task.get("cmd_type", "")
|
||||
accounts = ",".join(task.get("accounts", []))
|
||||
blueprint = task.get("blueprint", "")
|
||||
rounds = task.get("rounds", 1)
|
||||
|
||||
if cmd_type == "nurture":
|
||||
args = [python_path, "-m", "mc", "run",
|
||||
f"--accounts={accounts}", f"--blueprints={blueprint}",
|
||||
f"--rounds={rounds}", "--mix", "--interval=45-90"]
|
||||
return {"args": args, "cwd": scripts_dir, "env": env}
|
||||
elif cmd_type in ("interact", "comment"):
|
||||
url = task.get("params", {}).get("url", "")
|
||||
direction = task.get("params", {}).get("direction", "")
|
||||
args = [python_path, "-m", "mc", "run",
|
||||
f"--accounts={accounts}", f"--blueprints={blueprint}",
|
||||
"--rounds=1", f"--url={url}", f"--direction={direction}"]
|
||||
interval = task.get("interval", 0)
|
||||
if interval:
|
||||
args.append(f"--interval={interval}")
|
||||
return {"args": args, "cwd": scripts_dir, "env": env}
|
||||
elif cmd_type == "collect":
|
||||
args = [python_path, "-m", "mc", "run",
|
||||
f"--accounts={accounts}", f"--blueprints={blueprint}", "--rounds=1"]
|
||||
return {"args": args, "cwd": scripts_dir, "env": env}
|
||||
elif cmd_type == "login":
|
||||
args = [python_path, "-m", "mc", "smart-login", accounts]
|
||||
return {"args": args, "cwd": scripts_dir, "env": env}
|
||||
elif cmd_type == "record":
|
||||
platform = task.get("params", {}).get("platform", "douyin")
|
||||
args = [python_path, "-m", "mc", "record", "start",
|
||||
f"--accounts={accounts}", f"--platform={platform}"]
|
||||
return {"args": args, "cwd": scripts_dir, "env": env}
|
||||
return None
|
||||
|
||||
def _parse_and_update(self, task_id: str, account_id: str, browser_id: str, line: str):
|
||||
"""解析一行日志,更新任务/槽位进度"""
|
||||
if not self.slot_manager:
|
||||
return
|
||||
# 更新 step
|
||||
for pattern, step_name in [
|
||||
(r"(?:正在|开始|执行)\s*([\u4e00-\u9fa5_]+)", None),
|
||||
(r"step:\s*(\S+)", None),
|
||||
]:
|
||||
m = re.search(pattern, line)
|
||||
if m:
|
||||
self.slot_manager.update_step(browser_id, m.group(1))
|
||||
break
|
||||
# 更新 step_index(从 blueprint progress 提取)
|
||||
m = re.search(r"(?:步骤|step)\s*(\d+)\s*[/|]\s*(\d+)", line)
|
||||
if m:
|
||||
self.slot_manager.update_step(browser_id, line[:40],
|
||||
step_index=int(m.group(1)), total_steps=int(m.group(2)))
|
||||
|
||||
def kill(self, task_id: str) -> bool:
|
||||
proc = self._running_procs.get(task_id)
|
||||
if proc:
|
||||
try:
|
||||
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
||||
except Exception:
|
||||
proc.kill()
|
||||
return True
|
||||
return False
|
||||
@@ -0,0 +1,169 @@
|
||||
"""
|
||||
heartbeat.py — 增强心跳上报 (guardd 模块)
|
||||
|
||||
职责:
|
||||
- 每15秒采集系统状态 + 任务状态 + 槽位状态
|
||||
- 上报到 Dashboard (POST /api/push/heartbeat)
|
||||
- 本地写入 status/{hostname}/heartbeat.json
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from modules.task_store import TaskStore
|
||||
from modules.slot_manager import BrowserSlotManager
|
||||
from modules.scheduler import Scheduler
|
||||
from modules.account_monitor import AccountMonitor
|
||||
|
||||
logger = logging.getLogger("guardd.heartbeat")
|
||||
|
||||
|
||||
class HeartbeatReporter:
|
||||
"""增强心跳上报器"""
|
||||
|
||||
def __init__(self, task_store: TaskStore, slot_manager: BrowserSlotManager,
|
||||
scheduler: Scheduler, hostname: str, machine_uid: str,
|
||||
dashboard_url: str = None,
|
||||
account_monitor: AccountMonitor = None):
|
||||
self.task_store = task_store
|
||||
self.slot_manager = slot_manager
|
||||
self.scheduler = scheduler
|
||||
self.hostname = hostname
|
||||
self.machine_uid = machine_uid
|
||||
self.dashboard_url = dashboard_url or "http://127.0.0.1:9988"
|
||||
self.account_monitor = account_monitor
|
||||
|
||||
def collect(self) -> dict:
|
||||
"""采集完整状态(v2: 含 per-slot 详细进度)"""
|
||||
slot_usage = self.slot_manager.get_usage() if self.slot_manager else {"max": 0, "used": 0, "slots": []}
|
||||
|
||||
# 收集任务状态(v2: active_tasks 字典,支持多 slot 并行)
|
||||
active_tasks = list(self.scheduler.active_tasks.values()) if hasattr(self.scheduler, 'active_tasks') and self.scheduler.active_tasks else []
|
||||
active_list = []
|
||||
for t in active_tasks:
|
||||
at = self._task_to_heartbeat(t)
|
||||
if at:
|
||||
active_list.append(at)
|
||||
queued = self.scheduler.get_all_queued() if hasattr(self.scheduler, 'get_all_queued') else []
|
||||
task_counts = self.task_store.count()
|
||||
|
||||
# 增强 slot 信息:从 slot_manager 获取每个浏览器的当前步骤/时间
|
||||
slot_details = []
|
||||
for s in slot_usage.get("slots", []):
|
||||
if s.get("account_id"):
|
||||
slot_details.append({
|
||||
"slot_id": s.get("slot_id"),
|
||||
"account_id": s.get("account_id"),
|
||||
"nickname": s.get("nickname", ""),
|
||||
"platform": s.get("platform", ""),
|
||||
"current_step": s.get("current_step", ""),
|
||||
"step_index": s.get("step_index", 0),
|
||||
"total_steps": s.get("total_steps", 0),
|
||||
"elapsed_sec": s.get("elapsed_sec", 0),
|
||||
"health": s.get("health", "healthy"),
|
||||
"pid": s.get("pid"),
|
||||
})
|
||||
|
||||
# 系统状态
|
||||
cpu_load = -1
|
||||
try:
|
||||
cpu_load = round(os.getloadavg()[0], 2)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# 采集每个账号的登录状态
|
||||
account_status = {}
|
||||
if self.account_monitor:
|
||||
try:
|
||||
account_status = self.account_monitor.collect_status()
|
||||
except Exception as e:
|
||||
logger.debug(f"账号状态采集失败: {e}")
|
||||
|
||||
heartbeat = {
|
||||
"hostname": self.hostname,
|
||||
"machine_uid": self.machine_uid,
|
||||
"status": "online",
|
||||
"last_seen": datetime.now(timezone.utc).isoformat(),
|
||||
"version": "4.3.0",
|
||||
"account_status": account_status,
|
||||
"slots": {
|
||||
"max": slot_usage.get("max", 3),
|
||||
"used": slot_usage.get("used", 0),
|
||||
"list": slot_details,
|
||||
},
|
||||
"tasks": {
|
||||
"active": active_list,
|
||||
"queued": [
|
||||
{"task_id": q["task_id"], "priority": q.get("priority", 1),
|
||||
"estimated_at": q.get("scheduled_at", 0)}
|
||||
for q in queued[:20]
|
||||
],
|
||||
"counts": task_counts,
|
||||
},
|
||||
"system": {
|
||||
"cpu_load_1m": cpu_load,
|
||||
"browsers_open": slot_usage.get("used", 0),
|
||||
},
|
||||
}
|
||||
return heartbeat
|
||||
|
||||
def _task_to_heartbeat(self, task: dict) -> dict:
|
||||
"""把 task dict 转成心跳用的紧凑格式"""
|
||||
if not task:
|
||||
return None
|
||||
return {
|
||||
"task_id": task.get("task_id", ""),
|
||||
"cmd_type": task.get("cmd_type", ""),
|
||||
"type": task.get("task_type", task.get("cmd_type", "")),
|
||||
"account": (task.get("accounts") or [""])[0],
|
||||
"blueprint": task.get("blueprint", ""),
|
||||
"status": task.get("status", ""),
|
||||
"started_at": task.get("started_at", 0),
|
||||
"elapsed_sec": int(time.time() - task.get("started_at", time.time())),
|
||||
"progress": {
|
||||
"current_step": task.get("current_step", ""),
|
||||
"step_index": task.get("step_index", 0),
|
||||
"total_steps": task.get("total_steps", 0),
|
||||
},
|
||||
}
|
||||
|
||||
def send_to_dashboard(self, heartbeat: dict = None):
|
||||
"""发送心跳到 Dashboard"""
|
||||
if heartbeat is None:
|
||||
heartbeat = self.collect()
|
||||
|
||||
try:
|
||||
payload = json.dumps(heartbeat, ensure_ascii=False).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
f"{self.dashboard_url}/api/push/heartbeat",
|
||||
data=payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
urllib.request.urlopen(req, timeout=5)
|
||||
except Exception as e:
|
||||
logger.debug(f"心跳推送失败: {e}")
|
||||
|
||||
def write_local(self, heartbeat: dict = None):
|
||||
"""写入本地 heartbeat.json"""
|
||||
if heartbeat is None:
|
||||
heartbeat = self.collect()
|
||||
try:
|
||||
home = Path.home()
|
||||
status_dir = home / "workbuddy-agent-os" / "agent-local" / "runtime" / "guardd" / self.hostname
|
||||
status_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = status_dir / "heartbeat.json"
|
||||
path.write_text(json.dumps(heartbeat, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
# also write live file
|
||||
live_dir = status_dir.parent / "live"
|
||||
live_dir.mkdir(parents=True, exist_ok=True)
|
||||
(live_dir / f"{self.machine_uid}.json").write_text(
|
||||
json.dumps(heartbeat, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"本地心跳写入失败: {e}")
|
||||
@@ -0,0 +1,78 @@
|
||||
"""
|
||||
oracle_sync.py — ORACLE 定时任务同步 (guardd 模块)
|
||||
|
||||
职责:
|
||||
- guardd 启动时读取 ORACLE.yaml 的 schedules 节
|
||||
- 将定时任务导入到 task_store
|
||||
- 每6小时增量同步
|
||||
"""
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
from modules.task_store import TaskStore, STATUS_SCHEDULED
|
||||
|
||||
logger = logging.getLogger("guardd.oracle_sync")
|
||||
SYNC_INTERVAL = 6 * 3600 # 6小时
|
||||
|
||||
|
||||
class OracleSync:
|
||||
"""ORACLE.yaml 定时任务同步器"""
|
||||
|
||||
def __init__(self, task_store: TaskStore):
|
||||
self.task_store = task_store
|
||||
self.last_sync = 0
|
||||
self.home = Path.home()
|
||||
self.oracle_path = self.home / "workbuddy-agent-os" / "agent-sync" / "ORACLE.yaml"
|
||||
|
||||
def sync(self):
|
||||
"""执行同步(启动时 + 周期调用)"""
|
||||
now = time.time()
|
||||
if now - self.last_sync < SYNC_INTERVAL:
|
||||
return
|
||||
self.last_sync = now
|
||||
|
||||
if not self.oracle_path.exists():
|
||||
logger.info(" ORACLE.yaml 不存在,跳过定时任务同步")
|
||||
return
|
||||
|
||||
try:
|
||||
import yaml
|
||||
oracle = yaml.safe_load(self.oracle_path.read_text())
|
||||
schedules = oracle.get("schedules", [])
|
||||
count = 0
|
||||
for s in schedules:
|
||||
task_id = f"oracle_{s['name']}_{int(now)}"
|
||||
# 检查是否已存在
|
||||
existing = self.task_store.get(task_id)
|
||||
if existing:
|
||||
continue
|
||||
|
||||
blueprint = s.get("action", "").replace("mc run", "").strip()
|
||||
if not blueprint:
|
||||
blueprint = s.get("params", {}).get("blueprint", "douyin_daily")
|
||||
|
||||
task = {
|
||||
"task_id": task_id,
|
||||
"cmd_type": "nurture",
|
||||
"source": "oracle",
|
||||
"accounts": s.get("accounts", "all"),
|
||||
"machine": s.get("on_machines", "*"),
|
||||
"priority": 1,
|
||||
"schedule_type": "cron",
|
||||
"cron_expr": s["schedule"],
|
||||
"blueprint": blueprint,
|
||||
"rounds": s.get("params", {}).get("rounds", 10),
|
||||
"status": STATUS_SCHEDULED,
|
||||
"created_at": now,
|
||||
}
|
||||
self.task_store.save(task)
|
||||
count += 1
|
||||
logger.info(f" 📅 [{task_id}] 导入定时任务: {s['schedule']} {blueprint}")
|
||||
|
||||
if count:
|
||||
logger.info(f" ✅ 导入 {count} 条定时任务")
|
||||
else:
|
||||
logger.debug(" 没有新的定时任务需要导入")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f" ORACLE 同步异常: {e}")
|
||||
@@ -0,0 +1,110 @@
|
||||
"""
|
||||
priority_queue.py — 优先级队列 (guardd 模块)
|
||||
|
||||
基于 heapq 实现,支持:
|
||||
- 按优先级排序(0=最高)
|
||||
- 按时间调度(scheduled_at)
|
||||
- 取消/重排
|
||||
"""
|
||||
import heapq
|
||||
import time
|
||||
from typing import Optional, List
|
||||
from threading import Lock
|
||||
|
||||
|
||||
class PriorityQueue:
|
||||
"""优先级队列 — 支持时间调度"""
|
||||
|
||||
def __init__(self):
|
||||
self._heap = []
|
||||
self._task_map = {} # task_id → (priority, scheduled_at, task_id)
|
||||
self._lock = Lock()
|
||||
|
||||
def push(self, task: dict) -> None:
|
||||
"""推入任务,保留 accounts/cmd_type 等关键字段"""
|
||||
task_id = task["task_id"]
|
||||
priority = task.get("priority", 1)
|
||||
scheduled_at = task.get("scheduled_at", 0) or 0
|
||||
if isinstance(scheduled_at, str):
|
||||
try:
|
||||
scheduled_at = float(scheduled_at)
|
||||
except (ValueError, TypeError):
|
||||
scheduled_at = 0
|
||||
item = (priority, scheduled_at, task_id)
|
||||
with self._lock:
|
||||
heapq.heappush(self._heap, item)
|
||||
self._task_map[task_id] = {
|
||||
"priority": priority,
|
||||
"scheduled_at": scheduled_at,
|
||||
"accounts": task.get("accounts", []),
|
||||
"cmd_type": task.get("cmd_type", ""),
|
||||
"queued_at": task.get("queued_at", 0),
|
||||
"command_line": task.get("command_line", ""),
|
||||
"blueprint": task.get("blueprint", ""),
|
||||
}
|
||||
|
||||
def pop_ready(self, now: float = None) -> Optional[str]:
|
||||
"""取出到时间且优先级最高的任务,返回 task_id"""
|
||||
if now is None:
|
||||
now = time.time()
|
||||
with self._lock:
|
||||
while self._heap:
|
||||
priority, scheduled_at, task_id = self._heap[0]
|
||||
if scheduled_at <= now:
|
||||
heapq.heappop(self._heap)
|
||||
self._task_map.pop(task_id, None)
|
||||
return task_id
|
||||
else:
|
||||
return None # 堆顶还没到时间
|
||||
return None
|
||||
|
||||
def peek(self) -> Optional[dict]:
|
||||
"""查看堆顶但不取出"""
|
||||
with self._lock:
|
||||
if not self._heap:
|
||||
return None
|
||||
priority, scheduled_at, task_id = self._heap[0]
|
||||
return {"task_id": task_id, "priority": priority, "scheduled_at": scheduled_at}
|
||||
|
||||
def remove(self, task_id: str) -> bool:
|
||||
"""从队列中移除任务"""
|
||||
with self._lock:
|
||||
if task_id in self._task_map:
|
||||
# 标记移除(实际在 pop 时清理)
|
||||
self._task_map[task_id] = None
|
||||
return True
|
||||
return False
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
with self._lock:
|
||||
return len(self._heap) == 0
|
||||
|
||||
def size(self) -> int:
|
||||
with self._lock:
|
||||
return len(self._heap)
|
||||
|
||||
def clear(self):
|
||||
"""清空队列"""
|
||||
with self._lock:
|
||||
self._heap.clear()
|
||||
self._task_map.clear()
|
||||
|
||||
def get_all(self) -> List[dict]:
|
||||
"""返回所有队列中的任务信息(含 accounts/cmd_type 等字段)"""
|
||||
with self._lock:
|
||||
result = []
|
||||
for p, s, tid in self._heap:
|
||||
info = self._task_map.get(tid)
|
||||
if info is None:
|
||||
continue
|
||||
result.append({
|
||||
"task_id": tid,
|
||||
"priority": info.get("priority", p),
|
||||
"scheduled_at": info.get("scheduled_at", s),
|
||||
"accounts": info.get("accounts", []),
|
||||
"cmd_type": info.get("cmd_type", ""),
|
||||
"queued_at": info.get("queued_at", 0),
|
||||
"command_line": info.get("command_line", ""),
|
||||
"blueprint": info.get("blueprint", ""),
|
||||
})
|
||||
return result
|
||||
@@ -0,0 +1,135 @@
|
||||
"""
|
||||
schedule_bridge.py — 定时任务桥接 (guardd 模块)
|
||||
|
||||
职责:
|
||||
- 读取 mc/scheduler 的 schedule.yaml
|
||||
- 检查到期的定时任务
|
||||
- 提交到 guardd scheduler 执行(P1 优先级)
|
||||
- 记录最后执行时间,防止重复触发
|
||||
|
||||
配置路径: 05_tools/07_matrix/scripts/config/schedule.yaml
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from modules.task_store import TaskStore
|
||||
|
||||
logger = logging.getLogger("guardd.schedule_bridge")
|
||||
|
||||
HOME = Path.home()
|
||||
SCRIPTS_DIR = HOME / "workbuddy-agent-os" / "agent-sync" / "05_tools" / "07_matrix" / "scripts"
|
||||
SCHEDULE_YAML = SCRIPTS_DIR / "config" / "schedule.yaml"
|
||||
LAST_RUN_FILE = HOME / "workbuddy-agent-os" / "agent-local" / "runtime" / "guardd" / "schedule_last_run.json"
|
||||
|
||||
CHECK_INTERVAL = 60 # 每 60 秒检查一次
|
||||
|
||||
|
||||
class ScheduleBridge:
|
||||
"""定时任务 → guardd 调度器桥接"""
|
||||
|
||||
def __init__(self, task_store: TaskStore, scheduler=None, hostname: str = ""):
|
||||
self.task_store = task_store
|
||||
self.scheduler = scheduler
|
||||
self.hostname = hostname
|
||||
self.last_check = 0
|
||||
self._last_run_cache = {}
|
||||
|
||||
def load_last_runs(self):
|
||||
"""加载上次执行时间记录"""
|
||||
if LAST_RUN_FILE.exists():
|
||||
try:
|
||||
self._last_run_cache = json.loads(LAST_RUN_FILE.read_text())
|
||||
except Exception:
|
||||
self._last_run_cache = {}
|
||||
|
||||
def save_last_runs(self):
|
||||
"""保存上次执行时间"""
|
||||
LAST_RUN_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
LAST_RUN_FILE.write_text(json.dumps(self._last_run_cache, indent=2))
|
||||
|
||||
def check_schedules(self):
|
||||
"""检查是否有定时任务到期"""
|
||||
now = time.time()
|
||||
if now - self.last_check < CHECK_INTERVAL:
|
||||
return 0
|
||||
self.last_check = now
|
||||
|
||||
if not SCHEDULE_YAML.exists():
|
||||
return 0
|
||||
|
||||
try:
|
||||
import yaml
|
||||
data = yaml.safe_load(SCHEDULE_YAML.read_text())
|
||||
schedules = data.get("schedules", {}) if data else {}
|
||||
except Exception as e:
|
||||
logger.warning(f"schedule.yaml 读取失败: {e}")
|
||||
return 0
|
||||
|
||||
self.load_last_runs()
|
||||
triggered = 0
|
||||
now_dt = datetime.now()
|
||||
today_minutes = now_dt.hour * 60 + now_dt.minute
|
||||
weekday = now_dt.isoweekday() # 1=周一..7=周日
|
||||
|
||||
for sid, s in schedules.items():
|
||||
if not s.get("enabled", False):
|
||||
continue
|
||||
|
||||
# 检查星期匹配
|
||||
days_str = s.get("days", "1,2,3,4,5,6,7")
|
||||
days = [int(d.strip()) for d in days_str.split(",") if d.strip()]
|
||||
if weekday not in days:
|
||||
continue
|
||||
|
||||
# 检查时间匹配
|
||||
time_str = s.get("time", "")
|
||||
if not time_str:
|
||||
continue
|
||||
try:
|
||||
h, m = time_str.split(":")
|
||||
schedule_minutes = int(h) * 60 + int(m)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
|
||||
# 允许 1 分钟误差(防止跳过)
|
||||
if abs(today_minutes - schedule_minutes) > 1:
|
||||
continue
|
||||
|
||||
# 检查是否已触发过(防止重复)
|
||||
last_run = self._last_run_cache.get(sid, 0)
|
||||
if now - last_run < CHECK_INTERVAL * 2:
|
||||
continue
|
||||
|
||||
# 提交到 guardd scheduler
|
||||
account = s.get("account", "")
|
||||
blueprint = s.get("blueprint", "douyin_daily")
|
||||
rounds = s.get("rounds", 3)
|
||||
|
||||
task_id = f"schedule_{sid}_{int(now)}"
|
||||
task = {
|
||||
"task_id": task_id,
|
||||
"cmd_type": "nurture",
|
||||
"accounts": [account] if account else [],
|
||||
"blueprint": blueprint,
|
||||
"rounds": rounds,
|
||||
"priority": 1, # P1 日常
|
||||
"source": "schedule",
|
||||
"schedule_id": sid,
|
||||
"params": {},
|
||||
}
|
||||
|
||||
if self.scheduler and hasattr(self.scheduler, 'submit_task'):
|
||||
self.scheduler.submit_task(task)
|
||||
logger.info(f" 📅 [{task_id}] 定时任务触发: {sid} → {blueprint} @ {account}")
|
||||
else:
|
||||
logger.warning(f" ⚠️ scheduler 不可用,定时任务 {sid} 无法提交")
|
||||
|
||||
self._last_run_cache[sid] = now
|
||||
triggered += 1
|
||||
|
||||
if triggered:
|
||||
self.save_last_runs()
|
||||
|
||||
return triggered
|
||||
@@ -0,0 +1,350 @@
|
||||
"""
|
||||
scheduler.py — 调度引擎 v4(简化版)
|
||||
|
||||
核心变化(vs v3):
|
||||
1. 一条队列全局排序,3 slot 从队首取任务
|
||||
2. P0 插入规则: 找第一个"后面没有 P0"的 P1 → 插它后面(自动交替)
|
||||
3. 去掉 interval、decomposed_from、paused_slots 全部复杂逻辑
|
||||
4. executor 读输出改为非阻塞(select)
|
||||
5. _pop_next 增加 task_store 状态校验,过期自动清除
|
||||
"""
|
||||
import logging
|
||||
import threading
|
||||
from threading import RLock
|
||||
import time
|
||||
from typing import Optional
|
||||
from modules.task_store import (
|
||||
TaskStore, STATUS_QUEUED, STATUS_RUNNING,
|
||||
STATUS_COMPLETED, STATUS_FAILED, TERMINAL_STATUSES
|
||||
)
|
||||
from modules.priority_queue import PriorityQueue
|
||||
from modules.slot_manager import BrowserSlotManager
|
||||
from modules.executor import Executor
|
||||
|
||||
logger = logging.getLogger("guardd.scheduler")
|
||||
|
||||
|
||||
class Scheduler:
|
||||
"""调度引擎 v4 — 单队列 + 交替规则 + 事件驱动编排"""
|
||||
|
||||
def __init__(self, task_store: TaskStore, priority_queue: PriorityQueue,
|
||||
slot_manager: BrowserSlotManager, executor: Executor,
|
||||
on_task_event=None):
|
||||
self.task_store = task_store
|
||||
self.queue_priority = PriorityQueue() # P0
|
||||
self.queue_normal = PriorityQueue() # P1
|
||||
self.queue_filler = PriorityQueue() # P2
|
||||
|
||||
self.slot_manager = slot_manager
|
||||
self.executor = executor
|
||||
self.active_tasks: dict[int, dict] = {}
|
||||
self.account_slots: dict[str, int] = {}
|
||||
self.on_task_event = on_task_event
|
||||
|
||||
self.loop_interval = 15
|
||||
self.max_slots = slot_manager.max_slots if slot_manager else 3
|
||||
self._schedule_lock = RLock()
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# 主循环
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
def run_cycle(self):
|
||||
while True:
|
||||
try:
|
||||
self._check_all_active_tasks()
|
||||
self._schedule_all_slots()
|
||||
self.slot_manager.check_health()
|
||||
except Exception as e:
|
||||
logger.error(f"调度循环异常: {e}")
|
||||
time.sleep(self.loop_interval)
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# 任务提交
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
def submit_task(self, task: dict):
|
||||
with self._schedule_lock:
|
||||
self._submit_task_inner(task)
|
||||
|
||||
def _submit_task_inner(self, task: dict):
|
||||
task_id = task["task_id"]
|
||||
task["status"] = STATUS_QUEUED
|
||||
task["queued_at"] = time.time()
|
||||
self.task_store.save(task)
|
||||
|
||||
priority = task.get("priority", 1)
|
||||
if priority == 0:
|
||||
self._insert_p0(task)
|
||||
elif priority == 2:
|
||||
self.queue_filler.push(task)
|
||||
logger.info(f" 📥 [{task_id[:30]}] 入队 P2 (队尾)")
|
||||
else:
|
||||
self.queue_normal.push(task)
|
||||
logger.info(f" 📥 [{task_id[:30]}] 入队 P1 (日常)")
|
||||
|
||||
if self._has_free_slot():
|
||||
self._schedule_all_slots()
|
||||
|
||||
def _insert_p0(self, task: dict):
|
||||
"""P0 插入: 直接入 priority 队列"""
|
||||
task_id = task["task_id"]
|
||||
self.queue_priority.push(task)
|
||||
logger.info(f" 📥 [{task_id[:30]}] 入队 P0")
|
||||
|
||||
def _has_free_slot(self) -> bool:
|
||||
"""检查是否有空闲 slot。使用 active_tasks 长度作为主要依据,
|
||||
避免 executor 线程尚未调用 acquire() 时的竞态条件。"""
|
||||
# active_tasks 是同步设置的(在 _assign_task 中),比 slot_manager 更准确
|
||||
active_count = len(self.active_tasks)
|
||||
max_s = self.max_slots
|
||||
free = active_count < max_s
|
||||
return free
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# 活跃任务检查
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
def _check_all_active_tasks(self):
|
||||
if not self.active_tasks:
|
||||
return
|
||||
|
||||
for slot_id, task in list(self.active_tasks.items()):
|
||||
status = task.get("status", "")
|
||||
|
||||
if status not in TERMINAL_STATUSES:
|
||||
continue
|
||||
|
||||
logger.info(f" ✅ [{task['task_id'][:30]}] 完成 (status={status})")
|
||||
if self.on_task_event:
|
||||
try:
|
||||
self.on_task_event("completed" if status == STATUS_COMPLETED else "failed", task)
|
||||
except Exception:
|
||||
pass
|
||||
self._release_slot(slot_id)
|
||||
|
||||
def _release_slot(self, slot_id: int):
|
||||
task = self.active_tasks.pop(slot_id, None)
|
||||
if task:
|
||||
for acct in (task.get("accounts") or []):
|
||||
if acct and acct in self.account_slots:
|
||||
del self.account_slots[acct]
|
||||
logger.info(f" 🔓 slot {slot_id} 释放账号 {acct}")
|
||||
# 同步释放 slot_manager
|
||||
if self.slot_manager:
|
||||
try:
|
||||
self.slot_manager.release_by_account(acct)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# Slot 分配
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
def _schedule_all_slots(self):
|
||||
with self._schedule_lock:
|
||||
self._schedule_all_inner()
|
||||
|
||||
def _schedule_all_inner(self):
|
||||
usage = self.slot_manager.get_usage() if self.slot_manager else {"max": self.max_slots, "used": 0, "slots": []}
|
||||
max_s = usage.get("max", self.max_slots)
|
||||
|
||||
for slot_id in range(max_s):
|
||||
if slot_id in self.active_tasks:
|
||||
continue
|
||||
|
||||
if self.slot_manager:
|
||||
slots_info = usage.get("slots", [])
|
||||
slot_info = next((s for s in slots_info if s.get("slot_id") == slot_id), None)
|
||||
if slot_info and slot_info.get("account_id"):
|
||||
continue
|
||||
|
||||
task = self._pop_next(slot_id)
|
||||
if not task:
|
||||
continue
|
||||
|
||||
self._assign_task(slot_id, task)
|
||||
|
||||
def _pop_next(self, slot_id: int) -> Optional[dict]:
|
||||
"""取下一个可用任务,按 P0/P1 交替规则,跳过 busy 账号和过期条目。
|
||||
|
||||
交替规则: 优先取 P0,但已分配的 P0 数量>=P1 时取 P1,
|
||||
确保 P0 和 P1 交替执行,不会 3 个 slot 全跑 P0。
|
||||
"""
|
||||
# 统计本轮已分配给前几个 slot 的 P0/P1 数量
|
||||
p0_assigned = 0
|
||||
p1_assigned = 0
|
||||
for sid in range(slot_id):
|
||||
_t = self.active_tasks.get(sid)
|
||||
if _t:
|
||||
_p = _t.get("priority")
|
||||
if _p == 0:
|
||||
p0_assigned += 1
|
||||
elif _p == 1:
|
||||
p1_assigned += 1
|
||||
|
||||
# 交替策略:P0 >= P1 时优先取 P1(P0 永远跟在 P1 后面)
|
||||
prefer_p1 = (p0_assigned >= p1_assigned) and self.queue_normal.size() > 0
|
||||
|
||||
if prefer_p1:
|
||||
queue_order = [
|
||||
(self.queue_normal, "P1"),
|
||||
(self.queue_priority, "P0"),
|
||||
(self.queue_filler, "P2"),
|
||||
]
|
||||
else:
|
||||
queue_order = [
|
||||
(self.queue_priority, "P0"),
|
||||
(self.queue_normal, "P1"),
|
||||
(self.queue_filler, "P2"),
|
||||
]
|
||||
|
||||
for queue, _ in queue_order:
|
||||
candidates = queue.get_all()
|
||||
candidates.sort(key=lambda c: c.get("queued_at", 0))
|
||||
|
||||
for candidate in candidates:
|
||||
accounts = candidate.get("accounts", [])
|
||||
if any(acct in self.account_slots for acct in accounts):
|
||||
continue
|
||||
|
||||
task_id = candidate["task_id"]
|
||||
|
||||
# 校验 task_store 状态
|
||||
stored = self.task_store.get(task_id)
|
||||
if not stored:
|
||||
queue.remove(task_id)
|
||||
logger.warning(f" ⏭ [{task_id[:30]}] 过期清除(无记录)")
|
||||
continue
|
||||
if stored.get("status") != STATUS_QUEUED:
|
||||
queue.remove(task_id)
|
||||
logger.info(f" ⏭ [{task_id[:30]}] 过期清除(status={stored.get('status')})")
|
||||
continue
|
||||
|
||||
removed = queue.remove(task_id)
|
||||
if not removed:
|
||||
continue
|
||||
|
||||
candidate["status"] = STATUS_RUNNING
|
||||
self.task_store.save(candidate)
|
||||
return candidate
|
||||
|
||||
return None
|
||||
|
||||
def _assign_task(self, slot_id: int, task: dict):
|
||||
self.active_tasks[slot_id] = task
|
||||
task["started_at"] = time.time()
|
||||
task["slot_id"] = slot_id
|
||||
|
||||
for acct in (task.get("accounts") or []):
|
||||
if acct:
|
||||
self.account_slots[acct] = slot_id
|
||||
|
||||
self.task_store.save(task)
|
||||
accts_str = ",".join(task.get("accounts", []))
|
||||
logger.info(f" ▶️ [{task['task_id'][:30]}] slot {slot_id} 开始 (accounts={accts_str})")
|
||||
|
||||
if self.on_task_event:
|
||||
try:
|
||||
self.on_task_event("started", task)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
import threading
|
||||
threading.Thread(target=self.executor.execute, args=(task,), daemon=True).start()
|
||||
except Exception as e:
|
||||
logger.error(f" ❌ [{task['task_id'][:30]}] 执行失败: {e}")
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# 辅助
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
def kill_active(self):
|
||||
for slot_id, task in list(self.active_tasks.items()):
|
||||
try:
|
||||
self.executor.kill(task["task_id"])
|
||||
except Exception:
|
||||
pass
|
||||
self.active_tasks.clear()
|
||||
self.account_slots.clear()
|
||||
|
||||
def clear_all(self):
|
||||
"""清空所有队列 + 杀死活跃任务,完全还原"""
|
||||
self.kill_active()
|
||||
# 清空内存队列
|
||||
for q in [self.queue_priority, self.queue_normal, self.queue_filler]:
|
||||
q.clear()
|
||||
# 将 task_store 中所有 queued/pending 状态的任务标记为 cancelled
|
||||
for status in ("queued", "pending"):
|
||||
for t in self.task_store.get_by_status(status):
|
||||
t["status"] = "cancelled"
|
||||
self.task_store.save(t)
|
||||
if self.slot_manager:
|
||||
self.slot_manager.cleanup_orphans()
|
||||
|
||||
def queue_sizes(self) -> dict:
|
||||
return {
|
||||
"priority": len(self.queue_priority.get_all()),
|
||||
"normal": len(self.queue_normal.get_all()),
|
||||
"filler": len(self.queue_filler.get_all()),
|
||||
}
|
||||
|
||||
def get_all_queued(self) -> list:
|
||||
"""返回排队队列(P0/P1 交替排序,P2 附加末尾)"""
|
||||
p0_list = self.queue_priority.get_all()
|
||||
p1_list = self.queue_normal.get_all()
|
||||
p2_list = self.queue_filler.get_all()
|
||||
|
||||
# 按 queued_at 排序后交替合并
|
||||
p0_list.sort(key=lambda x: x.get("queued_at", 0))
|
||||
p1_list.sort(key=lambda x: x.get("queued_at", 0))
|
||||
|
||||
result = []
|
||||
p0_i, p1_i, p2_i = 0, 0, 0
|
||||
last_p0 = True # 从 P1 开始交替(P0 永远跟在 P1 后面)
|
||||
|
||||
while p0_i < len(p0_list) or p1_i < len(p1_list):
|
||||
if p0_i >= len(p0_list):
|
||||
# 只剩 P1
|
||||
p1_list[p1_i]["queue"] = "P1"
|
||||
result.append(p1_list[p1_i])
|
||||
p1_i += 1
|
||||
last_p0 = False
|
||||
elif p1_i >= len(p1_list):
|
||||
# 只剩 P0 → 用 P2 作间隔
|
||||
if last_p0 and p2_i < len(p2_list):
|
||||
p2_list[p2_i]["queue"] = "P2"
|
||||
result.append(p2_list[p2_i])
|
||||
p2_i += 1
|
||||
last_p0 = False
|
||||
else:
|
||||
p0_list[p0_i]["queue"] = "P0"
|
||||
result.append(p0_list[p0_i])
|
||||
p0_i += 1
|
||||
last_p0 = True
|
||||
# P0 后面没 P1 → 补一个 P2 作间隔
|
||||
if p2_i < len(p2_list):
|
||||
p2_list[p2_i]["queue"] = "P2"
|
||||
result.append(p2_list[p2_i])
|
||||
p2_i += 1
|
||||
last_p0 = False
|
||||
elif last_p0:
|
||||
# 上次取了 P0,这次取 P1(交替)
|
||||
p1_list[p1_i]["queue"] = "P1"
|
||||
result.append(p1_list[p1_i])
|
||||
p1_i += 1
|
||||
last_p0 = False
|
||||
else:
|
||||
# 上次取了 P1 或刚开始,取 P0
|
||||
p0_list[p0_i]["queue"] = "P0"
|
||||
result.append(p0_list[p0_i])
|
||||
p0_i += 1
|
||||
last_p0 = True
|
||||
|
||||
# 剩余的 P2 附加末尾
|
||||
for i in range(p2_i, len(p2_list)):
|
||||
p2_list[i]["queue"] = "P2"
|
||||
result.append(p2_list[i])
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,238 @@
|
||||
"""
|
||||
slot_manager.py — 浏览器槽位管理器 (guardd 模块)
|
||||
|
||||
职责:
|
||||
- 管理本机 Camoufox 浏览器实例(槽位)
|
||||
- 按 browser_id 识别,不依赖槽位编号(用户拖拽窗口不影响)
|
||||
- 同一账号不能同时在两个浏览器运行
|
||||
- 启动时扫描孤儿浏览器进程
|
||||
"""
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
from threading import Lock
|
||||
from typing import Optional, List
|
||||
|
||||
|
||||
class AccountBusyError(Exception):
|
||||
"""账号已在其他槽位运行"""
|
||||
pass
|
||||
|
||||
|
||||
class BrowserSlotManager:
|
||||
"""浏览器槽位管理器"""
|
||||
|
||||
def __init__(self, max_slots: int = 3):
|
||||
self.max_slots = max_slots
|
||||
self.slots = [None] * max_slots # list of dict or None
|
||||
self._lock = Lock()
|
||||
|
||||
def acquire(self, account_id: str, identity_dir: str, nickname: str = "",
|
||||
platform: str = "douyin") -> Optional[dict]:
|
||||
"""获取一个槽位,返回 slot_info dict
|
||||
Args:
|
||||
account_id: 账号ID (douyin_136)
|
||||
identity_dir: 身份目录名
|
||||
nickname: 账号昵称 (显示用)
|
||||
platform: 平台 (douyin / xiaohongshu)
|
||||
Raises: AccountBusyError 如果账号已在其他槽位运行
|
||||
"""
|
||||
with self._lock:
|
||||
for s in self.slots:
|
||||
if s and s["account_id"] == account_id:
|
||||
raise AccountBusyError(f"账号 {account_id} 已在 slot {s['slot_id']} 运行")
|
||||
|
||||
for i in range(self.max_slots):
|
||||
if self.slots[i] is None:
|
||||
info = {
|
||||
"slot_id": i,
|
||||
"account_id": account_id,
|
||||
"nickname": nickname,
|
||||
"platform": platform,
|
||||
"browser_id": identity_dir,
|
||||
"pid": None,
|
||||
"blueprint": None,
|
||||
"current_step": None,
|
||||
"step_index": 0,
|
||||
"total_steps": 0,
|
||||
"started_at": time.time(),
|
||||
"elapsed_sec": 0,
|
||||
"cpu_percent": 0.0,
|
||||
"memory_mb": 0.0,
|
||||
"health": "healthy",
|
||||
"last_heartbeat": time.time(),
|
||||
}
|
||||
self.slots[i] = info
|
||||
return info
|
||||
return None
|
||||
|
||||
def release(self, browser_id: str) -> bool:
|
||||
"""按 browser_id 释放槽位(不依赖槽位编号)"""
|
||||
with self._lock:
|
||||
for i in range(self.max_slots):
|
||||
if self.slots[i] and self.slots[i]["browser_id"] == browser_id:
|
||||
pid = self.slots[i].get("pid")
|
||||
if pid:
|
||||
try:
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
except (OSError, ProcessLookupError):
|
||||
pass
|
||||
self.slots[i] = None
|
||||
return True
|
||||
return False
|
||||
|
||||
def release_by_account(self, account_id: str) -> bool:
|
||||
"""按 account_id 释放槽位"""
|
||||
with self._lock:
|
||||
for i in range(self.max_slots):
|
||||
if self.slots[i] and self.slots[i]["account_id"] == account_id:
|
||||
self.slots[i] = None
|
||||
return True
|
||||
return False
|
||||
|
||||
def update_step(self, browser_id: str, step_name: str, step_index: int = None, total_steps: int = None):
|
||||
"""更新槽位中正在执行的步骤"""
|
||||
with self._lock:
|
||||
for s in self.slots:
|
||||
if s and s["browser_id"] == browser_id:
|
||||
s["current_step"] = step_name
|
||||
s["elapsed_sec"] = int(time.time() - s["started_at"])
|
||||
s["last_heartbeat"] = time.time()
|
||||
if step_index is not None:
|
||||
s["step_index"] = step_index
|
||||
if total_steps is not None:
|
||||
s["total_steps"] = total_steps
|
||||
return True
|
||||
return False
|
||||
|
||||
def set_pid(self, browser_id: str, pid: int):
|
||||
"""设置浏览器进程 PID"""
|
||||
with self._lock:
|
||||
for s in self.slots:
|
||||
if s and s["browser_id"] == browser_id:
|
||||
s["pid"] = pid
|
||||
return True
|
||||
return False
|
||||
|
||||
def find_account(self, account_id: str) -> Optional[dict]:
|
||||
"""查找账号在当前哪个槽位运行"""
|
||||
with self._lock:
|
||||
for s in self.slots:
|
||||
if s and s["account_id"] == account_id:
|
||||
return s
|
||||
return None
|
||||
|
||||
def find_by_browser(self, browser_id: str) -> Optional[dict]:
|
||||
with self._lock:
|
||||
for s in self.slots:
|
||||
if s and s["browser_id"] == browser_id:
|
||||
return s
|
||||
return None
|
||||
|
||||
def check_health(self):
|
||||
"""每轮 cycle 检查所有浏览器进程健康状态"""
|
||||
with self._lock:
|
||||
for s in self.slots:
|
||||
if not s or not s.get("pid"):
|
||||
continue
|
||||
try:
|
||||
os.kill(s["pid"], 0) # 空信号测存活
|
||||
except OSError:
|
||||
s["health"] = "crashed"
|
||||
s["account_id"] = None
|
||||
continue
|
||||
# 采集 CPU/内存
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["ps", "-p", str(s["pid"]), "-o", "%cpu=,%mem=,rss="],
|
||||
capture_output=True, text=True, timeout=3
|
||||
)
|
||||
parts = r.stdout.strip().split()
|
||||
if len(parts) >= 3:
|
||||
s["cpu_percent"] = float(parts[0])
|
||||
s["memory_mb"] = float(parts[2]) / 1024
|
||||
s["health"] = "warning" if float(parts[0]) > 80 else "healthy"
|
||||
except Exception:
|
||||
pass
|
||||
s["elapsed_sec"] = int(time.time() - s["started_at"])
|
||||
s["last_heartbeat"] = time.time()
|
||||
|
||||
def cleanup_orphans(self):
|
||||
"""启动时扫描孤儿浏览器进程"""
|
||||
print(" [SlotManager] 扫描孤儿浏览器进程...")
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["ps", "aux"],
|
||||
capture_output=True, text=True, timeout=5
|
||||
)
|
||||
orphan_count = 0
|
||||
for line in r.stdout.split("\n"):
|
||||
if "camoufox" in line.lower() or "firefox" in line.lower():
|
||||
parts = line.split()
|
||||
if len(parts) > 1:
|
||||
pid = parts[1]
|
||||
try:
|
||||
os.kill(int(pid), signal.SIGTERM)
|
||||
orphan_count += 1
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
if orphan_count:
|
||||
print(f" [SlotManager] 清理 {orphan_count} 个孤儿浏览器进程")
|
||||
except Exception as e:
|
||||
print(f" [SlotManager] 清理异常: {e}")
|
||||
|
||||
def check_processes(self):
|
||||
"""逐 slot 检查 PID 存活状态(5 秒线程调用)"""
|
||||
with self._lock:
|
||||
for s in self.slots:
|
||||
if not s or not s.get("pid"):
|
||||
continue
|
||||
pid = s["pid"]
|
||||
alive = True
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except OSError:
|
||||
alive = False
|
||||
if alive:
|
||||
s["health"] = "healthy"
|
||||
s["last_heartbeat"] = time.time()
|
||||
else:
|
||||
s["health"] = "crashed"
|
||||
s["pid"] = None
|
||||
s["account_id"] = None
|
||||
|
||||
def track_loop(self, interval: float = 5.0):
|
||||
"""后台线程:每 5 秒检查一次所有 slot 的进程存活"""
|
||||
logger = __import__("logging").getLogger("guardd.slot_manager")
|
||||
while True:
|
||||
try:
|
||||
self.check_processes()
|
||||
except Exception as e:
|
||||
logger.error(f"track_loop 异常: {e}")
|
||||
__import__("time").sleep(interval)
|
||||
|
||||
def get_usage(self) -> dict:
|
||||
with self._lock:
|
||||
slots_info = []
|
||||
for i, s in enumerate(self.slots):
|
||||
if s:
|
||||
slots_info.append({
|
||||
"slot_id": i,
|
||||
"account_id": s["account_id"],
|
||||
"nickname": s.get("nickname", ""),
|
||||
"platform": s.get("platform", ""),
|
||||
"browser_id": s["browser_id"],
|
||||
"pid": s["pid"],
|
||||
"blueprint": s.get("blueprint"),
|
||||
"current_step": s.get("current_step"),
|
||||
"step_index": s.get("step_index", 0),
|
||||
"total_steps": s.get("total_steps", 0),
|
||||
"elapsed_sec": s.get("elapsed_sec", 0),
|
||||
"health": s.get("health", "unknown"),
|
||||
})
|
||||
return {
|
||||
"max": self.max_slots,
|
||||
"used": sum(1 for s in self.slots if s is not None),
|
||||
"slots": slots_info,
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
"""
|
||||
task_store.py — 任务持久化存储 (guardd 模块)
|
||||
|
||||
职责:
|
||||
- 内存 dict + SQLite 双写
|
||||
- 内存保性能,SQLite 持久化
|
||||
- 启动时从未完成的任务恢复
|
||||
"""
|
||||
import json
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional, List
|
||||
from datetime import datetime
|
||||
|
||||
# Task status constants
|
||||
STATUS_PENDING = "pending"
|
||||
STATUS_SCHEDULED = "scheduled"
|
||||
STATUS_QUEUED = "queued"
|
||||
STATUS_PREFLIGHT = "preflight"
|
||||
STATUS_RUNNING = "running"
|
||||
STATUS_PAUSED = "paused"
|
||||
STATUS_WAITING_DEP = "waiting_dep"
|
||||
STATUS_COMPLETED = "completed"
|
||||
STATUS_FAILED = "failed"
|
||||
STATUS_CANCELLED = "cancelled"
|
||||
STATUS_SKIPPED = "skipped"
|
||||
|
||||
TERMINAL_STATUSES = {STATUS_COMPLETED, STATUS_FAILED, STATUS_CANCELLED, STATUS_SKIPPED}
|
||||
ACTIVE_STATUSES = {STATUS_RUNNING, STATUS_QUEUED, STATUS_PREFLIGHT, STATUS_PAUSED, STATUS_WAITING_DEP}
|
||||
|
||||
|
||||
class TaskStore:
|
||||
"""任务持久化存储 — 内存 dict + SQLite 双写"""
|
||||
|
||||
def __init__(self, db_path: str = None):
|
||||
if db_path is None:
|
||||
home = Path.home()
|
||||
db_path = str(home / "workbuddy-agent-os" / "agent-local" / "runtime" / "guardd" / "tasks.db")
|
||||
Path(db_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self.db_path = db_path
|
||||
self._lock = threading.Lock()
|
||||
self._mem: dict = {} # task_id → task dict
|
||||
self._init_db()
|
||||
self._load_from_db()
|
||||
|
||||
def _init_db(self):
|
||||
with sqlite3.connect(self.db_path) as conn:
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS tasks (
|
||||
task_id TEXT PRIMARY KEY,
|
||||
data TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
created_at REAL NOT NULL,
|
||||
updated_at REAL NOT NULL
|
||||
)
|
||||
""")
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS account_health (
|
||||
account_id TEXT PRIMARY KEY,
|
||||
data TEXT NOT NULL,
|
||||
updated_at REAL NOT NULL
|
||||
)
|
||||
""")
|
||||
conn.execute("""
|
||||
CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks(status)
|
||||
""")
|
||||
conn.commit()
|
||||
|
||||
def _load_from_db(self):
|
||||
"""启动时从 SQLite 加载所有任务到内存"""
|
||||
with sqlite3.connect(self.db_path) as conn:
|
||||
rows = conn.execute("SELECT task_id, data, status FROM tasks").fetchall()
|
||||
for task_id, data_json, status in rows:
|
||||
task = json.loads(data_json)
|
||||
task["status"] = status
|
||||
self._mem[task_id] = task
|
||||
|
||||
def save(self, task: dict) -> None:
|
||||
"""保存/更新任务(内存 + SQLite)"""
|
||||
task_id = task["task_id"]
|
||||
status = task.get("status", STATUS_PENDING)
|
||||
now = time.time()
|
||||
|
||||
with self._lock:
|
||||
self._mem[task_id] = task
|
||||
with sqlite3.connect(self.db_path) as conn:
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO tasks (task_id, data, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?)",
|
||||
(task_id, json.dumps(task, ensure_ascii=False), status, task.get("created_at", now), now),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def get(self, task_id: str) -> Optional[dict]:
|
||||
with self._lock:
|
||||
return self._mem.get(task_id)
|
||||
|
||||
def get_by_status(self, *statuses) -> List[dict]:
|
||||
with self._lock:
|
||||
return [t for t in self._mem.values() if t.get("status") in statuses]
|
||||
|
||||
def get_active(self) -> List[dict]:
|
||||
return self.get_by_status(*ACTIVE_STATUSES)
|
||||
|
||||
def find_dependents(self, task_id: str) -> List[str]:
|
||||
"""查找依赖指定 task_id 的所有任务"""
|
||||
result = []
|
||||
with self._lock:
|
||||
for tid, task in self._mem.items():
|
||||
deps = task.get("depends_on", [])
|
||||
if task_id in deps:
|
||||
result.append(tid)
|
||||
return result
|
||||
|
||||
def save_account_health(self, account_id: str, data: dict) -> None:
|
||||
data["updated_at"] = time.time()
|
||||
with self._lock:
|
||||
with sqlite3.connect(self.db_path) as conn:
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO account_health (account_id, data, updated_at) VALUES (?, ?, ?)",
|
||||
(account_id, json.dumps(data, ensure_ascii=False), time.time()),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def get_account_health(self, account_id: str) -> Optional[dict]:
|
||||
with sqlite3.connect(self.db_path) as conn:
|
||||
row = conn.execute("SELECT data FROM account_health WHERE account_id = ?", (account_id,)).fetchone()
|
||||
if row:
|
||||
return json.loads(row[0])
|
||||
return None
|
||||
|
||||
def get_unfinished(self) -> List[dict]:
|
||||
"""获取所有未完成的任务(启动恢复用)"""
|
||||
return self.get_by_status(*ACTIVE_STATUSES)
|
||||
|
||||
def reset_unfinished(self) -> int:
|
||||
"""启动时将 running 状态重置为 queued"""
|
||||
count = 0
|
||||
for task in self.get_unfinished():
|
||||
if task.get("status") == STATUS_RUNNING:
|
||||
task["status"] = STATUS_QUEUED
|
||||
task["message"] = "guardd 重启恢复"
|
||||
self.save(task)
|
||||
count += 1
|
||||
return count
|
||||
|
||||
def get_group_last_completed(self, group_id: str) -> Optional[float]:
|
||||
"""获取某组任务最后一次完成的时间戳
|
||||
|
||||
Args:
|
||||
group_id: 任务组ID (decomposed_from 或 task_id)
|
||||
Returns:
|
||||
最后完成时间的 Unix 时间戳,没有完成过的返回 None
|
||||
"""
|
||||
with self._lock:
|
||||
last_time = None
|
||||
for tid, task in self._mem.items():
|
||||
if task.get("decomposed_from") == group_id or tid == group_id:
|
||||
last_completed = task.get("completed_at")
|
||||
if last_completed and (last_time is None or last_completed > last_time):
|
||||
last_time = last_completed
|
||||
return last_time
|
||||
|
||||
def all_tasks(self) -> List[dict]:
|
||||
with self._lock:
|
||||
return list(self._mem.values())
|
||||
|
||||
def count(self) -> dict:
|
||||
with self._lock:
|
||||
counts = {}
|
||||
for t in self._mem.values():
|
||||
s = t.get("status", "unknown")
|
||||
counts[s] = counts.get(s, 0) + 1
|
||||
return counts
|
||||
Reference in New Issue
Block a user