feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程,直接发 HTTP
移植自 mac-agent-os 的 mediacrawler_adapter:不起子进程、不开页面,用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin,所以 ingest 那条链路一个字不用改。
**目前能用的(真环境实测,非推断)**:
profile/other : 200, 7075 字节 —— 博主主页指标(粉丝/获赞/作品数/昵称)
aweme/detail : 200, 45425 字节 —— 单条作品详情(含点赞/评论/收藏/分享)
**目前不能用的:作品列表 `aweme/post`。** 两个互相独立的原因:
1. 这个接口被抖音单独升级成了真校验:不带 x-tt-argus 回 403「Uifid Not Found」,
带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过,「真校验」过不了。
同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的,
挑中的恰好是「批量拉作品列表」这个最敏感的动作。
2. 改走页面截获也不行:CDP 浏览器打开博主主页会落到「验证码中间页」(当天大量探测的
代价,过几小时要重测)。
所以现在的边界是:**已知作品的指标刷新能做,自动发现新作品做不了**。
**排查中控住变量后得到的两条事实**(都写进注释了):
· `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
UA+client hints+Cookie 时是 200 但仅 121 字节的空壳,补上这三个头变 7074 字节。
(我先前猜的 sec-ch-ua 不是关键。)
· 因此 UA 与 client hints 必须**成套地取自同一个浏览器**,所以 BrowserIdentity 一次
从 CDP 取齐 cookie + UA + hints,而不是各自写死。
「200 + 空 body 必须当场报错」也是刻意写死的:放过去它会在下游变成「这个博主没作品」,
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的,还被翻译成「账号被封」。
测试 +11:cookie 解析、请求头成套性(含 uifid 缺失/回退)、产物键名与 store 对齐、
以及 _get 的三条失败路径(空 body / 403 带网关原话 / 正常返回)。
This commit is contained in:
@@ -0,0 +1,135 @@
|
||||
"""
|
||||
schedule_bridge.py — 定时任务桥接 (guardd 模块)
|
||||
|
||||
职责:
|
||||
- 读取 mc/scheduler 的 schedule.yaml
|
||||
- 检查到期的定时任务
|
||||
- 提交到 guardd scheduler 执行(P1 优先级)
|
||||
- 记录最后执行时间,防止重复触发
|
||||
|
||||
配置路径: 05_tools/07_matrix/scripts/config/schedule.yaml
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from modules.task_store import TaskStore
|
||||
|
||||
logger = logging.getLogger("guardd.schedule_bridge")
|
||||
|
||||
HOME = Path.home()
|
||||
SCRIPTS_DIR = HOME / "workbuddy-agent-os" / "agent-sync" / "05_tools" / "07_matrix" / "scripts"
|
||||
SCHEDULE_YAML = SCRIPTS_DIR / "config" / "schedule.yaml"
|
||||
LAST_RUN_FILE = HOME / "workbuddy-agent-os" / "agent-local" / "runtime" / "guardd" / "schedule_last_run.json"
|
||||
|
||||
CHECK_INTERVAL = 60 # 每 60 秒检查一次
|
||||
|
||||
|
||||
class ScheduleBridge:
|
||||
"""定时任务 → guardd 调度器桥接"""
|
||||
|
||||
def __init__(self, task_store: TaskStore, scheduler=None, hostname: str = ""):
|
||||
self.task_store = task_store
|
||||
self.scheduler = scheduler
|
||||
self.hostname = hostname
|
||||
self.last_check = 0
|
||||
self._last_run_cache = {}
|
||||
|
||||
def load_last_runs(self):
|
||||
"""加载上次执行时间记录"""
|
||||
if LAST_RUN_FILE.exists():
|
||||
try:
|
||||
self._last_run_cache = json.loads(LAST_RUN_FILE.read_text())
|
||||
except Exception:
|
||||
self._last_run_cache = {}
|
||||
|
||||
def save_last_runs(self):
|
||||
"""保存上次执行时间"""
|
||||
LAST_RUN_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
LAST_RUN_FILE.write_text(json.dumps(self._last_run_cache, indent=2))
|
||||
|
||||
def check_schedules(self):
|
||||
"""检查是否有定时任务到期"""
|
||||
now = time.time()
|
||||
if now - self.last_check < CHECK_INTERVAL:
|
||||
return 0
|
||||
self.last_check = now
|
||||
|
||||
if not SCHEDULE_YAML.exists():
|
||||
return 0
|
||||
|
||||
try:
|
||||
import yaml
|
||||
data = yaml.safe_load(SCHEDULE_YAML.read_text())
|
||||
schedules = data.get("schedules", {}) if data else {}
|
||||
except Exception as e:
|
||||
logger.warning(f"schedule.yaml 读取失败: {e}")
|
||||
return 0
|
||||
|
||||
self.load_last_runs()
|
||||
triggered = 0
|
||||
now_dt = datetime.now()
|
||||
today_minutes = now_dt.hour * 60 + now_dt.minute
|
||||
weekday = now_dt.isoweekday() # 1=周一..7=周日
|
||||
|
||||
for sid, s in schedules.items():
|
||||
if not s.get("enabled", False):
|
||||
continue
|
||||
|
||||
# 检查星期匹配
|
||||
days_str = s.get("days", "1,2,3,4,5,6,7")
|
||||
days = [int(d.strip()) for d in days_str.split(",") if d.strip()]
|
||||
if weekday not in days:
|
||||
continue
|
||||
|
||||
# 检查时间匹配
|
||||
time_str = s.get("time", "")
|
||||
if not time_str:
|
||||
continue
|
||||
try:
|
||||
h, m = time_str.split(":")
|
||||
schedule_minutes = int(h) * 60 + int(m)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
|
||||
# 允许 1 分钟误差(防止跳过)
|
||||
if abs(today_minutes - schedule_minutes) > 1:
|
||||
continue
|
||||
|
||||
# 检查是否已触发过(防止重复)
|
||||
last_run = self._last_run_cache.get(sid, 0)
|
||||
if now - last_run < CHECK_INTERVAL * 2:
|
||||
continue
|
||||
|
||||
# 提交到 guardd scheduler
|
||||
account = s.get("account", "")
|
||||
blueprint = s.get("blueprint", "douyin_daily")
|
||||
rounds = s.get("rounds", 3)
|
||||
|
||||
task_id = f"schedule_{sid}_{int(now)}"
|
||||
task = {
|
||||
"task_id": task_id,
|
||||
"cmd_type": "nurture",
|
||||
"accounts": [account] if account else [],
|
||||
"blueprint": blueprint,
|
||||
"rounds": rounds,
|
||||
"priority": 1, # P1 日常
|
||||
"source": "schedule",
|
||||
"schedule_id": sid,
|
||||
"params": {},
|
||||
}
|
||||
|
||||
if self.scheduler and hasattr(self.scheduler, 'submit_task'):
|
||||
self.scheduler.submit_task(task)
|
||||
logger.info(f" 📅 [{task_id}] 定时任务触发: {sid} → {blueprint} @ {account}")
|
||||
else:
|
||||
logger.warning(f" ⚠️ scheduler 不可用,定时任务 {sid} 无法提交")
|
||||
|
||||
self._last_run_cache[sid] = now
|
||||
triggered += 1
|
||||
|
||||
if triggered:
|
||||
self.save_last_runs()
|
||||
|
||||
return triggered
|
||||
Reference in New Issue
Block a user