feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程，直接发 HTTP
移植自 mac-agent-os 的 mediacrawler_adapter：不起子进程、不开页面，用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin，所以 ingest 那条链路一个字不用改。
**目前能用的（真环境实测，非推断）**：
profile/other : 200, 7075 字节 —— 博主主页指标（粉丝/获赞/作品数/昵称）
aweme/detail : 200, 45425 字节 —— 单条作品详情（含点赞/评论/收藏/分享）
**目前不能用的：作品列表 `aweme/post`。** 两个互相独立的原因：
1. 这个接口被抖音单独升级成了真校验：不带 x-tt-argus 回 403「Uifid Not Found」，
带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过，「真校验」过不了。
同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的，
挑中的恰好是「批量拉作品列表」这个最敏感的动作。
2. 改走页面截获也不行：CDP 浏览器打开博主主页会落到「验证码中间页」（当天大量探测的
代价，过几小时要重测）。
所以现在的边界是：**已知作品的指标刷新能做，自动发现新作品做不了**。
**排查中控住变量后得到的两条事实**（都写进注释了）：
· `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
UA+client hints+Cookie 时是 200 但仅 121 字节的空壳，补上这三个头变 7074 字节。
（我先前猜的 sec-ch-ua 不是关键。）
· 因此 UA 与 client hints 必须**成套地取自同一个浏览器**，所以 BrowserIdentity 一次
从 CDP 取齐 cookie + UA + hints，而不是各自写死。
「200 + 空 body 必须当场报错」也是刻意写死的：放过去它会在下游变成「这个博主没作品」，
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的，还被翻译成「账号被封」。
测试 +11：cookie 解析、请求头成套性（含 uifid 缺失/回退）、产物键名与 store 对齐、
以及 _get 的三条失败路径（空 body / 403 带网关原话 / 正常返回）。
This commit is contained in:
@@ -0,0 +1,420 @@
|
||||
"""
|
||||
ops/_base.py — 平台操作基类 v2.0
|
||||
|
||||
三段式操作模型 v2.0:
|
||||
每个原子操作 = 前置条件检查 + 执行动作 + 后置验证
|
||||
|
||||
1. _capture_state() → PageState 操作前/后页面状态指纹
|
||||
2. _get_pre_conditions(op) 每个操作的前置条件
|
||||
3. _do_execute(op, args) → OpResult 执行动作（子类实现，与原来一样）
|
||||
4. _get_post_conditions(op) 每个操作的后置验证条件
|
||||
5. execute() 统一入口 → 三段式执行，返回完整 OpResult
|
||||
|
||||
改造目标:
|
||||
- 录制时能记录操作前后的状态指纹
|
||||
- Analyzer 通过状态差异推断操作类型 → 不再依赖坐标
|
||||
- Exporter 输出三段式 JSON 蓝图
|
||||
"""
|
||||
import asyncio
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from typing import Optional
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════════
|
||||
# PageState — 页面状态指纹
|
||||
# ════════════════════════════════════════════════════════════
|
||||
|
||||
@dataclass
|
||||
class PageState:
|
||||
"""页面状态指纹 — 操作前后快照（供录制/分析/验证使用）
|
||||
|
||||
url_pattern: 归一化 URL 模式（去掉动态参数，保留路径结构）
|
||||
page_mode: grid / player / player_full / player_modal / search / profile / unknown
|
||||
selectors: {CSS选择器: 是否可见} — 关键元素的可见性状态
|
||||
markers: {标记名: 值} — 自定义状态标记（如 liked=true, like_count=1024）
|
||||
text_snippet: 页面文本片段（用于差异分析）
|
||||
"""
|
||||
url_pattern: str = ""
|
||||
page_mode: str = "unknown"
|
||||
selectors: dict = field(default_factory=dict)
|
||||
markers: dict = field(default_factory=dict)
|
||||
text_snippet: str = ""
|
||||
|
||||
def delta(self, other: 'PageState') -> dict:
|
||||
"""对比两个 PageState，返回差异摘要"""
|
||||
diffs = {}
|
||||
|
||||
if self.page_mode != other.page_mode:
|
||||
diffs["page_mode"] = f"{self.page_mode} → {other.page_mode}"
|
||||
|
||||
# 选择器状态变化
|
||||
sel_diffs = {}
|
||||
all_keys = set(self.selectors) | set(other.selectors)
|
||||
for k in all_keys:
|
||||
v1 = self.selectors.get(k)
|
||||
v2 = other.selectors.get(k)
|
||||
if v1 != v2:
|
||||
sel_diffs[k] = f"{v1} → {v2}"
|
||||
if sel_diffs:
|
||||
diffs["selectors"] = sel_diffs
|
||||
|
||||
# 标记变化
|
||||
mark_diffs = {}
|
||||
all_markers = set(self.markers) | set(other.markers)
|
||||
for k in all_markers:
|
||||
v1 = self.markers.get(k)
|
||||
v2 = other.markers.get(k)
|
||||
if v1 != v2:
|
||||
mark_diffs[k] = f"{v1} → {v2}"
|
||||
if mark_diffs:
|
||||
diffs["markers"] = mark_diffs
|
||||
|
||||
return diffs
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════════
|
||||
# Condition — 前置/后置条件
|
||||
# ════════════════════════════════════════════════════════════
|
||||
|
||||
@dataclass
|
||||
class Condition:
|
||||
"""执行条件（前置条件 / 后置验证）
|
||||
|
||||
条件类型 type:
|
||||
'page_mode' — 页面模式匹配（如 page_mode == 'player'）
|
||||
'selector' — DOM 元素可见性（如 '[data-e2e="digg"]' 可见）
|
||||
'marker' — 状态标记匹配（如 liked == False）
|
||||
'marker_diff' — 状态标记变化（如 like_count 增加了）
|
||||
soft: bool = False — 软条件：前置检查失败时不阻断执行（仅记录），用于底层有兜底的操作
|
||||
"""
|
||||
type: str = "page_mode"
|
||||
target: str = "" # 目标字段
|
||||
expected: any = None # 期望值
|
||||
actual: any = None # 运行时填充的实际值
|
||||
passed: bool = False # 是否通过
|
||||
message: str = "" # 检查结果描述
|
||||
soft: bool = False # 软条件（失败不阻断）
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"type": self.type,
|
||||
"target": self.target,
|
||||
"expected": self.expected,
|
||||
"actual": self.actual,
|
||||
"passed": self.passed,
|
||||
"message": self.message,
|
||||
"soft": self.soft,
|
||||
}
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════════
|
||||
# OpResult — 操作结果（v2.0 加入三段式字段）
|
||||
# ════════════════════════════════════════════════════════════
|
||||
|
||||
@dataclass
|
||||
class OpResult:
|
||||
"""单个操作的结构化结果（v2.0 新增 before/after 状态和条件检查）
|
||||
|
||||
原有字段不变，新增:
|
||||
before_state: 操作前页面状态指纹
|
||||
after_state: 操作后页面状态指纹
|
||||
pre_conditions: 前置条件检查结果列表
|
||||
post_conditions: 后置验证检查结果列表
|
||||
"""
|
||||
op: str # 操作名 (如 "like", "goto_home")
|
||||
step_id: int = 0 # 蓝图步骤编号
|
||||
success: bool = True
|
||||
detail: str = "" # 简短描述 (如 "👍", "profile_loaded", "no_btn")
|
||||
elapsed: float = 0.0 # 耗时(秒)
|
||||
error: str = "" # 失败原因
|
||||
|
||||
# v2.0 新增字段
|
||||
before_state: Optional[PageState] = None
|
||||
after_state: Optional[PageState] = None
|
||||
pre_conditions: Optional[list] = None
|
||||
post_conditions: Optional[list] = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
d = asdict(self)
|
||||
# 将 Optional 字段中的 None 去掉，保持 JSON 简洁
|
||||
if self.pre_conditions is None:
|
||||
d.pop("pre_conditions", None)
|
||||
if self.post_conditions is None:
|
||||
d.pop("post_conditions", None)
|
||||
if self.before_state is None:
|
||||
d.pop("before_state", None)
|
||||
if self.after_state is None:
|
||||
d.pop("after_state", None)
|
||||
return d
|
||||
|
||||
def __str__(self):
|
||||
icon = "✅" if self.success else "❌"
|
||||
extra = f" | {self.error}" if self.error else ""
|
||||
# 如果有条件检查结果，追加概要
|
||||
if self.pre_conditions:
|
||||
failed = [c for c in self.pre_conditions if not c.passed]
|
||||
if failed:
|
||||
extra += f" | pre_fail: {len(failed)}"
|
||||
if self.post_conditions:
|
||||
failed = [c for c in self.post_conditions if not c.passed]
|
||||
if failed:
|
||||
extra += f" | post_fail: {len(failed)}"
|
||||
return f"{icon} [{self.step_id:2d}] {self.op:18s} → {self.detail[:25]} ({self.elapsed:.1f}s){extra}"
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════════
|
||||
# PlatformOps — 平台操作基类（v2.0 三段式）
|
||||
# ════════════════════════════════════════════════════════════
|
||||
|
||||
class PlatformOps(ABC):
|
||||
"""平台操作基类（v2.0 三段式）
|
||||
|
||||
子类需要实现:
|
||||
- supported_ops() → list[str]
|
||||
- _do_execute(op, args, step_id) → OpResult (原有，实现动作)
|
||||
- _get_pre_conditions(op) → list[Condition] (新增，前置条件)
|
||||
- _get_post_conditions(op) → list[Condition] (新增，后置验证)
|
||||
|
||||
基类提供:
|
||||
- execute() → 三段式统一入口（子类不需要覆盖）
|
||||
- _capture_state() → 页面状态指纹采集（子类可重写加强）
|
||||
- _check_conditions() → 条件检查引擎
|
||||
"""
|
||||
|
||||
name: str = "base" # 平台标识 (douyin / xiaohongshu)
|
||||
retry_count: int = 1 # 默认重试次数
|
||||
|
||||
# 关键选择器（子类可覆盖，用于状态采集）
|
||||
STATE_SELECTORS: list = []
|
||||
"""状态采集时检查的关键元素选择器列表"""
|
||||
|
||||
# ── 公开接口 ──────────────────────────────────────────────
|
||||
|
||||
def supported_ops(self) -> list[str]:
|
||||
raise NotImplementedError
|
||||
|
||||
async def execute(self, op: str, args: dict = None,
|
||||
step_id: int = 0) -> OpResult:
|
||||
"""三段式执行入口（v2.0）
|
||||
|
||||
流程:
|
||||
1. 采集操作前状态（before_state）
|
||||
2. 检查前置条件（pre_conditions）
|
||||
3. 执行动作（_do_execute）
|
||||
4. 采集操作后状态（after_state）
|
||||
5. 检查后置条件（post_conditions）
|
||||
6. 返回完整 OpResult
|
||||
"""
|
||||
args = args or {}
|
||||
last_err = ""
|
||||
t0_start = time.time()
|
||||
|
||||
# 1. 采集操作前状态
|
||||
before = await self._capture_state()
|
||||
|
||||
# 2. 检查前置条件
|
||||
pre_conds = self._get_pre_conditions(op)
|
||||
pre_results = await self._check_conditions(pre_conds, before)
|
||||
# soft 条件失败不阻断（底层有兜底的操作放行）
|
||||
pre_all_pass = all(c.passed or c.soft for c in pre_results)
|
||||
pre_detail = f"pre:{sum(1 for c in pre_results if c.passed)}/{len(pre_results)}"
|
||||
|
||||
# 前置条件不满足 → 跳过（不是失败）；但全部为软条件失败时也放行
|
||||
if not pre_all_pass:
|
||||
elapsed = round(time.time() - t0_start, 2)
|
||||
result = OpResult(op, step_id, True, f"skipped({pre_detail})",
|
||||
elapsed, before_state=before,
|
||||
pre_conditions=pre_results)
|
||||
return result
|
||||
|
||||
for attempt in range(self.retry_count + 1):
|
||||
t0 = time.time()
|
||||
try:
|
||||
# 3. 执行动作
|
||||
result = await self._do_execute(op, args, step_id)
|
||||
if result is None:
|
||||
result = OpResult(op, step_id, True, "ok", time.time() - t0)
|
||||
result.elapsed = round(time.time() - t0, 2)
|
||||
|
||||
# 4. 采集操作后状态
|
||||
result.after_state = await self._capture_state()
|
||||
|
||||
# 5. 检查后置条件
|
||||
post_conds = self._get_post_conditions(op)
|
||||
result.post_conditions = await self._check_conditions(
|
||||
post_conds, result.after_state, before)
|
||||
post_all_pass = all(c.passed for c in result.post_conditions)
|
||||
post_detail = f"post:{sum(1 for c in result.post_conditions if c.passed)}/{len(result.post_conditions)}"
|
||||
|
||||
# 补充前置条件和 before_state
|
||||
result.pre_conditions = pre_results
|
||||
result.before_state = before
|
||||
|
||||
# 后置条件不满足且还有重试次数 → 重试
|
||||
if not post_all_pass and attempt < self.retry_count:
|
||||
result.success = False
|
||||
result.detail = f"retry({post_detail})"
|
||||
result.error = f"post_condition_failed: {post_detail}"
|
||||
await asyncio.sleep(1)
|
||||
continue
|
||||
|
||||
# 最终判定：后置条件通过或已用完重试
|
||||
result.success = post_all_pass or (attempt >= self.retry_count and result.success)
|
||||
result.detail = post_detail if post_all_pass else result.detail
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
last_err = f"{type(e).__name__}: {e}"
|
||||
if attempt < self.retry_count:
|
||||
await asyncio.sleep(1)
|
||||
|
||||
elapsed = round(time.time() - t0_start, 2)
|
||||
return OpResult(op, step_id, False, "retry_exhausted", elapsed, last_err,
|
||||
before_state=before, pre_conditions=pre_results)
|
||||
|
||||
# ── 状态采集（基类实现，子类可加强）──
|
||||
|
||||
async def _capture_state(self) -> PageState:
|
||||
"""采集当前页面状态指纹
|
||||
|
||||
子类可以重写此方法以加入平台特定的状态标记。
|
||||
基类实现采集:
|
||||
- URL 模式
|
||||
- 页面模式（通过 _detect_page_state）
|
||||
- 关键选择器的可见性
|
||||
- 页面文本片段
|
||||
"""
|
||||
state = PageState()
|
||||
try:
|
||||
page = getattr(self, 'page', None)
|
||||
if page is None:
|
||||
return state
|
||||
|
||||
# URL 模式
|
||||
url = page.url
|
||||
state.url_pattern = self._normalize_url(url)
|
||||
|
||||
# 页面模式（子类通常有 _detect_page_state 方法）
|
||||
detect = getattr(self, '_detect_page_state', None)
|
||||
if detect:
|
||||
state.page_mode = await detect()
|
||||
elif hasattr(self, '_detect_page_mode'):
|
||||
state.page_mode = await self._detect_page_mode()
|
||||
|
||||
# 关键选择器可见性
|
||||
for sel in self.STATE_SELECTORS:
|
||||
try:
|
||||
el = page.locator(sel)
|
||||
state.selectors[sel] = await el.count() > 0 and await el.first.is_visible()
|
||||
except Exception:
|
||||
state.selectors[sel] = False
|
||||
|
||||
# 页面文本片段
|
||||
try:
|
||||
text = await page.evaluate("(document.body.innerText || '').trim().slice(0, 2000)")
|
||||
state.text_snippet = text
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return state
|
||||
|
||||
def _normalize_url(self, url: str) -> str:
|
||||
"""归一化 URL，去掉动态参数，保留路径结构"""
|
||||
import re
|
||||
# 去掉查询参数
|
||||
url = re.sub(r'\?.*$', '', url)
|
||||
# 去掉 /video/ 后面的 ID
|
||||
url = re.sub(r'/video/[A-Za-z0-9]+', '/video/{id}', url)
|
||||
# 去掉 /user/ 后面的 ID
|
||||
url = re.sub(r'/user/[A-Za-z0-9]+', '/user/{id}', url)
|
||||
return url
|
||||
|
||||
# ── 条件检查引擎 ──────────────────────────────────────────
|
||||
|
||||
def _get_pre_conditions(self, op: str) -> list[Condition]:
|
||||
"""每个操作的前置条件——子类覆盖返回
|
||||
|
||||
默认返回空列表（不检查前置条件，兼容旧版）
|
||||
"""
|
||||
return []
|
||||
|
||||
def _get_post_conditions(self, op: str) -> list[Condition]:
|
||||
"""每个操作的后置验证条件——子类覆盖返回
|
||||
|
||||
默认返回空列表（不检查后置条件，兼容旧版）
|
||||
"""
|
||||
return []
|
||||
|
||||
async def _check_conditions(self, conditions: list[Condition],
|
||||
state: PageState,
|
||||
before_state: PageState = None) -> list[Condition]:
|
||||
"""检查一组条件，填充 actual/passed/message 字段"""
|
||||
for cond in conditions:
|
||||
try:
|
||||
if cond.type == 'page_mode':
|
||||
cond.actual = state.page_mode
|
||||
cond.passed = state.page_mode == cond.expected or (
|
||||
cond.expected == 'player' and state.page_mode in ('player', 'player_full', 'player_modal')
|
||||
)
|
||||
cond.message = f"page_mode={state.page_mode}" if cond.passed \
|
||||
else f"期望={cond.expected}, 实际={state.page_mode}"
|
||||
|
||||
elif cond.type == 'selector':
|
||||
visible = state.selectors.get(cond.target, False)
|
||||
cond.actual = visible
|
||||
cond.passed = (visible == cond.expected)
|
||||
cond.message = f"{cond.target} visible={visible}" if cond.passed \
|
||||
else f"期望可见={cond.expected}, 实际={visible}"
|
||||
|
||||
elif cond.type == 'marker':
|
||||
val = state.markers.get(cond.target)
|
||||
cond.actual = val
|
||||
cond.passed = (val == cond.expected)
|
||||
cond.message = f"{cond.target}={val}" if cond.passed \
|
||||
else f"期望={cond.expected}, 实际={val}"
|
||||
|
||||
elif cond.type == 'marker_diff' and before_state:
|
||||
before_val = before_state.markers.get(cond.target)
|
||||
after_val = state.markers.get(cond.target)
|
||||
cond.actual = (before_val, after_val)
|
||||
# marker_diff: expected 可以是 "increased"/"decreased"/"changed"
|
||||
if cond.expected == "increased":
|
||||
cond.passed = (isinstance(before_val, (int, float))
|
||||
and isinstance(after_val, (int, float))
|
||||
and after_val > before_val)
|
||||
elif cond.expected == "decreased":
|
||||
cond.passed = (isinstance(before_val, (int, float))
|
||||
and isinstance(after_val, (int, float))
|
||||
and after_val < before_val)
|
||||
else:
|
||||
cond.passed = (before_val != after_val)
|
||||
cond.message = f"{cond.target}: {before_val} → {after_val}"
|
||||
|
||||
else:
|
||||
cond.passed = True
|
||||
cond.message = f"unknown_condition({cond.type})"
|
||||
|
||||
except Exception as e:
|
||||
cond.passed = False
|
||||
cond.message = f"检查异常: {e}"
|
||||
|
||||
return conditions
|
||||
|
||||
# ── 子类实现 ──────────────────────────────────────────────
|
||||
|
||||
@abstractmethod
|
||||
async def _do_execute(self, op: str, args: dict, step_id: int) -> Optional[OpResult]:
|
||||
"""子类实现具体的操作分发。返回 OpResult 或 None（None=成功，无额外信息）
|
||||
|
||||
注意：execute() 方法在 _do_execute 之前/之后自动采集状态，
|
||||
所以 _do_execute 只需要做「动作」部分，不需要管条件检查。
|
||||
"""
|
||||
...
|
||||
Reference in New Issue
Block a user