feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程，直接发 HTTP

移植自 mac-agent-os 的 mediacrawler_adapter：不起子进程、不开页面，用浏览器里那份登录态直接调抖音 Web 接口。产物键名照抄 store/douyin，所以 ingest 那条链路一个字不用改。 **目前能用的（真环境实测，非推断）**： profile/other : 200, 7075 字节 —— 博主主页指标（粉丝/获赞/作品数/昵称） aweme/detail : 200, 45425 字节 —— 单条作品详情（含点赞/评论/收藏/分享） **目前不能用的：作品列表 `aweme/post`。** 两个互相独立的原因： 1. 这个接口被抖音单独升级成了真校验：不带 x-tt-argus 回 403「Uifid Not Found」，带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过，「真校验」过不了。同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的，挑中的恰好是「批量拉作品列表」这个最敏感的动作。 2. 改走页面截获也不行：CDP 浏览器打开博主主页会落到「验证码中间页」（当天大量探测的代价，过几小时要重测）。所以现在的边界是：**已知作品的指标刷新能做，自动发现新作品做不了**。 **排查中控住变量后得到的两条事实**（都写进注释了）： · `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有 UA+client hints+Cookie 时是 200 但仅 121 字节的空壳，补上这三个头变 7074 字节。（我先前猜的 sec-ch-ua 不是关键。） · 因此 UA 与 client hints 必须**成套地取自同一个浏览器**，所以 BrowserIdentity 一次从 CDP 取齐 cookie + UA + hints，而不是各自写死。「200 + 空 body 必须当场报错」也是刻意写死的：放过去它会在下游变成「这个博主没作品」，把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的，还被翻译成「账号被封」。测试 +11：cookie 解析、请求头成套性（含 uifid 缺失/回退）、产物键名与 store 对齐、以及 _get 的三条失败路径（空 body / 403 带网关原话 / 正常返回）。
2026-10-10 17:12:25 +08:00
parent 21ff01b894
commit 2112c1a870
1601 changed files with 367425 additions and 0 deletions
@@ -0,0 +1,335 @@
+"""
+exporter.py — 录制结果自动导出引擎 (v1.0)
+
+三阶段管线 Phase 3:
+  分析结果 → 生成 OP_GRAPH 条目 + 蓝图模板 + 代码片段
+
+用法:
+  from mc.exporter import export_all
+  result = export_all(analysis_result, recording_package)
+"""
+import json
+import re
+from pathlib import Path
+from typing import Optional
+
+SCRIPTS_DIR = Path(__file__).resolve().parent.parent
+TOOL_DIR = SCRIPTS_DIR.parent
+
+# 已存在的 OP_GRAPH 路径（追加用）
+MATRIX_MGMT = SCRIPTS_DIR / "matrix_mgmt.py"
+BLUEPRINTS_DIR = TOOL_DIR / "blueprints"
+
+
+def export_all(analysis: dict, package: dict) -> dict:
+    """完整导出管线——生成 OP_GRAPH + 蓝图 + 代码"""
+    return {
+        "op_graph_entries": _generate_op_graph_entries(analysis),
+        "blueprint": _generate_blueprint(analysis, package),
+        "code_snippet": _generate_code_snippet(analysis, package),
+        "summary": _generate_summary(analysis, package),
+    }
+
+
+def _generate_op_graph_entries(analysis: dict) -> list:
+    """从分析结果生成 OP_GRAPH 字典条目"""
+    actions = analysis.get("actions", [])
+    entries = []
+
+    for i, action in enumerate(actions):
+        at = action.get("action_type", "unknown")
+        desc = action.get("action_desc", "")
+
+        # 确定平台和分类
+        platform, category, label, requires, allows = _classify_action(at, desc)
+
+        # 生成 op name
+        op_name = _suggest_op_name(at, i)
+
+        # 检查是否已有同名操作
+        if _op_exists(op_name):
+            op_name = f"{op_name}_v{i+1}"
+
+        entry = {
+            "name": op_name,
+            "definition": {
+                "platform": platform,
+                "category": category,
+                "label": f"🎬 {desc}" if desc else f"🎬 自定义操作{i+1}",
+                "requires": requires,
+                "allows": allows,
+                "can_be_first": "goto_home" not in requires,
+                "desc": desc or f"录制步骤 {action.get('step_after', '?')}",
+                "_source": "recorded",  # recorded = AI录制, manual = 手工创建
+                "_action_type": at,
+            }
+        }
+
+        # 如果有点击坐标，记录
+        inferred = action.get("inferred_action", {})
+        if inferred.get("coords"):
+            entry["definition"]["_click_coords"] = inferred["coords"]
+
+        entries.append(entry)
+
+    return entries
+
+
+def _classify_action(action_type: str, desc: str) -> tuple:
+    """确定操作所属平台、分类、依赖"""
+    # 默认值
+    platform = "通用"
+    category = "custom"
+    requires = []
+    allows = ["rest", "go_back"]
+
+    if action_type == "like":
+        platform = "douyin"
+        category = "interact"
+        requires = ["goto_home"]
+        allows = ["rest", "go_back", "collect", "next_video"]
+    elif action_type == "comment" or action_type == "open_comment":
+        platform = "douyin"
+        category = "interact"
+        requires = ["goto_home"]
+        allows = ["rest", "go_back", "like"]
+    elif action_type in ("navigate_video", "enter_video"):
+        platform = "douyin"
+        category = "navigation"
+        requires = ["goto_home"]
+        allows = ["like", "collect", "comment", "next_video", "scroll_feed", "rest"]
+    elif action_type == "navigate_profile":
+        platform = "douyin"
+        category = "navigation"
+        requires = ["goto_home"]
+        allows = ["read_profile", "rest", "go_back"]
+    elif action_type == "scroll" or "scroll" in action_type:
+        platform = "douyin"
+        category = "navigation"
+        requires = ["goto_home"]
+        allows = ["rest", "go_back", "next_video"]
+    elif action_type == "search":
+        platform = "douyin"
+        category = "navigation"
+        requires = ["goto_home"]
+        allows = ["browse_feed", "rest", "go_back"]
+    elif action_type == "next_video":
+        platform = "douyin"
+        category = "navigation"
+        requires = ["goto_home"]
+        allows = ["like", "collect", "comment", "rest", "go_back"]
+    elif "xhs" in platform or "xiaohongshu" in desc.lower():
+        platform = "xiaohongshu"
+        category = "custom"
+        requires = ["xhs_goto_home"]
+        allows = ["rest", "go_back"]
+    elif "click" in action_type:
+        platform = "通用"
+        category = "interact"
+        requires = ["goto_home", "xhs_goto_home"]
+        allows = ["rest", "go_back"]
+
+    return platform, category, f"🎬 {desc[:15]}" if desc else "🎬 自定义", requires, allows
+
+
+def _suggest_op_name(action_type: str, index: int) -> str:
+    """生成操作名称"""
+    name_map = {
+        "like": "custom_like", "comment": "custom_comment", "follow": "custom_follow",
+        "navigate_video": "custom_open_video", "navigate_profile": "custom_goto_profile",
+        "scroll": "custom_scroll", "next_video": "custom_next",
+        "search": "custom_search", "enter_video": "custom_enter_video",
+        "click": "custom_click", "keypress": "custom_keypress",
+        "view_profile": "custom_view_profile", "unknown": "custom_action",
+    }
+    base = name_map.get(action_type, f"custom_{action_type}")
+    return base
+
+
+def _op_exists(op_name: str) -> bool:
+    """检查 OP_GRAPH 是否已有同名操作"""
+    if not MATRIX_MGMT.exists():
+        return False
+    content = MATRIX_MGMT.read_text()
+    return f'"{op_name}":' in content
+
+
+def _generate_blueprint(analysis: dict, package: dict) -> dict:
+    """从分析结果生成蓝图 JSON"""
+    actions = analysis.get("actions", [])
+    meta = package.get("meta", {})
+
+    # 确定平台
+    platform = meta.get("platform", "unknown")
+    for action in actions:
+        at = action.get("action_type", "")
+        if "xhs" in at or "profile" in at:
+            platform = "xiaohongshu" if "profile" in at else "douyin"
+            break
+
+    # 构建步骤
+    steps = []
+    seen_ops = set()
+    step_id = 1
+
+    # 如果是从首页开始的，先加 goto_home
+    if platform == "douyin":
+        steps.append({
+            "step_id": step_id, "op": "goto_home",
+            "args": {}, "wait_after": 3
+        })
+        seen_ops.add("goto_home")
+        step_id += 1
+
+    for action in actions:
+        at = action.get("action_type", "unknown")
+        op_name = _suggest_op_name(at, step_id)
+
+        # 跳过重复的导航步骤
+        if op_name in seen_ops:
+            continue
+        seen_ops.add(op_name)
+
+        step = {
+            "step_id": step_id,
+            "op": op_name,
+            "args": {},
+            "wait_after": 2,
+        }
+
+        # 如果是搜索操作，加默认关键词
+        if at == "search":
+            step["args"]["keyword"] = "热门"
+
+        steps.append(step)
+        step_id += 1
+
+    blueprint = {
+        "id": f"recorded_{meta.get('account_id', 'unknown')}_{len(actions)}steps",
+        "name": f"录制蓝图 ({len(actions)}步)",
+        "version": "1.0.0",
+        "platform": platform,
+        "description": f"由录制自动生成: {meta.get('account_id','?')} {len(actions)}步操作",
+        "steps": steps,
+    }
+
+    return blueprint
+
+
+def _generate_code_snippet(analysis: dict, package: dict) -> str:
+    """生成可在 engine.py 中使用的操作代码片段"""
+    actions = analysis.get("actions", [])
+    lines = []
+    lines.append("# ═══════════════════════════════════════════════")
+    lines.append(f"# 录制生成: {package.get('meta', {}).get('account_id', '?')}")
+    lines.append(f"# 步骤数: {len(actions)}")
+    lines.append("# ═══════════════════════════════════════════════")
+    lines.append("")
+
+    for i, action in enumerate(actions):
+        at = action.get("action_type", "unknown")
+        desc = action.get("action_desc", "")
+        inferred = action.get("inferred_action", {})
+
+        lines.append(f"    # 步骤 {i+1}: {desc}")
+        lines.append(f"    elif op == \"{_suggest_op_name(at, i)}\":")
+
+        if at == "like":
+            lines.append('        r = await conn.page.evaluate("""() => {')
+            lines.append('            const b = document.querySelector(\'[data-e2e="like-count"]\');')
+            lines.append('            return b ? (b.click(), \'👍\') : \'-\';')
+            lines.append('        }""")')
+            lines.append('        result = r')
+        elif at == "navigate_profile":
+            lines.append('        await conn.page.goto("https://www.douyin.com/user/self", timeout=20000)')
+            lines.append('        await asyncio.sleep(4)')
+            lines.append('        result = "profile_loaded"')
+        elif at == "scroll" or "scroll" in at:
+            lines.append('        await conn.page.evaluate("() => window.scrollBy(0, 600)")')
+            lines.append('        await asyncio.sleep(1)')
+            lines.append('        result = "scrolled"')
+        elif inferred.get("coords"):
+            coords = inferred["coords"]
+            lines.append(f'        # 点击坐标 ({coords["x"]}, {coords["y"]})')
+            lines.append(f'        await conn.page.mouse.click({coords["x"]}, {coords["y"]})')
+            lines.append('        await asyncio.sleep(2)')
+            lines.append('        result = "clicked"')
+        elif inferred.get("trigger", "").startswith("key:"):
+            key = inferred["trigger"].split(":")[1]
+            lines.append(f'        # 按键: {key}')
+            lines.append(f'        await conn.page.keyboard.press("{key}")')
+            lines.append('        await asyncio.sleep(2)')
+            lines.append('        result = "pressed"')
+        else:
+            lines.append(f'        # TODO: 实现 {desc} 的自动化操作')
+            lines.append('        await asyncio.sleep(2)')
+            lines.append('        result = f"TODO({at})"')
+
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+def _generate_summary(analysis: dict, package: dict) -> dict:
+    """生成操作汇总摘要"""
+    actions = analysis.get("actions", [])
+    meta = package.get("meta", {})
+
+    # 统计操作类型
+    type_counts = {}
+    for a in actions:
+        at = a.get("action_type", "unknown")
+        type_counts[at] = type_counts.get(at, 0) + 1
+
+    return {
+        "account": meta.get("account_id", "?"),
+        "total_steps": meta.get("total_steps", 0),
+        "actions_found": len(actions),
+        "action_types": type_counts,
+        "action_sequence": [a.get("action_type", "?") for a in actions],
+        "suggested_blueprint_id": f"recorded_{meta.get('account_id', '?')}_{len(actions)}steps",
+    }
+
+
+def export_recording(recording_path: str, output_dir: str = None) -> dict:
+    """一站式：加载 → 分析 → 导出"""
+    from mc.recorder import RecordingSession
+    from mc.analyzer import analyze_recording
+
+    pkg = RecordingSession.load_recording(recording_path)
+    analysis = analyze_recording(pkg)
+    result = export_all(analysis, pkg)
+
+    # 保存导出产物
+    out_dir = Path(output_dir) if output_dir else BLUEPRINTS_DIR
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    base_name = Path(recording_path).stem.replace("recording_", "export_")
+
+    # 保存蓝图
+    bp = result.get("blueprint", {})
+    if bp:
+        bp_file = out_dir / f"{base_name}.json"
+        with open(bp_file, "w", encoding="utf-8") as f:
+            json.dump(bp, f, ensure_ascii=False, indent=2)
+        result["saved_blueprint"] = str(bp_file)
+
+    # 保存代码片段
+    code = result.get("code_snippet", "")
+    if code:
+        code_file = out_dir / f"{base_name}.py"
+        with open(code_file, "w", encoding="utf-8") as f:
+            f.write(code)
+        result["saved_code"] = str(code_file)
+
+    return result
+
+
+if __name__ == "__main__":
+    import sys
+    path = sys.argv[1] if len(sys.argv) > 1 else ""
+    if path:
+        result = export_recording(path)
+        print(json.dumps(result, ensure_ascii=False, indent=2))
+    else:
+        print("用法: python -m mc.exporter <录制包路径>")