feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程，直接发 HTTP

移植自 mac-agent-os 的 mediacrawler_adapter：不起子进程、不开页面，用浏览器里那份登录态直接调抖音 Web 接口。产物键名照抄 store/douyin，所以 ingest 那条链路一个字不用改。 **目前能用的（真环境实测，非推断）**： profile/other : 200, 7075 字节 —— 博主主页指标（粉丝/获赞/作品数/昵称） aweme/detail : 200, 45425 字节 —— 单条作品详情（含点赞/评论/收藏/分享） **目前不能用的：作品列表 `aweme/post`。** 两个互相独立的原因： 1. 这个接口被抖音单独升级成了真校验：不带 x-tt-argus 回 403「Uifid Not Found」，带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过，「真校验」过不了。同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的，挑中的恰好是「批量拉作品列表」这个最敏感的动作。 2. 改走页面截获也不行：CDP 浏览器打开博主主页会落到「验证码中间页」（当天大量探测的代价，过几小时要重测）。所以现在的边界是：**已知作品的指标刷新能做，自动发现新作品做不了**。 **排查中控住变量后得到的两条事实**（都写进注释了）： · `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有 UA+client hints+Cookie 时是 200 但仅 121 字节的空壳，补上这三个头变 7074 字节。（我先前猜的 sec-ch-ua 不是关键。） · 因此 UA 与 client hints 必须**成套地取自同一个浏览器**，所以 BrowserIdentity 一次从 CDP 取齐 cookie + UA + hints，而不是各自写死。「200 + 空 body 必须当场报错」也是刻意写死的：放过去它会在下游变成「这个博主没作品」，把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的，还被翻译成「账号被封」。测试 +11：cookie 解析、请求头成套性（含 uifid 缺失/回退）、产物键名与 store 对齐、以及 _get 的三条失败路径（空 body / 403 带网关原话 / 正常返回）。
2026-10-10 17:12:25 +08:00
parent 21ff01b894
commit 2112c1a870
1601 changed files with 367425 additions and 0 deletions
@@ -0,0 +1,284 @@
+"""
+PageInspector — 三层页面状态检测
+
+L1: 页面类型 (grid/player/search/profile) — URL pattern + DOM特征
+L2: 鼠标所在区域 (feed/video/comment/bottom_bar) — 坐标区间判断
+L3: 鼠标指向元素详情 — tag/class/text/position
+
+基于 ghai 的人类视角判断逻辑:
+  1. 先看URL + 页面大结构 → 确定在什么页面
+  2. 再看鼠标周围区域 → 确定操作上下文
+  3. 最后看鼠标指向的具体元素 → 确定要操作什么
+
+Usage:
+  inspector = PageInspector(page, platform="douyin")
+  state = await inspector.inspect()
+  print(state.page_type)  # "player_modal"
+  print(state.has_video)  # True
+  print(state.nearby_elements)  # [element at cursor...]
+"""
+import asyncio
+from dataclasses import dataclass, field
+from typing import Optional
+
+
+@dataclass
+class PageState:
+    """完整页面状态快照"""
+    platform: str = ""
+    url: str = ""
+    title: str = ""
+
+    # L1: 页面类型
+    page_type: str = "unknown"          # grid/player_modal/player_full/search/profile/unknown
+    page_type_confidence: float = 0.0
+
+    # L1: DOM特征
+    video_count: int = 0
+    card_count: int = 0
+    has_search_bar: bool = False
+    has_nav: bool = False
+    has_comment_list: bool = False
+    is_self_page: bool = False          # 是否自己的主页
+    author_id: str = ""
+
+    # L2: 鼠标区域
+    mouse_x: int = 0
+    mouse_y: int = 0
+    mouse_region: str = "unknown"       # feed_area/video_area/bottom_bar/comment_area/nav_area
+
+    # L3: 附近元素
+    nearby_elements: list = field(default_factory=list)  # [{tag, class, text, x, y, ...}]
+    element_at_point: Optional[dict] = None
+
+    # 验证
+    errors: list = field(default_factory=list)
+    warnings: list = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        return {k: v for k, v in self.__dict__.items()}
+
+
+class PageInspector:
+    """三层页面状态检测器"""
+
+    def __init__(self, page, platform: str = "douyin"):
+        self.page = page
+        self.platform = platform
+
+    async def inspect(self, mouse_x: int = 0, mouse_y: int = 0) -> PageState:
+        """执行完整三层检测"""
+        state = PageState(platform=self.platform)
+
+        try:
+            # L1: 页面基本信息
+            state.url = self.page.url
+            state.title = await self.page.title()
+
+            # L1: DOM特征
+            dom = await self._capture_dom()
+            state.video_count = dom["video_count"]
+            state.card_count = dom["card_count"]
+            state.has_search_bar = dom["has_search_bar"]
+            state.has_nav = dom["has_nav"]
+            state.has_comment_list = dom["has_comment_list"]
+            state.author_id = dom["author_id"]
+            state.is_self_page = dom["is_self"]
+
+            # L1: 页面类型判定
+            state.page_type = self._classify_page_type(state)
+
+            # L2: 鼠标区域
+            if mouse_x > 0 and mouse_y > 0:
+                state.mouse_x = mouse_x
+                state.mouse_y = mouse_y
+                size = await self.page.evaluate(
+                    "() => ({ w: window.innerWidth, h: window.innerHeight })"
+                )
+                state.mouse_region = self._classify_region(
+                    mouse_x, mouse_y, size["w"], size["h"]
+                )
+
+            # L3: 鼠标处元素
+            if mouse_x > 0 and mouse_y > 0:
+                el_info = await self.page.evaluate(
+                    f"""() => {{
+                        const el = document.elementFromPoint({mouse_x}, {mouse_y});
+                        if (!el) return null;
+                        const rect = el.getBoundingClientRect();
+                        return {{
+                            tag: el.tagName,
+                            class: el.className || '',
+                            text: (el.textContent || '').trim().slice(0, 50),
+                            x: Math.round(rect.left), y: Math.round(rect.top),
+                            w: Math.round(rect.width), h: Math.round(rect.height),
+                            visible: el.offsetParent !== null
+                        }};
+                    }}"""
+                )
+                state.element_at_point = el_info
+
+            # L3: 附近区域元素（鼠标周围200px内的可交互元素）
+            if mouse_x > 0 and mouse_y > 0:
+                nearby = await self.page.evaluate(
+                    f"""() => {{
+                        const mx = {mouse_x}, my = {mouse_y};
+                        const els = document.querySelectorAll(
+                            'a, button, span, div, input, textarea, svg')
+                        const results = [];
+                        for (const el of els) {{
+                            if (!el.offsetParent) continue;
+                            const r = el.getBoundingClientRect();
+                            const cx = r.left + r.width/2;
+                            const cy = r.top + r.height/2;
+                            const dist = Math.sqrt((cx-mx)**2 + (cy-my)**2);
+                            if (dist < 200) {{
+                                results.push({{
+                                    tag: el.tagName,
+                                    class: (el.className || '').slice(0, 40),
+                                    text: (el.textContent || '').trim().slice(0, 30),
+                                    cx: Math.round(cx), cy: Math.round(cy), dist: Math.round(dist)
+                                }});
+                            }}
+                        }}
+                        return results.sort((a,b) => a.dist - b.dist).slice(0, 10);
+                    }}"""
+                )
+                state.nearby_elements = nearby
+
+        except Exception as e:
+            state.errors.append(str(e))
+
+        return state
+
+    async def _capture_dom(self) -> dict:
+        """捕获DOM特征"""
+        try:
+            return await self.page.evaluate("""() => {
+                const u = location.href;
+                const vc = document.querySelectorAll('video').length;
+
+                // cards: 视频卡片/笔记卡片
+                const cards = document.querySelectorAll(
+                    '.discover-video-card-item, [class*="video-card"], ' +
+                    'section.note-item, a[href*="/explore/"], [class*="note-item"]'
+                ).length;
+
+                // 搜索栏
+                const searchBar = document.querySelector(
+                    '[data-e2e="searchbar-input"], input[placeholder*="搜索"], textarea.textarea'
+                );
+
+                // 导航
+                const nav = document.querySelector(
+                    '[data-e2e="douyin-navigation"], [class*="nav"], .ulxic5B4'
+                );
+
+                // 评论列表
+                const commentList = document.querySelector(
+                    '[data-e2e="comment-list"], [class*="comment-list"]'
+                );
+
+                // 作者ID (从URL或个人信息区获取)
+                let authorId = '';
+                const userMatch = u.match(/\/user\/(\\d+)/);
+                if (userMatch) authorId = userMatch[1];
+                const exploreMatch = u.match(/\/explore\/(\\w+)/);
+                if (exploreMatch) authorId = exploreMatch[1];
+
+                // 是否自己的页面
+                const isSelf = u.includes('/user/self') ||
+                    !!document.querySelector('[data-e2e="user-info"]');
+
+                return {
+                    video_count: vc,
+                    card_count: cards,
+                    has_search_bar: !!searchBar,
+                    has_nav: !!nav,
+                    has_comment_list: !!commentList,
+                    author_id: authorId,
+                    is_self: isSelf
+                };
+            }""")
+        except:
+            return {"video_count":0, "card_count":0, "has_search_bar":False,
+                    "has_nav":False, "has_comment_list":False, "author_id":"", "is_self":False}
+
+    def _classify_page_type(self, state: PageState) -> str:
+        """L1: 页面类型判定"""
+        u = state.url.lower()
+
+        # 个人主页
+        if '/user/' in u:
+            return 'profile'
+
+        # 搜索页
+        if '/search' in u:
+            return 'search'
+
+        # 播放页: 有video
+        if state.video_count > 0:
+            if '/video/' in u and 'modal_id' not in u:
+                return 'player_full'
+            if 'modal_id' in u:
+                return 'player_modal'
+            return 'player'
+
+        # 笔记详情页 (小红书)
+        if '/explore/' in u and u.count('/') >= 4:
+            return 'note_detail'
+
+        # 首页/grid
+        if state.card_count > 0 or '/jingxuan' in u or '/explore' in u:
+            return 'grid'
+
+        return 'unknown'
+
+    def _classify_region(self, x: int, y: int, w: int, h: int) -> str:
+        """L2: 鼠标区域判定（基于百分比位置）"""
+        x_pct = x / w
+        y_pct = y / h
+
+        # 视频播放区（中央上部区域）
+        if 0.2 < x_pct < 0.8 and 0.1 < y_pct < 0.5:
+            return 'video_area'
+
+        # 底部互动按钮区
+        if 0.2 < x_pct < 0.8 and y_pct > 0.8:
+            return 'bottom_bar'
+
+        # 评论区（底部大区域）
+        if y_pct > 0.7:
+            return 'comment_area'
+
+        # 左侧导航区
+        if x_pct < 0.2:
+            return 'nav_area'
+
+        # 右侧推荐区
+        if x_pct > 0.8:
+            return 'sidebar'
+
+        # 内容feed区（中下部）
+        if 0.2 < x_pct < 0.8 and 0.5 < y_pct < 0.8:
+            return 'feed_area'
+
+        return 'unknown'
+
+
+async def demo(page):
+    """快速演示"""
+    inspector = PageInspector(page, "douyin")
+
+    # 全量检测
+    state = await inspector.inspect()
+    print(f"[{state.page_type}] v={state.video_count} c={state.card_count}")
+
+    # 带鼠标位置的检测（模拟鼠标在右下角互动区）
+    state2 = await inspector.inspect(mouse_x=479, mouse_y=687)
+    print(f"鼠标区域: {state2.mouse_region}")
+    print(f"指向元素: {state2.element_at_point}")
+    print(f"附近元素: {len(state2.nearby_elements)}个")
+    for el in state2.nearby_elements[:3]:
+        print(f"  {el['tag']} dist={el['dist']} {el['class'][:20]}")
+    return state2