feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程，直接发 HTTP

移植自 mac-agent-os 的 mediacrawler_adapter：不起子进程、不开页面，用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin，所以 ingest 那条链路一个字不用改。

**目前能用的（真环境实测，非推断）**：

    profile/other : 200, 7075 字节  —— 博主主页指标（粉丝/获赞/作品数/昵称）
    aweme/detail  : 200, 45425 字节 —— 单条作品详情（含点赞/评论/收藏/分享）

**目前不能用的：作品列表 `aweme/post`。** 两个互相独立的原因：
  1. 这个接口被抖音单独升级成了真校验：不带 x-tt-argus 回 403「Uifid Not Found」，
     带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过，「真校验」过不了。
     同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的，
     挑中的恰好是「批量拉作品列表」这个最敏感的动作。
  2. 改走页面截获也不行：CDP 浏览器打开博主主页会落到「验证码中间页」（当天大量探测的
     代价，过几小时要重测）。

  所以现在的边界是：**已知作品的指标刷新能做，自动发现新作品做不了**。

**排查中控住变量后得到的两条事实**（都写进注释了）：
  · `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
    UA+client hints+Cookie 时是 200 但仅 121 字节的空壳，补上这三个头变 7074 字节。
    （我先前猜的 sec-ch-ua 不是关键。）
  · 因此 UA 与 client hints 必须**成套地取自同一个浏览器**，所以 BrowserIdentity 一次
    从 CDP 取齐 cookie + UA + hints，而不是各自写死。

「200 + 空 body 必须当场报错」也是刻意写死的：放过去它会在下游变成「这个博主没作品」，
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的，还被翻译成「账号被封」。

测试 +11：cookie 解析、请求头成套性（含 uifid 缺失/回退）、产物键名与 store 对齐、
以及 _get 的三条失败路径（空 body / 403 带网关原话 / 正常返回）。

mac-agent-os-main/05_tools/10_dashboard/services/adapters/__init__.py

@@ -0,0 +1,192 @@
+"""
+抓取适配器基类 — 统一接口
+
+每个平台适配器实现 ScrapeAdapter 接口：
+  - collect_item(target, depth, tool_level) -> dict
+  - collect_user(user_id, limit) -> list[dict]
+  - collect_comments(item_id, limit) -> list[dict]
+  - collect_search(keyword, limit) -> list[dict]
+
+工具降级：
+  Level 1: OpenCLI（仅 CLI）
+  Level 2: OpenCLI → agent-browser（默认）
+  Level 3: OpenCLI → agent-browser → web_crawler
+"""
+
+import asyncio, json, logging, os
+from pathlib import Path
+
+logger = logging.getLogger("dashboard.scrape.adapter")
+
+
+class ScrapeAdapter:
+    """抓取适配器基类"""
+
+    platform = ""       # 子类覆写
+    adapter_name = ""
+
+    # OpenCLI 二进制路径
+    OPENCLI = os.environ.get("OPENCLI_PATH",
+        str(Path.home() / ".workbuddy" / "binaries" / "node" / "versions" / "22.22.2" / "bin" / "opencli"))
+
+    # ── 工具降级 ──
+
+    async def _try_tools(self, tool_level: int, funcs: list) -> tuple:
+        """
+        按降级顺序尝试工具。
+        funcs: [(tool_name, callable), ...]
+        返回: (success, result, used_tool)
+        """
+        tools = [f for f in funcs[:tool_level]]
+        for name, fn in tools:
+            try:
+                result = await fn()
+                if result is not None:
+                    return True, result, name
+            except Exception as e:
+                logger.warning(f"  ⚠️ [{self.platform}] 工具 {name} 失败: {e}")
+        return False, None, tools[-1][0] if tools else "none"
+
+    async def _run_opencli(self, args: list, timeout: int = 60):
+        """
+        执行 OpenCLI 命令并解析输出。
+
+        自动检测输出格式：
+          1. JSON（以 [ 或 { 开头）→ json.loads
+          2. YAML（PyYAML 可用时） → yaml.safe_load
+          3. 兜底 → 简易 key:value 行解析
+        """
+        cmd = [self.OPENCLI] + args
+        proc = await asyncio.create_subprocess_exec(
+            *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
+        try:
+            stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
+        except asyncio.TimeoutError:
+            proc.kill()
+            raise TimeoutError(f"OpenCLI 超时 ({timeout}s): {' '.join(args[:4])}")
+        if proc.returncode != 0:
+            err = stderr.decode().strip() or stdout.decode()[:200]
+            raise RuntimeError(f"OpenCLI 错误 ({proc.returncode}): {err}")
+        output = stdout.decode().strip()
+        if not output:
+            return {}
+        return self._parse_output(output)
+
+    def _parse_output(self, text: str):
+        """
+        解析 OpenCLI 输出，自动检测格式。
+        返回 list[dict] 或 dict。
+        """
+        text = text.strip()
+        if not text:
+            return {}
+
+        # 1. JSON 检测（以 [ 或 { 开头）
+        if text.startswith("[") or text.startswith("{"):
+            try:
+                return json.loads(text)
+            except json.JSONDecodeError:
+                logger.warning("  ⚠️ OpenCLI JSON 解析失败，尝试 YAML 兜底")
+
+        # 2. YAML 解析（PyYAML 在 requirements.txt 中）
+        try:
+            import yaml
+            parsed = yaml.safe_load(text)
+            if parsed is not None:
+                return parsed
+        except ImportError:
+            logger.debug("  PyYAML 未安装，跳过 YAML 解析")
+        except Exception as e:
+            logger.warning(f"  ⚠️ YAML 解析失败: {e}")
+
+        # 3. 兜底：简易 key:value 行解析（无依赖）
+        return self._parse_lines(text)
+
+    def _parse_lines(self, text: str) -> list:
+        """
+        最后一层兜底：按行解析 key:value 格式输出。
+        适用于没有 yaml 库或 JSON/YAML 都失败的情况。
+        """
+        items = []
+        current = {}
+        # 尝试识别表格/列表分隔符
+        in_top_comments = False
+        comment_buffer = []
+
+        def _maybe_flush():
+            """将当前记录推入 items"""
+            nonlocal current, comment_buffer, in_top_comments
+            # 处理缓存的评论
+            if comment_buffer:
+                current["top_comments"] = comment_buffer
+                comment_buffer = []
+            if current and any(v for v in current.values()):
+                items.append(current)
+            current = {}
+            in_top_comments = False
+
+        for line in text.split("\n"):
+            line_stripped = line.strip()
+            if not line_stripped:
+                _maybe_flush()
+                continue
+            # 分隔线（--- 或 =====）
+            if line_stripped.startswith("---") or line_stripped.startswith("==="):
+                _maybe_flush()
+                continue
+            # key: value 行
+            if ": " in line_stripped:
+                k, v = line_stripped.split(": ", 1)
+                k = k.strip()
+                v = v.strip().strip("'\"")
+                # 特殊字段: top_comments
+                if k == "top_comments":
+                    _maybe_flush()
+                    in_top_comments = True
+                    comment_buffer = []
+                elif k in ("nickname", "text", "digg_count") and in_top_comments:
+                    # 评论块中的字段
+                    if comment_buffer and k in comment_buffer[-1]:
+                        # 同 key 出现多次 → 新评论
+                        pass
+                    entry = comment_buffer[-1] if comment_buffer else {}
+                    entry[k] = v
+                    if not comment_buffer:
+                        comment_buffer.append(entry)
+                    else:
+                        # 如果当前 entry 已经有这个 key 了，说明是新记录
+                        if k in comment_buffer[-1] and comment_buffer[-1][k] != v:
+                            comment_buffer.append({k: v})
+                        else:
+                            comment_buffer[-1][k] = v
+                elif in_top_comments:
+                    # 在 top_comments 块里但字段不在三个已知中 → 只是浮层
+                    pass
+                else:
+                    current[k] = v
+            # 列表项（以 - 开头）
+            elif line_stripped.startswith("- "):
+                _maybe_flush()
+                val = line_stripped[2:].strip()
+                if val:
+                    current = {"_value": val}
+                    items.append(current)
+                    current = {}
+
+        _maybe_flush()
+        return items if items else [current] if current else []
+
+    # ── 接口方法（子类覆写）──
+
+    async def collect_item(self, target: str, depth: str = "light",
+                           tool_level: int = 2) -> dict:
+        raise NotImplementedError
+
+    async def collect_user(self, user_id: str, limit: int = 20) -> list[dict]:
+        raise NotImplementedError
+
+    async def collect_comments(self, item_id: str, limit: int = 20) -> list[dict]:
+        raise NotImplementedError
+
+    async def collect_search(self, keyword: str, limit: int = 20) -> list[dict]:
+        raise NotImplementedError

mac-agent-os-main/05_tools/10_dashboard/services/adapters/bilibili_scrape.py

@@ -0,0 +1,134 @@
+"""
+bilibili_scrape.py — B站抓取适配器
+
+工具：OpenCLI bilibili → agent-browser（Chrome DOM提取）
+"""
+import asyncio, json, logging
+from services.adapters import ScrapeAdapter
+
+logger = logging.getLogger("dashboard.scrape.bilibili")
+
+
+class BilibiliScrapeAdapter(ScrapeAdapter):
+    platform = "bilibili"
+    adapter_name = "bilibili_scrape"
+
+    async def collect_user(self, user_id: str, limit: int = 20) -> list[dict]:
+        """抓取指定 UP 主的视频列表"""
+        results = []
+        ok, data, tool = await self._try_tools(2, [
+            ("opencli", lambda: self._opencli_user_videos(user_id, limit)),
+        ])
+        if ok and data:
+            for item in (data if isinstance(data, list) else [data]):
+                results.append(self._to_schema(item))
+        return results
+
+    async def collect_item(self, target: str, depth: str = "light",
+                           tool_level: int = 2) -> dict:
+        """抓取单条视频详情"""
+        bvid = self._extract_bvid(target)
+        if not bvid:
+            return {"error": f"无法解析 BVID: {target}"}
+        ok, data, tool = await self._try_tools(tool_level, [
+            ("opencli", lambda: self._opencli_video(bvid)),
+        ])
+        if ok and data:
+            return self._to_schema(data)
+        return {"error": "所有工具均无法抓取"}
+
+    async def collect_comments(self, item_id: str, limit: int = 20) -> list[dict]:
+        """抓取视频评论"""
+        ok, data, tool = await self._try_tools(2, [
+            ("opencli", lambda: self._opencli_comments(item_id, limit)),
+        ])
+        if ok and data:
+            return [
+                {"author": c.get("nickname", c.get("user", "")),
+                 "text": c.get("text", c.get("content", "")),
+                 "likes": c.get("likes", c.get("like_count", 0))}
+                for c in (data if isinstance(data, list) else [data])
+            ]
+        return []
+
+    async def collect_search(self, keyword: str, limit: int = 20) -> list[dict]:
+        """搜索视频"""
+        results = []
+        ok, data, tool = await self._try_tools(2, [
+            ("opencli", lambda: self._opencli_search(keyword, limit)),
+        ])
+        if ok and data:
+            for item in (data if isinstance(data, list) else [data]):
+                results.append(self._to_schema(item))
+        return results
+
+    # ── OpenCLI 调用 ──
+
+    async def _opencli_user_videos(self, mid: str, limit: int):
+        return await self._run_opencli([
+            "bilibili", "user-videos", mid,
+            "--limit", str(limit), "-f", "json"
+        ])
+
+    async def _opencli_video(self, bvid: str):
+        return await self._run_opencli([
+            "bilibili", "video", bvid, "-f", "json"
+        ])
+
+    async def _opencli_comments(self, bvid: str, limit: int):
+        return await self._run_opencli([
+            "bilibili", "comments", bvid,
+            "--limit", str(limit), "-f", "json"
+        ])
+
+    async def _opencli_search(self, keyword: str, limit: int):
+        return await self._run_opencli([
+            "bilibili", "search", keyword,
+            "--limit", str(limit), "-f", "json"
+        ])
+
+    # ── URL 解析 ──
+
+    def _extract_bvid(self, target: str) -> str:
+        """从 URL 或文本中提取 BVID"""
+        import re
+        # bilibili.com/video/BVxxx
+        m = re.search(r'bilibili\.com/video/([a-zA-Z0-9]+)', target)
+        if m:
+            return m.group(1)
+        # 纯 BVID（BV 开头）
+        if target.startswith("BV") and len(target) >= 10:
+            return target
+        return ""
+
+    # ── 统一 Schema ──
+
+    def _to_schema(self, raw: dict) -> dict:
+        return {
+            "platform": "bilibili",
+            "item_id": raw.get("bvid", raw.get("aid", "")),
+            "url": raw.get("url", f"https://www.bilibili.com/video/{raw.get('bvid', '')}"),
+            "title": raw.get("title", ""),
+            "author_name": raw.get("author", raw.get("up", raw.get("owner_name", ""))),
+            "author_id": raw.get("mid", raw.get("owner_mid", "")),
+            "published_at": raw.get("pubdate", raw.get("pub_time", raw.get("created", ""))),
+            "text_content": raw.get("desc", raw.get("description", "")),
+            "tags": raw.get("tags", []),
+            "stats": {
+                "likes": raw.get("likes", raw.get("stat_like", raw.get("like_count", 0))),
+                "comments": raw.get("comments", raw.get("stat_reply", raw.get("reply_count", 0))),
+                "shares": raw.get("shares", raw.get("stat_share", raw.get("share_count", 0))),
+                "views": raw.get("views", raw.get("stat_view", raw.get("play_count", 0))),
+            },
+            "extra": {
+                "bvid": raw.get("bvid", ""),
+                "mid": raw.get("mid", raw.get("owner_mid", "")),
+                "duration": raw.get("duration", raw.get("v_duration", 0)),
+            },
+            "media": [{
+                "type": "video",
+                "url": raw.get("play_url", ""),
+                "duration": raw.get("duration", 0),
+            }] if raw.get("play_url") else [],
+            "comments": [],
+        }

mac-agent-os-main/05_tools/10_dashboard/services/adapters/browser_helpers.py

@@ -0,0 +1,191 @@
+"""
+browser_helpers.py — 抓取系统的浏览器辅助函数 v2
+
+用途：
+  当 OpenCLI 抓取失败时，通过真实 Chrome 浏览器降级提取数据。
+
+设计原则：
+  ⛔ 绝不使用 Camoufox（养号专用，Firefox 内核 + 特殊指纹）
+  ✅ 使用 Playwright 启动真实 Chrome（Chromium 内核，正常指纹）
+  ✅ 支持 headless 和 headed 两种模式
+  ✅ 浏览器实例复用，避免反复启动
+
+调用方式：
+  from services.adapters.browser_helpers import page_evaluate, page_extract
+
+  # 执行 JS 提取数据
+  data = await page_evaluate("https://www.douyin.com/video/xxx", js_code)
+
+  # 按 CSS 选择器提取
+  texts = await page_extract("https://www.douyin.com/video/xxx", ["h1.title", ".desc"])
+"""
+import asyncio, json, logging, os
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger("dashboard.scrape.browser")
+
+# ── 浏览器单例（复用实例，避免反复启动）──
+
+_browser = None
+_lock = asyncio.Lock()
+_USER_DATA_DIR = os.environ.get(
+    "SCRAPE_CHROME_USER_DATA",
+    str(Path.home() / "workbuddy-agent-os" / "agent-local" / "runtime" / "scrape_chrome_profile"),
+)
+
+# Chrome 可执行路径
+_CHROME_PATHS = [
+    "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",  # macOS 标准路径
+    "/Applications/Chromium.app/Contents/MacOS/Chromium",
+]
+_CHROME_PATH = None
+for p in _CHROME_PATHS:
+    if Path(p).exists():
+        _CHROME_PATH = p
+        break
+
+
+async def _get_browser(headless: bool = True):
+    """
+    获取/创建浏览器实例（单例）。
+    使用 Playwright 启动真实 Chrome，而非 Camoufox。
+    """
+    global _browser
+    async with _lock:
+        if _browser is not None and _browser.is_connected():
+            return _browser
+        try:
+            from playwright.async_api import async_playwright
+        except ImportError:
+            logger.error("Playwright 未安装，无法使用浏览器降级")
+            # 尝试安装
+            try:
+                proc = await asyncio.create_subprocess_exec(
+                    *["playwright", "install", "chromium"],
+                    stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+                )
+                await proc.communicate()
+                from playwright.async_api import async_playwright
+            except Exception:
+                raise RuntimeError("Playwright 不可用，无法启动浏览器降级")
+
+        p = await async_playwright().start()
+        launch_kwargs = {
+            "headless": headless,
+            "args": [
+                "--disable-blink-features=AutomationControlled",
+                "--no-sandbox",
+                "--disable-dev-shm-usage",
+                "--disable-gpu",
+                "--window-size=1280,720",
+            ],
+        }
+        if _CHROME_PATH:
+            launch_kwargs["executable_path"] = _CHROME_PATH
+            logger.info(f"使用系统 Chrome: {_CHROME_PATH}")
+        else:
+            logger.info("使用 Playwright 内置 Chromium")
+
+        try:
+            _browser = await p.chromium.launch(**launch_kwargs)
+            logger.info("✅ 浏览器降级: Chrome 启动成功")
+        except Exception as e:
+            logger.error(f"❌ 浏览器启动失败: {e}")
+            await p.stop()
+            raise
+        return _browser
+
+
+async def page_evaluate(url: str, js_code: str,
+                        timeout: int = 30000, headless: bool = True) -> dict:
+    """
+    打开 URL，执行 JavaScript 提取数据。
+
+    参数:
+      url:       目标页面地址
+      js_code:   要执行的 JS 代码（字符串形式的箭头函数或普通函数）
+      timeout:   页面加载超时（毫秒）
+      headless:  是否无头模式
+
+    返回:
+      字典。JS 返回值的 JSON 表示。失败时返回 {}。
+    """
+    browser = await _get_browser(headless=headless)
+    context = None
+    page = None
+    try:
+        context = await browser.new_context(
+            user_agent=(
+                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) "
+                "Chrome/125.0.0.0 Safari/537.36"
+            ),
+            viewport={"width": 1280, "height": 720},
+            locale="zh-CN",
+        )
+        page = await context.new_page()
+        await page.goto(url, wait_until="domcontentloaded", timeout=timeout)
+        # 等待页面稳定
+        await page.wait_for_load_state("networkidle", timeout=timeout)
+        # 额外等待，让动态渲染完成
+        await asyncio.sleep(1)
+
+        # 执行 JS
+        result = await page.evaluate(js_code)
+        if result is None:
+            logger.warning(f"  ⚠️ page_evaluate JS 返回 null: {url[:60]}")
+            return {}
+
+        # 序列化确保可 JSON 返回
+        if isinstance(result, (dict, list)):
+            return result
+        return {"_value": str(result)}
+
+    except Exception as e:
+        logger.warning(f"  ⚠️ page_evaluate 失败 [{url[:60]}]: {e}")
+        return {}
+    finally:
+        if page:
+            await page.close()
+        if context:
+            await context.close()
+
+
+async def page_extract(url: str, selectors: list,
+                       timeout: int = 30000, headless: bool = True) -> dict:
+    """
+    打开 URL，按 CSS 选择器提取 DOM 文本。
+
+    参数:
+      url:       目标页面
+      selectors: CSS 选择器列表，如 ["h1", ".video-title", "[data-e2e=comment-count]"]
+      timeout:   超时（毫秒）
+      headless:  是否无头
+
+    返回:
+      {selector: [text1, text2, ...], ...}
+    """
+    js = f"""() => {{
+        const results = {{}};
+        const sel = {json.dumps(selectors)};
+        for (const s of sel) {{
+            const els = document.querySelectorAll(s);
+            results[s] = Array.from(els).map(el => el.textContent.trim()).filter(t => t);
+        }}
+        return results;
+    }}"""
+    return await page_evaluate(url, js, timeout=timeout, headless=headless)
+
+
+async def close_browser():
+    """关闭浏览器实例（清理时调用）"""
+    global _browser
+    async with _lock:
+        if _browser:
+            try:
+                await _browser.close()
+                logger.info("浏览器降级: Chrome 已关闭")
+            except Exception as e:
+                logger.warning(f"关闭浏览器时出错: {e}")
+            _browser = None

mac-agent-os-main/05_tools/10_dashboard/services/adapters/douyin_scrape.py

@@ -0,0 +1,181 @@
+"""
+douyin_scrape.py — 抖音抓取适配器
+
+工具：OpenCLI douyin → agent-browser（Chrome DOM提取）
+"""
+import asyncio, json, logging
+from services.adapters import ScrapeAdapter
+
+logger = logging.getLogger("dashboard.scrape.douyin")
+
+
+class DouyinScrapeAdapter(ScrapeAdapter):
+    platform = "douyin"
+    adapter_name = "douyin_scrape"
+
+    # ── 公开接口 ──
+
+    async def collect_user(self, user_id: str, limit: int = 20) -> list[dict]:
+        """抓取指定用户的视频列表（含评论）"""
+        results = []
+        ok, data, tool = await self._try_tools(2, [
+            ("opencli", lambda: self._opencli_user_videos(user_id, limit)),
+            ("agent-browser", lambda: self._browser_user_profile(user_id, limit)),
+        ])
+        if ok and data:
+            for item in data:
+                results.append(self._to_schema(item))
+        return results
+
+    async def collect_item(self, target: str, depth: str = "light",
+                           tool_level: int = 2) -> dict:
+        """抓取单条视频详情"""
+        aweme_id = self._extract_aweme_id(target)
+        if not aweme_id:
+            return {"error": f"无法解析视频ID: {target}"}
+
+        ok, data, tool = await self._try_tools(tool_level, [
+            ("opencli", lambda: self._opencli_stats(aweme_id)),
+            ("agent-browser", lambda: self._browser_video_page(aweme_id)),
+        ])
+        if ok and data:
+            return self._to_schema(data)
+        return {"error": "所有工具均无法抓取"}
+
+    async def collect_comments(self, item_id: str, limit: int = 20) -> list[dict]:
+        """抓取评论区（需先有作者 sec_uid，目前返回空表）"""
+        return []
+
+    async def collect_search(self, keyword: str, limit: int = 20) -> list[dict]:
+        """搜索视频"""
+        results = []
+        ok, data, tool = await self._try_tools(2, [
+            ("opencli", lambda: self._opencli_search(keyword, limit)),
+        ])
+        if ok and data:
+            for item in data:
+                results.append(self._to_schema(item))
+        return results
+
+    # ── OpenCLI 调用 ──
+
+    async def _opencli_user_videos(self, sec_uid: str, limit: int):
+        data = await self._run_opencli([
+            "douyin", "user-videos", sec_uid,
+            "--limit", str(limit), "--with_comments", "true",
+            "-f", "json"
+        ])
+        # OpenCLI JSON 输出直接是数组
+        if isinstance(data, list):
+            return data
+        if isinstance(data, dict) and "items" in data:
+            return data["items"]
+        # YAML 格式回退
+        if isinstance(data, list) and len(data) > 0 and "aweme_id" in data[0]:
+            return data
+        return []
+
+    async def _opencli_stats(self, aweme_id: str):
+        try:
+            return await self._run_opencli([
+                "douyin", "stats", aweme_id, "-f", "json"
+            ])
+        except Exception as e:
+            logger.warning(f"  ⚠️ douyin stats 失败: {e}")
+            return None
+
+    async def _opencli_search(self, keyword: str, limit: int):
+        # OpenCLI 暂无抖音搜索命令，返回空
+        return []
+
+    # ── agent-browser 降级 ──
+
+    async def _browser_video_page(self, aweme_id: str):
+        """通过 Chrome 打开视频页提取数据"""
+        from services.adapters.browser_helpers import page_evaluate
+        url = f"https://www.douyin.com/video/{aweme_id}"
+        js = """() => {
+            const body = document.body.innerText || '';
+            const title = document.title || '';
+            const url = window.location.href;
+            const uidM = body.match(/抖音号[：:]\\s*(\\S+)/);
+            const nickM = body.match(/@(\\S+)/);
+            function extractNum(label) {
+                var m = body.match(new RegExp('(\\\\d+(?:\\\\.\\\\d+)?[万w]?)\\\\s*' + label));
+                if (m) return m[1];
+                m = body.match(new RegExp(label + '\\\\s*(\\\\d+(?:\\\\.\\\\d+)?[万w]?)'));
+                return m ? m[1] : null;
+            }
+            return {
+                aweme_id: '"' + url.match(/\\/video\\/(\\d+)/)?.[1] + '"',
+                title: title,
+                author_nickname: nickM ? nickM[1] : '',
+                douyin_id: uidM ? uidM[1] : '',
+                digg_count: extractNum('获赞') || '',
+                fans: extractNum('粉丝') || '',
+                following: extractNum('关注') || '',
+            };
+        }"""
+        data = await page_evaluate(url, js)
+        return data
+
+    async def _browser_user_profile(self, user_id: str, limit: int):
+        return []
+
+    # ── URL 解析 ──
+
+    def _extract_aweme_id(self, target: str) -> str:
+        """从 URL 或文本中提取 aweme_id"""
+        import re
+        # 视频页: douyin.com/video/{id}
+        m = re.search(r'douyin\.com/video/(\d+)', target)
+        if m:
+            return m.group(1)
+        # iesdouyin.com/share/video/{id}
+        m = re.search(r'iesdouyin\.com/share/video/(\d+)', target)
+        if m:
+            return m.group(1)
+        # 纯数字
+        if target.isdigit():
+            return target
+        return ""
+
+    # ── 统一 Schema 转换 ──
+
+    def _to_schema(self, raw: dict) -> dict:
+        """OpenCLI 原始数据 → 统一 Schema"""
+        return {
+            "platform": "douyin",
+            "item_id": raw.get("aweme_id", ""),
+            "url": f"https://www.douyin.com/video/{raw.get('aweme_id', '')}",
+            "title": raw.get("title", ""),
+            "author_name": raw.get("author_nickname", raw.get("nickname", "")),
+            "author_id": raw.get("author_id", raw.get("douyin_id", "")),
+            "published_at": raw.get("create_time", raw.get("published_at", "")),
+            "text_content": raw.get("title", ""),
+            "tags": [],
+            "stats": {
+                "likes": raw.get("digg_count", raw.get("likes", 0)),
+                "comments": raw.get("comment_count", 0),
+                "shares": raw.get("share_count", 0),
+                "views": raw.get("play_count", raw.get("views", 0)),
+            },
+            "extra": {
+                "sec_uid": raw.get("sec_uid", ""),
+                "duration": raw.get("duration", 0),
+                "status": raw.get("status", ""),
+            },
+            "media": [{
+                "type": "video",
+                "url": raw.get("play_url", ""),
+                "duration": raw.get("duration", 0),
+            }] if raw.get("play_url") else [],
+            "comments": [
+                {
+                    "author": c.get("nickname", ""),
+                    "text": c.get("text", ""),
+                    "likes": c.get("digg_count", 0),
+                }
+                for c in (raw.get("top_comments") or [])
+            ] if raw.get("top_comments") else [],
+        }

mac-agent-os-main/05_tools/10_dashboard/services/adapters/web_scrape.py

@@ -0,0 +1,115 @@
+"""
+web_scrape.py — 通用网页抓取适配器
+
+工具：web_crawler skill → agent-browser（Chrome DOM提取）
+
+用途：抓取非平台特定网页内容（文章、博客、新闻等）
+"""
+import asyncio, json, logging
+from services.adapters import ScrapeAdapter
+
+logger = logging.getLogger("dashboard.scrape.web")
+
+
+class WebScrapeAdapter(ScrapeAdapter):
+    platform = "web"
+    adapter_name = "web_scrape"
+
+    async def collect_item(self, target: str, depth: str = "light",
+                           tool_level: int = 2) -> dict:
+        """抓取单条网页内容"""
+        ok, data, tool = await self._try_tools(tool_level, [
+            ("web_crawler", lambda: self._web_crawl(target)),
+            ("agent-browser", lambda: self._browser_extract(target)),
+        ])
+        if ok and data:
+            return self._to_schema(data)
+        return {"error": "所有工具均无法抓取"}
+
+    async def collect_user(self, user_id: str, limit: int = 20) -> list[dict]:
+        return []
+
+    async def collect_comments(self, item_id: str, limit: int = 20) -> list[dict]:
+        return []
+
+    async def collect_search(self, keyword: str, limit: int = 20) -> list[dict]:
+        return []
+
+    # ── web_crawler 调用 ──
+
+    async def _web_crawl(self, url: str):
+        """通过 web_crawler skill 爬取网页"""
+        try:
+            from services.crawl_service import crawl_url
+            result = await crawl_url(url)
+            return result
+        except ImportError:
+            logger.warning("  ⚠️ web_crawler skill 未加载，尝试 Crawl4AI 直调")
+            return await self._crawl4ai_fallback(url)
+        except Exception as e:
+            logger.warning(f"  ⚠️ web_crawler 失败: {e}")
+            return await self._crawl4ai_fallback(url)
+
+    async def _crawl4ai_fallback(self, url: str) -> dict:
+        """通过 Crawl4AI 直接爬取"""
+        try:
+            import subprocess, json
+            # 使用 crawl4ai 命令行
+            proc = await asyncio.create_subprocess_exec(
+                *["python3", "-m", "crawl4ai", url, "--json"],
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+            )
+            stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=30)
+            if stdout:
+                result = json.loads(stdout.decode().strip())
+                return result
+        except Exception as e:
+            logger.warning(f"  ⚠️ Crawl4AI 降级失败: {e}")
+        return None
+
+    async def _browser_extract(self, url: str) -> dict:
+        """通过 Chrome 浏览器提取页面内容"""
+        from services.adapters.browser_helpers import page_extract
+        selectors = ["article", "main", ".content", ".post-content", ".article-content",
+                     "[class*='content']", "body"]
+        data = await page_extract(url, selectors, headless=True)
+        if data:
+            # 提取最长文本块作为正文
+            texts = []
+            for sel, items in data.items():
+                for t in items:
+                    if len(t) > 100:
+                        texts.append(t)
+            content = max(texts, key=len) if texts else list(data.values())[0][0] if any(data.values()) else ""
+            return {
+                "url": url,
+                "title": (await self._browser_title(url)).get("title", ""),
+                "content": content,
+            }
+        return {}
+
+    async def _browser_title(self, url: str) -> dict:
+        """提取页面标题"""
+        from services.adapters.browser_helpers import page_evaluate
+        js = "() => ({title: document.title})"
+        return await page_evaluate(url, js, headless=True)
+
+    # ── 统一 Schema ──
+
+    def _to_schema(self, raw: dict) -> dict:
+        return {
+            "platform": "web",
+            "item_id": raw.get("url", ""),
+            "url": raw.get("url", ""),
+            "title": raw.get("title", ""),
+            "author_name": raw.get("author", raw.get("site_name", "")),
+            "author_id": "",
+            "published_at": raw.get("published_at", raw.get("date", "")),
+            "text_content": raw.get("content", raw.get("text", raw.get("markdown", ""))),
+            "tags": raw.get("tags", []),
+            "stats": {},
+            "extra": {},
+            "media": [],
+            "comments": [],
+        }

mac-agent-os-main/05_tools/10_dashboard/services/adapters/xhs_scrape.py

@@ -0,0 +1,130 @@
+"""
+xhs_scrape.py — 小红书抓取适配器
+
+工具：OpenCLI xiaohongshu → agent-browser
+"""
+import asyncio, json, logging
+from services.adapters import ScrapeAdapter
+
+logger = logging.getLogger("dashboard.scrape.xhs")
+
+
+class XhsScrapeAdapter(ScrapeAdapter):
+    platform = "xiaohongshu"
+    adapter_name = "xhs_scrape"
+
+    async def collect_user(self, user_id: str, limit: int = 20) -> list[dict]:
+        """抓取指定用户的笔记列表"""
+        results = []
+        ok, data, tool = await self._try_tools(2, [
+            ("opencli", lambda: self._opencli_user_notes(user_id, limit)),
+        ])
+        if ok and data:
+            for item in (data if isinstance(data, list) else [data]):
+                results.append(self._to_schema(item))
+        return results
+
+    async def collect_item(self, target: str, depth: str = "light",
+                           tool_level: int = 2) -> dict:
+        """抓取单篇笔记详情"""
+        note_id = self._extract_note_id(target)
+        if not note_id:
+            return {"error": f"无法解析笔记ID: {target}"}
+        ok, data, tool = await self._try_tools(tool_level, [
+            ("opencli", lambda: self._opencli_note(note_id)),
+        ])
+        if ok and data:
+            return self._to_schema(data)
+        return {"error": "所有工具均无法抓取"}
+
+    async def collect_comments(self, item_id: str, limit: int = 20) -> list[dict]:
+        """抓取笔记评论"""
+        ok, data, tool = await self._try_tools(2, [
+            ("opencli", lambda: self._opencli_comments(item_id, limit)),
+        ])
+        if ok and data:
+            return [
+                {"author": c.get("nickname", ""), "text": c.get("text", ""),
+                 "likes": c.get("likes", 0)}
+                for c in (data if isinstance(data, list) else [data])
+            ]
+        return []
+
+    async def collect_search(self, keyword: str, limit: int = 20) -> list[dict]:
+        """搜索笔记"""
+        results = []
+        ok, data, tool = await self._try_tools(2, [
+            ("opencli", lambda: self._opencli_search(keyword, limit)),
+        ])
+        if ok and data:
+            for item in (data if isinstance(data, list) else [data]):
+                results.append(self._to_schema(item))
+        return results
+
+    # ── OpenCLI 调用 ──
+
+    async def _opencli_user_notes(self, user_id: str, limit: int):
+        return await self._run_opencli([
+            "xiaohongshu", "user", user_id,
+            "--limit", str(limit), "-f", "json"
+        ])
+
+    async def _opencli_note(self, note_id: str):
+        return await self._run_opencli([
+            "xiaohongshu", "note", note_id, "-f", "json"
+        ])
+
+    async def _opencli_comments(self, note_id: str, limit: int):
+        return await self._run_opencli([
+            "xiaohongshu", "comments", note_id,
+            "--limit", str(limit), "-f", "json"
+        ])
+
+    async def _opencli_search(self, keyword: str, limit: int):
+        return await self._run_opencli([
+            "xiaohongshu", "search", keyword,
+            "--limit", str(limit), "-f", "json"
+        ])
+
+    # ── URL 解析 ──
+
+    def _extract_note_id(self, target: str) -> str:
+        import re
+        m = re.search(r'xiaohongshu\.com/explore/([a-f0-9]+)', target)
+        if m:
+            return m.group(1)
+        m = re.search(r'xiaohongshu\.com/discovery/item/([a-f0-9]+)', target)
+        if m:
+            return m.group(1)
+        if target.isalnum() and len(target) >= 10:
+            return target
+        return ""
+
+    # ── 统一 Schema ──
+
+    def _to_schema(self, raw: dict) -> dict:
+        return {
+            "platform": "xiaohongshu",
+            "item_id": raw.get("note_id", raw.get("id", "")),
+            "url": raw.get("url", f"https://www.xiaohongshu.com/explore/{raw.get('note_id', '')}"),
+            "title": raw.get("title", ""),
+            "author_name": raw.get("author", raw.get("nickname", raw.get("user_name", ""))),
+            "author_id": raw.get("user_id", raw.get("author_id", "")),
+            "published_at": raw.get("create_time", raw.get("time", "")),
+            "text_content": raw.get("desc", raw.get("description", raw.get("text", ""))),
+            "tags": raw.get("tags", []),
+            "stats": {
+                "likes": raw.get("likes", raw.get("liked_count", 0)),
+                "comments": raw.get("comments", raw.get("comment_count", 0)),
+                "shares": raw.get("shares", raw.get("share_count", 0)),
+                "views": raw.get("views", 0),
+            },
+            "extra": {
+                "note_id": raw.get("note_id", ""),
+                "user_id": raw.get("user_id", ""),
+            },
+            "media": [
+                {"type": "image", "url": u} for u in (raw.get("images") or [])
+            ] if raw.get("images") else [],
+            "comments": [],
+        }

mac-agent-os-main/05_tools/10_dashboard/services/adapters/zhihu_scrape.py

@@ -0,0 +1,115 @@
+"""
+zhihu_scrape.py — 知乎抓取适配器
+
+工具：OpenCLI zhihu → agent-browser（Chrome DOM提取）
+"""
+import asyncio, json, logging, re
+from services.adapters import ScrapeAdapter
+
+logger = logging.getLogger("dashboard.scrape.zhihu")
+
+
+class ZhihuScrapeAdapter(ScrapeAdapter):
+    platform = "zhihu"
+    adapter_name = "zhihu_scrape"
+
+    async def collect_user(self, user_id: str, limit: int = 20) -> list[dict]:
+        """抓取指定用户的回答/文章列表"""
+        results = []
+        ok, data, tool = await self._try_tools(2, [
+            ("opencli", lambda: self._opencli_user_answers(user_id, limit)),
+        ])
+        if ok and data:
+            for item in (data if isinstance(data, list) else [data]):
+                results.append(self._to_schema(item))
+        return results
+
+    async def collect_item(self, target: str, depth: str = "light",
+                           tool_level: int = 2) -> dict:
+        """抓取单条回答/文章详情"""
+        item_id = self._extract_id(target)
+        if not item_id:
+            return {"error": f"无法解析内容ID: {target}"}
+        ok, data, tool = await self._try_tools(tool_level, [
+            ("opencli", lambda: self._opencli_answer(item_id)),
+        ])
+        if ok and data:
+            return self._to_schema(data)
+        return {"error": "所有工具均无法抓取"}
+
+    async def collect_comments(self, item_id: str, limit: int = 20) -> list[dict]:
+        return []
+
+    async def collect_search(self, keyword: str, limit: int = 20) -> list[dict]:
+        """搜索问题/回答"""
+        results = []
+        ok, data, tool = await self._try_tools(2, [
+            ("opencli", lambda: self._opencli_search(keyword, limit)),
+        ])
+        if ok and data:
+            for item in (data if isinstance(data, list) else [data]):
+                results.append(self._to_schema(item))
+        return results
+
+    # ── OpenCLI 调用 ──
+
+    async def _opencli_user_answers(self, user_id: str, limit: int):
+        return await self._run_opencli([
+            "zhihu", "user-answers", user_id,
+            "--limit", str(limit), "-f", "json"
+        ])
+
+    async def _opencli_answer(self, answer_id: str):
+        return await self._run_opencli([
+            "zhihu", "answer", answer_id, "-f", "json"
+        ])
+
+    async def _opencli_search(self, keyword: str, limit: int):
+        return await self._run_opencli([
+            "zhihu", "search", keyword,
+            "--limit", str(limit), "-f", "json"
+        ])
+
+    # ── URL 解析 ──
+
+    def _extract_id(self, target: str) -> str:
+        """从 URL 提取 answer_id / question_id"""
+        # zhihu.com/answer/{id}
+        m = re.search(r'zhihu\.com/answer/(\d+)', target)
+        if m:
+            return m.group(1)
+        # zhihu.com/question/{id}/answer/{id}
+        m = re.search(r'zhihu\.com/question/\d+/answer/(\d+)', target)
+        if m:
+            return m.group(1)
+        # 纯数字
+        if target.isdigit():
+            return target
+        return ""
+
+    # ── 统一 Schema ──
+
+    def _to_schema(self, raw: dict) -> dict:
+        return {
+            "platform": "zhihu",
+            "item_id": raw.get("answer_id", raw.get("id", raw.get("url_token", ""))),
+            "url": raw.get("url", ""),
+            "title": raw.get("title", raw.get("question_title", raw.get("excerpt", ""))),
+            "author_name": raw.get("author", raw.get("user_name", raw.get("name", ""))),
+            "author_id": raw.get("user_id", raw.get("author_id", "")),
+            "published_at": raw.get("created_time", raw.get("created_at", raw.get("updated_time", ""))),
+            "text_content": raw.get("content", raw.get("excerpt", "")),
+            "tags": raw.get("topics", []),
+            "stats": {
+                "likes": raw.get("voteup_count", raw.get("likes", 0)),
+                "comments": raw.get("comment_count", raw.get("comments", 0)),
+                "shares": 0,
+                "views": 0,
+            },
+            "extra": {
+                "question_id": raw.get("question_id", ""),
+                "answer_id": raw.get("answer_id", ""),
+            },
+            "media": [],
+            "comments": [],
+        }