MediaCrawler/mac-agent-os-main/05_tools/10_dashboard/services/adapters/browser_helpers.py

"""
browser_helpers.py — 抓取系统的浏览器辅助函数 v2

用途：
  当 OpenCLI 抓取失败时，通过真实 Chrome 浏览器降级提取数据。

设计原则：
  ⛔ 绝不使用 Camoufox（养号专用，Firefox 内核 + 特殊指纹）
  ✅ 使用 Playwright 启动真实 Chrome（Chromium 内核，正常指纹）
  ✅ 支持 headless 和 headed 两种模式
  ✅ 浏览器实例复用，避免反复启动

调用方式：
  from services.adapters.browser_helpers import page_evaluate, page_extract

  # 执行 JS 提取数据
  data = await page_evaluate("https://www.douyin.com/video/xxx", js_code)

  # 按 CSS 选择器提取
  texts = await page_extract("https://www.douyin.com/video/xxx", ["h1.title", ".desc"])
"""
import asyncio, json, logging, os
from pathlib import Path
from typing import Optional

logger = logging.getLogger("dashboard.scrape.browser")

# ── 浏览器单例（复用实例，避免反复启动）──

_browser = None
_lock = asyncio.Lock()
_USER_DATA_DIR = os.environ.get(
    "SCRAPE_CHROME_USER_DATA",
    str(Path.home() / "workbuddy-agent-os" / "agent-local" / "runtime" / "scrape_chrome_profile"),
)

# Chrome 可执行路径
_CHROME_PATHS = [
    "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",  # macOS 标准路径
    "/Applications/Chromium.app/Contents/MacOS/Chromium",
]
_CHROME_PATH = None
for p in _CHROME_PATHS:
    if Path(p).exists():
        _CHROME_PATH = p
        break


async def _get_browser(headless: bool = True):
    """
    获取/创建浏览器实例（单例）。
    使用 Playwright 启动真实 Chrome，而非 Camoufox。
    """
    global _browser
    async with _lock:
        if _browser is not None and _browser.is_connected():
            return _browser
        try:
            from playwright.async_api import async_playwright
        except ImportError:
            logger.error("Playwright 未安装，无法使用浏览器降级")
            # 尝试安装
            try:
                proc = await asyncio.create_subprocess_exec(
                    *["playwright", "install", "chromium"],
                    stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
                )
                await proc.communicate()
                from playwright.async_api import async_playwright
            except Exception:
                raise RuntimeError("Playwright 不可用，无法启动浏览器降级")

        p = await async_playwright().start()
        launch_kwargs = {
            "headless": headless,
            "args": [
                "--disable-blink-features=AutomationControlled",
                "--no-sandbox",
                "--disable-dev-shm-usage",
                "--disable-gpu",
                "--window-size=1280,720",
            ],
        }
        if _CHROME_PATH:
            launch_kwargs["executable_path"] = _CHROME_PATH
            logger.info(f"使用系统 Chrome: {_CHROME_PATH}")
        else:
            logger.info("使用 Playwright 内置 Chromium")

        try:
            _browser = await p.chromium.launch(**launch_kwargs)
            logger.info("✅ 浏览器降级: Chrome 启动成功")
        except Exception as e:
            logger.error(f"❌ 浏览器启动失败: {e}")
            await p.stop()
            raise
        return _browser


async def page_evaluate(url: str, js_code: str,
                        timeout: int = 30000, headless: bool = True) -> dict:
    """
    打开 URL，执行 JavaScript 提取数据。

    参数:
      url:       目标页面地址
      js_code:   要执行的 JS 代码（字符串形式的箭头函数或普通函数）
      timeout:   页面加载超时（毫秒）
      headless:  是否无头模式

    返回:
      字典。JS 返回值的 JSON 表示。失败时返回 {}。
    """
    browser = await _get_browser(headless=headless)
    context = None
    page = None
    try:
        context = await browser.new_context(
            user_agent=(
                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
                "AppleWebKit/537.36 (KHTML, like Gecko) "
                "Chrome/125.0.0.0 Safari/537.36"
            ),
            viewport={"width": 1280, "height": 720},
            locale="zh-CN",
        )
        page = await context.new_page()
        await page.goto(url, wait_until="domcontentloaded", timeout=timeout)
        # 等待页面稳定
        await page.wait_for_load_state("networkidle", timeout=timeout)
        # 额外等待，让动态渲染完成
        await asyncio.sleep(1)

        # 执行 JS
        result = await page.evaluate(js_code)
        if result is None:
            logger.warning(f"  ⚠️ page_evaluate JS 返回 null: {url[:60]}")
            return {}

        # 序列化确保可 JSON 返回
        if isinstance(result, (dict, list)):
            return result
        return {"_value": str(result)}

    except Exception as e:
        logger.warning(f"  ⚠️ page_evaluate 失败 [{url[:60]}]: {e}")
        return {}
    finally:
        if page:
            await page.close()
        if context:
            await context.close()


async def page_extract(url: str, selectors: list,
                       timeout: int = 30000, headless: bool = True) -> dict:
    """
    打开 URL，按 CSS 选择器提取 DOM 文本。

    参数:
      url:       目标页面
      selectors: CSS 选择器列表，如 ["h1", ".video-title", "[data-e2e=comment-count]"]
      timeout:   超时（毫秒）
      headless:  是否无头

    返回:
      {selector: [text1, text2, ...], ...}
    """
    js = f"""() => {{
        const results = {{}};
        const sel = {json.dumps(selectors)};
        for (const s of sel) {{
            const els = document.querySelectorAll(s);
            results[s] = Array.from(els).map(el => el.textContent.trim()).filter(t => t);
        }}
        return results;
    }}"""
    return await page_evaluate(url, js, timeout=timeout, headless=headless)


async def close_browser():
    """关闭浏览器实例（清理时调用）"""
    global _browser
    async with _lock:
        if _browser:
            try:
                await _browser.close()
                logger.info("浏览器降级: Chrome 已关闭")
            except Exception as e:
                logger.warning(f"关闭浏览器时出错: {e}")
            _browser = None