feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程，直接发 HTTP

移植自 mac-agent-os 的 mediacrawler_adapter：不起子进程、不开页面，用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin，所以 ingest 那条链路一个字不用改。

**目前能用的（真环境实测，非推断）**：

    profile/other : 200, 7075 字节  —— 博主主页指标（粉丝/获赞/作品数/昵称）
    aweme/detail  : 200, 45425 字节 —— 单条作品详情（含点赞/评论/收藏/分享）

**目前不能用的：作品列表 `aweme/post`。** 两个互相独立的原因：
  1. 这个接口被抖音单独升级成了真校验：不带 x-tt-argus 回 403「Uifid Not Found」，
     带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过，「真校验」过不了。
     同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的，
     挑中的恰好是「批量拉作品列表」这个最敏感的动作。
  2. 改走页面截获也不行：CDP 浏览器打开博主主页会落到「验证码中间页」（当天大量探测的
     代价，过几小时要重测）。

  所以现在的边界是：**已知作品的指标刷新能做，自动发现新作品做不了**。

**排查中控住变量后得到的两条事实**（都写进注释了）：
  · `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
    UA+client hints+Cookie 时是 200 但仅 121 字节的空壳，补上这三个头变 7074 字节。
    （我先前猜的 sec-ch-ua 不是关键。）
  · 因此 UA 与 client hints 必须**成套地取自同一个浏览器**，所以 BrowserIdentity 一次
    从 CDP 取齐 cookie + UA + hints，而不是各自写死。

「200 + 空 body 必须当场报错」也是刻意写死的：放过去它会在下游变成「这个博主没作品」，
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的，还被翻译成「账号被封」。

测试 +11：cookie 解析、请求头成套性（含 uifid 缺失/回退）、产物键名与 store 对齐、
以及 _get 的三条失败路径（空 body / 403 带网关原话 / 正常返回）。

mac-agent-os-main/05_tools/09_ave/scripts/asset_manager/cache.py

@@ -0,0 +1,367 @@
+"""
+asset_manager/cache — 缓存管理与自动入库
+
+职责:
+  1. 增量扫描: 跟踪目录 mtime, 只扫描新文件
+  2. 自动入库: 发现新文件 → 调用 AssetIndex 入库
+  3. 缓存清理: 按策略清理孤立/过期文件
+  4. 生产者钩子: Pexels/Kling 下载完成后自动通知
+
+用法:
+  from asset_manager.cache import CacheManager
+  cm = CacheManager()
+  cm.scan_new()               # 扫描所有目录的新文件
+  cm.cleanup(days=30)         # 清理 30 天前无关联的素材
+  cm.register_scan("/tmp/")   # 注册额外扫描目录
+
+设计原则:
+  - 无持续进程/守护: 每次 CLI 调用时触发扫描
+  - mtime 记录在 SQLite meta_json 中, 跨会话持久
+  - 重要素材 (有 production_id 关联) 不会被清理
+"""
+import json
+import os
+import shutil
+import sqlite3
+import time
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Callable, Optional, Union
+
+from lib.logger import get_logger
+from asset_manager.index import AssetIndex
+
+logger = get_logger("asset_cache")
+
+# ── 全局路径 (复用 index.py) ─────────────────────────────
+_LOCAL_ROOT = Path(os.environ.get(
+    "AVE_LOCAL_DIR",
+    str(Path.home() / "workbuddy-agent-os/agent-local"),
+))
+_CACHE_DIR = _LOCAL_ROOT / "tools" / "ave" / "cache"
+_DB_PATH = _LOCAL_ROOT / "tools" / "ave" / "data" / "ave.db"
+
+
+# ═══════════════════════════════════════════════════════════
+# 核心类
+# ═══════════════════════════════════════════════════════════
+
+class CacheManager:
+    """缓存管理器
+
+    属性:
+      index:         AssetIndex 实例 (共享 DB)
+      scan_records: {dir_path: last_mtime} 记录上次扫描时间
+    """
+
+    def __init__(self, cache_dir: Union[str, Path] = _CACHE_DIR,
+                 db_path: Union[str, Path] = _DB_PATH):
+        self.cache_dir = Path(cache_dir)
+        self.db_path = Path(db_path)
+        self.index = AssetIndex(cache_dir=cache_dir, db_path=db_path)
+        # 额外扫描目录注册表: {路径: 回调函数}
+        self._extra_scans: list[tuple[str, Optional[Callable]]] = []
+
+    # ── 注册额外扫描目录 ─────────────────────────────────
+
+    def register_scan(self, directory: str,
+                      callback: Optional[Callable] = None):
+        """注册额外目录到扫描列表 (如外部导入的素材目录)"""
+        self._extra_scans.append((directory, callback))
+        logger.info(f"注册扫描目录: {directory}")
+
+    # ── 增量扫描 ─────────────────────────────────────────
+
+    def _load_scan_record(self, subdir: str) -> float:
+        """从 meta_json 读取上次扫描记录的 mtime"""
+        conn = sqlite3.connect(str(self.db_path))
+        try:
+            # 用特殊 hash 值 '_scan_record' 标记扫描记录
+            row = conn.execute(
+                "SELECT meta_json FROM assets WHERE hash = ? AND source = ?",
+                (f"_scan_{subdir}", "__cache_mgr__"),
+            ).fetchone()
+            if row:
+                rec = json.loads(row[0])
+                return float(rec.get("last_mtime", 0))
+        except Exception:
+            pass
+        finally:
+            conn.close()
+        return 0.0
+
+    def _save_scan_record(self, subdir: str, mtime: float):
+        """持久化扫描记录到 DB"""
+        conn = sqlite3.connect(str(self.db_path))
+        try:
+            # 先删除旧记录
+            conn.execute(
+                "DELETE FROM assets WHERE hash = ? AND source = ?",
+                (f"_scan_{subdir}", "__cache_mgr__"),
+            )
+            conn.execute(
+                """INSERT INTO assets (asset_type, name, file_path, file_size, hash, source, meta_json)
+                   VALUES (?, ?, ?, ?, ?, ?, ?)""",
+                ("__meta__", f"scan_record_{subdir}", "",
+                 0, f"_scan_{subdir}", "__cache_mgr__",
+                 json.dumps({"last_mtime": mtime, "updated_at": datetime.now().isoformat()})),
+            )
+            conn.commit()
+        except Exception as e:
+            logger.debug(f"保存扫描记录失败: {e}")
+        finally:
+            conn.close()
+
+    def scan_new(self, force_rescan: bool = False) -> dict:
+        """
+        增量扫描: 只扫描 mtime 超过上次记录的文件。
+
+        参数:
+          force_rescan: 强制全量重新扫描
+
+        返回:
+          {"total_new": int, "per_dir": {subdir: new_count}}
+        """
+        from asset_manager.index import WATCHED_SUBDIRS, EXT_TYPE_MAP
+
+        results: dict = {"total_new": 0, "per_dir": {}}
+        valid_exts = set(EXT_TYPE_MAP.keys())
+
+        for subdir in WATCHED_SUBDIRS:
+            target = self.cache_dir / subdir
+            if not target.is_dir():
+                continue
+
+            last_mtime = 0.0 if force_rescan else self._load_scan_record(subdir)
+            new_count = 0
+
+            for fpath in sorted(target.rglob("*")):
+                if not fpath.is_file():
+                    continue
+                if fpath.suffix.lower() not in valid_exts:
+                    continue
+
+                file_mtime = fpath.stat().st_mtime
+                if file_mtime > last_mtime:
+                    ret = self.index.index_file(str(fpath))
+                    if ret > 0:
+                        new_count += 1
+
+            # 更新扫描记录 (用当前最大 mtime)
+            current_max = 0.0
+            for fpath in target.rglob("*"):
+                if fpath.is_file():
+                    current_max = max(current_max, fpath.stat().st_mtime)
+            self._save_scan_record(subdir, current_max)
+
+            if new_count > 0:
+                logger.info(f"  [{subdir}] 新增 {new_count} 个素材")
+            results["per_dir"][subdir] = new_count
+            results["total_new"] += new_count
+
+        # 额外扫描目录
+        for extra_dir, callback in self._extra_scans:
+            extra_path = Path(extra_dir)
+            if not extra_path.is_dir():
+                continue
+            for fpath in extra_path.rglob("*"):
+                if not fpath.is_file():
+                    continue
+                if fpath.suffix.lower() not in valid_exts:
+                    continue
+                ret = self.index.index_file(str(fpath))
+                if ret > 0:
+                    results["total_new"] += 1
+                    if callback:
+                        try:
+                            callback(str(fpath))
+                        except Exception as e:
+                            logger.warning(f"回调失败 {fpath}: {e}")
+
+        logger.info(f"增量扫描完成, 共新增 {results['total_new']} 个素材")
+        return results
+
+    # ── 全量扫描 ─────────────────────────────────────────
+
+    def scan_all(self, production_id: int = 0) -> dict:
+        """全量扫描并重置所有扫描记录"""
+        results = self.index.scan_all(production_id)
+
+        # 重置扫描记录为最新 mtime
+        from asset_manager.index import WATCHED_SUBDIRS
+        for subdir in WATCHED_SUBDIRS:
+            target = self.cache_dir / subdir
+            if target.is_dir():
+                current_max = max(
+                    (f.stat().st_mtime for f in target.rglob("*") if f.is_file()),
+                    default=0.0,
+                )
+                self._save_scan_record(subdir, current_max)
+
+        return results
+
+    # ── 缓存清理 ─────────────────────────────────────────
+
+    def cleanup(self, days: int = 30, dry_run: bool = False) -> dict:
+        """
+        清理过期缓存。
+
+        清理规则:
+          - 无 production_id 关联
+          - 文件创建时间 > days 天前
+          - 保留有 production 引用的素材
+
+        参数:
+          days:    保留天数
+          dry_run: 仅预览不删除
+
+        返回:
+          {"deleted": int, "freed_bytes": int, "files": [路径]}
+        """
+        conn = sqlite3.connect(str(self.db_path))
+        cutoff = (datetime.now() - timedelta(days=days)).isoformat()
+        result: dict = {"deleted": 0, "freed_bytes": 0, "files": []}
+
+        try:
+            # 查找无关联的旧资产
+            rows = conn.execute(
+                """SELECT id, file_path, file_size FROM assets
+                   WHERE production_id IS NULL
+                     AND created_at < ?
+                     AND asset_type != '__meta__'
+                   ORDER BY created_at ASC""",
+                (cutoff,),
+            ).fetchall()
+
+            for row in rows:
+                aid, fpath, fsize = row["id"], row["file_path"], row["file_size"] or 0
+
+                if dry_run:
+                    result["deleted"] += 1
+                    result["freed_bytes"] += fsize
+                    result["files"].append(f"{fpath} ({_human_size(fsize)})")
+                    continue
+
+                # 删除磁盘文件
+                if fpath and os.path.isfile(fpath):
+                    try:
+                        os.remove(fpath)
+                        logger.debug(f"  删除文件: {fpath}")
+                    except OSError as e:
+                        logger.warning(f"  删除失败: {fpath}: {e}")
+
+                # 删除 DB 记录
+                conn.execute("DELETE FROM asset_tags WHERE asset_id = ?", (aid,))
+                conn.execute("DELETE FROM production_assets WHERE asset_id = ?", (aid,))
+                conn.execute("DELETE FROM assets WHERE id = ?", (aid,))
+
+                result["deleted"] += 1
+                result["freed_bytes"] += fsize
+                result["files"].append(fpath)
+
+            conn.commit()
+        except Exception as e:
+            logger.error(f"cleanup 失败: {e}")
+        finally:
+            conn.close()
+
+        action = "预览" if dry_run else "清理"
+        logger.info(f"缓存{action}: {result['deleted']} 个文件, "
+                     f"释放 {_human_size(result['freed_bytes'])}")
+        return result
+
+    # ── 目录统计 ─────────────────────────────────────────
+
+    def disk_usage(self) -> dict:
+        """统计每个缓存目录的磁盘占用"""
+        usage: dict = {}
+        for subdir in os.listdir(self.cache_dir):
+            target = self.cache_dir / subdir
+            if not target.is_dir():
+                continue
+            total = 0
+            count = 0
+            for fpath in target.rglob("*"):
+                if fpath.is_file():
+                    total += fpath.stat().st_size
+                    count += 1
+            usage[subdir] = {
+                "count": count,
+                "size_bytes": total,
+                "size_human": _human_size(total),
+            }
+        return usage
+
+    def print_disk_usage(self):
+        """打印磁盘占用表"""
+        usage = self.disk_usage()
+        print(f"\n{'='*50}")
+        print(f"  缓存磁盘占用")
+        print(f"{'='*50}")
+        print(f"  {'目录':20s} {'文件数':>8s} {'大小':>12s}")
+        print(f"  {'-'*40}")
+        total_size = 0
+        for subdir, info in sorted(usage.items(), key=lambda x: -x[1]["size_bytes"]):
+            print(f"  {subdir:20s} {info['count']:>8d} {info['size_human']:>12s}")
+            total_size += info["size_bytes"]
+        print(f"  {'-'*40}")
+        print(f"  {'总计':20s} {'':>8s} {_human_size(total_size):>12s}")
+        print(f"{'='*50}\n")
+
+
+# ═══════════════════════════════════════════════════════════
+# 生产者自动入库钩子
+# ═══════════════════════════════════════════════════════════
+
+def auto_index_on_download(file_path: str, production_id: int = 0) -> int:
+    """
+    生产者在下载/生成素材后, 调用此函数自动入库。
+
+    用法:
+      from asset_manager.cache import auto_index_on_download
+
+      path = pexels_search.download(url)
+      auto_index_on_download(path, production_id=pid)
+    """
+    idx = AssetIndex()
+    return idx.index_file(file_path, production_id=production_id)
+
+
+def index_output_video(output_path: str, production_id: int) -> int:
+    """
+    最终成片入库 (特殊标记 source=output)。
+    """
+    if not os.path.isfile(output_path):
+        return -1
+    idx = AssetIndex()
+    # 强制标记 source 为 output
+    conn = sqlite3.connect(str(idx.db_path))
+    try:
+        # 先通过 index_file 入库
+        aid = idx.index_file(output_path, production_id=production_id)
+        if aid > 0:
+            # 覆盖 source
+            conn.execute(
+                "UPDATE assets SET source = 'production_output' WHERE id = ?",
+                (aid,),
+            )
+            conn.commit()
+            return aid
+        return aid
+    except Exception as e:
+        logger.error(f"index_output 失败: {e}")
+        return -1
+    finally:
+        conn.close()
+
+
+# ── 辅助函数 ─────────────────────────────────────────────
+
+def _human_size(bytes_: int) -> str:
+    if bytes_ < 1024:
+        return f"{bytes_} B"
+    elif bytes_ < 1024**2:
+        return f"{bytes_/1024:.1f} KB"
+    elif bytes_ < 1024**3:
+        return f"{bytes_/(1024**2):.1f} MB"
+    return f"{bytes_/(1024**3):.2f} GB"