feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程，直接发 HTTP

移植自 mac-agent-os 的 mediacrawler_adapter：不起子进程、不开页面，用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin，所以 ingest 那条链路一个字不用改。

**目前能用的（真环境实测，非推断）**：

    profile/other : 200, 7075 字节  —— 博主主页指标（粉丝/获赞/作品数/昵称）
    aweme/detail  : 200, 45425 字节 —— 单条作品详情（含点赞/评论/收藏/分享）

**目前不能用的：作品列表 `aweme/post`。** 两个互相独立的原因：
  1. 这个接口被抖音单独升级成了真校验：不带 x-tt-argus 回 403「Uifid Not Found」，
     带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过，「真校验」过不了。
     同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的，
     挑中的恰好是「批量拉作品列表」这个最敏感的动作。
  2. 改走页面截获也不行：CDP 浏览器打开博主主页会落到「验证码中间页」（当天大量探测的
     代价，过几小时要重测）。

  所以现在的边界是：**已知作品的指标刷新能做，自动发现新作品做不了**。

**排查中控住变量后得到的两条事实**（都写进注释了）：
  · `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
    UA+client hints+Cookie 时是 200 但仅 121 字节的空壳，补上这三个头变 7074 字节。
    （我先前猜的 sec-ch-ua 不是关键。）
  · 因此 UA 与 client hints 必须**成套地取自同一个浏览器**，所以 BrowserIdentity 一次
    从 CDP 取齐 cookie + UA + hints，而不是各自写死。

「200 + 空 body 必须当场报错」也是刻意写死的：放过去它会在下游变成「这个博主没作品」，
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的，还被翻译成「账号被封」。

测试 +11：cookie 解析、请求头成套性（含 uifid 缺失/回退）、产物键名与 store 对齐、
以及 _get 的三条失败路径（空 body / 403 带网关原话 / 正常返回）。

mac-agent-os-main/05_tools/09_ave/scripts/asset_manager/__init__.py

@@ -0,0 +1,14 @@
+"""
+AVE asset_manager — 素材资产管理包
+
+提供素材索引、缓存管理、标签搜索三大能力。
+
+模块:
+  index.py   — AssetIndex: 扫描目录、提取元数据、入库 SQLite
+  cache.py   — CacheManager: 增量扫描、缓存清理、孤立文件回收
+  tags.py    — AssetSearch: 自动标签、模糊搜索、CLI 查询
+
+依赖:
+  - lib/dashboard.py (assets 表 schema 已存在)
+  - ffprobe (FFmpeg, 用于提取视频元数据)
+"""

mac-agent-os-main/05_tools/09_ave/scripts/asset_manager/cache.py

@@ -0,0 +1,367 @@
+"""
+asset_manager/cache — 缓存管理与自动入库
+
+职责:
+  1. 增量扫描: 跟踪目录 mtime, 只扫描新文件
+  2. 自动入库: 发现新文件 → 调用 AssetIndex 入库
+  3. 缓存清理: 按策略清理孤立/过期文件
+  4. 生产者钩子: Pexels/Kling 下载完成后自动通知
+
+用法:
+  from asset_manager.cache import CacheManager
+  cm = CacheManager()
+  cm.scan_new()               # 扫描所有目录的新文件
+  cm.cleanup(days=30)         # 清理 30 天前无关联的素材
+  cm.register_scan("/tmp/")   # 注册额外扫描目录
+
+设计原则:
+  - 无持续进程/守护: 每次 CLI 调用时触发扫描
+  - mtime 记录在 SQLite meta_json 中, 跨会话持久
+  - 重要素材 (有 production_id 关联) 不会被清理
+"""
+import json
+import os
+import shutil
+import sqlite3
+import time
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Callable, Optional, Union
+
+from lib.logger import get_logger
+from asset_manager.index import AssetIndex
+
+logger = get_logger("asset_cache")
+
+# ── 全局路径 (复用 index.py) ─────────────────────────────
+_LOCAL_ROOT = Path(os.environ.get(
+    "AVE_LOCAL_DIR",
+    str(Path.home() / "workbuddy-agent-os/agent-local"),
+))
+_CACHE_DIR = _LOCAL_ROOT / "tools" / "ave" / "cache"
+_DB_PATH = _LOCAL_ROOT / "tools" / "ave" / "data" / "ave.db"
+
+
+# ═══════════════════════════════════════════════════════════
+# 核心类
+# ═══════════════════════════════════════════════════════════
+
+class CacheManager:
+    """缓存管理器
+
+    属性:
+      index:         AssetIndex 实例 (共享 DB)
+      scan_records: {dir_path: last_mtime} 记录上次扫描时间
+    """
+
+    def __init__(self, cache_dir: Union[str, Path] = _CACHE_DIR,
+                 db_path: Union[str, Path] = _DB_PATH):
+        self.cache_dir = Path(cache_dir)
+        self.db_path = Path(db_path)
+        self.index = AssetIndex(cache_dir=cache_dir, db_path=db_path)
+        # 额外扫描目录注册表: {路径: 回调函数}
+        self._extra_scans: list[tuple[str, Optional[Callable]]] = []
+
+    # ── 注册额外扫描目录 ─────────────────────────────────
+
+    def register_scan(self, directory: str,
+                      callback: Optional[Callable] = None):
+        """注册额外目录到扫描列表 (如外部导入的素材目录)"""
+        self._extra_scans.append((directory, callback))
+        logger.info(f"注册扫描目录: {directory}")
+
+    # ── 增量扫描 ─────────────────────────────────────────
+
+    def _load_scan_record(self, subdir: str) -> float:
+        """从 meta_json 读取上次扫描记录的 mtime"""
+        conn = sqlite3.connect(str(self.db_path))
+        try:
+            # 用特殊 hash 值 '_scan_record' 标记扫描记录
+            row = conn.execute(
+                "SELECT meta_json FROM assets WHERE hash = ? AND source = ?",
+                (f"_scan_{subdir}", "__cache_mgr__"),
+            ).fetchone()
+            if row:
+                rec = json.loads(row[0])
+                return float(rec.get("last_mtime", 0))
+        except Exception:
+            pass
+        finally:
+            conn.close()
+        return 0.0
+
+    def _save_scan_record(self, subdir: str, mtime: float):
+        """持久化扫描记录到 DB"""
+        conn = sqlite3.connect(str(self.db_path))
+        try:
+            # 先删除旧记录
+            conn.execute(
+                "DELETE FROM assets WHERE hash = ? AND source = ?",
+                (f"_scan_{subdir}", "__cache_mgr__"),
+            )
+            conn.execute(
+                """INSERT INTO assets (asset_type, name, file_path, file_size, hash, source, meta_json)
+                   VALUES (?, ?, ?, ?, ?, ?, ?)""",
+                ("__meta__", f"scan_record_{subdir}", "",
+                 0, f"_scan_{subdir}", "__cache_mgr__",
+                 json.dumps({"last_mtime": mtime, "updated_at": datetime.now().isoformat()})),
+            )
+            conn.commit()
+        except Exception as e:
+            logger.debug(f"保存扫描记录失败: {e}")
+        finally:
+            conn.close()
+
+    def scan_new(self, force_rescan: bool = False) -> dict:
+        """
+        增量扫描: 只扫描 mtime 超过上次记录的文件。
+
+        参数:
+          force_rescan: 强制全量重新扫描
+
+        返回:
+          {"total_new": int, "per_dir": {subdir: new_count}}
+        """
+        from asset_manager.index import WATCHED_SUBDIRS, EXT_TYPE_MAP
+
+        results: dict = {"total_new": 0, "per_dir": {}}
+        valid_exts = set(EXT_TYPE_MAP.keys())
+
+        for subdir in WATCHED_SUBDIRS:
+            target = self.cache_dir / subdir
+            if not target.is_dir():
+                continue
+
+            last_mtime = 0.0 if force_rescan else self._load_scan_record(subdir)
+            new_count = 0
+
+            for fpath in sorted(target.rglob("*")):
+                if not fpath.is_file():
+                    continue
+                if fpath.suffix.lower() not in valid_exts:
+                    continue
+
+                file_mtime = fpath.stat().st_mtime
+                if file_mtime > last_mtime:
+                    ret = self.index.index_file(str(fpath))
+                    if ret > 0:
+                        new_count += 1
+
+            # 更新扫描记录 (用当前最大 mtime)
+            current_max = 0.0
+            for fpath in target.rglob("*"):
+                if fpath.is_file():
+                    current_max = max(current_max, fpath.stat().st_mtime)
+            self._save_scan_record(subdir, current_max)
+
+            if new_count > 0:
+                logger.info(f"  [{subdir}] 新增 {new_count} 个素材")
+            results["per_dir"][subdir] = new_count
+            results["total_new"] += new_count
+
+        # 额外扫描目录
+        for extra_dir, callback in self._extra_scans:
+            extra_path = Path(extra_dir)
+            if not extra_path.is_dir():
+                continue
+            for fpath in extra_path.rglob("*"):
+                if not fpath.is_file():
+                    continue
+                if fpath.suffix.lower() not in valid_exts:
+                    continue
+                ret = self.index.index_file(str(fpath))
+                if ret > 0:
+                    results["total_new"] += 1
+                    if callback:
+                        try:
+                            callback(str(fpath))
+                        except Exception as e:
+                            logger.warning(f"回调失败 {fpath}: {e}")
+
+        logger.info(f"增量扫描完成, 共新增 {results['total_new']} 个素材")
+        return results
+
+    # ── 全量扫描 ─────────────────────────────────────────
+
+    def scan_all(self, production_id: int = 0) -> dict:
+        """全量扫描并重置所有扫描记录"""
+        results = self.index.scan_all(production_id)
+
+        # 重置扫描记录为最新 mtime
+        from asset_manager.index import WATCHED_SUBDIRS
+        for subdir in WATCHED_SUBDIRS:
+            target = self.cache_dir / subdir
+            if target.is_dir():
+                current_max = max(
+                    (f.stat().st_mtime for f in target.rglob("*") if f.is_file()),
+                    default=0.0,
+                )
+                self._save_scan_record(subdir, current_max)
+
+        return results
+
+    # ── 缓存清理 ─────────────────────────────────────────
+
+    def cleanup(self, days: int = 30, dry_run: bool = False) -> dict:
+        """
+        清理过期缓存。
+
+        清理规则:
+          - 无 production_id 关联
+          - 文件创建时间 > days 天前
+          - 保留有 production 引用的素材
+
+        参数:
+          days:    保留天数
+          dry_run: 仅预览不删除
+
+        返回:
+          {"deleted": int, "freed_bytes": int, "files": [路径]}
+        """
+        conn = sqlite3.connect(str(self.db_path))
+        cutoff = (datetime.now() - timedelta(days=days)).isoformat()
+        result: dict = {"deleted": 0, "freed_bytes": 0, "files": []}
+
+        try:
+            # 查找无关联的旧资产
+            rows = conn.execute(
+                """SELECT id, file_path, file_size FROM assets
+                   WHERE production_id IS NULL
+                     AND created_at < ?
+                     AND asset_type != '__meta__'
+                   ORDER BY created_at ASC""",
+                (cutoff,),
+            ).fetchall()
+
+            for row in rows:
+                aid, fpath, fsize = row["id"], row["file_path"], row["file_size"] or 0
+
+                if dry_run:
+                    result["deleted"] += 1
+                    result["freed_bytes"] += fsize
+                    result["files"].append(f"{fpath} ({_human_size(fsize)})")
+                    continue
+
+                # 删除磁盘文件
+                if fpath and os.path.isfile(fpath):
+                    try:
+                        os.remove(fpath)
+                        logger.debug(f"  删除文件: {fpath}")
+                    except OSError as e:
+                        logger.warning(f"  删除失败: {fpath}: {e}")
+
+                # 删除 DB 记录
+                conn.execute("DELETE FROM asset_tags WHERE asset_id = ?", (aid,))
+                conn.execute("DELETE FROM production_assets WHERE asset_id = ?", (aid,))
+                conn.execute("DELETE FROM assets WHERE id = ?", (aid,))
+
+                result["deleted"] += 1
+                result["freed_bytes"] += fsize
+                result["files"].append(fpath)
+
+            conn.commit()
+        except Exception as e:
+            logger.error(f"cleanup 失败: {e}")
+        finally:
+            conn.close()
+
+        action = "预览" if dry_run else "清理"
+        logger.info(f"缓存{action}: {result['deleted']} 个文件, "
+                     f"释放 {_human_size(result['freed_bytes'])}")
+        return result
+
+    # ── 目录统计 ─────────────────────────────────────────
+
+    def disk_usage(self) -> dict:
+        """统计每个缓存目录的磁盘占用"""
+        usage: dict = {}
+        for subdir in os.listdir(self.cache_dir):
+            target = self.cache_dir / subdir
+            if not target.is_dir():
+                continue
+            total = 0
+            count = 0
+            for fpath in target.rglob("*"):
+                if fpath.is_file():
+                    total += fpath.stat().st_size
+                    count += 1
+            usage[subdir] = {
+                "count": count,
+                "size_bytes": total,
+                "size_human": _human_size(total),
+            }
+        return usage
+
+    def print_disk_usage(self):
+        """打印磁盘占用表"""
+        usage = self.disk_usage()
+        print(f"\n{'='*50}")
+        print(f"  缓存磁盘占用")
+        print(f"{'='*50}")
+        print(f"  {'目录':20s} {'文件数':>8s} {'大小':>12s}")
+        print(f"  {'-'*40}")
+        total_size = 0
+        for subdir, info in sorted(usage.items(), key=lambda x: -x[1]["size_bytes"]):
+            print(f"  {subdir:20s} {info['count']:>8d} {info['size_human']:>12s}")
+            total_size += info["size_bytes"]
+        print(f"  {'-'*40}")
+        print(f"  {'总计':20s} {'':>8s} {_human_size(total_size):>12s}")
+        print(f"{'='*50}\n")
+
+
+# ═══════════════════════════════════════════════════════════
+# 生产者自动入库钩子
+# ═══════════════════════════════════════════════════════════
+
+def auto_index_on_download(file_path: str, production_id: int = 0) -> int:
+    """
+    生产者在下载/生成素材后, 调用此函数自动入库。
+
+    用法:
+      from asset_manager.cache import auto_index_on_download
+
+      path = pexels_search.download(url)
+      auto_index_on_download(path, production_id=pid)
+    """
+    idx = AssetIndex()
+    return idx.index_file(file_path, production_id=production_id)
+
+
+def index_output_video(output_path: str, production_id: int) -> int:
+    """
+    最终成片入库 (特殊标记 source=output)。
+    """
+    if not os.path.isfile(output_path):
+        return -1
+    idx = AssetIndex()
+    # 强制标记 source 为 output
+    conn = sqlite3.connect(str(idx.db_path))
+    try:
+        # 先通过 index_file 入库
+        aid = idx.index_file(output_path, production_id=production_id)
+        if aid > 0:
+            # 覆盖 source
+            conn.execute(
+                "UPDATE assets SET source = 'production_output' WHERE id = ?",
+                (aid,),
+            )
+            conn.commit()
+            return aid
+        return aid
+    except Exception as e:
+        logger.error(f"index_output 失败: {e}")
+        return -1
+    finally:
+        conn.close()
+
+
+# ── 辅助函数 ─────────────────────────────────────────────
+
+def _human_size(bytes_: int) -> str:
+    if bytes_ < 1024:
+        return f"{bytes_} B"
+    elif bytes_ < 1024**2:
+        return f"{bytes_/1024:.1f} KB"
+    elif bytes_ < 1024**3:
+        return f"{bytes_/(1024**2):.1f} MB"
+    return f"{bytes_/(1024**3):.2f} GB"

mac-agent-os-main/05_tools/09_ave/scripts/asset_manager/index.py

@@ -0,0 +1,469 @@
+"""
+asset_manager/index — 素材索引引擎
+
+职责:
+  1. 扫描所有已知素材目录，发现新文件
+  2. 通过 ffprobe 提取视频/音频元数据 (时长/分辨率/编码/FPS)
+  3. 计算文件 hash (首 64KB + 文件大小, 快速去重)
+  4. 写入 SQLite assets 表, 避免重复入库
+
+用法:
+  from asset_manager.index import AssetIndex
+
+  idx = AssetIndex()
+  stats = idx.scan_all()           # 全量扫描所有目录
+  stats = idx.scan_dir("materials")  # 扫描单个子目录
+  print(idx.summarize())             # 打印统计
+
+AssetIndex 发现策略:
+  - 只扫描已知目录 (materials/kling/wan2_2/bgm/bgm_library/lipsync/character_sheet)
+  - 按 hash 去重: 同一文件即使出现在不同目录也只入库一次
+  - 增量模式: 只扫描 mtime > 上次扫描时间的文件
+"""
+import hashlib
+import json
+import os
+import sqlite3
+import subprocess
+import time
+from datetime import datetime
+from pathlib import Path
+from typing import Optional, Union
+
+from lib.logger import get_logger
+
+logger = get_logger("asset_index")
+
+# ── 全局路径 ───────────────────────────────────────────────
+_LOCAL_ROOT = Path(os.environ.get(
+    "AVE_LOCAL_DIR",
+    str(Path.home() / "workbuddy-agent-os/agent-local"),
+))
+_CACHE_DIR = _LOCAL_ROOT / "tools" / "ave" / "cache"
+_DB_PATH = _LOCAL_ROOT / "tools" / "ave" / "data" / "ave.db"
+
+# ── 已知素材目录 (相对于 cache) ──────────────────────────
+WATCHED_SUBDIRS: dict[str, str] = {
+    "materials":       "pexels 下载素材",
+    "kling":           "Kling 生成视频",
+    "wan2_2":          "Wan2.2 数字人视频",
+    "bgm":             "BGM 缓存",
+    "bgm_library":     "BGM 素材库",
+    "lipsync":         "唇形同步产出",
+    "character_sheet": "定妆照/角色图",
+    "outputs":         "最终成片",
+    "beat_sync":       "节拍分析缓存",
+}
+
+# ── 已知扩展名 → asset_type ─────────────────────────────
+EXT_TYPE_MAP = {
+    ".mp4":  "video",
+    ".mov":  "video",
+    ".webm": "video",
+    ".avi":  "video",
+    ".mkv":  "video",
+    ".wav":  "audio",
+    ".mp3":  "audio",
+    ".aac":  "audio",
+    ".m4a":  "audio",
+    ".flac": "audio",
+    ".ogg":  "audio",
+    ".png":  "image",
+    ".jpg":  "image",
+    ".jpeg": "image",
+    ".webp": "image",
+    ".gif":  "image",
+}
+
+
+# ═══════════════════════════════════════════════════════════
+# 核心类
+# ═══════════════════════════════════════════════════════════
+
+class AssetIndex:
+    """素材索引引擎"""
+
+    def __init__(self, cache_dir: Union[str, Path] = _CACHE_DIR,
+                 db_path: Union[str, Path] = _DB_PATH):
+        self.cache_dir = Path(cache_dir)
+        self.db_path = Path(db_path)
+        self._init_db()
+
+    # ── DB 初始化 ──────────────────────────────────────────
+
+    def _init_db(self):
+        """确保 assets 相关表存在（复用 dashboard.py 的 schema）"""
+        try:
+            conn = sqlite3.connect(str(self.db_path))
+            conn.executescript("""
+                CREATE TABLE IF NOT EXISTS assets (
+                    id              INTEGER PRIMARY KEY AUTOINCREMENT,
+                    asset_type      TEXT NOT NULL,
+                    name            TEXT,
+                    file_path       TEXT,
+                    file_size       INTEGER,
+                    hash            TEXT,
+                    source          TEXT,
+                    meta_json       TEXT,
+                    production_id   INTEGER,
+                    created_at      TEXT DEFAULT (datetime('now','localtime'))
+                );
+                CREATE TABLE IF NOT EXISTS asset_tags (
+                    asset_id        INTEGER NOT NULL REFERENCES assets(id),
+                    tag             TEXT NOT NULL,
+                    PRIMARY KEY (asset_id, tag)
+                );
+                CREATE TABLE IF NOT EXISTS production_assets (
+                    production_id   INTEGER NOT NULL REFERENCES productions(id),
+                    asset_id        INTEGER NOT NULL REFERENCES assets(id),
+                    role            TEXT,
+                    PRIMARY KEY (production_id, asset_id)
+                );
+                CREATE INDEX IF NOT EXISTS idx_assets_hash ON assets(hash);
+                CREATE INDEX IF NOT EXISTS idx_assets_type ON assets(asset_type);
+                CREATE INDEX IF NOT EXISTS idx_assets_path ON assets(file_path);
+                CREATE INDEX IF NOT EXISTS idx_asset_tags_tag ON asset_tags(tag);
+            """)
+            conn.commit()
+            conn.close()
+        except Exception as e:
+            logger.error(f"DB 初始化失败: {e}")
+
+    # ── 文件级操作 ────────────────────────────────────────
+
+    def _compute_hash(self, file_path: str) -> str:
+        """快速 hash: 读首 64KB + 文件大小 (比全量 MD5 快 100x)"""
+        try:
+            sz = os.path.getsize(file_path)
+            with open(file_path, "rb") as f:
+                head = f.read(65536)  # 64KB
+            h = hashlib.md5(head).hexdigest()[:16]
+            return f"{h}_{sz}"
+        except OSError:
+            return ""
+
+    def _extract_media_info(self, file_path: str) -> dict:
+        """通过 ffprobe 提取媒体元数据"""
+        meta: dict = {}
+        ext = Path(file_path).suffix.lower()
+
+        # 非视频/音频文件跳过 ffprobe
+        if ext not in (".mp4", ".mov", ".webm", ".avi", ".mkv",
+                       ".wav", ".mp3", ".aac", ".m4a", ".flac", ".ogg"):
+            return meta
+
+        try:
+            result = subprocess.run(
+                ["ffprobe", "-v", "quiet", "-print_format", "json",
+                 "-show_format", "-show_streams", file_path],
+                capture_output=True, text=True, timeout=30,
+            )
+            if result.returncode != 0:
+                return meta
+
+            data = json.loads(result.stdout)
+
+            # 格式信息
+            fmt = data.get("format", {})
+            meta["format"] = fmt.get("format_name", "")
+            meta["bitrate"] = int(fmt.get("bit_rate", 0))
+            meta["duration_sec"] = round(float(fmt.get("duration", 0)), 2)
+            meta["size_bytes"] = int(fmt.get("size", 0))
+
+            # 视频流
+            for stream in data.get("streams", []):
+                if stream.get("codec_type") == "video":
+                    meta["video_codec"] = stream.get("codec_name", "")
+                    meta["width"] = stream.get("width", 0)
+                    meta["height"] = stream.get("height", 0)
+                    meta["fps"] = _parse_fps(stream.get("r_frame_rate", ""))
+                    meta["pixel_format"] = stream.get("pix_fmt", "")
+                    break
+
+            # 音频流
+            for stream in data.get("streams", []):
+                if stream.get("codec_type") == "audio":
+                    meta["audio_codec"] = stream.get("codec_name", "")
+                    meta["sample_rate"] = int(stream.get("sample_rate", 0))
+                    meta["channels"] = stream.get("channels", 0)
+                    break
+
+        except (subprocess.TimeoutExpired, json.JSONDecodeError,
+                FileNotFoundError) as e:
+            logger.debug(f"ffprobe 失败 {file_path}: {e}")
+
+        return meta
+
+    def _source_from_path(self, file_path: str) -> str:
+        """从文件路径推断来源"""
+        p = Path(file_path)
+        try:
+            rel = p.relative_to(self.cache_dir)
+            parts = rel.parts
+            if len(parts) >= 1:
+                return parts[0]  # 顶级子目录名 = source
+        except ValueError:
+            pass
+        return "unknown"
+
+    def _infer_tags(self, file_path: str, meta: dict, source: str) -> list[str]:
+        """基于文件属性自动生成标签"""
+        tags: list[str] = [source]
+        ext = Path(file_path).suffix.lower()
+
+        # 文件类型标签
+        asset_type = EXT_TYPE_MAP.get(ext, "other")
+        tags.append(asset_type)
+
+        # 尺寸标签 (视频)
+        w = meta.get("width", 0)
+        h = meta.get("height", 0)
+        if w and h:
+            if w >= 3840 or h >= 2160:
+                tags.append("4k")
+            elif w >= 1920 or h >= 1080:
+                tags.append("1080p")
+            elif w >= 1280 or h >= 720:
+                tags.append("720p")
+            else:
+                tags.append("sd")
+
+        # 时长标签
+        dur = meta.get("duration_sec", 0)
+        if dur > 0:
+            if dur < 2:
+                tags.append("ultra_short")
+            elif dur < 5:
+                tags.append("short")
+            elif dur < 15:
+                tags.append("medium")
+            elif dur < 60:
+                tags.append("long")
+            else:
+                tags.append("extra_long")
+
+        # 编码标签
+        vcodec = meta.get("video_codec", "")
+        if vcodec:
+            tags.append(f"codec_{vcodec}")
+
+        return tags
+
+    # ── 单文件入库 ────────────────────────────────────────
+
+    def index_file(self, file_path: str, production_id: int = 0,
+                   force: bool = False) -> int:
+        """
+        索引单个文件并入库。
+
+        参数:
+          file_path:     文件绝对路径
+          production_id: 关联的 production ID (0 = 不关联)
+          force:         强制重新索引 (即使 hash 已存在)
+
+        返回:
+          asset_id (失败返回 -1, 已存在且未 force 返回 -2)
+        """
+        if not os.path.isfile(file_path):
+            logger.warning(f"文件不存在: {file_path}")
+            return -1
+
+        ext = Path(file_path).suffix.lower()
+        asset_type = EXT_TYPE_MAP.get(ext, "other")
+        name = Path(file_path).name
+        file_size = os.path.getsize(file_path)
+        file_hash = self._compute_hash(file_path)
+        source = self._source_from_path(file_path)
+        meta = self._extract_media_info(file_path)
+        tags = self._infer_tags(file_path, meta, source)
+
+        if not file_hash:
+            return -1
+
+        conn = sqlite3.connect(str(self.db_path))
+        try:
+            # 去重检查
+            if not force and file_hash:
+                existing = conn.execute(
+                    "SELECT id FROM assets WHERE hash = ? LIMIT 1",
+                    (file_hash,),
+                ).fetchone()
+                if existing:
+                    conn.close()
+                    return -2  # 已存在
+
+            # 插入资产
+            cur = conn.execute(
+                """INSERT INTO assets
+                   (asset_type, name, file_path, file_size, hash, source, meta_json, production_id)
+                   VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
+                (asset_type, name, file_path, file_size, file_hash, source,
+                 json.dumps(meta, ensure_ascii=False),
+                 production_id if production_id > 0 else None),
+            )
+            aid = cur.lastrowid
+
+            # 标签
+            for tag in set(tags):  # set 去重
+                conn.execute(
+                    "INSERT OR IGNORE INTO asset_tags (asset_id, tag) VALUES (?, ?)",
+                    (aid, tag),
+                )
+
+            # 关联 production
+            if production_id > 0:
+                conn.execute(
+                    """INSERT OR IGNORE INTO production_assets
+                       (production_id, asset_id, role) VALUES (?, ?, ?)""",
+                    (production_id, aid, "output" if source in ("materials", "kling", "wan2_2")
+                     else "cache"),
+                )
+
+            conn.commit()
+            logger.info(f"  入库: {name} (type={asset_type}, hash={file_hash[:12]}...)")
+            return aid
+        except Exception as e:
+            logger.error(f"index_file 失败: {file_path}: {e}")
+            return -1
+        finally:
+            conn.close()
+
+    # ── 目录扫描 ──────────────────────────────────────────
+
+    def scan_dir(self, subdir: str, production_id: int = 0,
+                 progress: bool = True) -> dict:
+        """
+        扫描一个子目录, 入库所有新文件。
+
+        参数:
+          subdir:        WATCHED_SUBDIRS 中的 key (如 "materials")
+          production_id: 可选关联 production
+          progress:      打印进度
+
+        返回:
+          {"dir": str, "total": int, "new": int, "skipped": int, "errors": int}
+        """
+        target_dir = self.cache_dir / subdir
+        stat = {"dir": subdir, "total": 0, "new": 0, "skipped": 0, "errors": 0}
+
+        if not target_dir.is_dir():
+            logger.warning(f"目录不存在: {target_dir}")
+            return stat
+
+        files = sorted(target_dir.rglob("*"))
+        valid_exts = set(EXT_TYPE_MAP.keys())
+
+        for i, fpath in enumerate(files):
+            if not fpath.is_file():
+                continue
+            if fpath.suffix.lower() not in valid_exts:
+                continue
+
+            stat["total"] += 1
+            ret = self.index_file(str(fpath), production_id)
+            if ret == -1:
+                stat["errors"] += 1
+            elif ret == -2:
+                stat["skipped"] += 1
+            else:
+                stat["new"] += 1
+
+            if progress and i % 20 == 0 and i > 0:
+                logger.info(f"  [{subdir}] {i} 文件已处理...")
+
+        logger.info(f"  [{subdir}] 共 {stat['total']} 文件, "
+                     f"新增 {stat['new']}, 跳过 {stat['skipped']}, 错误 {stat['errors']}")
+        return stat
+
+    def scan_all(self, production_id: int = 0, progress: bool = True) -> dict:
+        """
+        扫描所有已知素材目录。
+
+        返回:
+          {"total_new": int, "dirs": {subdir: stat_dict}}
+        """
+        results: dict = {"total_new": 0, "dirs": {}}
+        for subdir in WATCHED_SUBDIRS:
+            stat = self.scan_dir(subdir, production_id, progress)
+            results["dirs"][subdir] = stat
+            results["total_new"] += stat["new"]
+
+        logger.info(f"全量扫描完成, 新增 {results['total_new']} 个素材")
+        return results
+
+    # ── 统计查询 ──────────────────────────────────────────
+
+    def summarize(self) -> dict:
+        """汇总当前素材库统计信息"""
+        conn = sqlite3.connect(str(self.db_path))
+        try:
+            total = conn.execute("SELECT COUNT(*) FROM assets").fetchone()[0]
+            by_type = conn.execute(
+                "SELECT asset_type, COUNT(*) as cnt FROM assets GROUP BY asset_type"
+            ).fetchall()
+            by_source = conn.execute(
+                "SELECT source, COUNT(*) as cnt FROM assets GROUP BY source"
+            ).fetchall()
+            total_size = conn.execute(
+                "SELECT COALESCE(SUM(file_size), 0) FROM assets"
+            ).fetchone()[0]
+            orphan = conn.execute(
+                "SELECT COUNT(*) FROM assets WHERE production_id IS NULL"
+            ).fetchone()[0]
+
+            return {
+                "total_assets": total,
+                "by_type": dict(by_type),
+                "by_source": dict(by_source),
+                "total_size_bytes": total_size,
+                "total_size_mb": round(total_size / (1024 * 1024), 1),
+                "orphan_count": orphan,
+                "watched_dirs": list(WATCHED_SUBDIRS.keys()),
+            }
+        finally:
+            conn.close()
+
+    def print_summary(self):
+        """打印人类可读的统计"""
+        s = self.summarize()
+        print(f"\n{'='*50}")
+        print(f"  素材库统计")
+        print(f"{'='*50}")
+        print(f"  素材总数:     {s['total_assets']}")
+        print(f"  总大小:       {s['total_size_mb']} MB")
+        print(f"  无关联素材:    {s['orphan_count']}")
+        print(f"\n  ─ 按类型 ─")
+        for t, c in sorted(s["by_type"].items(), key=lambda x: -x[1]):
+            print(f"    {t:12s}  {c}")
+        print(f"\n  ─ 按来源 ─")
+        for src, c in sorted(s["by_source"].items(), key=lambda x: -x[1]):
+            print(f"    {src:16s}  {c}")
+        print(f"\n  ─ 监控目录 ─")
+        for d in s["watched_dirs"]:
+            print(f"    {d}")
+        print(f"{'='*50}\n")
+
+
+# ═══════════════════════════════════════════════════════════
+# 辅助函数
+# ═══════════════════════════════════════════════════════════
+
+def _parse_fps(r_frame_rate: str) -> float:
+    """解析 ffprobe 的 r_frame_rate (如 '30000/1001' → 29.97)"""
+    try:
+        if "/" in r_frame_rate:
+            num, den = r_frame_rate.split("/")
+            return round(float(num) / float(den), 2)
+        return float(r_frame_rate)
+    except (ValueError, ZeroDivisionError):
+        return 0.0
+
+
+def _human_size(bytes_: int) -> str:
+    if bytes_ < 1024:
+        return f"{bytes_} B"
+    elif bytes_ < 1024**2:
+        return f"{bytes_/1024:.1f} KB"
+    elif bytes_ < 1024**3:
+        return f"{bytes_/(1024**2):.1f} MB"
+    return f"{bytes_/(1024**3):.2f} GB"

mac-agent-os-main/05_tools/09_ave/scripts/asset_manager/tags.py

@@ -0,0 +1,317 @@
+"""
+asset_manager/tags — 标签系统 + 素材搜索
+
+职责:
+  1. 自动标签: 根据文件属性和路径生成标签 (复用 index.py 的 _infer_tags)
+  2. 标签管理: 查看所有标签、标签频次、批量标记
+  3. 搜素引擎: 按名称/路径/标签/类型/来源 多维度过滤
+  4. 搜索结果格式化输出 (表格/JSON)
+
+用法:
+  from asset_manager.tags import AssetSearch
+  srch = AssetSearch()
+
+  # 搜索
+  results = srch.search("人物", tags=["video"])
+  results = srch.search_by_tag("pexels", limit=20)
+
+  # 标签管理
+  srch.tag_stats()           # 所有标签频次
+  srch.tag_asset(42, ["人物", "户外"])  # 给资产打标签
+"""
+import json
+import os
+import sqlite3
+from datetime import datetime
+from pathlib import Path
+from typing import Optional, Union
+
+from lib.logger import get_logger
+
+logger = get_logger("asset_tags")
+
+# ── 全局路径 ─────────────────────────────────────────────
+_LOCAL_ROOT = Path(os.environ.get(
+    "AVE_LOCAL_DIR",
+    str(Path.home() / "workbuddy-agent-os/agent-local"),
+))
+_DB_PATH = _LOCAL_ROOT / "tools" / "ave" / "data" / "ave.db"
+
+
+# ═══════════════════════════════════════════════════════════
+# 搜索类
+# ═══════════════════════════════════════════════════════════
+
+class AssetSearch:
+    """素材搜索引擎 (基于 SQLite LIKE + 标签过滤)"""
+
+    def __init__(self, db_path: Union[str, Path] = _DB_PATH):
+        self.db_path = Path(db_path)
+
+    def _connect(self) -> sqlite3.Connection:
+        conn = sqlite3.connect(str(self.db_path))
+        conn.row_factory = sqlite3.Row
+        return conn
+
+    # ── 搜索 ─────────────────────────────────────────────
+
+    def search(
+        self,
+        keyword: str = "",
+        *,
+        asset_type: Optional[str] = None,
+        source: Optional[str] = None,
+        tags: Optional[list[str]] = None,
+        min_size: int = 0,
+        max_size: int = 0,
+        min_duration: float = 0.0,
+        max_duration: float = 0.0,
+        production_id: Optional[int] = None,
+        limit: int = 50,
+        offset: int = 0,
+        order_by: str = "id DESC",
+    ) -> list[dict]:
+        """
+        多维度素材搜索。
+
+        参数:
+          keyword:      关键字 (模糊匹配 name/file_path)
+          asset_type:   文件类型 (video/audio/image/other)
+          source:       来源 (materials/kling/wan2_2/bgm/...)
+          tags:         标签列表 (AND 逻辑, 需同时命中)
+          min_size:     最小文件大小 (bytes)
+          max_size:     最大文件大小 (0 = 不限制)
+          min_duration: 最小时长 (秒)
+          max_duration: 最大时长 (0 = 不限制)
+          production_id:关联 production ID
+          limit/offset: 分页
+          order_by:     排序 (SQL 片段, 如 "file_size DESC")
+
+        返回:
+          [{
+            "id", "asset_type", "name", "file_path", "file_size",
+            "hash", "source", "meta", "tags", "created_at",
+            "production_id"
+          }, ...]
+        """
+        conn = self._connect()
+        try:
+            where = []
+            params: list = []
+
+            # 关键字
+            if keyword:
+                like = f"%{keyword}%"
+                where.append("(a.name LIKE ? OR a.file_path LIKE ?)")
+                params.extend([like, like])
+
+            # 类型
+            if asset_type:
+                where.append("a.asset_type = ?")
+                params.append(asset_type)
+
+            # 来源
+            if source:
+                where.append("a.source = ?")
+                params.append(source)
+
+            # 标签 (AND: 需要所有指定标签)
+            if tags:
+                tag_conditions = []
+                for tag in tags:
+                    tag_conditions.append(
+                        "EXISTS (SELECT 1 FROM asset_tags WHERE asset_id = a.id AND tag = ?)"
+                    )
+                    params.append(tag.lower())
+                where.append("(" + " AND ".join(tag_conditions) + ")")
+
+            # 文件大小
+            if min_size > 0:
+                where.append("a.file_size >= ?")
+                params.append(min_size)
+            if max_size > 0:
+                where.append("a.file_size <= ?")
+                params.append(max_size)
+
+            # 时长 (meta_json 中的 duration_sec)
+            if min_duration > 0:
+                where.append(
+                    "json_extract(a.meta_json, '$.duration_sec') >= ?"
+                )
+                params.append(min_duration)
+            if max_duration > 0:
+                where.append(
+                    "json_extract(a.meta_json, '$.duration_sec') <= ?"
+                )
+                params.append(max_duration)
+
+            # production
+            if production_id is not None:
+                where.append("a.production_id = ?")
+                params.append(production_id)
+
+            # 排除内部记录
+            where.append("a.asset_type != '__meta__'")
+
+            where_clause = " AND ".join(where) if where else "1=1"
+
+            # 安全校验 order_by (防止 SQL 注入)
+            allowed_orders = {
+                "id", "id DESC", "id ASC",
+                "file_size", "file_size DESC", "file_size ASC",
+                "created_at", "created_at DESC", "created_at ASC",
+                "name", "name DESC", "name ASC",
+            }
+            safe_order = order_by if order_by in allowed_orders else "id DESC"
+
+            rows = conn.execute(
+                f"""SELECT a.*, GROUP_CONCAT(at.tag, ',') as tag_list
+                    FROM assets a
+                    LEFT JOIN asset_tags at ON at.asset_id = a.id
+                    WHERE {where_clause}
+                    GROUP BY a.id
+                    ORDER BY {safe_order}
+                    LIMIT ? OFFSET ?""",
+                params + [limit, offset],
+            ).fetchall()
+
+            results = []
+            for r in rows:
+                d = dict(r)
+                # 解析 meta_json
+                try:
+                    d["meta"] = json.loads(d.pop("meta_json", "{}") or "{}")
+                except json.JSONDecodeError:
+                    d["meta"] = {}
+                # 标签列表
+                d["tags"] = d.pop("tag_list", "").split(",") if d.get("tag_list") else []
+                d["file_size_human"] = _human_size(d.get("file_size", 0))
+                results.append(d)
+
+            return results
+
+        except Exception as e:
+            logger.error(f"search 失败: {e}")
+            return []
+        finally:
+            conn.close()
+
+    # ── 标签搜索 ─────────────────────────────────────────
+
+    def search_by_tag(self, tag: str, limit: int = 50) -> list[dict]:
+        """按单个标签搜索"""
+        return self.search(tags=[tag], limit=limit)
+
+    def search_by_type(self, asset_type: str, limit: int = 50) -> list[dict]:
+        """按类型搜索"""
+        return self.search(asset_type=asset_type, limit=limit)
+
+    # ── 标签管理 ─────────────────────────────────────────
+
+    def tag_stats(self) -> list[dict]:
+        """返回所有标签及出现频次, 按频次降序"""
+        conn = self._connect()
+        try:
+            rows = conn.execute(
+                """SELECT tag, COUNT(*) as count
+                   FROM asset_tags
+                   GROUP BY tag
+                   ORDER BY count DESC"""
+            ).fetchall()
+            return [dict(r) for r in rows]
+        finally:
+            conn.close()
+
+    def tag_asset(self, asset_id: int, tags: list[str]) -> bool:
+        """给指定资产打标签"""
+        conn = self._connect()
+        try:
+            for tag in tags:
+                tag_clean = tag.strip().lower()
+                if tag_clean:
+                    conn.execute(
+                        "INSERT OR IGNORE INTO asset_tags (asset_id, tag) VALUES (?, ?)",
+                        (asset_id, tag_clean),
+                    )
+            conn.commit()
+            logger.info(f"资产 {asset_id} 标记: {tags}")
+            return True
+        except Exception as e:
+            logger.error(f"标记失败 asset {asset_id}: {e}")
+            return False
+        finally:
+            conn.close()
+
+    def untag_asset(self, asset_id: int, tags: list[str]) -> bool:
+        """移除标签"""
+        conn = self._connect()
+        try:
+            for tag in tags:
+                conn.execute(
+                    "DELETE FROM asset_tags WHERE asset_id = ? AND tag = ?",
+                    (asset_id, tag.strip().lower()),
+                )
+            conn.commit()
+            return True
+        except Exception as e:
+            logger.error(f"移除标签失败: {e}")
+            return False
+        finally:
+            conn.close()
+
+    def print_results(self, results: list[dict], show_meta: bool = False):
+        """打印搜索结果表格"""
+        if not results:
+            print("  未找到匹配的素材")
+            return
+
+        print(f"\n  {'ID':>4s} {'类型':8s} {'来源':12s} {'大小':10s} {'名称'}")
+        print(f"  {'-'*80}")
+        for r in results:
+            print(f"  {r['id']:>4d} {r['asset_type']:8s} {r['source'] or '?':12s} "
+                  f"{r.get('file_size_human', ''):>10s} {r['name'] or '?'}")
+            if show_meta and r.get("meta"):
+                meta = r["meta"]
+                parts = []
+                if meta.get("duration_sec"):
+                    parts.append(f"dur={meta['duration_sec']}s")
+                if meta.get("width"):
+                    parts.append(f"{meta['width']}x{meta['height']}")
+                if meta.get("fps"):
+                    parts.append(f"{meta['fps']}fps")
+                if parts:
+                    print(f"  {'':>4s} {'':8s} {'':12s} {'':10s} [{', '.join(parts)}]")
+            if r.get("tags"):
+                print(f"  {'':>4s} {'':8s} {'':12s} {'':10s} #{', #'.join(r['tags'][:5])}")
+        print(f"  {'-'*80}")
+        print(f"  共 {len(results)} 条结果\n")
+
+    def print_tag_cloud(self):
+        """打印标签云"""
+        stats = self.tag_stats()
+        if not stats:
+            print("  暂无标签")
+            return
+        max_count = max(s["count"] for s in stats) if stats else 1
+
+        print(f"\n  {'标签':24s} {'频次':>6s}  分布")
+        print(f"  {'-'*60}")
+        for s in stats:
+            bar_len = int(s["count"] / max_count * 20)
+            bar = "█" * bar_len + "░" * (20 - bar_len)
+            print(f"  {s['tag']:24s} {s['count']:>6d}  {bar}")
+        print(f"  共 {len(stats)} 个标签\n")
+
+
+# ═══════════════════════════════════════════════════════════
+# 辅助函数
+# ═══════════════════════════════════════════════════════════
+
+def _human_size(bytes_: int) -> str:
+    if bytes_ < 1024:
+        return f"{bytes_} B"
+    elif bytes_ < 1024**2:
+        return f"{bytes_/1024:.1f} KB"
+    elif bytes_ < 1024**3:
+        return f"{bytes_/(1024**2):.1f} MB"
+    return f"{bytes_/(1024**3):.2f} GB"