butubb
2112c1a870
移植自 mac-agent-os 的 mediacrawler_adapter：不起子进程、不开页面，用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin，所以 ingest 那条链路一个字不用改。
**目前能用的（真环境实测，非推断）**：
profile/other : 200, 7075 字节 —— 博主主页指标（粉丝/获赞/作品数/昵称）
aweme/detail : 200, 45425 字节 —— 单条作品详情（含点赞/评论/收藏/分享）
**目前不能用的：作品列表 `aweme/post`。** 两个互相独立的原因：
1. 这个接口被抖音单独升级成了真校验：不带 x-tt-argus 回 403「Uifid Not Found」，
带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过，「真校验」过不了。
同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的，
挑中的恰好是「批量拉作品列表」这个最敏感的动作。
2. 改走页面截获也不行：CDP 浏览器打开博主主页会落到「验证码中间页」（当天大量探测的
代价，过几小时要重测）。
所以现在的边界是：**已知作品的指标刷新能做，自动发现新作品做不了**。
**排查中控住变量后得到的两条事实**（都写进注释了）：
· `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
UA+client hints+Cookie 时是 200 但仅 121 字节的空壳，补上这三个头变 7074 字节。
（我先前猜的 sec-ch-ua 不是关键。）
· 因此 UA 与 client hints 必须**成套地取自同一个浏览器**，所以 BrowserIdentity 一次
从 CDP 取齐 cookie + UA + hints，而不是各自写死。
「200 + 空 body 必须当场报错」也是刻意写死的：放过去它会在下游变成「这个博主没作品」，
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的，还被翻译成「账号被封」。
测试 +11：cookie 解析、请求头成套性（含 uifid 缺失/回退）、产物键名与 store 对齐、
以及 _get 的三条失败路径（空 body / 403 带网关原话 / 正常返回）。
368 lines
14 KiB
Python
368 lines
14 KiB
Python
"""
|
|
asset_manager/cache — 缓存管理与自动入库
|
|
|
|
职责:
|
|
1. 增量扫描: 跟踪目录 mtime, 只扫描新文件
|
|
2. 自动入库: 发现新文件 → 调用 AssetIndex 入库
|
|
3. 缓存清理: 按策略清理孤立/过期文件
|
|
4. 生产者钩子: Pexels/Kling 下载完成后自动通知
|
|
|
|
用法:
|
|
from asset_manager.cache import CacheManager
|
|
cm = CacheManager()
|
|
cm.scan_new() # 扫描所有目录的新文件
|
|
cm.cleanup(days=30) # 清理 30 天前无关联的素材
|
|
cm.register_scan("/tmp/") # 注册额外扫描目录
|
|
|
|
设计原则:
|
|
- 无持续进程/守护: 每次 CLI 调用时触发扫描
|
|
- mtime 记录在 SQLite meta_json 中, 跨会话持久
|
|
- 重要素材 (有 production_id 关联) 不会被清理
|
|
"""
|
|
import json
|
|
import os
|
|
import shutil
|
|
import sqlite3
|
|
import time
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Callable, Optional, Union
|
|
|
|
from lib.logger import get_logger
|
|
from asset_manager.index import AssetIndex
|
|
|
|
logger = get_logger("asset_cache")
|
|
|
|
# ── 全局路径 (复用 index.py) ─────────────────────────────
|
|
_LOCAL_ROOT = Path(os.environ.get(
|
|
"AVE_LOCAL_DIR",
|
|
str(Path.home() / "workbuddy-agent-os/agent-local"),
|
|
))
|
|
_CACHE_DIR = _LOCAL_ROOT / "tools" / "ave" / "cache"
|
|
_DB_PATH = _LOCAL_ROOT / "tools" / "ave" / "data" / "ave.db"
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════
|
|
# 核心类
|
|
# ═══════════════════════════════════════════════════════════
|
|
|
|
class CacheManager:
|
|
"""缓存管理器
|
|
|
|
属性:
|
|
index: AssetIndex 实例 (共享 DB)
|
|
scan_records: {dir_path: last_mtime} 记录上次扫描时间
|
|
"""
|
|
|
|
def __init__(self, cache_dir: Union[str, Path] = _CACHE_DIR,
|
|
db_path: Union[str, Path] = _DB_PATH):
|
|
self.cache_dir = Path(cache_dir)
|
|
self.db_path = Path(db_path)
|
|
self.index = AssetIndex(cache_dir=cache_dir, db_path=db_path)
|
|
# 额外扫描目录注册表: {路径: 回调函数}
|
|
self._extra_scans: list[tuple[str, Optional[Callable]]] = []
|
|
|
|
# ── 注册额外扫描目录 ─────────────────────────────────
|
|
|
|
def register_scan(self, directory: str,
|
|
callback: Optional[Callable] = None):
|
|
"""注册额外目录到扫描列表 (如外部导入的素材目录)"""
|
|
self._extra_scans.append((directory, callback))
|
|
logger.info(f"注册扫描目录: {directory}")
|
|
|
|
# ── 增量扫描 ─────────────────────────────────────────
|
|
|
|
def _load_scan_record(self, subdir: str) -> float:
|
|
"""从 meta_json 读取上次扫描记录的 mtime"""
|
|
conn = sqlite3.connect(str(self.db_path))
|
|
try:
|
|
# 用特殊 hash 值 '_scan_record' 标记扫描记录
|
|
row = conn.execute(
|
|
"SELECT meta_json FROM assets WHERE hash = ? AND source = ?",
|
|
(f"_scan_{subdir}", "__cache_mgr__"),
|
|
).fetchone()
|
|
if row:
|
|
rec = json.loads(row[0])
|
|
return float(rec.get("last_mtime", 0))
|
|
except Exception:
|
|
pass
|
|
finally:
|
|
conn.close()
|
|
return 0.0
|
|
|
|
def _save_scan_record(self, subdir: str, mtime: float):
|
|
"""持久化扫描记录到 DB"""
|
|
conn = sqlite3.connect(str(self.db_path))
|
|
try:
|
|
# 先删除旧记录
|
|
conn.execute(
|
|
"DELETE FROM assets WHERE hash = ? AND source = ?",
|
|
(f"_scan_{subdir}", "__cache_mgr__"),
|
|
)
|
|
conn.execute(
|
|
"""INSERT INTO assets (asset_type, name, file_path, file_size, hash, source, meta_json)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?)""",
|
|
("__meta__", f"scan_record_{subdir}", "",
|
|
0, f"_scan_{subdir}", "__cache_mgr__",
|
|
json.dumps({"last_mtime": mtime, "updated_at": datetime.now().isoformat()})),
|
|
)
|
|
conn.commit()
|
|
except Exception as e:
|
|
logger.debug(f"保存扫描记录失败: {e}")
|
|
finally:
|
|
conn.close()
|
|
|
|
def scan_new(self, force_rescan: bool = False) -> dict:
|
|
"""
|
|
增量扫描: 只扫描 mtime 超过上次记录的文件。
|
|
|
|
参数:
|
|
force_rescan: 强制全量重新扫描
|
|
|
|
返回:
|
|
{"total_new": int, "per_dir": {subdir: new_count}}
|
|
"""
|
|
from asset_manager.index import WATCHED_SUBDIRS, EXT_TYPE_MAP
|
|
|
|
results: dict = {"total_new": 0, "per_dir": {}}
|
|
valid_exts = set(EXT_TYPE_MAP.keys())
|
|
|
|
for subdir in WATCHED_SUBDIRS:
|
|
target = self.cache_dir / subdir
|
|
if not target.is_dir():
|
|
continue
|
|
|
|
last_mtime = 0.0 if force_rescan else self._load_scan_record(subdir)
|
|
new_count = 0
|
|
|
|
for fpath in sorted(target.rglob("*")):
|
|
if not fpath.is_file():
|
|
continue
|
|
if fpath.suffix.lower() not in valid_exts:
|
|
continue
|
|
|
|
file_mtime = fpath.stat().st_mtime
|
|
if file_mtime > last_mtime:
|
|
ret = self.index.index_file(str(fpath))
|
|
if ret > 0:
|
|
new_count += 1
|
|
|
|
# 更新扫描记录 (用当前最大 mtime)
|
|
current_max = 0.0
|
|
for fpath in target.rglob("*"):
|
|
if fpath.is_file():
|
|
current_max = max(current_max, fpath.stat().st_mtime)
|
|
self._save_scan_record(subdir, current_max)
|
|
|
|
if new_count > 0:
|
|
logger.info(f" [{subdir}] 新增 {new_count} 个素材")
|
|
results["per_dir"][subdir] = new_count
|
|
results["total_new"] += new_count
|
|
|
|
# 额外扫描目录
|
|
for extra_dir, callback in self._extra_scans:
|
|
extra_path = Path(extra_dir)
|
|
if not extra_path.is_dir():
|
|
continue
|
|
for fpath in extra_path.rglob("*"):
|
|
if not fpath.is_file():
|
|
continue
|
|
if fpath.suffix.lower() not in valid_exts:
|
|
continue
|
|
ret = self.index.index_file(str(fpath))
|
|
if ret > 0:
|
|
results["total_new"] += 1
|
|
if callback:
|
|
try:
|
|
callback(str(fpath))
|
|
except Exception as e:
|
|
logger.warning(f"回调失败 {fpath}: {e}")
|
|
|
|
logger.info(f"增量扫描完成, 共新增 {results['total_new']} 个素材")
|
|
return results
|
|
|
|
# ── 全量扫描 ─────────────────────────────────────────
|
|
|
|
def scan_all(self, production_id: int = 0) -> dict:
|
|
"""全量扫描并重置所有扫描记录"""
|
|
results = self.index.scan_all(production_id)
|
|
|
|
# 重置扫描记录为最新 mtime
|
|
from asset_manager.index import WATCHED_SUBDIRS
|
|
for subdir in WATCHED_SUBDIRS:
|
|
target = self.cache_dir / subdir
|
|
if target.is_dir():
|
|
current_max = max(
|
|
(f.stat().st_mtime for f in target.rglob("*") if f.is_file()),
|
|
default=0.0,
|
|
)
|
|
self._save_scan_record(subdir, current_max)
|
|
|
|
return results
|
|
|
|
# ── 缓存清理 ─────────────────────────────────────────
|
|
|
|
def cleanup(self, days: int = 30, dry_run: bool = False) -> dict:
|
|
"""
|
|
清理过期缓存。
|
|
|
|
清理规则:
|
|
- 无 production_id 关联
|
|
- 文件创建时间 > days 天前
|
|
- 保留有 production 引用的素材
|
|
|
|
参数:
|
|
days: 保留天数
|
|
dry_run: 仅预览不删除
|
|
|
|
返回:
|
|
{"deleted": int, "freed_bytes": int, "files": [路径]}
|
|
"""
|
|
conn = sqlite3.connect(str(self.db_path))
|
|
cutoff = (datetime.now() - timedelta(days=days)).isoformat()
|
|
result: dict = {"deleted": 0, "freed_bytes": 0, "files": []}
|
|
|
|
try:
|
|
# 查找无关联的旧资产
|
|
rows = conn.execute(
|
|
"""SELECT id, file_path, file_size FROM assets
|
|
WHERE production_id IS NULL
|
|
AND created_at < ?
|
|
AND asset_type != '__meta__'
|
|
ORDER BY created_at ASC""",
|
|
(cutoff,),
|
|
).fetchall()
|
|
|
|
for row in rows:
|
|
aid, fpath, fsize = row["id"], row["file_path"], row["file_size"] or 0
|
|
|
|
if dry_run:
|
|
result["deleted"] += 1
|
|
result["freed_bytes"] += fsize
|
|
result["files"].append(f"{fpath} ({_human_size(fsize)})")
|
|
continue
|
|
|
|
# 删除磁盘文件
|
|
if fpath and os.path.isfile(fpath):
|
|
try:
|
|
os.remove(fpath)
|
|
logger.debug(f" 删除文件: {fpath}")
|
|
except OSError as e:
|
|
logger.warning(f" 删除失败: {fpath}: {e}")
|
|
|
|
# 删除 DB 记录
|
|
conn.execute("DELETE FROM asset_tags WHERE asset_id = ?", (aid,))
|
|
conn.execute("DELETE FROM production_assets WHERE asset_id = ?", (aid,))
|
|
conn.execute("DELETE FROM assets WHERE id = ?", (aid,))
|
|
|
|
result["deleted"] += 1
|
|
result["freed_bytes"] += fsize
|
|
result["files"].append(fpath)
|
|
|
|
conn.commit()
|
|
except Exception as e:
|
|
logger.error(f"cleanup 失败: {e}")
|
|
finally:
|
|
conn.close()
|
|
|
|
action = "预览" if dry_run else "清理"
|
|
logger.info(f"缓存{action}: {result['deleted']} 个文件, "
|
|
f"释放 {_human_size(result['freed_bytes'])}")
|
|
return result
|
|
|
|
# ── 目录统计 ─────────────────────────────────────────
|
|
|
|
def disk_usage(self) -> dict:
|
|
"""统计每个缓存目录的磁盘占用"""
|
|
usage: dict = {}
|
|
for subdir in os.listdir(self.cache_dir):
|
|
target = self.cache_dir / subdir
|
|
if not target.is_dir():
|
|
continue
|
|
total = 0
|
|
count = 0
|
|
for fpath in target.rglob("*"):
|
|
if fpath.is_file():
|
|
total += fpath.stat().st_size
|
|
count += 1
|
|
usage[subdir] = {
|
|
"count": count,
|
|
"size_bytes": total,
|
|
"size_human": _human_size(total),
|
|
}
|
|
return usage
|
|
|
|
def print_disk_usage(self):
|
|
"""打印磁盘占用表"""
|
|
usage = self.disk_usage()
|
|
print(f"\n{'='*50}")
|
|
print(f" 缓存磁盘占用")
|
|
print(f"{'='*50}")
|
|
print(f" {'目录':20s} {'文件数':>8s} {'大小':>12s}")
|
|
print(f" {'-'*40}")
|
|
total_size = 0
|
|
for subdir, info in sorted(usage.items(), key=lambda x: -x[1]["size_bytes"]):
|
|
print(f" {subdir:20s} {info['count']:>8d} {info['size_human']:>12s}")
|
|
total_size += info["size_bytes"]
|
|
print(f" {'-'*40}")
|
|
print(f" {'总计':20s} {'':>8s} {_human_size(total_size):>12s}")
|
|
print(f"{'='*50}\n")
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════
|
|
# 生产者自动入库钩子
|
|
# ═══════════════════════════════════════════════════════════
|
|
|
|
def auto_index_on_download(file_path: str, production_id: int = 0) -> int:
|
|
"""
|
|
生产者在下载/生成素材后, 调用此函数自动入库。
|
|
|
|
用法:
|
|
from asset_manager.cache import auto_index_on_download
|
|
|
|
path = pexels_search.download(url)
|
|
auto_index_on_download(path, production_id=pid)
|
|
"""
|
|
idx = AssetIndex()
|
|
return idx.index_file(file_path, production_id=production_id)
|
|
|
|
|
|
def index_output_video(output_path: str, production_id: int) -> int:
|
|
"""
|
|
最终成片入库 (特殊标记 source=output)。
|
|
"""
|
|
if not os.path.isfile(output_path):
|
|
return -1
|
|
idx = AssetIndex()
|
|
# 强制标记 source 为 output
|
|
conn = sqlite3.connect(str(idx.db_path))
|
|
try:
|
|
# 先通过 index_file 入库
|
|
aid = idx.index_file(output_path, production_id=production_id)
|
|
if aid > 0:
|
|
# 覆盖 source
|
|
conn.execute(
|
|
"UPDATE assets SET source = 'production_output' WHERE id = ?",
|
|
(aid,),
|
|
)
|
|
conn.commit()
|
|
return aid
|
|
return aid
|
|
except Exception as e:
|
|
logger.error(f"index_output 失败: {e}")
|
|
return -1
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
# ── 辅助函数 ─────────────────────────────────────────────
|
|
|
|
def _human_size(bytes_: int) -> str:
|
|
if bytes_ < 1024:
|
|
return f"{bytes_} B"
|
|
elif bytes_ < 1024**2:
|
|
return f"{bytes_/1024:.1f} KB"
|
|
elif bytes_ < 1024**3:
|
|
return f"{bytes_/(1024**2):.1f} MB"
|
|
return f"{bytes_/(1024**3):.2f} GB"
|