feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程,直接发 HTTP
Deploy VitePress site to Pages / build (push) Canceled after 0s
Deploy VitePress site to Pages / Deploy (push) Canceled after 0s

移植自 mac-agent-os 的 mediacrawler_adapter:不起子进程、不开页面,用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin,所以 ingest 那条链路一个字不用改。

**目前能用的(真环境实测,非推断)**:

    profile/other : 200, 7075 字节  —— 博主主页指标(粉丝/获赞/作品数/昵称)
    aweme/detail  : 200, 45425 字节 —— 单条作品详情(含点赞/评论/收藏/分享)

**目前不能用的:作品列表 `aweme/post`。** 两个互相独立的原因:
  1. 这个接口被抖音单独升级成了真校验:不带 x-tt-argus 回 403「Uifid Not Found」,
     带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过,「真校验」过不了。
     同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的,
     挑中的恰好是「批量拉作品列表」这个最敏感的动作。
  2. 改走页面截获也不行:CDP 浏览器打开博主主页会落到「验证码中间页」(当天大量探测的
     代价,过几小时要重测)。

  所以现在的边界是:**已知作品的指标刷新能做,自动发现新作品做不了**。

**排查中控住变量后得到的两条事实**(都写进注释了):
  · `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
    UA+client hints+Cookie 时是 200 但仅 121 字节的空壳,补上这三个头变 7074 字节。
    (我先前猜的 sec-ch-ua 不是关键。)
  · 因此 UA 与 client hints 必须**成套地取自同一个浏览器**,所以 BrowserIdentity 一次
    从 CDP 取齐 cookie + UA + hints,而不是各自写死。

「200 + 空 body 必须当场报错」也是刻意写死的:放过去它会在下游变成「这个博主没作品」,
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的,还被翻译成「账号被封」。

测试 +11:cookie 解析、请求头成套性(含 uifid 缺失/回退)、产物键名与 store 对齐、
以及 _get 的三条失败路径(空 body / 403 带网关原话 / 正常返回)。
This commit is contained in:
2026-10-10 17:12:25 +08:00
parent 21ff01b894
commit 2112c1a870
1601 changed files with 367425 additions and 0 deletions
@@ -0,0 +1,421 @@
"""
assets_manager — 视频工厂资源库(数据层)
==========================================
按《DCS 说明说明书 v2.0》第五节「资产库规范」实现。
目录结构(默认 `agent-local/tools/ave/assets/`):
characters/{char_id}/
meta.json 元数据(名字/类型/用途/标签/一致性种子…)
ref_front.png 正面参考
ref_side.png 侧面
ref_back.png 背面
ref_full.png 全身
expressions/ 表情图(happy/sad/angry…)
wardrobe/{w_id}.png 服装变体
voice_sample.mp3 音色样本
locations/{loc_id}/ AI 生成场景
real_locations/{loc_id}/ 真实场地
products/{prod_id}/ 产品(含 must_keep)
props/{prop_id}/ 道具
voices/{voice_id}/ 音色
bgm/ sfx/ vfx_presets/
设计要点:
· 每个资产 = 一个目录(图/音/元数据分离,便于直接喂给 API)
· meta.json 是唯一真相源(列表/筛选都读它)
· 支持旧数据迁移(character_registry/registry.yaml → characters/)
"""
from __future__ import annotations
import json
import logging
import os
import re
import shutil
import time
from pathlib import Path
from typing import Any, Optional
log = logging.getLogger("assets_manager")
# 资产类型 → 目录名
ASSET_DIRS = {
"characters": "角色",
"locations": "场景",
"real_locations": "真实场地",
"products": "产品",
"props": "道具",
"wardrobe": "服装",
"voices": "音色",
"bgm": "BGM",
"sfx": "音效",
"ambience": "环境音",
"vfx_presets": "特效预设",
"materials": "素材",
}
# 角色:类型 / 用途枚举
CHAR_TYPES = {"real": "真人", "anime": "动漫", "3d": "3D"}
CHAR_ROLES = {"lead": "主角", "support": "配角", "extra": "群演"}
def assets_root() -> Path:
"""资源库根目录(可用 AVE_ASSETS_DIR 覆盖)"""
p = os.environ.get("AVE_ASSETS_DIR")
if p:
return Path(p)
base = Path(os.environ.get("AGENT_LOCAL")
or (Path.home() / "workbuddy-agent-os" / "agent-local"))
return base / "tools" / "ave" / "assets"
# 🔒 锁定后受保护的字段(关键一致性字段:改动会破坏角色一致性 / 版本可追溯)
LOCK_PROTECTED_FIELDS = (
"reference_images", "generation_history", "active_version",
"consistency_seed", "voice", "locked",
)
def _slug(name: str) -> str:
"""中文/特殊字符 → 安全 ID(保留字母数字与中文,其余转下划线)"""
s = re.sub(r"[^\w\u4e00-\u9fff]+", "_", (name or "").strip())
return s.strip("_") or f"item_{int(time.time())}"
class AssetStore:
"""通用资产仓库(一个子目录 = 一个资产)"""
def __init__(self, kind: str, root: Optional[Path] = None):
if kind not in ASSET_DIRS:
raise ValueError(f"未知资产类型: {kind}(可选: {list(ASSET_DIRS)})")
self.kind = kind
self.dir = (root or assets_root()) / kind
self.dir.mkdir(parents=True, exist_ok=True)
# ── 读 ─────────────────────────────────────────────
def _meta_path(self, aid: str) -> Path:
return self.dir / aid / "meta.json"
def kind_dir(self, aid: str) -> Path:
"""某资产的目录(`assets/<kind>/<aid>/`)
供外部写入文件用:参考图(`ref_front.png`)/ 音色(`voice_sample.mp3`)等。
"""
d = self.dir / aid
d.mkdir(parents=True, exist_ok=True)
return d
def get(self, aid: str) -> Optional[dict]:
p = self._meta_path(aid)
if not p.exists():
return None
try:
d = json.loads(p.read_text(encoding="utf-8"))
d.setdefault("id", aid)
d["files"] = self._list_files(aid)
return d
except Exception as e:
log.warning("读取 %s 失败: %s", p, e)
return None
def _list_files(self, aid: str) -> dict:
"""列出资产目录下的文件(供前端展示)"""
d = self.dir / aid
out: dict[str, list[str]] = {}
if not d.exists():
return out
for f in sorted(d.rglob("*")):
if f.is_file() and f.name != "meta.json":
rel = str(f.relative_to(d))
out.setdefault(f.parent.name if f.parent != d else "root", []).append(rel)
return out
def list(self, q: str = "", filters: Optional[dict] = None,
sort: str = "updated_desc", limit: int = 500) -> list[dict]:
"""列表(带搜索/筛选/排序)
q: 关键词(匹配 name / tags / description)
filters: 字段等值筛选(如 {"type": "real", "role": "lead"})
sort: updated_desc | updated_asc | name | usage
"""
filters = {k: v for k, v in (filters or {}).items() if v not in (None, "", [])}
out: list[dict] = []
if not self.dir.exists():
return out
for sub in sorted(self.dir.iterdir()):
if not sub.is_dir():
continue
m = self.get(sub.name)
if not m:
continue
if filters and any(str(m.get(k, "")) != str(v) for k, v in filters.items()):
continue
if q:
hay = " ".join([
str(m.get("name", "")), str(m.get("description", "")),
" ".join(m.get("tags") or []), str(m.get("project", "")),
]).lower()
if q.lower() not in hay:
continue
out.append(m)
if sort == "name":
out.sort(key=lambda x: str(x.get("name", "")))
elif sort == "usage":
out.sort(key=lambda x: -int(x.get("usage_count") or 0))
elif sort == "updated_asc":
out.sort(key=lambda x: str(x.get("updated_at", "")))
else: # updated_desc(默认)
out.sort(key=lambda x: str(x.get("updated_at", "")), reverse=True)
return out[:limit]
# ── 写 ─────────────────────────────────────────────
def create(self, data: dict, aid: Optional[str] = None) -> dict:
# 优先用显式 id(调用方可指定稳定标识),否则由 name 生成
# ⚠️ 原实现写成 name 优先 → 传入的 id 被忽略(曾导致上传时按 id 找不到资产)
aid = aid or _slug(data.get("id") or data.get("name") or "")
d = self.dir / aid
d.mkdir(parents=True, exist_ok=True)
now = time.strftime("%Y-%m-%d %H:%M:%S")
meta = {**data}
meta["id"] = aid
meta.setdefault("created_at", now)
meta["updated_at"] = now
meta.setdefault("usage_count", 0)
(d / "meta.json").write_text(
json.dumps(meta, ensure_ascii=False, indent=2), encoding="utf-8")
log.info(" ➕ 新建 %s: %s", ASSET_DIRS[self.kind], aid)
return self.get(aid) or meta
def update(self, aid: str, patch: dict) -> Optional[dict]:
cur = self.get(aid)
if not cur:
return None
# 🔒 锁定保护:已锁定的资产不允许修改「关键字段」(保证角色一致性 / 版本可追溯)
if cur.get("locked"):
blocked = [k for k in patch if k in LOCK_PROTECTED_FIELDS]
if blocked:
raise PermissionError(
f"资产已锁定({cur.get('locked_at') or ''}),不能修改:{'、'.join(blocked)}。"
"如需修改请先解锁。")
cur.pop("files", None)
cur.update({k: v for k, v in patch.items() if k not in ("id", "created_at")})
cur["updated_at"] = time.strftime("%Y-%m-%d %H:%M:%S")
(self.dir / aid / "meta.json").write_text(
json.dumps(cur, ensure_ascii=False, indent=2), encoding="utf-8")
log.info(" ✏️ 更新 %s: %s", ASSET_DIRS[self.kind], aid)
return self.get(aid)
# ── 🔒 锁定 / 版本记录(角色一致性保障)────────────────
def lock(self, aid: str, reason: str = "", by: str = "user") -> Optional[dict]:
"""锁定资产 → 冻结关键字段(reference_images / 生成历史 / seed / 音色)
何时锁定:
· 用户手动点「🔒 锁定」
· 首次被流程引用(`bump_usage` 自动锁定)
锁定后如需修改,必须先 `unlock()`(会记入 `unlock_history`,便于追溯)。
"""
cur = self.get(aid)
if not cur:
return None
cur.pop("files", None)
if cur.get("locked"):
return self.get(aid)
cur["locked"] = True
cur["locked_at"] = time.strftime("%Y-%m-%d %H:%M:%S")
cur["locked_reason"] = reason or "手动锁定"
cur["locked_by"] = by
(self.dir / aid / "meta.json").write_text(
json.dumps(cur, ensure_ascii=False, indent=2), encoding="utf-8")
log.info(" 🔒 锁定 %s: %s(%s)", ASSET_DIRS[self.kind], aid, cur["locked_reason"])
return self.get(aid)
def unlock(self, aid: str, reason: str = "") -> Optional[dict]:
"""解锁(改动会记入解锁历史,供版本追溯)"""
cur = self.get(aid)
if not cur or not cur.get("locked"):
return self.get(aid) if cur else None
cur.pop("files", None)
hist = list(cur.get("unlock_history") or [])
hist.append({
"at": time.strftime("%Y-%m-%d %H:%M:%S"),
"was_locked_at": cur.get("locked_at"),
"reason": reason or "手动解锁",
})
cur["unlock_history"] = hist
cur["locked"] = False
cur.pop("locked_at", None)
cur.pop("locked_reason", None)
(self.dir / aid / "meta.json").write_text(
json.dumps(cur, ensure_ascii=False, indent=2), encoding="utf-8")
log.info(" 🔓 解锁 %s: %s", ASSET_DIRS[self.kind], aid)
return self.get(aid)
def add_generation(self, aid: str, record: dict) -> Optional[dict]:
"""追加一次**生成记录**(版本历史)—— 记录 prompt / seed / 模型 / 引擎
record 建议字段:
prompt, negative_prompt, seed, engine, model, slot, file, params, at
→ 自动编号 version,并更新 active_version + last_generation
"""
cur = self.get(aid)
if not cur:
return None
cur.pop("files", None)
hist = list(cur.get("generation_history") or [])
rec = {**record}
rec.setdefault("at", time.strftime("%Y-%m-%d %H:%M:%S"))
rec["version"] = len(hist) + 1
hist.append(rec)
cur["generation_history"] = hist
cur["active_version"] = rec["version"]
cur["last_generation"] = rec
# 若 record 带 slot + file → 同步写入参考图(**锁定状态下也允许新增/更新该槽位**,
# 因为这是"追加新版本产物",不是篡改已有设定)
if rec.get("slot") and rec.get("file"):
refs = dict(cur.get("reference_images") or {})
refs[rec["slot"]] = rec["file"]
cur["reference_images"] = refs
# 记录 seed(供后续生成保持一致)
if rec.get("seed") is not None and not cur.get("consistency_seed"):
cur["consistency_seed"] = rec["seed"]
cur["updated_at"] = time.strftime("%Y-%m-%d %H:%M:%S")
(self.dir / aid / "meta.json").write_text(
json.dumps(cur, ensure_ascii=False, indent=2), encoding="utf-8")
log.info(" 📌 %s 生成记录 v%d(%s / %s / seed=%s)",
aid, rec["version"], rec.get("engine"), rec.get("model"), rec.get("seed"))
return self.get(aid)
def delete(self, aid: str, hard: bool = False) -> bool:
d = self.dir / aid
if not d.exists():
return False
if hard:
shutil.rmtree(d)
else: # 软删:移到 _trash/
t = self.dir / "_trash"
t.mkdir(exist_ok=True)
shutil.move(str(d), str(t / f"{aid}_{int(time.time())}"))
log.info(" 🗑 删除 %s: %s", ASSET_DIRS[self.kind], aid)
return True
def bump_usage(self, aid: str) -> None:
"""使用次数 +1(流程引用资产时调用)"""
m = self.get(aid)
if not m:
return
m.pop("files", None)
m["usage_count"] = int(m.get("usage_count") or 0) + 1
# 🆕 首次被使用 → 自动锁定(用户要求:"角色开始使用就要锁定",冻结关键字段保证一致性)
if not m.get("locked"):
m["locked"] = True
m["locked_at"] = time.strftime("%Y-%m-%d %H:%M:%S")
m["locked_reason"] = f"已被流程引用(第 {m['usage_count']} 次,自动锁定以保证角色一致性)"
m["locked_by"] = "auto"
(self.dir / aid / "meta.json").write_text(
json.dumps(m, ensure_ascii=False, indent=2), encoding="utf-8")
def save_file(self, aid: str, filename: str, content: bytes) -> Path:
"""保存资产文件(参考图/音色等)"""
d = self.dir / aid
d.mkdir(parents=True, exist_ok=True)
sub = d / filename
sub.parent.mkdir(parents=True, exist_ok=True)
sub.write_bytes(content)
return sub
class CharacterStore(AssetStore):
"""角色库(AssetStore 的特化:带类型/用途枚举、使用统计)"""
def __init__(self, root: Optional[Path] = None):
super().__init__("characters", root)
def stats(self) -> dict:
"""统计(给 UI 显示)"""
items = self.list(limit=10000)
by_type: dict[str, int] = {}
by_role: dict[str, int] = {}
with_voice = 0
for m in items:
by_type[str(m.get("type") or "real")] = by_type.get(str(m.get("type") or "real"), 0) + 1
by_role[str(m.get("role") or "lead")] = by_role.get(str(m.get("role") or "lead"), 0) + 1
if (m.get("voice") or {}).get("sample"):
with_voice += 1
return {"total": len(items), "by_type": by_type, "by_role": by_role,
"with_voice": with_voice}
def migrate_from_registry(registry_path: Optional[str] = None,
root: Optional[Path] = None) -> dict:
"""旧角色库(character_registry/registry.yaml)→ 新目录结构
保留原字段,映射到新 schema;已存在的角色不覆盖(除非 force)。
"""
import yaml
scripts = Path(__file__).resolve().parent.parent # 09_ave/scripts/
p = Path(registry_path) if registry_path else (scripts / "character_registry" / "registry.yaml")
if not p.exists():
return {"ok": False, "reason": f"旧角色库不存在: {p}"}
raw = yaml.safe_load(p.read_text(encoding="utf-8")) or {}
chars = raw.get("characters") or {}
store = CharacterStore(root)
created, skipped = 0, 0
reg_dir = p.parent
for name, c in chars.items():
aid = _slug(name)
if (store.dir / aid / "meta.json").exists():
skipped += 1
continue
# 旧 reference_images → 新 reference_images
ri = c.get("reference_images") or {}
refs: dict[str, str] = {}
for k in ("front", "side", "back", "full", "portrait", "full_body"):
if ri.get(k):
refs[k] = str(ri[k])
body = ri.get("body") or {}
for k, v in (body.items() if isinstance(body, dict) else []):
if v:
refs[f"body_{k}"] = str(v)
store.create({
"name": name,
"type": "anime", # 旧库多为漫剧风,默认 anime(可改)
"role": "lead",
"project": "",
"tags": list(c.get("preferred_bgm_tags") or []),
"description": c.get("description") or "",
"appearance": list(c.get("appearance") or []),
"personality": c.get("personality") or "",
"art_style": c.get("art_style") or "",
"voice_style": c.get("voice_style") or "",
"bgm_style": c.get("default_bgm_style") or "",
"prompt_template": c.get("prompt_template") or {},
"reference_images": refs,
"consistency_seed": 42,
"voice": {},
"wardrobe": [],
"lora": None,
"_migrated_from": str(p),
}, aid=aid)
created += 1
# 同时把旧图拷进新目录(如果存在)
try:
for k, rel in refs.items():
src = (reg_dir / rel) if not Path(rel).is_absolute() else Path(rel)
if src.exists() and src.is_file():
store.save_file(aid, f"ref_{k}{src.suffix}", src.read_bytes())
except Exception as e:
log.debug("拷贝参考图失败 %s: %s", name, e)
log.info(" 迁移完成: 新建 %d / 跳过 %d(共 %d)", created, skipped, len(chars))
return {"ok": True, "created": created, "skipped": skipped, "total": len(chars)}
# 便捷入口
def get_store(kind: str = "characters") -> AssetStore:
return CharacterStore() if kind == "characters" else AssetStore(kind)