feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程,直接发 HTTP
移植自 mac-agent-os 的 mediacrawler_adapter:不起子进程、不开页面,用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin,所以 ingest 那条链路一个字不用改。
**目前能用的(真环境实测,非推断)**:
profile/other : 200, 7075 字节 —— 博主主页指标(粉丝/获赞/作品数/昵称)
aweme/detail : 200, 45425 字节 —— 单条作品详情(含点赞/评论/收藏/分享)
**目前不能用的:作品列表 `aweme/post`。** 两个互相独立的原因:
1. 这个接口被抖音单独升级成了真校验:不带 x-tt-argus 回 403「Uifid Not Found」,
带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过,「真校验」过不了。
同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的,
挑中的恰好是「批量拉作品列表」这个最敏感的动作。
2. 改走页面截获也不行:CDP 浏览器打开博主主页会落到「验证码中间页」(当天大量探测的
代价,过几小时要重测)。
所以现在的边界是:**已知作品的指标刷新能做,自动发现新作品做不了**。
**排查中控住变量后得到的两条事实**(都写进注释了):
· `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
UA+client hints+Cookie 时是 200 但仅 121 字节的空壳,补上这三个头变 7074 字节。
(我先前猜的 sec-ch-ua 不是关键。)
· 因此 UA 与 client hints 必须**成套地取自同一个浏览器**,所以 BrowserIdentity 一次
从 CDP 取齐 cookie + UA + hints,而不是各自写死。
「200 + 空 body 必须当场报错」也是刻意写死的:放过去它会在下游变成「这个博主没作品」,
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的,还被翻译成「账号被封」。
测试 +11:cookie 解析、请求头成套性(含 uifid 缺失/回退)、产物键名与 store 对齐、
以及 _get 的三条失败路径(空 body / 403 带网关原话 / 正常返回)。
This commit is contained in:
@@ -0,0 +1,421 @@
|
||||
"""
|
||||
assets_manager — 视频工厂资源库(数据层)
|
||||
==========================================
|
||||
|
||||
按《DCS 说明说明书 v2.0》第五节「资产库规范」实现。
|
||||
|
||||
目录结构(默认 `agent-local/tools/ave/assets/`):
|
||||
characters/{char_id}/
|
||||
meta.json 元数据(名字/类型/用途/标签/一致性种子…)
|
||||
ref_front.png 正面参考
|
||||
ref_side.png 侧面
|
||||
ref_back.png 背面
|
||||
ref_full.png 全身
|
||||
expressions/ 表情图(happy/sad/angry…)
|
||||
wardrobe/{w_id}.png 服装变体
|
||||
voice_sample.mp3 音色样本
|
||||
locations/{loc_id}/ AI 生成场景
|
||||
real_locations/{loc_id}/ 真实场地
|
||||
products/{prod_id}/ 产品(含 must_keep)
|
||||
props/{prop_id}/ 道具
|
||||
voices/{voice_id}/ 音色
|
||||
bgm/ sfx/ vfx_presets/
|
||||
|
||||
设计要点:
|
||||
· 每个资产 = 一个目录(图/音/元数据分离,便于直接喂给 API)
|
||||
· meta.json 是唯一真相源(列表/筛选都读它)
|
||||
· 支持旧数据迁移(character_registry/registry.yaml → characters/)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
log = logging.getLogger("assets_manager")
|
||||
|
||||
# 资产类型 → 目录名
|
||||
ASSET_DIRS = {
|
||||
"characters": "角色",
|
||||
"locations": "场景",
|
||||
"real_locations": "真实场地",
|
||||
"products": "产品",
|
||||
"props": "道具",
|
||||
"wardrobe": "服装",
|
||||
"voices": "音色",
|
||||
"bgm": "BGM",
|
||||
"sfx": "音效",
|
||||
"ambience": "环境音",
|
||||
"vfx_presets": "特效预设",
|
||||
"materials": "素材",
|
||||
}
|
||||
|
||||
# 角色:类型 / 用途枚举
|
||||
CHAR_TYPES = {"real": "真人", "anime": "动漫", "3d": "3D"}
|
||||
CHAR_ROLES = {"lead": "主角", "support": "配角", "extra": "群演"}
|
||||
|
||||
|
||||
def assets_root() -> Path:
|
||||
"""资源库根目录(可用 AVE_ASSETS_DIR 覆盖)"""
|
||||
p = os.environ.get("AVE_ASSETS_DIR")
|
||||
if p:
|
||||
return Path(p)
|
||||
base = Path(os.environ.get("AGENT_LOCAL")
|
||||
or (Path.home() / "workbuddy-agent-os" / "agent-local"))
|
||||
return base / "tools" / "ave" / "assets"
|
||||
|
||||
|
||||
# 🔒 锁定后受保护的字段(关键一致性字段:改动会破坏角色一致性 / 版本可追溯)
|
||||
LOCK_PROTECTED_FIELDS = (
|
||||
"reference_images", "generation_history", "active_version",
|
||||
"consistency_seed", "voice", "locked",
|
||||
)
|
||||
|
||||
|
||||
def _slug(name: str) -> str:
|
||||
"""中文/特殊字符 → 安全 ID(保留字母数字与中文,其余转下划线)"""
|
||||
s = re.sub(r"[^\w\u4e00-\u9fff]+", "_", (name or "").strip())
|
||||
return s.strip("_") or f"item_{int(time.time())}"
|
||||
|
||||
|
||||
class AssetStore:
|
||||
"""通用资产仓库(一个子目录 = 一个资产)"""
|
||||
|
||||
def __init__(self, kind: str, root: Optional[Path] = None):
|
||||
if kind not in ASSET_DIRS:
|
||||
raise ValueError(f"未知资产类型: {kind}(可选: {list(ASSET_DIRS)})")
|
||||
self.kind = kind
|
||||
self.dir = (root or assets_root()) / kind
|
||||
self.dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# ── 读 ─────────────────────────────────────────────
|
||||
|
||||
def _meta_path(self, aid: str) -> Path:
|
||||
return self.dir / aid / "meta.json"
|
||||
|
||||
def kind_dir(self, aid: str) -> Path:
|
||||
"""某资产的目录(`assets/<kind>/<aid>/`)
|
||||
|
||||
供外部写入文件用:参考图(`ref_front.png`)/ 音色(`voice_sample.mp3`)等。
|
||||
"""
|
||||
d = self.dir / aid
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
return d
|
||||
|
||||
def get(self, aid: str) -> Optional[dict]:
|
||||
p = self._meta_path(aid)
|
||||
if not p.exists():
|
||||
return None
|
||||
try:
|
||||
d = json.loads(p.read_text(encoding="utf-8"))
|
||||
d.setdefault("id", aid)
|
||||
d["files"] = self._list_files(aid)
|
||||
return d
|
||||
except Exception as e:
|
||||
log.warning("读取 %s 失败: %s", p, e)
|
||||
return None
|
||||
|
||||
def _list_files(self, aid: str) -> dict:
|
||||
"""列出资产目录下的文件(供前端展示)"""
|
||||
d = self.dir / aid
|
||||
out: dict[str, list[str]] = {}
|
||||
if not d.exists():
|
||||
return out
|
||||
for f in sorted(d.rglob("*")):
|
||||
if f.is_file() and f.name != "meta.json":
|
||||
rel = str(f.relative_to(d))
|
||||
out.setdefault(f.parent.name if f.parent != d else "root", []).append(rel)
|
||||
return out
|
||||
|
||||
def list(self, q: str = "", filters: Optional[dict] = None,
|
||||
sort: str = "updated_desc", limit: int = 500) -> list[dict]:
|
||||
"""列表(带搜索/筛选/排序)
|
||||
|
||||
q: 关键词(匹配 name / tags / description)
|
||||
filters: 字段等值筛选(如 {"type": "real", "role": "lead"})
|
||||
sort: updated_desc | updated_asc | name | usage
|
||||
"""
|
||||
filters = {k: v for k, v in (filters or {}).items() if v not in (None, "", [])}
|
||||
out: list[dict] = []
|
||||
if not self.dir.exists():
|
||||
return out
|
||||
for sub in sorted(self.dir.iterdir()):
|
||||
if not sub.is_dir():
|
||||
continue
|
||||
m = self.get(sub.name)
|
||||
if not m:
|
||||
continue
|
||||
if filters and any(str(m.get(k, "")) != str(v) for k, v in filters.items()):
|
||||
continue
|
||||
if q:
|
||||
hay = " ".join([
|
||||
str(m.get("name", "")), str(m.get("description", "")),
|
||||
" ".join(m.get("tags") or []), str(m.get("project", "")),
|
||||
]).lower()
|
||||
if q.lower() not in hay:
|
||||
continue
|
||||
out.append(m)
|
||||
if sort == "name":
|
||||
out.sort(key=lambda x: str(x.get("name", "")))
|
||||
elif sort == "usage":
|
||||
out.sort(key=lambda x: -int(x.get("usage_count") or 0))
|
||||
elif sort == "updated_asc":
|
||||
out.sort(key=lambda x: str(x.get("updated_at", "")))
|
||||
else: # updated_desc(默认)
|
||||
out.sort(key=lambda x: str(x.get("updated_at", "")), reverse=True)
|
||||
return out[:limit]
|
||||
|
||||
# ── 写 ─────────────────────────────────────────────
|
||||
|
||||
def create(self, data: dict, aid: Optional[str] = None) -> dict:
|
||||
# 优先用显式 id(调用方可指定稳定标识),否则由 name 生成
|
||||
# ⚠️ 原实现写成 name 优先 → 传入的 id 被忽略(曾导致上传时按 id 找不到资产)
|
||||
aid = aid or _slug(data.get("id") or data.get("name") or "")
|
||||
d = self.dir / aid
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
now = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
meta = {**data}
|
||||
meta["id"] = aid
|
||||
meta.setdefault("created_at", now)
|
||||
meta["updated_at"] = now
|
||||
meta.setdefault("usage_count", 0)
|
||||
(d / "meta.json").write_text(
|
||||
json.dumps(meta, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
log.info(" ➕ 新建 %s: %s", ASSET_DIRS[self.kind], aid)
|
||||
return self.get(aid) or meta
|
||||
|
||||
def update(self, aid: str, patch: dict) -> Optional[dict]:
|
||||
cur = self.get(aid)
|
||||
if not cur:
|
||||
return None
|
||||
# 🔒 锁定保护:已锁定的资产不允许修改「关键字段」(保证角色一致性 / 版本可追溯)
|
||||
if cur.get("locked"):
|
||||
blocked = [k for k in patch if k in LOCK_PROTECTED_FIELDS]
|
||||
if blocked:
|
||||
raise PermissionError(
|
||||
f"资产已锁定({cur.get('locked_at') or ''}),不能修改:{'、'.join(blocked)}。"
|
||||
"如需修改请先解锁。")
|
||||
cur.pop("files", None)
|
||||
cur.update({k: v for k, v in patch.items() if k not in ("id", "created_at")})
|
||||
cur["updated_at"] = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
(self.dir / aid / "meta.json").write_text(
|
||||
json.dumps(cur, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
log.info(" ✏️ 更新 %s: %s", ASSET_DIRS[self.kind], aid)
|
||||
return self.get(aid)
|
||||
|
||||
# ── 🔒 锁定 / 版本记录(角色一致性保障)────────────────
|
||||
|
||||
def lock(self, aid: str, reason: str = "", by: str = "user") -> Optional[dict]:
|
||||
"""锁定资产 → 冻结关键字段(reference_images / 生成历史 / seed / 音色)
|
||||
|
||||
何时锁定:
|
||||
· 用户手动点「🔒 锁定」
|
||||
· 首次被流程引用(`bump_usage` 自动锁定)
|
||||
锁定后如需修改,必须先 `unlock()`(会记入 `unlock_history`,便于追溯)。
|
||||
"""
|
||||
cur = self.get(aid)
|
||||
if not cur:
|
||||
return None
|
||||
cur.pop("files", None)
|
||||
if cur.get("locked"):
|
||||
return self.get(aid)
|
||||
cur["locked"] = True
|
||||
cur["locked_at"] = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
cur["locked_reason"] = reason or "手动锁定"
|
||||
cur["locked_by"] = by
|
||||
(self.dir / aid / "meta.json").write_text(
|
||||
json.dumps(cur, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
log.info(" 🔒 锁定 %s: %s(%s)", ASSET_DIRS[self.kind], aid, cur["locked_reason"])
|
||||
return self.get(aid)
|
||||
|
||||
def unlock(self, aid: str, reason: str = "") -> Optional[dict]:
|
||||
"""解锁(改动会记入解锁历史,供版本追溯)"""
|
||||
cur = self.get(aid)
|
||||
if not cur or not cur.get("locked"):
|
||||
return self.get(aid) if cur else None
|
||||
cur.pop("files", None)
|
||||
hist = list(cur.get("unlock_history") or [])
|
||||
hist.append({
|
||||
"at": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"was_locked_at": cur.get("locked_at"),
|
||||
"reason": reason or "手动解锁",
|
||||
})
|
||||
cur["unlock_history"] = hist
|
||||
cur["locked"] = False
|
||||
cur.pop("locked_at", None)
|
||||
cur.pop("locked_reason", None)
|
||||
(self.dir / aid / "meta.json").write_text(
|
||||
json.dumps(cur, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
log.info(" 🔓 解锁 %s: %s", ASSET_DIRS[self.kind], aid)
|
||||
return self.get(aid)
|
||||
|
||||
def add_generation(self, aid: str, record: dict) -> Optional[dict]:
|
||||
"""追加一次**生成记录**(版本历史)—— 记录 prompt / seed / 模型 / 引擎
|
||||
|
||||
record 建议字段:
|
||||
prompt, negative_prompt, seed, engine, model, slot, file, params, at
|
||||
→ 自动编号 version,并更新 active_version + last_generation
|
||||
"""
|
||||
cur = self.get(aid)
|
||||
if not cur:
|
||||
return None
|
||||
cur.pop("files", None)
|
||||
hist = list(cur.get("generation_history") or [])
|
||||
rec = {**record}
|
||||
rec.setdefault("at", time.strftime("%Y-%m-%d %H:%M:%S"))
|
||||
rec["version"] = len(hist) + 1
|
||||
hist.append(rec)
|
||||
cur["generation_history"] = hist
|
||||
cur["active_version"] = rec["version"]
|
||||
cur["last_generation"] = rec
|
||||
# 若 record 带 slot + file → 同步写入参考图(**锁定状态下也允许新增/更新该槽位**,
|
||||
# 因为这是"追加新版本产物",不是篡改已有设定)
|
||||
if rec.get("slot") and rec.get("file"):
|
||||
refs = dict(cur.get("reference_images") or {})
|
||||
refs[rec["slot"]] = rec["file"]
|
||||
cur["reference_images"] = refs
|
||||
# 记录 seed(供后续生成保持一致)
|
||||
if rec.get("seed") is not None and not cur.get("consistency_seed"):
|
||||
cur["consistency_seed"] = rec["seed"]
|
||||
cur["updated_at"] = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
(self.dir / aid / "meta.json").write_text(
|
||||
json.dumps(cur, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
log.info(" 📌 %s 生成记录 v%d(%s / %s / seed=%s)",
|
||||
aid, rec["version"], rec.get("engine"), rec.get("model"), rec.get("seed"))
|
||||
return self.get(aid)
|
||||
|
||||
def delete(self, aid: str, hard: bool = False) -> bool:
|
||||
d = self.dir / aid
|
||||
if not d.exists():
|
||||
return False
|
||||
if hard:
|
||||
shutil.rmtree(d)
|
||||
else: # 软删:移到 _trash/
|
||||
t = self.dir / "_trash"
|
||||
t.mkdir(exist_ok=True)
|
||||
shutil.move(str(d), str(t / f"{aid}_{int(time.time())}"))
|
||||
log.info(" 🗑 删除 %s: %s", ASSET_DIRS[self.kind], aid)
|
||||
return True
|
||||
|
||||
def bump_usage(self, aid: str) -> None:
|
||||
"""使用次数 +1(流程引用资产时调用)"""
|
||||
m = self.get(aid)
|
||||
if not m:
|
||||
return
|
||||
m.pop("files", None)
|
||||
m["usage_count"] = int(m.get("usage_count") or 0) + 1
|
||||
# 🆕 首次被使用 → 自动锁定(用户要求:"角色开始使用就要锁定",冻结关键字段保证一致性)
|
||||
if not m.get("locked"):
|
||||
m["locked"] = True
|
||||
m["locked_at"] = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
m["locked_reason"] = f"已被流程引用(第 {m['usage_count']} 次,自动锁定以保证角色一致性)"
|
||||
m["locked_by"] = "auto"
|
||||
(self.dir / aid / "meta.json").write_text(
|
||||
json.dumps(m, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
def save_file(self, aid: str, filename: str, content: bytes) -> Path:
|
||||
"""保存资产文件(参考图/音色等)"""
|
||||
d = self.dir / aid
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
sub = d / filename
|
||||
sub.parent.mkdir(parents=True, exist_ok=True)
|
||||
sub.write_bytes(content)
|
||||
return sub
|
||||
|
||||
|
||||
class CharacterStore(AssetStore):
|
||||
"""角色库(AssetStore 的特化:带类型/用途枚举、使用统计)"""
|
||||
|
||||
def __init__(self, root: Optional[Path] = None):
|
||||
super().__init__("characters", root)
|
||||
|
||||
def stats(self) -> dict:
|
||||
"""统计(给 UI 显示)"""
|
||||
items = self.list(limit=10000)
|
||||
by_type: dict[str, int] = {}
|
||||
by_role: dict[str, int] = {}
|
||||
with_voice = 0
|
||||
for m in items:
|
||||
by_type[str(m.get("type") or "real")] = by_type.get(str(m.get("type") or "real"), 0) + 1
|
||||
by_role[str(m.get("role") or "lead")] = by_role.get(str(m.get("role") or "lead"), 0) + 1
|
||||
if (m.get("voice") or {}).get("sample"):
|
||||
with_voice += 1
|
||||
return {"total": len(items), "by_type": by_type, "by_role": by_role,
|
||||
"with_voice": with_voice}
|
||||
|
||||
|
||||
def migrate_from_registry(registry_path: Optional[str] = None,
|
||||
root: Optional[Path] = None) -> dict:
|
||||
"""旧角色库(character_registry/registry.yaml)→ 新目录结构
|
||||
|
||||
保留原字段,映射到新 schema;已存在的角色不覆盖(除非 force)。
|
||||
"""
|
||||
import yaml
|
||||
scripts = Path(__file__).resolve().parent.parent # 09_ave/scripts/
|
||||
p = Path(registry_path) if registry_path else (scripts / "character_registry" / "registry.yaml")
|
||||
if not p.exists():
|
||||
return {"ok": False, "reason": f"旧角色库不存在: {p}"}
|
||||
|
||||
raw = yaml.safe_load(p.read_text(encoding="utf-8")) or {}
|
||||
chars = raw.get("characters") or {}
|
||||
store = CharacterStore(root)
|
||||
created, skipped = 0, 0
|
||||
|
||||
reg_dir = p.parent
|
||||
for name, c in chars.items():
|
||||
aid = _slug(name)
|
||||
if (store.dir / aid / "meta.json").exists():
|
||||
skipped += 1
|
||||
continue
|
||||
# 旧 reference_images → 新 reference_images
|
||||
ri = c.get("reference_images") or {}
|
||||
refs: dict[str, str] = {}
|
||||
for k in ("front", "side", "back", "full", "portrait", "full_body"):
|
||||
if ri.get(k):
|
||||
refs[k] = str(ri[k])
|
||||
body = ri.get("body") or {}
|
||||
for k, v in (body.items() if isinstance(body, dict) else []):
|
||||
if v:
|
||||
refs[f"body_{k}"] = str(v)
|
||||
store.create({
|
||||
"name": name,
|
||||
"type": "anime", # 旧库多为漫剧风,默认 anime(可改)
|
||||
"role": "lead",
|
||||
"project": "",
|
||||
"tags": list(c.get("preferred_bgm_tags") or []),
|
||||
"description": c.get("description") or "",
|
||||
"appearance": list(c.get("appearance") or []),
|
||||
"personality": c.get("personality") or "",
|
||||
"art_style": c.get("art_style") or "",
|
||||
"voice_style": c.get("voice_style") or "",
|
||||
"bgm_style": c.get("default_bgm_style") or "",
|
||||
"prompt_template": c.get("prompt_template") or {},
|
||||
"reference_images": refs,
|
||||
"consistency_seed": 42,
|
||||
"voice": {},
|
||||
"wardrobe": [],
|
||||
"lora": None,
|
||||
"_migrated_from": str(p),
|
||||
}, aid=aid)
|
||||
created += 1
|
||||
# 同时把旧图拷进新目录(如果存在)
|
||||
try:
|
||||
for k, rel in refs.items():
|
||||
src = (reg_dir / rel) if not Path(rel).is_absolute() else Path(rel)
|
||||
if src.exists() and src.is_file():
|
||||
store.save_file(aid, f"ref_{k}{src.suffix}", src.read_bytes())
|
||||
except Exception as e:
|
||||
log.debug("拷贝参考图失败 %s: %s", name, e)
|
||||
|
||||
log.info(" 迁移完成: 新建 %d / 跳过 %d(共 %d)", created, skipped, len(chars))
|
||||
return {"ok": True, "created": created, "skipped": skipped, "total": len(chars)}
|
||||
|
||||
|
||||
# 便捷入口
|
||||
def get_store(kind: str = "characters") -> AssetStore:
|
||||
return CharacterStore() if kind == "characters" else AssetStore(kind)
|
||||
Reference in New Issue
Block a user