butubb
2112c1a870
移植自 mac-agent-os 的 mediacrawler_adapter：不起子进程、不开页面，用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin，所以 ingest 那条链路一个字不用改。
**目前能用的（真环境实测，非推断）**：
profile/other : 200, 7075 字节 —— 博主主页指标（粉丝/获赞/作品数/昵称）
aweme/detail : 200, 45425 字节 —— 单条作品详情（含点赞/评论/收藏/分享）
**目前不能用的：作品列表 `aweme/post`。** 两个互相独立的原因：
1. 这个接口被抖音单独升级成了真校验：不带 x-tt-argus 回 403「Uifid Not Found」，
带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过，「真校验」过不了。
同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的，
挑中的恰好是「批量拉作品列表」这个最敏感的动作。
2. 改走页面截获也不行：CDP 浏览器打开博主主页会落到「验证码中间页」（当天大量探测的
代价，过几小时要重测）。
所以现在的边界是：**已知作品的指标刷新能做，自动发现新作品做不了**。
**排查中控住变量后得到的两条事实**（都写进注释了）：
· `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
UA+client hints+Cookie 时是 200 但仅 121 字节的空壳，补上这三个头变 7074 字节。
（我先前猜的 sec-ch-ua 不是关键。）
· 因此 UA 与 client hints 必须**成套地取自同一个浏览器**，所以 BrowserIdentity 一次
从 CDP 取齐 cookie + UA + hints，而不是各自写死。
「200 + 空 body 必须当场报错」也是刻意写死的：放过去它会在下游变成「这个博主没作品」，
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的，还被翻译成「账号被封」。
测试 +11：cookie 解析、请求头成套性（含 uifid 缺失/回退）、产物键名与 store 对齐、
以及 _get 的三条失败路径（空 body / 403 带网关原话 / 正常返回）。
1275 lines
64 KiB
Python
1275 lines
64 KiB
Python
"""
|
||
composer.pipeline — 分镜 → 成片流水线（S2-6）
|
||
================================================
|
||
|
||
统一分镜 Schema 进 → 成片出：
|
||
|
||
① 逐镜头生成（按 shot.type 选能力 → 调生成引擎）
|
||
② 每片段去 AI 化（composer.de_ai）
|
||
③ 拼接 + 输出（composer.ffmpeg.concat_segments）
|
||
|
||
已可用的部分：
|
||
· `render_from_clips()` —— 已有片段 → 去AI化 → 拼接 → 成片（**可独立测试**）
|
||
· `render()` —— 完整链路（生成环节依赖生成引擎就绪：可灵 / 即梦等）
|
||
|
||
进度回调 `progress_cb(step, shot_id, status, message)` 供前端轮询展示。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
import sys
|
||
from pathlib import Path
|
||
from typing import Callable, Optional
|
||
|
||
_SCRIPTS = Path(__file__).resolve().parent.parent
|
||
if str(_SCRIPTS) not in sys.path:
|
||
sys.path.insert(0, str(_SCRIPTS))
|
||
|
||
from schemas import Storyboard # noqa: E402
|
||
from composer.de_ai import DeAIProcessor # noqa: E402
|
||
from composer.ffmpeg import concat_segments # noqa: E402
|
||
|
||
log = logging.getLogger("composer.pipeline")
|
||
|
||
ProgressCB = Optional[Callable[[str, str, str, str], None]]
|
||
|
||
# ════════════════════════════════════════════════════════════
|
||
# 影棚词清理（研究结论落地，2026-10-03）
|
||
# AI 感的头号来源之一：资产描述里混入「柔光箱均匀布光 / 中焦85mm / 纯灰色专业背景」
|
||
# 这类专业影棚语言 —— 生活场景/自拍 Vlog 一旦带上，画面立刻变假。
|
||
# 用户实测：角色「中国」的描述末尾就带着这三个词 → 生成画面影棚味很重。
|
||
# ════════════════════════════════════════════════════════════
|
||
_STUDIO_JARGON = (
|
||
"柔光箱均匀布光", "柔光箱", "均匀布光", "布光", "打光", "主光", "轮廓光",
|
||
"纯灰色专业背景", "纯灰色背景", "纯色背景", "灰色背景", "专业背景", "纯灰背景",
|
||
"中焦85mm", "85mm", "压缩空间", "反光板", "灯架", "摄影棚", "影棚",
|
||
"studio background", "key light", "rim light", "softbox",
|
||
)
|
||
|
||
|
||
def _clean_studio_jargon(text: str) -> str:
|
||
"""剔除影棚语言（保留其余描述），并清理残留的分隔符
|
||
|
||
例：'【神态&场景】柔光箱均匀布光、中焦85mm、纯灰色专业背景'
|
||
→ ''（清理后只剩标签 → 整体丢弃）
|
||
"""
|
||
import re as _re
|
||
out = str(text or "")
|
||
for w in _STUDIO_JARGON:
|
||
out = out.replace(w, "")
|
||
out = _re.sub(r"[、,，]\s*(?=[、,，。；）\)])", "", out) # 连续分隔符
|
||
out = _re.sub(r"[（(]\s*[)）]", "", out) # 空括号
|
||
out = _re.sub(r"[、,，。；]\s*$", "", out) # 结尾多余标点
|
||
out = out.strip()
|
||
# ★ 清理后若只剩标签/符号（无实质内容）→ 整体丢弃，避免"【神态&场景】"这种空壳
|
||
# 判据：**先剔除【标签】本身**，再看剩余内容长度
|
||
# （曾误把 \w 也算作标签——Python 的 \w 匹配中文 → 导致正常描述被清空）
|
||
_no_tags = _re.sub(r"【[^】]*】", "", out)
|
||
_content = _re.sub(r"[\[\]()（）、,，。；：&/|\-—_ \t\n]", "", _no_tags)
|
||
if len(_content) < 2:
|
||
return ""
|
||
return out
|
||
|
||
|
||
|
||
# ── 出片引擎分流辅助 ──────────────────────────────────────────
|
||
_VIDEO_MODELS_CACHE = None
|
||
|
||
|
||
def video_channel_of(model: str) -> str:
|
||
"""查 config/video_models.yaml → 该模型走哪条通道（kling / dashscope）
|
||
|
||
留空 = "kling"（可灵，向后兼容 —— 老分镜没这个字段就还是走可灵）。
|
||
"""
|
||
global _VIDEO_MODELS_CACHE
|
||
if not model:
|
||
return "kling"
|
||
if _VIDEO_MODELS_CACHE is None:
|
||
try:
|
||
import yaml as _y
|
||
p = Path(__file__).resolve().parents[2] / "config" / "video_models.yaml"
|
||
d = _y.safe_load(p.read_text(encoding="utf-8")) or {}
|
||
_VIDEO_MODELS_CACHE = {m.get("id"): m for m in (d.get("models") or [])}
|
||
except Exception as e:
|
||
log.warning("读 video_models.yaml 失败: %s", e)
|
||
_VIDEO_MODELS_CACHE = {}
|
||
return (_VIDEO_MODELS_CACHE.get(model) or {}).get("channel") or "kling"
|
||
|
||
class PipelineError(RuntimeError):
|
||
pass
|
||
|
||
|
||
|
||
def _with_ai_negatives(p: str, ga) -> str:
|
||
"""追加「关闭 AI 默认审美」的负面约束（幂等；realism_boost=off 时跳过）
|
||
|
||
来源：anti-ai-checklist.md —— AI 感的头号来源是模型的默认审美
|
||
（电影化运镜 / 稳定画面 / 商业调色 / 完美构图 / 磨皮皮肤），必须显式关闭。
|
||
"""
|
||
try:
|
||
from composer.realism import ai_negative_block
|
||
boost = (getattr(ga, "realism_boost", "mild") or "mild")
|
||
if boost == "off":
|
||
return p
|
||
block = ai_negative_block(hard=(boost == "strong"))
|
||
if block and "关闭 AI 默认审美" not in p:
|
||
return p.rstrip("。. ") + "。" + block
|
||
except Exception:
|
||
pass
|
||
return p
|
||
|
||
|
||
class StoryboardPipeline:
|
||
"""分镜 → 成片"""
|
||
|
||
def __init__(self, work_dir: str | Path | None = None):
|
||
self.work_dir = Path(work_dir or "/tmp/ave_render")
|
||
self.work_dir.mkdir(parents=True, exist_ok=True)
|
||
self.de_ai = DeAIProcessor()
|
||
|
||
# ── 可独立测试：已有片段 → 成片 ────────────────────
|
||
|
||
def _normalize_clip(self, src: str, dest: str,
|
||
storyboard: Storyboard | None = None,
|
||
max_duration: float = 8.0) -> str:
|
||
"""规范化片段：统一分辨率（默认竖屏 1080x1920）+ 裁到最长 max_duration 秒
|
||
|
||
素材可能来自 Pexels（4K / 超长）或生成引擎（规格不一），
|
||
拼接前统一规格可避免「成片体积失控」（实测 5 段原始素材 → 37 MB）。
|
||
"""
|
||
import subprocess
|
||
from lib.ffmpeg import FFMPEG
|
||
|
||
res = "1080x1920"
|
||
if storyboard and storyboard.shots:
|
||
res = (storyboard.shots[0].output.resolution or res)
|
||
try:
|
||
w, h = (int(x) for x in str(res).lower().split("x"))
|
||
except Exception:
|
||
w, h = 1080, 1920
|
||
|
||
cmd = [FFMPEG, "-y", "-i", str(src),
|
||
"-t", str(max_duration),
|
||
"-vf", (f"scale={w}:{h}:force_original_aspect_ratio=increase,"
|
||
f"crop={w}:{h},setsar=1"),
|
||
"-c:v", "libx264", "-preset", "medium", "-crf", "23",
|
||
"-pix_fmt", "yuv420p", "-r", "30", "-an", str(dest)]
|
||
r = subprocess.run(cmd, capture_output=True, text=True)
|
||
if r.returncode != 0:
|
||
raise PipelineError(f"规范化失败: {(r.stderr or '')[-200:]}")
|
||
return str(dest)
|
||
|
||
def render_from_clips(self, clip_paths: list[str], output_path: str,
|
||
storyboard: Storyboard | None = None,
|
||
apply_deai: bool = True,
|
||
progress_cb: ProgressCB = None) -> dict:
|
||
"""已有片段 → 去 AI 化 → 拼接 → 成片
|
||
|
||
返回 {output, clips, total_duration_est}
|
||
"""
|
||
if not clip_paths:
|
||
raise PipelineError("没有可用片段")
|
||
|
||
processed: list[str] = []
|
||
for i, c in enumerate(clip_paths, 1):
|
||
src = Path(c)
|
||
if not src.exists():
|
||
log.warning(" ⚠️ 片段不存在，跳过: %s", c)
|
||
continue
|
||
if progress_cb:
|
||
progress_cb("deai", f"clip_{i:02d}", "running", src.name)
|
||
# ① 先规范化（统一分辨率 + 裁到最长 N 秒）—— 避免 Pexels 等原始素材把成片撑大
|
||
norm = self.work_dir / f"norm_{i:02d}_{src.stem}.mp4"
|
||
try:
|
||
base = self._normalize_clip(str(src), str(norm), storyboard)
|
||
except Exception as e:
|
||
log.warning(" ⚠️ 规范化失败，用原片: %s", e)
|
||
base = str(src)
|
||
# ② 去 AI 化
|
||
if apply_deai:
|
||
out = self.work_dir / f"deai_{i:02d}_{src.stem}.mp4"
|
||
try:
|
||
self.de_ai.apply_to_storyboard(base, str(out), storyboard)
|
||
processed.append(str(out))
|
||
except Exception as e:
|
||
log.warning(" ⚠️ 去AI化失败，用规范化片: %s", e)
|
||
processed.append(base)
|
||
else:
|
||
processed.append(base)
|
||
if progress_cb:
|
||
progress_cb("deai", f"clip_{i:02d}", "done", "")
|
||
|
||
if not processed:
|
||
raise PipelineError("所有片段都不可用")
|
||
|
||
if progress_cb:
|
||
progress_cb("concat", "-", "running", f"{len(processed)} 段")
|
||
out_path = Path(output_path)
|
||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||
concat_segments(processed, str(out_path))
|
||
if progress_cb:
|
||
progress_cb("concat", "-", "done", str(out_path))
|
||
|
||
log.info(" 🎬 成片输出: %s（%d 段）", out_path.name, len(processed))
|
||
return {"output": str(out_path), "clips": processed,
|
||
"segments": len(processed)}
|
||
|
||
# ── 完整链路：分镜 → 成片 ─────────────────────────
|
||
|
||
def render(self, sb: Storyboard, output_path: str,
|
||
progress_cb: ProgressCB = None,
|
||
generate_fn: Optional[Callable] = None,
|
||
max_workers: int = 3,
|
||
reuse_existing: bool = False,
|
||
audio_first: bool = False,
|
||
lipsync: bool = False,
|
||
align_to_audio: bool = False,
|
||
sfx_from_video: bool = False) -> dict:
|
||
"""分镜 → 逐镜头生成 → 去AI化 → 拼接 → 成片
|
||
|
||
generate_fn(shot, storyboard, index) -> 片段路径
|
||
生成回调（由调用方注入具体引擎：可灵 / 即梦 / 素材库…）。
|
||
不传则用内置生成（可灵引擎）。
|
||
max_workers: 并发生成镜头数（默认 3）
|
||
reuse_existing: 续跑模式 —— work_dir 里已存在的 shot_NN.mp4 直接复用
|
||
audio_first: **音频先行** —— 先生成台词音频、按音频真实时长调整镜头时长，
|
||
再生成画面（口型/节奏才能对得上）；音频存 pre_audio/ 供混流复用
|
||
lipsync: **对口型** —— 仅对「有台词 + 有出场人物」的镜头做口型同步
|
||
（百炼 VideoRetalk，¥0.08/秒，1800 秒免费额度）；失败自动退回原片段
|
||
align_to_audio: **画面跟随音频对齐**（"剪画面"策略）—— 把「有台词音频、
|
||
但没做对口型」的镜头变速到音频长度；对口型镜头不动（口型已绑定）
|
||
"""
|
||
import threading
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
|
||
# ★ 音频先行：先生成台词音频 → 按**音频真实时长**调整镜头时长 → 再生成画面
|
||
# 这样口型与节奏才对得上（画面跟着声音走，而不是反过来）
|
||
# 生成的音频存到 work_dir/pre_audio/，后续混流直接复用（不重复 TTS）
|
||
if audio_first:
|
||
try:
|
||
from composer.dubbing import pre_synthesize
|
||
pre = pre_synthesize(sb, str(self.work_dir / "pre_audio"),
|
||
progress_cb=progress_cb)
|
||
log.info(" 🎙 音频先行：%d 个镜头时长已按台词音频调整（总时长 %.1fs）",
|
||
pre.get("updated", 0), sb.total_duration())
|
||
for _n in (pre.get("notes") or [])[:10]:
|
||
log.info(" · %s", _n)
|
||
except Exception as e:
|
||
log.warning(" ⚠️ 音频先行失败（继续按原时长生成画面）: %s", e)
|
||
|
||
results: dict[str, str] = {}
|
||
failed: list[str] = []
|
||
clips_by_idx: dict[int, str] = {} # 按镜头序号存，保证成片顺序正确
|
||
lock = threading.Lock()
|
||
|
||
# ★ 首尾帧衔接的依赖同步：链式镜头必须等上一镜生成完（拿它的尾帧）
|
||
_chain_events: dict = {i: threading.Event() for i in range(1, len(sb.shots) + 1)}
|
||
|
||
def _chain_required(idx: int, shot) -> bool:
|
||
"""本镜是否要求承接上一镜（tech_requirements.chain_from_prev）"""
|
||
if idx <= 1:
|
||
return False
|
||
tr = getattr(shot, "tech_requirements", None) or {}
|
||
if isinstance(tr, dict):
|
||
return bool(tr.get("chain_from_prev"))
|
||
return bool(getattr(tr, "chain_from_prev", False))
|
||
|
||
def _gen_one(idx: int, shot) -> None:
|
||
"""包装：链式依赖等待 + 完成后发信号（供下一镜取尾帧）"""
|
||
if _chain_required(idx, shot):
|
||
log.info(" 🔗 %s 需承接上一镜，等待其生成完成…", getattr(shot, "shot_id", idx))
|
||
_chain_events[idx - 1].wait(timeout=1800)
|
||
try:
|
||
_gen_one_inner(idx, shot)
|
||
finally:
|
||
_chain_events[idx].set()
|
||
|
||
def _gen_one_inner(idx: int, shot) -> None:
|
||
if progress_cb:
|
||
progress_cb("generate", shot.shot_id, "running", f"{idx}/{len(sb.shots)}")
|
||
# 续跑：复用已生成的镜头片段（跳过生成 = 不产生费用）
|
||
if reuse_existing:
|
||
# 优先复用「对口型后」的片段（若已做过），否则用原始生成片段
|
||
_cands = [self.work_dir / f"lipsync_{idx:02d}.mp4",
|
||
self.work_dir / f"shot_{idx:02d}.mp4"]
|
||
exist = next((c for c in _cands
|
||
if c.exists() and c.stat().st_size > 10_000), _cands[-1])
|
||
try:
|
||
if exist.exists() and exist.stat().st_size > 10_000:
|
||
shot.status = "done"
|
||
shot.artifact = str(exist)
|
||
with lock:
|
||
results[shot.shot_id] = str(exist)
|
||
clips_by_idx[idx] = str(exist)
|
||
log.info(" ♻️ 复用已有镜头 %s：%s", shot.shot_id, exist.name)
|
||
if progress_cb:
|
||
progress_cb("generate", shot.shot_id, "done", "复用已有片段")
|
||
return
|
||
except Exception:
|
||
pass
|
||
try:
|
||
clip = (generate_fn or self._generate_shot)(shot, sb, idx)
|
||
if not clip or not Path(clip).exists():
|
||
raise PipelineError("生成未返回有效片段")
|
||
shot.status = "done"
|
||
shot.artifact = str(clip)
|
||
with lock:
|
||
results[shot.shot_id] = str(clip)
|
||
clips_by_idx[idx] = str(clip) # 并发结果按序号归位
|
||
if progress_cb:
|
||
progress_cb("generate", shot.shot_id, "done", "")
|
||
except Exception as e:
|
||
shot.status = "failed"
|
||
shot.error = str(e)
|
||
with lock:
|
||
failed.append(f"{shot.shot_id}: {e}")
|
||
log.warning(" ⚠️ 镜头生成失败 %s: %s", shot.shot_id, e)
|
||
if progress_cb:
|
||
progress_cb("generate", shot.shot_id, "failed", str(e))
|
||
|
||
n = len(sb.shots)
|
||
if max_workers > 1 and n > 1:
|
||
log.info(" ⚡ 并发生成 %d 个镜头（并发度 %d）", n, min(max_workers, n))
|
||
with ThreadPoolExecutor(max_workers=min(max_workers, n)) as ex:
|
||
list(ex.map(lambda x: _gen_one(*x), list(enumerate(sb.shots, 1))))
|
||
else:
|
||
for i, shot in enumerate(sb.shots, 1):
|
||
_gen_one(i, shot)
|
||
|
||
clips = [clips_by_idx[i] for i in sorted(clips_by_idx)] # 恢复镜头顺序
|
||
if not clips:
|
||
raise PipelineError("所有镜头都生成失败：" + "; ".join(failed[:3]))
|
||
|
||
# ★ 对口型（仅「有台词 + 有出场人物」的镜头）—— 在做去AI化/拼接之前
|
||
if lipsync:
|
||
clips = self._apply_lipsync(sb, clips, progress_cb)
|
||
|
||
# ★ 视频生音效（可灵 video-to-audio · ¥0.25/镜头）—— 给静音成片补环境音
|
||
# 产出存 sfx_audio/{shot}.mp3，由 dubbing 按镜头时长混入
|
||
if sfx_from_video:
|
||
try:
|
||
self._apply_sfx_from_video(clips, progress_cb)
|
||
except Exception as e:
|
||
log.warning(" ⚠️ 音效流程失败（不影响出片）: %s", e)
|
||
|
||
# ★ 画面跟随音频对齐（"剪画面"策略）——仅对**没做对口型**的镜头
|
||
# 用户洞察：音画不同步时应动画面而非声音（音频变速会变调）
|
||
if align_to_audio:
|
||
try:
|
||
from composer.align import align_clips_to_audio
|
||
_ls_idx = set()
|
||
if lipsync:
|
||
for i, _s in enumerate(list(sb.shots), 1):
|
||
if ((_s.text or "").strip()
|
||
and [x for x in (getattr(_s, "cast_slots", None) or []) if x]):
|
||
_ls_idx.add(i) # 与 _apply_lipsync 的判据一致
|
||
clips = align_clips_to_audio(self.work_dir, sb, clips,
|
||
lipsync_indexes=_ls_idx,
|
||
progress_cb=progress_cb)
|
||
except Exception as e:
|
||
log.warning(" ⚠️ 画面跟随音频对齐失败（不影响出片）: %s", e)
|
||
|
||
# 去AI化 + 拼接
|
||
r = self.render_from_clips(clips, output_path, storyboard=sb, progress_cb=progress_cb)
|
||
sb.status = "done" if not failed else "partial"
|
||
sb.output_path = r["output"]
|
||
# 交付状态：诚实标注（未经完整播放审查，一律不算「合格成片」）
|
||
sb.delivery_status, sb.delivery_note = self.delivery_status(sb, failed)
|
||
return {**r, "failed": failed, "shot_artifacts": results,
|
||
"total_duration": sb.total_duration(),
|
||
"delivery_status": sb.delivery_status, "delivery_note": sb.delivery_note}
|
||
|
||
@staticmethod
|
||
def delivery_status(sb: Storyboard, failed: list, reviewed: bool = False) -> tuple[str, str]:
|
||
"""按 produce-ai-video 验收方法论判定交付状态（**诚实标注，不夸大**）
|
||
|
||
合格成片 —— 全部镜头成功 + 已完整播放审查（用户确认）
|
||
候选成片 —— 可播放，但有镜头失败 或 未经完整播放审查
|
||
未完成/待验证 —— 未生成 / 拼接失败
|
||
"""
|
||
if not sb.output_path:
|
||
return "未完成/待验证", "尚未生成成片"
|
||
if failed:
|
||
return "候选成片", f"{len(failed)} 个镜头失败（其余已出片，可重试失败镜头）"
|
||
if not reviewed:
|
||
return "候选成片", "已生成拼接，尚未完整播放审查"
|
||
return "合格成片", "全部镜头成功 + 已完整播放审查"
|
||
|
||
# ── 混流（视频 + 人声 + BGM + 字幕）────────────────
|
||
|
||
@staticmethod
|
||
def mux_final(video_path: str, output_path: str,
|
||
voice_path: str = "", bgm_path: str = "",
|
||
subtitle_path: str = "", bgm_volume: float = 0.15,
|
||
burn_subtitle: bool = True,
|
||
narration_path: str = "", narration_volume: float = 1.0) -> str:
|
||
"""把成片素材混流为最终视频（**两步法**）
|
||
|
||
⚠️ 实测发现：本机 ffmpeg 6.0 在「视频重编码(libx264) + 音频映射」同一条命令里
|
||
会**丢弃音频**（输出流声明了 aac 但编码 0 字节）；而 `-c:v copy` + 音频可正常保留。
|
||
|
||
故采用两步法（稳定）：
|
||
第 1 步：视频处理（烧字幕/重编码，**无音频**）→ 中间文件
|
||
第 2 步：把音频（人声 + BGM）以 `-c:v copy` 方式混入 → 最终文件
|
||
|
||
video_path 拼接后的画面（必填）
|
||
voice_path TTS 人声（可选）
|
||
bgm_path BGM（可选，自动压低 + 淡入淡出）
|
||
subtitle_path ASS 字幕（可选，烧录进画面）
|
||
"""
|
||
import subprocess
|
||
from lib.ffmpeg import FFMPEG
|
||
|
||
out = Path(output_path)
|
||
out.parent.mkdir(parents=True, exist_ok=True)
|
||
|
||
# ── 第 1 步：视频（可选烧字幕）→ 中间文件 ──
|
||
need_video_pass = bool(burn_subtitle and subtitle_path and Path(subtitle_path).exists())
|
||
if need_video_pass:
|
||
tmp_video = out.parent / f"_{out.stem}_v.mp4"
|
||
sub_esc = str(subtitle_path).replace("\\", "/").replace(":", "\\:")
|
||
cmd1 = [FFMPEG, "-y", "-i", str(video_path),
|
||
"-vf", f"ass='{sub_esc}'",
|
||
"-c:v", "libx264", "-preset", "medium", "-crf", "23",
|
||
"-pix_fmt", "yuv420p", "-an", str(tmp_video)]
|
||
r1 = subprocess.run(cmd1, capture_output=True, text=True)
|
||
if r1.returncode != 0:
|
||
raise PipelineError(f"字幕烧录失败: {(r1.stderr or '')[-300:]}")
|
||
src_video = str(tmp_video)
|
||
else:
|
||
src_video = str(video_path)
|
||
|
||
# ── 若无音频需求：直接返回（第 1 步产物或原片）──
|
||
has_voice = bool(voice_path and Path(voice_path).exists())
|
||
has_narr = bool(narration_path and Path(narration_path).exists())
|
||
has_bgm = bool(bgm_path and Path(bgm_path).exists())
|
||
if not (has_voice or has_narr or has_bgm):
|
||
if src_video != str(out):
|
||
import shutil
|
||
shutil.copy2(src_video, out)
|
||
return str(out)
|
||
|
||
# ── 第 2 步：混音频（视频 **copy**，音频重编码）──
|
||
# 支持三路：台词（voice）+ 旁白（narration，独立轨）+ BGM
|
||
cmd2 = [FFMPEG, "-y", "-i", src_video]
|
||
idx = 1
|
||
v_i = n_i = b_i = -1
|
||
if has_voice:
|
||
cmd2 += ["-i", str(voice_path)]
|
||
v_i = idx; idx += 1
|
||
if has_narr:
|
||
cmd2 += ["-i", str(narration_path)]
|
||
n_i = idx; idx += 1
|
||
if has_bgm:
|
||
cmd2 += ["-i", str(bgm_path)]
|
||
b_i = idx; idx += 1
|
||
cmd2 += ["-map", "0:v:0"]
|
||
|
||
parts: list[str] = []
|
||
labels: list[str] = []
|
||
if v_i > 0:
|
||
parts.append(f"[{v_i}:a]volume=1.0[v]")
|
||
labels.append("[v]")
|
||
if n_i > 0:
|
||
parts.append(f"[{n_i}:a]volume={narration_volume}[n]")
|
||
labels.append("[n]")
|
||
if b_i > 0:
|
||
parts.append(f"[{b_i}:a]volume={bgm_volume},afade=t=in:st=0:d=1[b]")
|
||
labels.append("[b]")
|
||
|
||
if len(labels) > 1:
|
||
parts.append(f"{''.join(labels)}amix=inputs={len(labels)}:duration=first:normalize=0[aout]")
|
||
cmd2 += ["-filter_complex", ";".join(parts), "-map", "[aout]"]
|
||
elif len(labels) == 1:
|
||
# 单路：也走 filter_complex，统一音量处理
|
||
cmd2 += ["-filter_complex", ";".join(parts + [f"{labels[0]}anull[aout]"]),
|
||
"-map", "[aout]"]
|
||
else:
|
||
cmd2 += ["-map", "1:a"]
|
||
|
||
cmd2 += ["-c:v", "copy", "-c:a", "aac", "-b:a", "192k",
|
||
"-movflags", "+faststart", str(out)]
|
||
r2 = subprocess.run(cmd2, capture_output=True, text=True)
|
||
if r2.returncode != 0:
|
||
raise PipelineError(f"音频混流失败: {(r2.stderr or '')[-300:]}")
|
||
|
||
# 校验音轨确实写入了（防 ffmpeg 静默丢音轨）—— 用 ffmpeg -i 输出判断（最兼容）
|
||
try:
|
||
chk = subprocess.run([FFMPEG, "-i", str(out)],
|
||
capture_output=True, text=True, timeout=30)
|
||
info = (chk.stderr or "") + (chk.stdout or "")
|
||
if "Audio:" not in info:
|
||
raise PipelineError("音频混流后未检测到音轨（可尝试重新生成 TTS 音频）")
|
||
except FileNotFoundError:
|
||
pass # 无 ffmpeg 时跳过校验
|
||
|
||
# 清理中间文件（烧字幕的临时视频），保持产物目录干净
|
||
if need_video_pass:
|
||
try:
|
||
Path(tmp_video).unlink(missing_ok=True)
|
||
except Exception:
|
||
pass
|
||
|
||
log.info(" 🎬 混流完成: %s（人声=%s BGM=%s 字幕=%s）",
|
||
out.name, has_voice, has_bgm, need_video_pass)
|
||
return str(out)
|
||
|
||
# ── 生成环节（待接入引擎）──────────────────────────
|
||
|
||
def _generate_via_dashscope(self, shot, storyboard: Storyboard, prompt: str, dest: str,
|
||
ref_image: str, tail_image: str, want_image: bool) -> str:
|
||
"""走百炼（万相 KF2V）生成这一镜 —— ¥0.1/秒，可灵的 1/6
|
||
|
||
⚠️ 约束：
|
||
· 首尾帧模型**必须有尾帧**（没有就报错，不静默降级 —— 免得花了钱效果不对）
|
||
· 时长固定 5 秒（官方规格）
|
||
· 不支持 camera_control / aspect_ratio（这些是可灵参数）
|
||
"""
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||
from capabilities import dashscope_video as dv
|
||
if not dv.available():
|
||
raise PipelineError("百炼通道未配置 key（agent-local/tools/ave/config/local.yaml → aliyun.api_key）")
|
||
if not want_image or not ref_image:
|
||
raise PipelineError("百炼 KF2V 需要首帧 —— 本镜没有可用的参考图/关键帧")
|
||
if not tail_image:
|
||
raise PipelineError(
|
||
"百炼 KF2V 必须有尾帧。请在分镜里给这一镜设置 tail_mode，"
|
||
"或改用可灵（tech_requirements.video_model 留空）")
|
||
log.info(" ☁️ %s 走百炼 KF2V（¥0.1/秒）", shot.shot_id)
|
||
dv.generate_kf2v(ref_image, tail_image, dest, prompt=prompt,
|
||
progress_cb=lambda s: log.info(" %s", s))
|
||
return dest
|
||
|
||
def _generate_shot(self, shot, storyboard: Storyboard, index: int) -> str:
|
||
"""按镜头生成片段（可灵引擎）
|
||
|
||
生成能力决定（`shot.generation_capability` 优先，为空则自动判定）：
|
||
· `image_to_video` 或（自动 + 绑了角色）→ **图生视频**（用角色参考图保住人脸一致性）
|
||
· 其他 → **文生视频**
|
||
"""
|
||
try:
|
||
from capabilities.kling_video import KlingVideoClient, KlingError
|
||
except ImportError as e:
|
||
raise PipelineError(f"生成能力未就绪: {e}")
|
||
|
||
client = KlingVideoClient()
|
||
if not client.available:
|
||
raise PipelineError(
|
||
"可灵未配置 —— 请到 Dashboard → ⚙️ 服务 → 🔑 API 配置 填入可灵 api_key"
|
||
)
|
||
|
||
dest = self.work_dir / f"shot_{index:02d}.mp4"
|
||
# 可灵 duration 仅支持 5 / 10 秒
|
||
# ★ 时长档位：可灵只有 5 秒 / 10 秒两档 → 超过 5 秒必须用 10 秒档
|
||
# （原实现阈值是 7 秒 → 6 秒的镜头会生成 5 秒，比脚本短 1 秒 → 拼接时穿帮）
|
||
dur = "10" if (shot.duration or 0) > 5.2 else "5"
|
||
|
||
# ── 决定 图生视频 / 文生视频 ──
|
||
cap = (getattr(shot, "generation_capability", "") or "").lower()
|
||
# ★ 是否"绑了角色"要同时看多角色演员表（cast_slots）
|
||
# 旧实现只看 character_id → 用演员表的分镜 has_char=False → 不走图生视频
|
||
_cast_slots = [s for s in (getattr(shot, "cast_slots", None) or []) if s]
|
||
has_char = bool((getattr(shot, "character_id", "") or "").strip() or _cast_slots)
|
||
# ★ 是否用参考图：
|
||
# · 明确指定 image_to_video → 用
|
||
# · 明确指定 text_to_video / vfx / comfyui → 不用（尊重用户选择）
|
||
# · talking_head 或未指定 → 只要有角色就用（口播镜头人脸一致性最关键！
|
||
# 且后续对口型也基于这个画面）
|
||
if cap in ("text_to_video", "vfx", "comfyui"):
|
||
want_image = False
|
||
else:
|
||
want_image = (cap == "image_to_video") or has_char
|
||
ref_image = ""
|
||
# ★ 环境镜头也走图生视频（用场景参考图保环境）—— 解决"画面没有环境"的问题
|
||
_loc_ref = self._resolve_location_ref(shot, storyboard)
|
||
|
||
# ★★ 关键帧优先（第②步 → 第③步的衔接点）★★
|
||
# 这一镜若在「关键帧工作台」里有【已采用】的关键帧，直接拿它当首帧 ——
|
||
# 关键帧是用户审过的静帧，比角色定妆照更接近成片，也让用户的审核真正生效。
|
||
_kf, _kf_src = self._keyframe_of_shot(getattr(storyboard, "project_id", ""), shot.shot_id)
|
||
if _kf:
|
||
ref_image = _kf
|
||
want_image = True
|
||
_src_txt = "已采用的关键帧" if _kf_src == "adopted" else "关键帧（未点采用·取最新版）"
|
||
log.info(" 🖼️ 镜头 %s 用【%s】作首帧：%s", shot.shot_id, _src_txt, Path(_kf).name)
|
||
elif want_image or _loc_ref:
|
||
ref_image = self._resolve_reference_image(shot, storyboard)
|
||
if not ref_image:
|
||
if cap == "image_to_video":
|
||
log.warning(" ⚠️ 镜头 %s 指定了图生视频，但未取到参考图 → 退化为文生视频",
|
||
shot.shot_id)
|
||
want_image = False
|
||
else:
|
||
_kind = "场景图" if (_loc_ref and ref_image == _loc_ref) else "角色图"
|
||
log.info(" 🖼️ 镜头 %s 用图生视频（%s：%s）", shot.shot_id, _kind, Path(ref_image).name)
|
||
|
||
# ── 特效（vfx）→ 运镜控制 ──
|
||
vfx_type, vfx_params = "", None
|
||
_vfx = getattr(shot, "vfx", None)
|
||
if _vfx is not None:
|
||
vfx_type = (getattr(_vfx, "vfx_type", "") or "").strip()
|
||
vfx_params = getattr(_vfx, "params", None)
|
||
if not vfx_type:
|
||
# 兜底：镜头语言里出现"环绕/推/拉"等词 → 映射成运镜预设
|
||
_move = (getattr(getattr(shot, "shot_contract", None), "camera_move", "") or "")
|
||
if "环绕" in _move:
|
||
vfx_type = "orbit"
|
||
elif "推" in _move:
|
||
vfx_type = "forward_up"
|
||
elif "拉" in _move:
|
||
vfx_type = "down_back"
|
||
|
||
# ★ 提示词编译（分支不同：图生视频只写动作，不重描述视觉 —— 可灵官方守则）
|
||
prompt = self.build_shot_prompt(shot, storyboard, has_ref_image=bool(want_image))
|
||
|
||
# ★ 去 AI 感：按「拍摄风格包 + 真实感强度」增强提示词
|
||
# mild → 追加真实质感锚点（噪点/毛孔/不匀光）
|
||
# strong→ 额外剔除「4K/超清/无瑕/大师作品」等反真实词 + 反完美总纲
|
||
try:
|
||
from composer.realism import apply_style_pack
|
||
_ga = getattr(storyboard, "global_anchors", None)
|
||
_style = (getattr(_ga, "shooting_style", "") or "")
|
||
_boost = (getattr(_ga, "realism_boost", "mild") or "mild")
|
||
if _style:
|
||
prompt, _rnotes = apply_style_pack(prompt, _style, _boost)
|
||
for _rn in _rnotes:
|
||
log.info(" ✨ 去AI感[%s/%s]: %s", _style, _boost, _rn)
|
||
# ★ 关闭「AI 默认审美」的负面约束（研究：AI 感的头号来源）
|
||
try:
|
||
from composer.realism import ai_negative_block
|
||
_neg = ai_negative_block(hard=(_boost == "strong"))
|
||
if _neg and "关闭 AI 默认审美" not in prompt:
|
||
prompt = prompt.rstrip("。. ") + "。" + _neg
|
||
except Exception as _ne:
|
||
log.debug("负面约束注入失败: %s", _ne)
|
||
except Exception as _e:
|
||
log.debug("风格包应用失败（忽略）: %s", _e)
|
||
|
||
# ★ 首尾帧衔接（chain_from_prev）：让本镜**从上一镜的收尾画面开始** → 无缝接续
|
||
# ⚠️ 语义澄清（2026-10-03 修正）：
|
||
# 可灵的 `image_tail` = 指定视频的**结束画面**，用它做接续是反的 ✗
|
||
# 正确做法：把上一镜的**尾帧当作本镜的首帧**（image 参数）✓
|
||
_tail_image = ""
|
||
try:
|
||
_tr = getattr(shot, "tech_requirements", None) or {}
|
||
_chain = bool(_tr.get("chain_from_prev")) if isinstance(_tr, dict) else False
|
||
if _chain and index > 1:
|
||
from composer.frames import extract_tail_frame
|
||
_prev = None
|
||
# 优先取"已后处理"的片段（对口型 > 去AI化 > 原始），衔接点更准
|
||
for _c in (self.work_dir / f"lipsync_{index-1:02d}.mp4",
|
||
self.work_dir / f"deai_{index-1:02d}.mp4",
|
||
self.work_dir / f"shot_{index-1:02d}.mp4"):
|
||
if _c.exists() and _c.stat().st_size > 10_000:
|
||
_prev = _c
|
||
break
|
||
if _prev:
|
||
_prev_tail = extract_tail_frame(
|
||
str(_prev), str(self.work_dir / "chain" / f"tail_{index-1:02d}.jpg"))
|
||
if ref_image and Path(ref_image).exists():
|
||
# 有角色参考图：人脸一致优先，不用尾帧当首帧（两者冲突，只能二选一）
|
||
log.info(" 🔗 %s 需承接上一镜，但已有角色参考图 → 优先保人脸（未用尾帧衔接）",
|
||
shot.shot_id)
|
||
else:
|
||
# 无参考图：用上一镜尾帧当**首帧** → 画面从上一镜收尾处接着走 ✓
|
||
ref_image, want_image = _prev_tail, True
|
||
log.info(" 🔗 %s 以上一镜尾帧为首帧衔接：%s",
|
||
shot.shot_id, Path(_prev_tail).name)
|
||
except Exception as e:
|
||
log.warning(" ⚠️ 首尾帧衔接失败（按普通方式生成）: %s", e)
|
||
|
||
# ★ 原生有声（sound=on）：一次生成「带语音的视频」→ 音画天然同步，
|
||
# 不再需要后期 TTS / 对口型。按镜头开：tech_requirements.native_sound
|
||
_tr_sound = (getattr(shot, "tech_requirements", None) or {}).get("native_sound")
|
||
_sound = "on" if _tr_sound else "off"
|
||
if _sound == "on":
|
||
log.info(" 🔊 %s 使用「原生有声」生成（无需后期配音/口型）", shot.shot_id)
|
||
|
||
# ★ 出片引擎分流（tech_requirements.video_model，留空 = 可灵默认）
|
||
# 为什么：可灵 ¥0.6/秒，百炼 kf2v ¥0.1/秒 —— 同一部片子差 6 倍成本。
|
||
# 分流逻辑与「🎞 视频」标签保持一致，都用 config/video_models.yaml 的 channel。
|
||
_tr_model = (getattr(shot, "tech_requirements", None) or {}).get("video_model") or ""
|
||
_channel = video_channel_of(_tr_model)
|
||
if _channel == "dashscope":
|
||
out_path = self._generate_via_dashscope(
|
||
shot, storyboard, prompt, dest, ref_image, _tail_image, want_image)
|
||
else:
|
||
try:
|
||
out_path = client.generate_video(prompt, dest, image_path=ref_image if want_image else "",
|
||
duration=dur,
|
||
aspect_ratio=self._aspect_ratio(storyboard),
|
||
vfx_type=vfx_type, vfx_params=vfx_params,
|
||
tail_image_path=_tail_image,
|
||
sound=_sound)
|
||
except KlingError as e:
|
||
raise PipelineError(str(e))
|
||
|
||
# ── 变速（慢动作 / 快进）：**零成本**本地 FFmpeg 处理 ──
|
||
speed = float(getattr(getattr(shot, "speed", None), "factor", 1.0) or 1.0)
|
||
if abs(speed - 1.0) > 0.01:
|
||
try:
|
||
from composer.speed_ramp import speed_ramp
|
||
ramped = self.work_dir / f"shot_{index:02d}_x{speed}.mp4"
|
||
speed_ramp(str(out_path), str(ramped), speed=speed)
|
||
log.info(" 🎞️ 镜头 %s 变速 %.2fx → %s", shot.shot_id, speed, ramped.name)
|
||
return str(ramped)
|
||
except Exception as e:
|
||
log.warning(" ⚠️ 变速失败（用原片）: %s", e)
|
||
return str(out_path)
|
||
|
||
def _apply_sfx_from_video(self, clips: list, progress_cb: ProgressCB = None) -> list:
|
||
"""给镜头生成**音效**（可灵 video-to-audio · ¥0.25/镜头）
|
||
|
||
背景：我们的成片默认**完全静音** —— 这是最大的"AI 感"来源之一。
|
||
可灵提供「视频生音效」接口：给视频自动生成环境音 + BGM。
|
||
|
||
· 依赖镜头生成时记录的 task_id（sidecar `{shot}.mp4.taskid`）
|
||
· 产出存 `sfx_audio/{shot_id}.mp3`（由 dubbing 在混音时按镜头时长混入）
|
||
· 已有产物则跳过（断点续跑，不重复扣费）
|
||
· 任一镜头失败不影响其他镜头（失败仅告警）
|
||
|
||
⚠️ 成本 ¥0.25 × 镜头数（5 镜头 ≈ ¥1.25）；每镜头耗时 30~90 秒。
|
||
"""
|
||
sfx_dir = self.work_dir / "sfx_audio"
|
||
sfx_dir.mkdir(parents=True, exist_ok=True)
|
||
try:
|
||
from capabilities.kling_video import KlingVideoClient
|
||
except Exception as e:
|
||
log.warning(" ⚠️ 音效模块不可用（跳过）: %s", e)
|
||
return clips
|
||
|
||
client = KlingVideoClient()
|
||
done = skipped = failed = 0
|
||
for idx, clip in enumerate(clips, 1):
|
||
tid = KlingVideoClient.load_task_id(clip)
|
||
if not tid:
|
||
log.debug(" 镜头%d 无 task_id，跳过音效", idx)
|
||
continue
|
||
dest = sfx_dir / f"shot_{idx:02d}.mp3"
|
||
if dest.exists() and dest.stat().st_size > 1000:
|
||
skipped += 1
|
||
continue
|
||
if progress_cb:
|
||
progress_cb("sfx", f"shot_{idx:02d}", "running", "生成音效")
|
||
try:
|
||
client.generate_sfx_for_video(tid, str(dest))
|
||
done += 1
|
||
if progress_cb:
|
||
progress_cb("sfx", f"shot_{idx:02d}", "done", "音效就绪")
|
||
except Exception as e:
|
||
failed += 1
|
||
log.warning(" ⚠️ 镜头%d 音效生成失败（不影响出片）: %s", idx, str(e)[:120])
|
||
|
||
if done or skipped:
|
||
log.info(" 🎧 音效：新增 %d · 复用 %d · 失败 %d（¥%.2f）",
|
||
done, skipped, failed, done * 0.25)
|
||
return clips
|
||
|
||
def _apply_lipsync(self, sb, clips: list, progress_cb: ProgressCB = None) -> list:
|
||
"""对口型：对「有台词 + 有出场人物」的镜头做口型同步（百炼 VideoRetalk）
|
||
|
||
· 仅 A 类镜头（有台词且有出场人物）；旁白/空镜不动
|
||
· 音频复用「音频先行」产物（pre_audio/{shot_id}.mp3）—— 不重复 TTS
|
||
· 参考图用角色资产（提升人脸匹配）
|
||
· **失败自动退回原片段**（不影响出片）
|
||
· 成本 ¥0.08/秒（1800 秒免费额度内免费）
|
||
"""
|
||
try:
|
||
from composer.lipsync import lipsync_videoretalk, vr_estimate_cost
|
||
except Exception as e:
|
||
log.warning(" ⚠️ 对口型模块不可用（跳过）: %s", e)
|
||
return clips
|
||
pre_dir = self.work_dir / "pre_audio"
|
||
out = list(clips)
|
||
done = 0
|
||
for idx, shot in enumerate(list(getattr(sb, "shots", []) or []), 1):
|
||
text = (getattr(shot, "text", "") or "").strip()
|
||
slots = [s for s in (getattr(shot, "cast_slots", None) or []) if s]
|
||
if not (text and slots):
|
||
continue # 非 A 类 → 跳过
|
||
if idx - 1 >= len(out):
|
||
continue
|
||
src = Path(out[idx - 1])
|
||
sid = getattr(shot, "shot_id", f"shot_{idx:02d}")
|
||
audio = pre_dir / f"{sid}.mp3"
|
||
if not (src.exists() and audio.exists()):
|
||
log.warning(" ⚠️ 对口型跳过 %s（缺视频或先行音频）", src.name)
|
||
continue
|
||
ref = self._resolve_character_ref(shot, sb)
|
||
dur = float(getattr(shot, "duration", 0) or 0)
|
||
dst = self.work_dir / f"lipsync_{idx:02d}.mp4"
|
||
log.info(" 🎭 对口型 %s（%.1fs · 约 ¥%.2f · 走免费额度优先）",
|
||
src.name, dur, vr_estimate_cost(dur))
|
||
if progress_cb:
|
||
progress_cb("lipsync", sid, "running", "口型同步中")
|
||
try:
|
||
lipsync_videoretalk(str(src), str(audio), str(dst),
|
||
ref_image_path=ref,
|
||
progress_cb=lambda p, n: progress_cb("lipsync", sid, "running", n)
|
||
if progress_cb else None)
|
||
out[idx - 1] = str(dst) # 用对口型后的片段
|
||
done += 1
|
||
if progress_cb:
|
||
progress_cb("lipsync", sid, "done", "已对口型")
|
||
log.info(" ✅ %s 对口型完成", sid)
|
||
except Exception as e:
|
||
log.warning(" ⚠️ 对口型失败（保留原片段）: %s", str(e)[:120])
|
||
if progress_cb:
|
||
progress_cb("lipsync", sid, "failed", str(e)[:60])
|
||
log.info(" 🎭 对口型完成：%d 个镜头", done)
|
||
return out
|
||
|
||
# ── 关键帧（第②步产物）────────────────────────────────
|
||
@staticmethod
|
||
def _keyframe_root() -> Path:
|
||
"""关键帧根目录（agent-local/runtime/ave/keyframes）
|
||
|
||
⚠️ 路径层级：__file__ = agent-sync/05_tools/09_ave/scripts/composer/pipeline.py
|
||
parents[0]=composer [1]=scripts [2]=09_ave [3]=05_tools [4]=agent-sync
|
||
→ agent-local 与 agent-sync 同级，即 parents[4].parent/"agent-local"
|
||
"""
|
||
import os as _os
|
||
base = _os.environ.get("AGENT_LOCAL")
|
||
root = Path(base) if base else (Path(__file__).resolve().parents[4].parent / "agent-local")
|
||
return root / "runtime" / "ave" / "keyframes"
|
||
|
||
@classmethod
|
||
def _keyframe_of_shot(cls, project_id: str, shot_id: str,
|
||
fallback: bool = True) -> tuple[str, str]:
|
||
"""取该镜头的关键帧 → (图片路径, 来源)
|
||
|
||
来源取值：
|
||
"adopted" 用户明确「采用」的那一版（最可信）
|
||
"latest" 未采用，回退到最新一版（用户生成了但没点采用）
|
||
"" 完全没有关键帧
|
||
|
||
fallback=True 时，"生成了但没点采用" 也算可用 ——
|
||
符合用户直觉（我生成了就该能用），调用方可根据来源决定是否提示。
|
||
"""
|
||
if not (project_id and shot_id):
|
||
return "", ""
|
||
sdir = cls._keyframe_root() / str(project_id) / str(shot_id)
|
||
if not sdir.is_dir():
|
||
return "", ""
|
||
import json as _json
|
||
latest = ""
|
||
for jf in sorted(sdir.glob("v*.json")):
|
||
try:
|
||
m = _json.loads(jf.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
continue
|
||
png = jf.with_suffix(".png")
|
||
if not png.exists():
|
||
continue
|
||
if m.get("adopted"):
|
||
return str(png), "adopted"
|
||
latest = str(png) # 保留最后一个（最新的）作为回退
|
||
if fallback and latest:
|
||
return latest, "latest"
|
||
return "", ""
|
||
|
||
@classmethod
|
||
def _adopted_keyframe(cls, project_id: str, shot_id: str) -> str:
|
||
"""兼容旧调用：只要路径（优先 adopted，无则回退最新版）"""
|
||
p, _ = cls._keyframe_of_shot(project_id, shot_id, fallback=True)
|
||
return p
|
||
|
||
@staticmethod
|
||
def _ref_image_of_character(cid: str, shot_size: str = "", angle: str = "") -> str:
|
||
"""从角色资产取参考图 —— **按镜头景别/角度选最合适的那张**
|
||
|
||
★ 2026-10-04（用户指出）：套装生成了 5~7 张，但旧实现只取 front 一张，
|
||
其余全部闲置。现在按镜头类型选：
|
||
· 全景/远景 → 全身图 · 特写 → 面部特写
|
||
· 侧面/45° → 侧视图 · 背面/回头 → 回头斜侧
|
||
· 其他 → 半身正面（主定妆）
|
||
这样在"可灵只能吃一张图"的限制下把一致性做到最好。
|
||
"""
|
||
cid = (cid or "").strip()
|
||
if not cid:
|
||
return ""
|
||
try:
|
||
from assets_manager import AssetStore
|
||
st = AssetStore("characters")
|
||
item = st.get(cid) or {}
|
||
refs = item.get("reference_images") or {}
|
||
base = Path(st.kind_dir(cid))
|
||
|
||
# ★ 先按镜头类型选（让套装每张都发挥作用）
|
||
try:
|
||
from character_generator.portrait_set import pick_reference_for_shot
|
||
slot = pick_reference_for_shot(shot_size, angle, refs)
|
||
if slot and refs.get(slot):
|
||
p = base / Path(str(refs[slot])).name
|
||
if p.exists():
|
||
log.debug(" 参考图按镜头选择: %s（景别=%s 角度=%s）",
|
||
slot, shot_size or "-", angle or "-")
|
||
return str(p)
|
||
except Exception as e:
|
||
log.debug("按镜头选参考图失败，回退通用顺序: %s", e)
|
||
|
||
# 回退：通用槽位顺序（向后兼容旧角色资产）
|
||
for key in ("front", "front_face", "portrait", "side", "full"):
|
||
if refs.get(key):
|
||
p = base / Path(str(refs[key])).name
|
||
if p.exists():
|
||
return str(p)
|
||
for fn in refs.values():
|
||
if fn:
|
||
p = base / Path(str(fn)).name
|
||
if p.exists():
|
||
return str(p)
|
||
except Exception as e:
|
||
log.debug("取角色参考图失败（%s）: %s", cid, e)
|
||
return ""
|
||
|
||
@classmethod
|
||
def _resolve_character_ref(cls, shot, storyboard=None) -> str:
|
||
"""取镜头所绑角色的**参考图本地路径**（用于图生视频，保人脸一致性）
|
||
|
||
★ 支持多角色演员表（2026-10-03 修复）
|
||
旧实现只看 shot.character_id → 用「演员表」的分镜 character_id 为空
|
||
→ 每个镜头都退化为文生视频 → **每次生成的人脸随机**（用户实测发现）
|
||
解析顺序：
|
||
① shot.cast_slots + storyboard.anchors.cast → 按演员表顺序取 asset_id（主角通常在前）
|
||
② 回退 shot.character_id（单角色，向后兼容）
|
||
多角色同框时图生视频只能喂一张参考图 → 取**第一个有参考图的出场角色**
|
||
"""
|
||
slots = [s for s in (getattr(shot, "cast_slots", None) or []) if s]
|
||
ga = getattr(storyboard, "global_anchors", None) if storyboard is not None else None
|
||
cast = (getattr(ga, "cast", None) or []) if ga is not None else []
|
||
cids: list[str] = []
|
||
if slots and cast:
|
||
for m in cast: # 按演员表顺序（主角在前）
|
||
if getattr(m, "slot", "") in slots and getattr(m, "asset_id", ""):
|
||
cids.append(m.asset_id)
|
||
if not cids and (getattr(shot, "character_id", "") or "").strip():
|
||
cids.append(shot.character_id.strip())
|
||
# 镜头景别/角度 → 供「按镜头类型选参考图」使用
|
||
_sc = getattr(shot, "shot_contract", None)
|
||
_size = str(getattr(_sc, "shot_size", "") or "") if _sc else ""
|
||
_angle = str(getattr(_sc, "angle", "") or "") if _sc else ""
|
||
for cid in cids:
|
||
p = cls._ref_image_of_character(cid, shot_size=_size, angle=_angle)
|
||
if p:
|
||
return p
|
||
if slots and not cids:
|
||
log.debug("镜头 %s 出场 %s 但未绑定角色资产 → 无参考图（退化为文生视频）",
|
||
getattr(shot, "shot_id", "?"), slots)
|
||
return ""
|
||
|
||
@classmethod
|
||
def _resolve_reference_image(cls, shot, storyboard=None) -> str:
|
||
"""★ 按镜头类型选参考图（零成本，不依赖多图 API）
|
||
|
||
背景（2026-10-03 研究结论）：可灵官方守则要求"给每张参考图指定明确角色"，
|
||
而我们的 API 只能传一张 → 改为**按镜头类型分配**：
|
||
· 环境镜头（无出场人物 / 远景·全景）→ 优先**场景参考图**（保环境）
|
||
· 人物镜头（有出场 / 近景·特写） → **角色参考图**（保人脸）
|
||
两侧都没有 → 返回空（走文生视频）
|
||
|
||
判据顺序：
|
||
① 有出场人物 且 景别是近景/特写 → 角色图
|
||
② 无出场人物 或 景别是远景/全景 → 场地图（无则退回角色图）
|
||
③ 其余 → 角色图
|
||
"""
|
||
size = ""
|
||
sc = getattr(shot, "shot_contract", None)
|
||
if sc is not None:
|
||
size = str(getattr(sc, "shot_size", "") or "")
|
||
slots = [s for s in (getattr(shot, "cast_slots", None) or []) if s]
|
||
has_char = bool(slots) or bool((getattr(shot, "character_id", "") or "").strip())
|
||
|
||
_CLOSE = ("特写", "近景", "close", "extreme")
|
||
_WIDE = ("全景", "远景", "wide", "long", "establish")
|
||
is_close = any(k in size for k in _CLOSE)
|
||
is_wide = any(k in size for k in _WIDE)
|
||
|
||
char_ref = cls._resolve_character_ref(shot, storyboard)
|
||
loc_ref = cls._resolve_location_ref(shot, storyboard)
|
||
|
||
# ① 人物特写/近景 → 角色图（保脸最重要）
|
||
if has_char and is_close and char_ref:
|
||
return char_ref
|
||
# ② 空镜 / 远景全景 → 场地图（保环境）
|
||
if (not has_char) or is_wide:
|
||
if loc_ref:
|
||
return loc_ref
|
||
if char_ref:
|
||
return char_ref
|
||
# ③ 其余 → 角色图优先，次选场地图
|
||
return char_ref or loc_ref
|
||
|
||
@classmethod
|
||
def _resolve_location_ref(cls, shot, storyboard=None) -> str:
|
||
"""取镜头所绑场景的**参考图本地路径**（保环境一致性）
|
||
|
||
场景来源：shot.location_id 优先，其次 storyboard.global_anchors.location_id
|
||
（支持 locations / real_locations 两类资产）
|
||
"""
|
||
import sys as _sys
|
||
if str(_SCRIPTS) not in _sys.path:
|
||
_sys.path.insert(0, str(_SCRIPTS))
|
||
ga = getattr(storyboard, "global_anchors", None) if storyboard is not None else None
|
||
lid = ((getattr(shot, "location_id", "") or "").strip()
|
||
or ((getattr(ga, "location_id", "") or "").strip() if ga is not None else ""))
|
||
if not lid:
|
||
return ""
|
||
try:
|
||
from assets_manager import AssetStore
|
||
except Exception:
|
||
return ""
|
||
for kind in ("locations", "real_locations"):
|
||
try:
|
||
st = AssetStore(kind)
|
||
it = st.get(lid)
|
||
if not it:
|
||
# ★ 兼容 id 规范化差异（2026-10-04）：
|
||
# 资产 slug 会把下划线转成横杠（"病房_302" → "病房-302"）
|
||
# → 分镜里手写的 location_id 会查不到，这里再兜一层
|
||
_norm = lambda s: str(s or "").replace("-", "_") # noqa: E731
|
||
it = st.get(str(lid).replace("_", "-"))
|
||
if not it:
|
||
for _x in st.list():
|
||
if (_norm(_x.get("id")) == _norm(lid)
|
||
or str(_x.get("name") or "") == str(lid)):
|
||
it = _x
|
||
break
|
||
if not it:
|
||
continue
|
||
_aid = str(it.get("id") or lid) # 用资产**实际 id** 拼目录
|
||
refs = it.get("reference_images") or {}
|
||
# 槽位优先级：wide（广角全景）> main > front > scene
|
||
# ⚠️ 元数据里存的是**相对文件名**（如 ref_wide.png），需拼上资产目录
|
||
def _abs_ref(v):
|
||
p = Path(str(v))
|
||
if p.is_absolute() and p.exists():
|
||
return p
|
||
q = Path(st.kind_dir(_aid)) / p.name
|
||
return q if q.exists() else None
|
||
|
||
# ★ 场景图也按镜头机位角度选（2026-10-04）：
|
||
# 俯视镜头 → top 图；侧向镜头 → side 图；正面 → front；默认 → wide（广角全景）
|
||
# 场景资产可上传多角度图：wide / top / front / side / back / detail
|
||
_ang = ""
|
||
try:
|
||
_sc2 = getattr(shot, "shot_contract", None)
|
||
_ang = str(getattr(_sc2, "angle", "") or "") if _sc2 else ""
|
||
except Exception:
|
||
_ang = ""
|
||
_LOC_ANGLE_SLOTS = (
|
||
(("俯视", "顶", "鸟瞰", "top", "bird"), ("top", "wide")),
|
||
(("仰视", "low"), ("front", "wide")),
|
||
(("侧", "45", "三分", "side", "profile"), ("side", "left", "right", "wide")),
|
||
(("过肩", "背后", "behind", "从后"), ("back", "wide")),
|
||
)
|
||
_pref: tuple = ()
|
||
# ★ 景别优先：特写/近景 → detail 图（否则会被 wide 抢先）
|
||
_size_s = ""
|
||
try:
|
||
_sc3 = getattr(shot, "shot_contract", None)
|
||
_size_s = str(getattr(_sc3, "shot_size", "") or "") if _sc3 else ""
|
||
except Exception:
|
||
_size_s = ""
|
||
if any(k in _size_s for k in ("特写", "近景", "close", "macro")):
|
||
_pref = ("detail",)
|
||
else:
|
||
for _keys, _slots in _LOC_ANGLE_SLOTS:
|
||
if any(k in _ang for k in _keys):
|
||
_pref = _slots
|
||
break
|
||
for slot in (_pref + ("wide", "main", "front", "scene", "side", "top", "detail")):
|
||
if refs.get(slot):
|
||
p = _abs_ref(refs[slot])
|
||
if p:
|
||
log.debug(" 场景参考图 %s/%s → %s（机位 %s）",
|
||
kind, slot, p.name, _ang or "默认")
|
||
return str(p)
|
||
for slot, v in refs.items():
|
||
p = _abs_ref(v)
|
||
if p:
|
||
return str(p)
|
||
except Exception:
|
||
continue
|
||
return ""
|
||
|
||
@staticmethod
|
||
def _aspect_ratio(sb: Storyboard) -> str:
|
||
"""按输出分辨率推断画幅（默认竖屏 9:16）"""
|
||
res = (sb.shots[0].output.resolution if sb.shots else "") or "1080x1920"
|
||
try:
|
||
w, h = (int(x) for x in res.lower().split("x"))
|
||
return "16:9" if w > h else "9:16"
|
||
except Exception:
|
||
return "9:16"
|
||
|
||
@staticmethod
|
||
def build_shot_prompt(shot, sb: Storyboard, has_ref_image: bool = False) -> str:
|
||
"""把分镜编译成生成提示词（六模块母提示词 + 写实感增强）
|
||
|
||
六模块：① 生成任务 ② 主要主体 ③ 场景状态 ④ 情绪目标 ⑤ 视觉风格 ⑥ 分段脚本
|
||
|
||
⚠️ has_ref_image=True（图生视频）时**改写策略**：
|
||
可灵官方守则 —— 图生视频"只描述运动，不重描述视觉"，
|
||
因为模型会照参考图重画，此时再描述一遍视觉 = 两套冲突指令 → AI 感。
|
||
故图生视频分支只保留：动作演化 / 情绪外化 / 摄影特征 / 质感锚点 / 约束。
|
||
|
||
写实感增强（composer/realism.py）：
|
||
· 情绪外化：抽象情绪词 → 具体身体动作
|
||
· 写实锚点：按拍摄风格给"设备/光源/皮肤/微瑕疵"具体锚点
|
||
· 矛盾自检：检测「画面稳定 vs 手持」这类相互抵消的组合并告警
|
||
"""
|
||
try:
|
||
from composer.realism import (externalize_emotion, get_anchors,
|
||
compose_constraints, check_conflicts)
|
||
except Exception: # 极端情况下退回基础行为
|
||
externalize_emotion = lambda s: s or ""
|
||
get_anchors = lambda st: {"device": "", "hand": "", "light": "",
|
||
"skin": "", "flaws": "", "motion": "", "_no": ""}
|
||
compose_constraints = lambda extra="", with_anti_twin=True: \
|
||
"负向约束：不要运镜特效痕迹、不要 AI 塑料质感、不要字幕水印"
|
||
check_conflicts = lambda t: []
|
||
|
||
ga = sb.global_anchors
|
||
sc = shot.shot_contract
|
||
res = (shot.output.resolution if shot.output else "") or "1080x1920"
|
||
anchors = get_anchors(getattr(ga, "shooting_style", "") or "")
|
||
parts: list[str] = []
|
||
|
||
# ══ 分支 A：图生视频（有参考图）—— 只写动作，不重描述视觉 ══
|
||
if has_ref_image:
|
||
parts.append(
|
||
f"【生成任务】{shot.duration:.0f} 秒连续镜头；"
|
||
f"**严格以参考图中的人物外貌、服装为准；"
|
||
f"不要使用参考图的背景、环境、构图与分区布局**"
|
||
f"—— 背景环境以本提示词描述的场景为准"
|
||
)
|
||
act = (getattr(shot, "action", "") or "").strip() or \
|
||
(shot.visual_description or "").strip() or "自然表演"
|
||
parts.append(f"【动作演化】{act}")
|
||
emo = externalize_emotion(getattr(shot, "sentiment", "") or "")
|
||
if emo:
|
||
parts.append(f"【情绪外化】{emo}")
|
||
cam = "；".join(x for x in [
|
||
anchors.get("device", ""), anchors.get("hand", ""), anchors.get("light", ""),
|
||
] if x)
|
||
if cam:
|
||
parts.append(f"【摄影特征】{cam}")
|
||
tex = "；".join(x for x in [anchors.get("skin", ""), anchors.get("flaws", "")] if x)
|
||
if tex:
|
||
parts.append(f"【质感锚点】{tex}")
|
||
if anchors.get("motion"):
|
||
parts.append(f"【表演细节】{anchors['motion']}")
|
||
if shot.text:
|
||
# 台词用 {} 包裹（特殊字符规范：区分"要说的话"与画面描述）
|
||
parts.append(f"【台词】{{{shot.text}}}")
|
||
# 音效用 <> 标注（规范）；这些音效同时会进音效库匹配（sfx 链路）
|
||
_sfx = []
|
||
try:
|
||
_a = getattr(shot, "audio", None) or {}
|
||
_raw = (_a.get("sfx", []) if isinstance(_a, dict)
|
||
else getattr(_a, "sfx", [])) or []
|
||
if isinstance(_raw, (str, dict)):
|
||
_raw = [_raw]
|
||
for _s in _raw:
|
||
if isinstance(_s, dict):
|
||
_d = str(_s.get("desc") or "").strip()
|
||
else:
|
||
# SFX 对象（pydantic）→ 取 desc 属性
|
||
_d = str(getattr(_s, "desc", "") or _s).strip()
|
||
if _d:
|
||
_sfx.append(_d)
|
||
except Exception:
|
||
pass
|
||
if _sfx:
|
||
parts.append("音效：" + "".join(f"<{x}>" for x in _sfx))
|
||
parts.append(compose_constraints(anchors.get("_no", "")))
|
||
full = "\n".join(parts)
|
||
for w in check_conflicts(full):
|
||
log.warning(" ⚠️ 提示词矛盾：%s", w)
|
||
return _strip_shot_md(_with_ai_negatives(full, sb.global_anchors))
|
||
|
||
# ══ 分支 B：文生视频（无参考图）—— 完整视觉描述 + 写实锚点 ══
|
||
move = sc.camera_move or "自然运镜"
|
||
parts.append(
|
||
f"【生成任务】生成一个 {shot.duration:.0f} 秒的连续镜头（类型：{shot.type}，"
|
||
f"画幅 {res}）；{'固定机位、自然表演' if '固定' in move else move}"
|
||
)
|
||
# ② 主要主体
|
||
# ★ 用资产**详情**而不是 ID（"角色 中国"这种 ID 对模型毫无信息量）
|
||
# 研究结论：文字只写参考图**没有**的，但主体身份/场景环境必须具体
|
||
_char_ctx = _clean_studio_jargon((getattr(ga, "character_context", "") or "").strip())
|
||
subj = "；".join(x for x in [
|
||
_char_ctx if _char_ctx else (f"角色 {shot.character_id}" if shot.character_id else ""),
|
||
f"服装 {ga.wardrobe}" if ga.wardrobe else "",
|
||
f"动作：{shot.action}" if shot.action else "",
|
||
] if x)
|
||
if subj:
|
||
parts.append(f"【主要主体】{subj}")
|
||
# ③ 场景状态（同样用详情）
|
||
_loc_ctx = _clean_studio_jargon((getattr(ga, "location_context", "") or "").strip())
|
||
scene = "；".join(x for x in [
|
||
_loc_ctx if _loc_ctx else (f"地点 {shot.location_id}" if shot.location_id else ""),
|
||
f"光线 {ga.lighting_state}" if ga.lighting_state else anchors.get("light", ""),
|
||
f"镜头语言 {ga.camera_lens}" if ga.camera_lens else "",
|
||
f"道具 {', '.join(shot.props)}" if shot.props else "",
|
||
] if x)
|
||
if scene:
|
||
parts.append(f"【场景状态】{scene}")
|
||
# ④ 情绪目标 → 外化为具体动作
|
||
emo = externalize_emotion(shot.sentiment or "")
|
||
if emo:
|
||
parts.append(f"【情绪外化】{emo}")
|
||
# ⑤ 视觉风格（用写实锚点替换抽象形容词）
|
||
style = "；".join(x for x in [
|
||
f"拍摄风格 {ga.shooting_style}" if ga.shooting_style else "伪纪录片写实",
|
||
anchors.get("device", ""),
|
||
anchors.get("skin", ""),
|
||
anchors.get("flaws", ""),
|
||
] if x)
|
||
if style:
|
||
parts.append(f"【视觉风格】{style}")
|
||
# ⑥ 分段脚本（本镜头）
|
||
script = "；".join(x for x in [
|
||
f"景别 {sc.shot_size}" if sc.shot_size else "",
|
||
f"机位 {sc.angle}" if sc.angle else "",
|
||
f"摄影机 {sc.camera_move}" if sc.camera_move else "",
|
||
f"景深 {sc.depth_of_field}" if sc.depth_of_field else "",
|
||
f"光学 {sc.lens_feel}" if sc.lens_feel else "",
|
||
f"画面与动作：{shot.visual_description}" if shot.visual_description else "",
|
||
f"台词：{shot.text}" if shot.text else "",
|
||
] if x)
|
||
if script:
|
||
parts.append(f"【分段脚本】{script}")
|
||
# 约束（护栏）
|
||
parts.append(compose_constraints(anchors.get("_no", "")))
|
||
|
||
full = "\n".join(parts)
|
||
for w in check_conflicts(full):
|
||
log.warning(" ⚠️ 提示词矛盾：%s", w)
|
||
return _strip_shot_md(_with_ai_negatives(full, sb.global_anchors))
|
||
|
||
|
||
# 便捷入口
|
||
def _strip_shot_md(p: str) -> str:
|
||
"""去掉提示词里的 markdown 符号（** __ ` 等）
|
||
|
||
⚠️ 生成模型读的是**自然语言**，markdown 标记对它只是噪音。
|
||
实测：提示词里出现过 `**严格以参考图中的人物外貌...**` 这种写法。
|
||
"""
|
||
import re as _re
|
||
if not p:
|
||
return p
|
||
s = _re.sub(r"\*{1,3}([^*\n]+)\*{1,3}", r"\1", p) # **粗体** / *斜体*
|
||
s = _re.sub(r"__([^_\n]+)__", r"\1", s) # __粗体__
|
||
s = _re.sub(r"`([^`\n]+)`", r"\1", s) # `代码`
|
||
s = _re.sub(r"[ \t]{2,}", " ", s)
|
||
return s
|
||
|
||
|
||
def render_storyboard(sb: Storyboard, output_path: str,
|
||
generate_fn: Optional[Callable] = None,
|
||
progress_cb: ProgressCB = None) -> dict:
|
||
return StoryboardPipeline().render(sb, output_path, progress_cb=progress_cb,
|
||
generate_fn=generate_fn)
|