""" composer/dubbing.py — 配音合成(TTS 人声 + 环境音铺底 + 时间轴对齐) 背景(2026-10-02 实测确认): · 可灵生成的镜头**没有音轨**(payload 无音频参数) · 分镜的 Shot.text(台词)此前只拼进提示词,不产生任何声音 · 导演台出片链路(pipeline.render)没有音频环节 → 成片无声 本模块补齐「声音」: 按镜头时间轴,把台词逐句合成为一条完整音轨;每个镜头一段,段长严格等于 镜头时长(短则补静音、长则加速),因此天然对齐画面。 音色来源(优先级): 1. 演员表指定的 voice_id(CastMember.voice_id) 2. 按角色代号稳定分配的路人音色池(同一角色全片同一音色,不会听起来换人) 环境音来源: 分镜的 Shot.audio_cue(如"远处车流声")→ 与环境音库资产做关键词匹配 → 该镜头期间低音量循环铺底。 输出:一条 44.1kHz 立体声 wav(可直接交给 StoryboardPipeline.mux_final 的 voice_path) """ from __future__ import annotations import os import re import subprocess import sys from pathlib import Path from typing import Callable, Optional _SCRIPTS = Path(__file__).resolve().parent.parent if str(_SCRIPTS) not in sys.path: sys.path.insert(0, str(_SCRIPTS)) from lib.logger import get_logger # noqa: E402 from lib.ffmpeg import FFMPEG # noqa: E402 # 统一 ffmpeg 定位(绝对路径) log = get_logger("dubbing") API_BASE_SR = 44100 DEFAULT_SFX_VOLUME = 0.85 # 音效默认音量(非语言发声为主,比环境音高) DEFAULT_AMBIENCE_VOLUME = 0.15 DEFAULT_VOICE_VOLUME = 1.0 ProgressCB = Optional[Callable[[str, str, str, str], None]] # ════════════════════════════════════════════════════════════ # 音色分配 # ════════════════════════════════════════════════════════════ # 路人音色池(未指定音色的角色按代号稳定分配;来自 config/cosyvoice_voices.yaml 的 pools) FALLBACK_MALE = ["longze_v3", "longzhe_v3", "longanlang_v3"] FALLBACK_FEMALE = ["longxing_v3", "longmiao_v3", "longanqin_v3"] NARRATOR_VOICE = "longxiu_v3" def _is_female(slot: str, note: str = "") -> bool: """粗判角色性别(用于选路人音色池):代号/备注里出现女/她/妈/姐/妹/婆 → 女声""" t = f"{slot} {note}" return bool(re.search(r"女|她|妈|姐|妹|婆|娘|太太|女士|girl|woman", t)) def resolve_voice_map(sb, cast_voice_map: dict | None = None) -> dict: """角色代号 → voice_id(确定性分配) cast_voice_map: 演员表里显式指定的 {slot: voice_id} 未指定的角色 → 按代号 hash 从路人池稳定取一个(同名永远同音色) """ cast_voice_map = cast_voice_map or {} out: dict[str, str] = {} # ⚠️ 字段名是 global_anchors(不是 anchors)—— 写错会导致演员表里的 voice_id 读不到, # 角色音色全部退化为"路人池按 hash 分配"(用户实测发现) _ga = getattr(sb, "global_anchors", None) _cast = [] if _ga is not None and hasattr(_ga, "effective_cast"): try: _cast = _ga.effective_cast() or [] except Exception: _cast = [] for m in _cast: slot = getattr(m, "slot", "") or "" if not slot: continue vid = (cast_voice_map.get(slot) or getattr(m, "voice_id", "") or "").strip() if not vid: pool = FALLBACK_FEMALE if _is_female(slot, getattr(m, "note", "")) else FALLBACK_MALE vid = pool[sum(ord(c) for c in slot) % len(pool)] out[slot] = vid # 演员表未覆盖的出场代号(导演写的)也要有音色 for shot in getattr(sb, "shots", []) or []: for slot in (getattr(shot, "cast_slots", None) or []): if slot and slot not in out: pool = FALLBACK_FEMALE if _is_female(slot) else FALLBACK_MALE out[slot] = pool[sum(ord(c) for c in slot) % len(pool)] return out def pick_voice_for_shot(shot, voice_map: dict, narrator: str = NARRATOR_VOICE) -> str: """该镜头台词用谁的音色:出场人物第一顺位;无出场 → 旁白音色""" slots = getattr(shot, "cast_slots", None) or [] for s in slots: if s in voice_map: return voice_map[s] if slots: # 演员表里没有的代号 → 用其自身 hash 分配 pool = FALLBACK_FEMALE if _is_female(slots[0]) else FALLBACK_MALE return pool[sum(ord(c) for c in slots[0]) % len(pool)] return narrator # ════════════════════════════════════════════════════════════ # 音频工具 # ════════════════════════════════════════════════════════════ def _run(cmd: list[str], timeout: int = 600) -> subprocess.CompletedProcess: return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) def probe_audio_duration(path: str) -> float: """音频/视频时长(秒);用 ffmpeg -i 解析(本机无真 ffprobe,见 lib/ffmpeg.py)""" from lib.ffmpeg import probe_duration d = probe_duration(path) return float(d or 0.0) def synthesize_line(text: str, voice_id: str, out_path: str, api_key: str = "", speed: float = 1.0) -> str: """单句 TTS(阿里 CosyVoice)""" from voice_synthesizer.aliyun import synthesize os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True) synthesize(text, out_path, api_key=api_key, voice_id=voice_id, speed=speed) return out_path def make_silence(duration_sec: float, dst: str) -> str: """生成指定时长的静音段(44.1k 立体声 wav)""" r = _run([FFMPEG, "-y", "-f", "lavfi", "-i", f"anullsrc=channel_layout=stereo:sample_rate={API_BASE_SR}", "-t", f"{max(0.05, duration_sec):.3f}", "-c:a", "pcm_s16le", dst], timeout=120) if r.returncode != 0: raise RuntimeError(f"生成静音失败: {r.stderr[-300:]}") return dst def fit_audio_to_duration(src: str, dst: str, target_sec: float, max_speedup: float = 1.6) -> dict: """把音频适配到目标时长 · 短于目标 → 补静音(apad),保持原速 · 长于目标 → 加速(atempo,最多 max_speedup 倍);仍超长则截断 返回 {out, src_dur, target_dur, action} """ src_dur = probe_audio_duration(src) target = max(0.3, float(target_sec)) action = "keep" afilters: list[str] = [] if src_dur <= 0: # 探不到时长 → 直接按目标截断 + 补静音,交给 ffmpeg afilters = ["apad"] action = "unknown-duration" elif src_dur > target * 1.02: ratio = src_dur / target if ratio <= max_speedup: afilters = [f"atempo={ratio:.4f}"] action = f"加速×{ratio:.2f}" else: afilters = [f"atempo={max_speedup:.4f}", "apad"] action = f"加速×{max_speedup:.2f}+截断" else: afilters = ["apad"] action = "补静音" af = ",".join(afilters) if afilters else "anull" r = _run([FFMPEG, "-y", "-i", src, "-af", af, "-t", f"{target:.3f}", "-ar", str(API_BASE_SR), "-ac", "2", "-c:a", "pcm_s16le", dst], timeout=300) if r.returncode != 0: raise RuntimeError(f"音频适配失败: {r.stderr[-300:]}") return {"out": dst, "src_dur": round(src_dur, 2), "target_dur": round(target, 2), "action": action} def mix_segment(voice_path: str, ambience_path: str, duration_sec: float, dst: str, ambience_volume: float = DEFAULT_AMBIENCE_VOLUME, sfx_items: list | None = None) -> str: """把「人声 + 环境音(循环)+ 音效(按时点插入)」混成一个镜头段,严格等于目标时长 sfx_items: [(音效文件路径, 插入秒数, 音量)] —— 非语言发声(惨叫/闷哼/关门声…) """ dur = max(0.3, float(duration_sec)) sfx_items = [x for x in (sfx_items or []) if x and x[0] and Path(x[0]).exists()] has_v = bool(voice_path and Path(voice_path).exists()) has_a = bool(ambience_path and Path(ambience_path).exists()) if not (has_v or has_a or sfx_items): return make_silence(dur, dst) inputs: list[str] = [] if has_v: inputs += ["-i", voice_path] if has_a: inputs += ["-stream_loop", "-1", "-i", ambience_path] for f, _at, _vol in sfx_items: inputs += ["-i", f] parts: list[str] = [] labels: list[str] = [] idx = 0 if has_v: parts.append(f"[{idx}:a]volume={DEFAULT_VOICE_VOLUME},apad[v]") labels.append("[v]"); idx += 1 if has_a: parts.append(f"[{idx}:a]volume={ambience_volume},atrim=0:{dur:.3f},asetpts=N/SR/TB[a]") labels.append("[a]"); idx += 1 for k, (f, at, vol) in enumerate(sfx_items): ms = int(max(0.0, min(float(at), dur - 0.05)) * 1000) parts.append(f"[{idx}:a]volume={vol},adelay={ms}|{ms}[s{k}]") labels.append(f"[s{k}]"); idx += 1 if len(labels) == 1: parts.append(f"{labels[0]}anull[out]") else: parts.append(f"{''.join(labels)}amix=inputs={len(labels)}:duration=first:normalize=0[out]") r = _run([FFMPEG, "-y", *inputs, "-filter_complex", ";".join(parts), "-map", "[out]", "-t", f"{dur:.3f}", "-ar", str(API_BASE_SR), "-ac", "2", "-c:a", "pcm_s16le", dst], timeout=300) if r.returncode != 0: raise RuntimeError(f"段混音失败: {r.stderr[-300:]}") return dst def concat_audio(seg_paths: list[str], out_path: str) -> str: """按顺序拼接音频段(同规格 wav → concat demuxer)""" if not seg_paths: raise ValueError("没有音频段可拼接") lst = Path(out_path).with_suffix(".txt") lst.write_text("".join(f"file '{Path(p).resolve()}'\n" for p in seg_paths), encoding="utf-8") r = _run([FFMPEG, "-y", "-f", "concat", "-safe", "0", "-i", str(lst), "-c:a", "pcm_s16le", "-ar", str(API_BASE_SR), "-ac", "2", out_path], timeout=600) lst.unlink(missing_ok=True) if r.returncode != 0: raise RuntimeError(f"音频拼接失败: {r.stderr[-300:]}") return out_path # ════════════════════════════════════════════════════════════ # 环境音匹配(分镜 audio_cue ↔ 环境音库) # ════════════════════════════════════════════════════════════ def load_ambience_assets() -> list[dict]: """读取环境音库资产(含本地文件路径)""" try: from assets_manager import AssetStore st = AssetStore("ambience") out = [] for a in st.list(): f = "" for slot in ("audio", "file", "sample"): v = a.get(slot) or "" if v: try: f = str(Path(st.kind_dir(a["id"])) / Path(v).name) except Exception: f = "" break if not f or not Path(f).exists(): # 退一步:目录下第一个音频文件 try: d = Path(st.kind_dir(a["id"])) cands = [p for p in d.glob("*") if p.suffix.lower() in (".mp3", ".wav", ".m4a", ".aac", ".ogg")] f = str(cands[0]) if cands else "" except Exception: f = "" a["_file"] = f out.append(a) return out except Exception as e: log.warning("环境音库读取失败: %s", e) return [] def load_sfx_assets() -> list[dict]: """读取音效库资产(含本地文件路径)—— 结构同环境音库""" try: from assets_manager import AssetStore st = AssetStore("sfx") out = [] for a in st.list(): f = "" for slot in ("audio", "file", "sample"): v = a.get(slot) or "" if v: try: f = str(Path(st.kind_dir(a["id"])) / Path(v).name) except Exception: f = "" break if not f or not Path(f).exists(): try: d = Path(st.kind_dir(a["id"])) cands = [p for p in d.glob("*") if p.suffix.lower() in (".mp3", ".wav", ".m4a", ".aac", ".ogg")] f = str(cands[0]) if cands else "" except Exception: f = "" a["_file"] = f out.append(a) return out except Exception as e: log.warning("音效库读取失败: %s", e) return [] def match_sfx(desc: str, assets: list[dict]) -> str: """用音效描述匹配音效资产(名称/标签最长子串命中)→ 文件路径 例:desc「痛苦的闷哼」→ 命中音效「闷哼」/「痛呼」 """ if not desc or not assets: return "" d = str(desc) best, best_len = "", 0 for a in assets: f = a.get("_file") or "" if not f: continue for key in [a.get("name", "")] + list(a.get("tags") or []): k = str(key or "").strip() if len(k) >= 2 and (k in d or d in k) and len(k) > best_len: best, best_len = f, len(k) return best def match_ambience(cue: str, assets: list[dict]) -> str: """用 audio_cue 文本匹配环境音资产(名称/标签最长子串命中)→ 文件路径""" if not cue or not assets: return "" best, best_len = "", 0 for a in assets: f = a.get("_file") or "" if not f: continue for key in [a.get("name", "")] + list(a.get("tags") or []): k = str(key or "").strip() if len(k) >= 2 and k in cue and len(k) > best_len: best, best_len = f, len(k) return best # ════════════════════════════════════════════════════════════ # 主入口 # ════════════════════════════════════════════════════════════ def pre_synthesize(sb, work_dir: str, api_key: str = "", cast_voice_map: dict | None = None, min_dur: float = 2.0, max_dur: float = 10.0, pad: float = 0.4, progress_cb: ProgressCB = None) -> dict: """【音频先行】逐镜合成台词 → 按音频**真实时长**更新镜头时长 为什么:画面时长若拍脑袋定,台词长了就得加速/截断 → 口型节奏必然对不上。 音频先行 = 让画面跟着声音走。 · 有台词的镜头:TTS → 探测时长 → shot.duration = clamp(音频时长 + 气口, min, max) · 无台词镜头:保持原时长 · 音频存到 {work_dir}/{shot_id}.mp3 → 后续混流**直接复用,不重复 TTS** 返回 {updated, shots:{shot_id:{audio,audio_dur,old_dur,new_dur,voice_id}}, notes} """ work = Path(work_dir) work.mkdir(parents=True, exist_ok=True) from lib.config import load_config cfg = load_config() api_key = api_key or (cfg.get("aliyun", {}) or {}).get("api_key", "") if not api_key: raise RuntimeError("音频先行需要百炼 API Key(TTS)") voice_map = resolve_voice_map(sb, cast_voice_map) out: dict = {} notes: list = [] updated = 0 shots = list(getattr(sb, "shots", []) or []) for idx, shot in enumerate(shots, 1): sid = getattr(shot, "shot_id", f"shot_{idx:02d}") text = (getattr(shot, "text", "") or "").strip() if not text: continue vid = pick_voice_for_shot(shot, voice_map) mp3 = work / f"{sid}.mp3" if not (mp3.exists() and mp3.stat().st_size > 1000): # 已有则复用 synthesize_line(text, vid, str(mp3), api_key=api_key) adur = probe_audio_duration(str(mp3)) if adur <= 0: notes.append(f"{sid}: 音频时长探测失败,保持原时长") continue old = float(getattr(shot, "duration", 5) or 5) new = max(min_dur, min(max_dur, round(adur + pad, 2))) shot.duration = new if abs(new - old) > 0.01: updated += 1 notes.append(f"{sid}: {old}s → {new}s(音频 {adur:.2f}s · {vid})") out[sid] = {"audio": str(mp3), "audio_dur": round(adur, 2), "old_dur": old, "new_dur": new, "voice_id": vid, "has_cast": bool([s for s in (getattr(shot, "cast_slots", None) or []) if s])} if progress_cb: progress_cb("tts", sid, "done", f"{adur:.1f}s → 镜头 {new}s") log.info(" 🎙 音频先行:%d/%d 个镜头时长按台词音频调整", updated, len(shots)) return {"updated": updated, "shots": out, "notes": notes, "voice_map": voice_map} def build_voice_track(sb, out_path: str, work_dir: str, api_key: str = "", cast_voice_map: dict | None = None, use_ambience: bool = True, ambience_volume: float = DEFAULT_AMBIENCE_VOLUME, progress_cb: ProgressCB = None, reuse_dir: str = "") -> dict: """按分镜时间轴合成完整音轨 返回 {output, segments, voice_seconds, ambience_hits, voices:[...], notes:[...]} """ work = Path(work_dir) work.mkdir(parents=True, exist_ok=True) if str(_SCRIPTS) not in sys.path: sys.path.insert(0, str(_SCRIPTS)) from lib.config import load_config cfg = load_config() api_key = api_key or (cfg.get("aliyun", {}) or {}).get("api_key", "") if not api_key: raise RuntimeError("未配置百炼 API Key(TTS 需要)") voice_map = resolve_voice_map(sb, cast_voice_map) amb_assets = load_ambience_assets() if use_ambience else [] sfx_assets = load_sfx_assets() # 音效库(非语言发声:惨叫/闷哼/关门…) log.info("配音: 音色映射 %s | 环境音库 %d 条 | 音效库 %d 条", voice_map, len(amb_assets), len(sfx_assets)) segs: list[str] = [] notes: list[str] = [] voice_seconds = 0.0 amb_hits = 0 sfx_hits = 0 # ★ 分轨(用户 2026-10-03 提出的声音模型): # · 台词轨(dialogue):有出场人物的镜头 → 与画面强绑定(还要用于对口型) # · 旁白轨(narration):无出场人物 → 独立一条声轨,允许与画面有轻微差异 dialogue_segs: list[str] = [] narration_segs: list[str] = [] shots = list(getattr(sb, "shots", []) or []) for idx, shot in enumerate(shots, 1): sid = getattr(shot, "shot_id", f"shot_{idx:02d}") dur = float(getattr(shot, "duration", 3) or 3) text = (getattr(shot, "text", "") or "").strip() if progress_cb: progress_cb("dub", sid, "running", f"{idx}/{len(shots)}") voice_path = "" if text: vid = pick_voice_for_shot(shot, voice_map) # ★ 优先复用「音频先行」已合成的音频(不重复 TTS = 不重复计费) _pre = Path(reuse_dir) / f"{sid}.mp3" if reuse_dir else None if _pre and _pre.exists() and _pre.stat().st_size > 1000: raw = _pre notes.append(f"{sid}: 复用先行音频") else: raw = work / f"tts_{idx:02d}.mp3" synthesize_line(text, vid, str(raw), api_key=api_key) try: fitted = work / f"voice_{idx:02d}.wav" info = fit_audio_to_duration(str(raw), str(fitted), dur) voice_path = str(fitted) voice_seconds += min(info["src_dur"], dur) notes.append(f"{sid}: {vid} · {info['action']}({len(text)}字)") except Exception as e: log.warning(" ⚠️ %s 台词合成失败: %s", sid, e) notes.append(f"{sid}: 台词合成失败({str(e)[:40]})") voice_path = "" # 环境音(该镜头的 audio_cue) amb_path = "" if amb_assets: _a = getattr(shot, "audio", None) cue = "" if _a is not None: cue = (_a.get("audio_cue", "") if isinstance(_a, dict) else getattr(_a, "audio_cue", "")) or "" cue = str(cue).strip() if cue: amb_path = match_ambience(cue, amb_assets) if amb_path: amb_hits += 1 notes.append(f"{sid}: 环境音「{cue}」匹配 ✓") # 音效(非语言发声):按 desc 匹配素材,按 at_second 插入该镜头内 sfx_items: list = [] if sfx_assets: _a = getattr(shot, "audio", None) raw_sfx = [] if _a is not None: raw_sfx = ((_a.get("sfx", []) if isinstance(_a, dict) else getattr(_a, "sfx", [])) or []) if isinstance(raw_sfx, (str, dict)): raw_sfx = [raw_sfx] for s in raw_sfx: if isinstance(s, dict): desc = str(s.get("desc") or s.get("name") or "").strip() at = float(s.get("at_second") or 0) vol = float(s.get("volume") or DEFAULT_SFX_VOLUME) else: desc, at, vol = str(s or "").strip(), 0.0, DEFAULT_SFX_VOLUME if not desc: continue f = match_sfx(desc, sfx_assets) if f: sfx_items.append((f, at, vol)) sfx_hits += 1 notes.append(f"{sid}: 音效「{desc}」@{at}s ✓") else: notes.append(f"{sid}: 音效「{desc}」未匹配到素材(音效库为空?)") # ★ 分轨:判断这一段是「角色台词」还是「旁白」 # 台词(有出场人物)→ 主轨;旁白(无出场人物)→ 单独轨(不重复放进主轨) _slots = [x for x in (getattr(shot, "cast_slots", None) or []) if x] is_narration = bool(text) and not _slots voice_for_main = "" if is_narration else voice_path seg = work / f"seg_{idx:02d}.wav" mix_segment(voice_for_main, amb_path, dur, str(seg), ambience_volume=ambience_volume, sfx_items=sfx_items) segs.append(str(seg)) # 旁白轨:该镜头是旁白 → 只放人声;否则留空(拼接时补等长静音) if is_narration: narr_seg = work / f"narr_{idx:02d}.wav" try: mix_segment(voice_path, "", dur, str(narr_seg)) narration_segs.append(str(narr_seg)) notes.append(f"{sid}: 旁白(单独轨)") except Exception as e: log.warning(" ⚠️ %s 旁白轨生成失败: %s", sid, e) narration_segs.append("") else: narration_segs.append("") if progress_cb: progress_cb("dub", sid, "done", "旁白" if is_narration else ("角色台词" if text else "无台词")) concat_audio(segs, out_path) # ★ 旁白轨(独立输出):无旁白的镜头补等长静音,保证与主轨时间轴对齐 narr_out = "" if any(narration_segs): narr_parts = [] for i, (n, shot) in enumerate(zip(narration_segs, shots), 1): if n: narr_parts.append(n) else: dur_i = float(getattr(shot, "duration", 3) or 3) sil = work / f"narr_sil_{i:02d}.wav" try: if not sil.exists(): make_silence(dur_i, str(sil)) narr_parts.append(str(sil)) except Exception: pass if narr_parts: narr_out = str(Path(out_path).with_name("narration_track.wav")) try: concat_audio(narr_parts, narr_out) log.info(" 🎙 旁白轨单独输出: %s(%d 段)", Path(narr_out).name, len([x for x in narration_segs if x])) except Exception as e: log.warning(" ⚠️ 旁白轨拼接失败: %s", e) narr_out = "" total = sum(float(getattr(s, "duration", 3) or 3) for s in shots) log.info(" 🎙 配音完成: %s(%d 段 / %.1fs 音轨)", Path(out_path).name, len(segs), total) return {"output": out_path, "narration": narr_out, "segments": len(segs), "voice_seconds": round(voice_seconds, 2), "ambience_hits": amb_hits, "sfx_hits": sfx_hits, "narration_segments": len([x for x in narration_segs if x]), "total_seconds": round(total, 2), "voices": voice_map, "notes": notes} # ── CLI 自测:从已保存分镜生成音轨 ── if __name__ == "__main__": import argparse, json ap = argparse.ArgumentParser(description="从分镜生成配音音轨(有 TTS 费用)") ap.add_argument("--storyboard", required=True, help="分镜 JSON 路径或 project_id") ap.add_argument("--output", default="", help="输出 wav 路径") ap.add_argument("--no-ambience", action="store_true", help="不使用环境音") args = ap.parse_args() p = Path(args.storyboard) if not p.exists(): p = Path.home() / "workbuddy-agent-os/agent-local/runtime/ave/storyboards" / f"{args.storyboard}.json" data = json.loads(p.read_text(encoding="utf-8")) from schemas import Storyboard sb = Storyboard.from_dict(data) out = args.output or str(Path.home() / "workbuddy-agent-os/agent-local/runtime/ave/dub_test.wav") r = build_voice_track(sb, out, str(Path(out).parent / "dub_work"), use_ambience=not args.no_ambience) print(json.dumps({k: v for k, v in r.items() if k != "notes"}, ensure_ascii=False, indent=1)) for n in r["notes"]: print(" ·", n)