feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程，直接发 HTTP

移植自 mac-agent-os 的 mediacrawler_adapter：不起子进程、不开页面，用浏览器里那份登录态直接调抖音 Web 接口。产物键名照抄 store/douyin，所以 ingest 那条链路一个字不用改。 **目前能用的（真环境实测，非推断）**： profile/other : 200, 7075 字节 —— 博主主页指标（粉丝/获赞/作品数/昵称） aweme/detail : 200, 45425 字节 —— 单条作品详情（含点赞/评论/收藏/分享） **目前不能用的：作品列表 `aweme/post`。** 两个互相独立的原因： 1. 这个接口被抖音单独升级成了真校验：不带 x-tt-argus 回 403「Uifid Not Found」，带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过，「真校验」过不了。同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的，挑中的恰好是「批量拉作品列表」这个最敏感的动作。 2. 改走页面截获也不行：CDP 浏览器打开博主主页会落到「验证码中间页」（当天大量探测的代价，过几小时要重测）。所以现在的边界是：**已知作品的指标刷新能做，自动发现新作品做不了**。 **排查中控住变量后得到的两条事实**（都写进注释了）： · `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有 UA+client hints+Cookie 时是 200 但仅 121 字节的空壳，补上这三个头变 7074 字节。（我先前猜的 sec-ch-ua 不是关键。） · 因此 UA 与 client hints 必须**成套地取自同一个浏览器**，所以 BrowserIdentity 一次从 CDP 取齐 cookie + UA + hints，而不是各自写死。「200 + 空 body 必须当场报错」也是刻意写死的：放过去它会在下游变成「这个博主没作品」，把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的，还被翻译成「账号被封」。测试 +11：cookie 解析、请求头成套性（含 uifid 缺失/回退）、产物键名与 store 对齐、以及 _get 的三条失败路径（空 body / 403 带网关原话 / 正常返回）。
2026-10-10 17:12:25 +08:00
parent 21ff01b894
commit 2112c1a870
1601 changed files with 367425 additions and 0 deletions
@@ -0,0 +1,192 @@
+"""
+
+AVE 01_director_parser — 文案 → 导演脚本 YAML
+
+流程:
+  1. 读取文案文本
+  2. 调用本地 oMLX (Qwen3.5) 解析 → 结构化 segments
+  3. 调用 schemas.py 校验
+  4. 输出 director_script.yaml
+
+本地 LLM: http://localhost:8000/v1/chat/completions
+"""
+import sys, os; sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import json
+import os
+import yaml
+from pathlib import Path
+from typing import Optional
+
+from lib.logger import get_logger
+from .schemas import DirectorScript, validate_script
+
+logger = get_logger("director_parser")
+
+# 本地 oMLX 配置
+OMLX_URL = "http://localhost:8000/v1/chat/completions"
+OMLX_MODEL = "Qwen3.5-4B-MLX-4bit"
+OMLX_API_KEY = "omlx"
+
+
+def parse_script(
+    script_path: str,
+    style: str = "knowledge_lecture",
+    output_path: str = "director_script.yaml",
+    use_llm: bool = True,
+) -> str:
+    """
+    文案 → 导演脚本 YAML
+
+    - script_path: 文案文件路径 (.txt)
+    - style: 视频风格 (knowledge_lecture | bedtime_story | funny_talk | tech_review)
+    - output_path: 输出的 YAML 路径
+    - use_llm: 是否使用本地 LLM 解析 (False = 手动模板)
+
+    返回: 生成的 YAML 路径
+    """
+    if not os.path.exists(script_path):
+        raise FileNotFoundError(f"文案文件不存在: {script_path}")
+
+    with open(script_path, "r", encoding="utf-8") as f:
+        script_text = f.read().strip()
+
+    if not script_text:
+        raise ValueError(f"文案内容为空: {script_path}")
+
+    if use_llm:
+        # 用本地 oMLX 解析
+        script_data = _parse_with_llm(script_text, style)
+    else:
+        # 手动模板: 整段文案作为单 segment
+        script_data = _manual_parse(script_text, style)
+
+    # Pydantic 校验
+    validated = validate_script(script_data)
+
+    # 输出 YAML
+    with open(output_path, "w", encoding="utf-8") as f:
+        yaml.dump(
+            validated.model_dump(mode="json"),
+            f,
+            allow_unicode=True,
+            default_flow_style=False,
+            sort_keys=False,
+        )
+
+    logger.info(f"导演脚本已生成: {output_path} ({len(validated.segments)} 段)")
+    return output_path
+
+
+def _parse_with_llm(script_text: str, style: str) -> dict:
+    """调用本地 oMLX 解析文案为结构化数据"""
+    import httpx
+
+    prompt = f"""你是一个视频导演脚本生成器。将以下文案解析为结构化导演脚本。
+
+风格: {style}
+
+要求:
+1. 按自然语义停顿分段 (每段6-30秒)
+2. 为每段分配情绪描述
+3. 为每段设计视觉素材搜索词 (中文关键词)
+4. 为每段指定 BGM 段落 (intro/main/climax/outro)
+5. BGM结构: intro(0-8s) → main(8-50s) → climax(50-65s) → outro(65-75s)
+6. 输出纯 JSON，不要 markdown 包裹
+
+文案:
+{script_text}
+
+输出 JSON 格式:
+{{
+  "meta": {{
+    "style": "{style}",
+    "voice": {{"provider": "volcano", "voice_id": "default", "default_emotion": "专业沉稳"}},
+    "bgm": {{
+      "genre": "cinematic", "tempo": 110, "mood": "inspirational",
+      "structure": [{{"section": "intro"}}, {{"section": "main"}}, {{"section": "climax"}}, {{"section": "outro"}}]
+    }},
+    "output": {{"resolution": "1080x1920", "fps": 30}}
+  }},
+  "segments": [
+    {{
+      "id": 1,
+      "text": "分段文案",
+      "duration_sec": 8,
+      "voice_emotion": "开场悬念，压低声音",
+      "camera": "slow_zoom_in",
+      "bgm_section": "intro",
+      "material": {{"source": "pexels", "search": "中文搜索关键词"}},
+      "subtitles": true
+    }}
+  ]
+}}"""
+
+    try:
+        resp = httpx.post(
+            OMLX_URL,
+            json={
+                "model": OMLX_MODEL,
+                "messages": [{"role": "user", "content": prompt}],
+                "temperature": 0.3,
+                "max_tokens": 4096,
+            },
+            headers={"Authorization": f"Bearer {OMLX_API_KEY}"},
+            timeout=120,
+        )
+        resp.raise_for_status()
+        result = resp.json()
+        content = result["choices"][0]["message"]["content"]
+
+        # 清理可能的 markdown 包裹
+        content = content.strip()
+        if content.startswith("```"):
+            content = content.split("\n", 1)[1]
+            content = content.rsplit("```", 1)[0]
+        if content.startswith("json"):
+            content = content[4:]
+
+        data = json.loads(content)
+        data["meta"]["style"] = style
+        logger.info(f"oMLX 解析完成: {len(data.get('segments', []))} 段")
+        return data
+
+    except httpx.ConnectError:
+        logger.warning("oMLX 未运行，降级到手动模板模式")
+        return _manual_parse(script_text, style)
+    except (json.JSONDecodeError, KeyError) as e:
+        logger.warning(f"LLM 解析失败 ({e})，降级到手动模板")
+        return _manual_parse(script_text, style)
+
+
+def _manual_parse(script_text: str, style: str) -> dict:
+    """手动模板: 整段文案作为一个 segment（降级方案）"""
+    return {
+        "meta": {
+            "style": style,
+            "voice": {"provider": "volcano", "voice_id": "default", "default_emotion": "专业沉稳"},
+            "bgm": {
+                "genre": "cinematic" if style != "funny_talk" else "upbeat",
+                "tempo": 110,
+                "mood": "inspirational",
+                "structure": [
+                    {"section": "intro"},
+                    {"section": "main"},
+                    {"section": "climax"},
+                    {"section": "outro"},
+                ],
+            },
+            "output": {"resolution": "1080x1920", "fps": 30},
+        },
+        "segments": [
+            {
+                "id": 1,
+                "text": script_text,
+                "duration_sec": max(15, min(len(script_text) * 0.2, 120)),
+                "voice_emotion": "正常讲述",
+                "camera": "static",
+                "bgm_section": "main",
+                "material": {"source": "pexels", "search": style},
+                "subtitles": True,
+            }
+        ],
+    }
@@ -0,0 +1,131 @@
+"""
+
+AVE 01_director_parser — 导演脚本解析器
+
+职责:
+  1. schemas.py: YAML 结构定义 + Pydantic 校验
+  2. parser.py: 文案 → director_script.yaml (调用本地 oMLX)
+
+本地 LLM (oMLX) 路径:
+  http://localhost:8000/v1/chat/completions
+  模型: Qwen3.5-4B-MLX-4bit
+  API Key: omlx
+"""
+import sys, os; sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from pydantic import BaseModel, Field, field_validator
+from typing import Optional
+from enum import Enum
+
+
+class StyleEnum(str, Enum):
+    knowledge_lecture = "knowledge_lecture"
+    bedtime_story = "bedtime_story"
+    funny_talk = "funny_talk"
+    tech_review = "tech_review"
+
+
+class VoiceProvider(str, Enum):
+    volcano = "volcano"
+    aliyun = "aliyun"
+
+
+class BGMSection(BaseModel):
+    section: str
+    description: Optional[str] = None
+
+
+class BGMMeta(BaseModel):
+    provider: str = "ace_step"
+    genre: str = "cinematic"
+    tempo: int = 110
+    mood: str = "inspirational"
+    structure: list[BGMSection] = []
+
+    @field_validator("tempo")
+    @classmethod
+    def tempo_range(cls, v):
+        if v < 40 or v > 220:
+            raise ValueError("tempo 应在 40-220 之间")
+        return v
+
+
+class OutputMeta(BaseModel):
+    resolution: str = "1080x1920"
+    fps: int = 30
+
+    @field_validator("resolution")
+    @classmethod
+    def valid_resolution(cls, v):
+        valid = ["1080x1920", "1920x1080", "720x1280", "1280x720"]
+        if v not in valid:
+            raise ValueError(f"分辨率必须是 {valid} 之一")
+        return v
+
+
+class MaterialSource(str, Enum):
+    pexels = "pexels"
+    ai_generate = "ai_generate"
+    local = "local"
+
+
+class MaterialConfig(BaseModel):
+    source: MaterialSource = MaterialSource.pexels
+    search: str = ""
+    transition_in: str = "fade_in"
+    transition_out: str = "dissolve"
+    fallback: bool = True
+
+
+class CharacterRef(BaseModel):
+    """角色引用（定妆照锁定）"""
+    name: str = Field(description="角色名")
+    description: str = Field(description="角色描述块（跨场景一致性注入）")
+    grid_path: Optional[str] = Field(default="", description="定妆照网格图路径")
+    lip_sync: bool = Field(default=False, description="是否对角色视频做 LipSync")
+
+
+class AvatarConfig(BaseModel):
+    image_url: Optional[str] = None
+    provider: str = "wan2.2_s2v"
+    default_position: str = "bottom_right"
+
+
+class Segment(BaseModel):
+    id: int = Field(ge=1)
+    text: str = Field(min_length=1)
+    duration_sec: float = Field(default=10, ge=1, le=300)
+    voice_emotion: str = "正常讲述"
+    camera: str = "static"
+    bgm_section: str = "main"
+    avatar: Optional[dict] = None
+    material: MaterialConfig = MaterialConfig()
+    subtitles: bool = True
+    character_ref: Optional[str] = Field(default=None, description="引用角色名（对应 meta.character_refs）")
+
+
+class VoiceMeta(BaseModel):
+    provider: VoiceProvider = VoiceProvider.volcano
+    voice_id: str = "default"
+    default_emotion: str = "专业沉稳"
+
+
+class Meta(BaseModel):
+    project_id: str = ""
+    style: StyleEnum = StyleEnum.knowledge_lecture
+    voice: VoiceMeta = VoiceMeta()
+    bgm: BGMMeta = BGMMeta()
+    output: OutputMeta = OutputMeta()
+    avatar: Optional[AvatarConfig] = None
+    material: dict = {"fallback": True}
+    character_refs: list[CharacterRef] = Field(default=[], description="角色引用列表（定妆照一致性）")
+
+
+class DirectorScript(BaseModel):
+    """完整导演脚本"""
+    meta: Meta = Meta()
+    segments: list[Segment] = Field(min_length=1)
+
+
+def validate_script(data: dict) -> DirectorScript:
+    """校验并返回结构化的导演脚本"""
+    return DirectorScript.model_validate(data)