feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程，直接发 HTTP

移植自 mac-agent-os 的 mediacrawler_adapter：不起子进程、不开页面，用浏览器里那份登录态直接调抖音 Web 接口。产物键名照抄 store/douyin，所以 ingest 那条链路一个字不用改。 **目前能用的（真环境实测，非推断）**： profile/other : 200, 7075 字节 —— 博主主页指标（粉丝/获赞/作品数/昵称） aweme/detail : 200, 45425 字节 —— 单条作品详情（含点赞/评论/收藏/分享） **目前不能用的：作品列表 `aweme/post`。** 两个互相独立的原因： 1. 这个接口被抖音单独升级成了真校验：不带 x-tt-argus 回 403「Uifid Not Found」，带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过，「真校验」过不了。同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的，挑中的恰好是「批量拉作品列表」这个最敏感的动作。 2. 改走页面截获也不行：CDP 浏览器打开博主主页会落到「验证码中间页」（当天大量探测的代价，过几小时要重测）。所以现在的边界是：**已知作品的指标刷新能做，自动发现新作品做不了**。 **排查中控住变量后得到的两条事实**（都写进注释了）： · `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有 UA+client hints+Cookie 时是 200 但仅 121 字节的空壳，补上这三个头变 7074 字节。（我先前猜的 sec-ch-ua 不是关键。） · 因此 UA 与 client hints 必须**成套地取自同一个浏览器**，所以 BrowserIdentity 一次从 CDP 取齐 cookie + UA + hints，而不是各自写死。「200 + 空 body 必须当场报错」也是刻意写死的：放过去它会在下游变成「这个博主没作品」，把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的，还被翻译成「账号被封」。测试 +11：cookie 解析、请求头成套性（含 uifid 缺失/回退）、产物键名与 store 对齐、以及 _get 的三条失败路径（空 body / 403 带网关原话 / 正常返回）。
2026-10-10 17:12:25 +08:00
parent 21ff01b894
commit 2112c1a870
1601 changed files with 367425 additions and 0 deletions
@@ -0,0 +1,343 @@
+#!/usr/bin/env python3
+"""一次性：给 capabilities.yaml 增加「图像」类能力 + 视频类新能力（含 docs 关联）
+
+用法：python3 scripts/tools/extend_capabilities.py --dry   # 只看，不写
+      python3 scripts/tools/extend_capabilities.py          # 写入
+"""
+from __future__ import annotations
+
+import argparse
+import shutil
+import sys
+from datetime import datetime
+from pathlib import Path
+
+import yaml
+
+ROOT = Path(__file__).resolve().parents[2]
+CFG = ROOT / "config" / "capabilities.yaml"
+
+# ── 新增分类 ──────────────────────────────────────────────
+NEW_CATEGORIES = {
+    "image": {"name": "图像", "icon": "🖼️", "order": 2},   # 插在视觉之前
+    "visual": {"name": "视觉", "icon": "🎨", "order": 3},
+    "video": {"name": "视频", "icon": "🎞️", "order": 4},
+    "audio": {"name": "音频", "icon": "🔊", "order": 5},
+    "analysis": {"name": "分析", "icon": "🧠", "order": 6},
+    "external": {"name": "外部", "icon": "🧩", "order": 7},
+}
+
+# ── 图像类能力（关键帧环节的主力）────────────────────────
+Q = "qwen"          # 千问AI平台（阿里 DashScope）
+IMG_CAPS = [
+    {
+        "id": "text_to_image", "name": "文生图", "category": "image",
+        "description": "文本 → 图片（角色定妆照 / 概念图 / 场景图 / 关键帧草图）",
+        "input": {"prompt": "string"}, "output": {"image_path": "string"},
+        "params_schema": {
+            "size": {"type": "enum", "options": ["1K", "2K", "4K"], "default": "2K", "label": "分辨率档"},
+            "n": {"type": "number", "min": 1, "max": 12, "default": 1, "label": "输出张数"},
+            "negative_prompt": {"type": "string", "default": "", "label": "负向提示词（排除不要的）"},
+        },
+        "providers": [
+            {"id": Q, "name": "千问AI平台·万相2.7", "api_key_ref": "aliyun.api_key",
+             "model": "wan2.7-image-pro", "default": True},
+            {"id": Q, "name": "千问AI平台·Qwen-Image", "api_key_ref": "aliyun.api_key",
+             "model": "qwen-image-3.0-pro"},
+            {"id": Q, "name": "千问AI平台·Z-Image（快/便宜）", "api_key_ref": "aliyun.api_key",
+             "model": "z-image-turbo"},
+            {"id": "kling", "name": "可灵", "api_key_ref": "kling.api_key", "model": "kling-v3"},
+            {"id": "jimeng", "name": "即梦", "api_key_ref": "volcano.ark_api_key", "model": "seedream"},
+        ],
+        "defaults": {"provider": Q, "model": "wan2.7-image-pro", "size": "2K", "n": 1},
+        "execution": "remote", "status": "ready",
+        "impl": "character_sheet.py / 待接 qwen-image",
+        "docs": [{"source": Q, "slug": "api-reference-image-generation-wan27-image-gen-edit-synchronous",
+                  "title": "Wan 2.7 — 同步调用"}],
+    },
+    {
+        "id": "ref_to_image", "name": "参考图生图（多图融合）★", "category": "image",
+        "description": "1~9 张参考图（角色多角度 + 场景多机位 + 道具）+ 文本指令 → 融合成一张画面"
+                       " —— 关键帧创作的核心能力：角色与场景不再二选一",
+        "input": {"prompt": "string", "ref_images": "list[file]"},
+        "output": {"image_path": "string"},
+        "params_schema": {
+            "size": {"type": "enum", "options": ["1K", "2K", "4K"], "default": "2K", "label": "输出档"},
+            "n": {"type": "number", "min": 1, "max": 6, "default": 1, "label": "输出张数"},
+            "mode": {"type": "enum", "options": ["single", "sequential"], "default": "single",
+                     "label": "单张 / 顺序模式（一次多张且角色一致）"},
+        },
+        "providers": [
+            {"id": Q, "name": "千问AI平台·万相2.7 Pro（最多 9 图 + 顺序 12 张）",
+             "api_key_ref": "aliyun.api_key", "model": "wan2.7-image-pro", "default": True},
+            {"id": Q, "name": "千问AI平台·Qwen-Image-2.1 Pro（最多 10 图 + 透明通道）",
+             "api_key_ref": "aliyun.api_key", "model": "qwen-image-2.1-pro"},
+            {"id": "kling", "name": "可灵（主体1-4 + 场景 + 风格分离）",
+             "api_key_ref": "kling.api_key", "model": "kling-v2-1"},
+        ],
+        "defaults": {"provider": Q, "model": "wan2.7-image-pro", "mode": "single", "n": 1},
+        "execution": "remote", "status": "partial",
+        "impl": "待接（capabilities/kling_video.submit_multi_image2image 已有可灵版）",
+        "note": "关键帧创作的主力；可灵版接口已实现但从未接入流程",
+        "docs": [{"source": Q, "slug": "developer-guides-image-generation-wan-image-editing",
+                  "title": "图像编辑 - 万相2.7/2.6/2.5（多图输入多图输出）"},
+                 {"source": "kling", "slug": "multiImage2Image", "title": "可灵·多图参考生图"}],
+    },
+    {
+        "id": "image_edit", "name": "图像编辑（指令）", "category": "image",
+        "description": "文本指令修改图片（换主体 / 换环境 / 改细节），保留其余部分",
+        "input": {"prompt": "string", "ref_images": "list[file]"},
+        "output": {"image_path": "string"},
+        "params_schema": {"size": {"type": "enum", "options": ["1K", "2K"], "default": "2K", "label": "分辨率档"}},
+        "providers": [
+            {"id": Q, "name": "千问AI平台·Qwen-Image-Edit", "api_key_ref": "aliyun.api_key",
+             "model": "qwen-image-3.0-pro", "default": True},
+            {"id": Q, "name": "千问AI平台·Wan 2.5 图像编辑", "api_key_ref": "aliyun.api_key",
+             "model": "wan2.5-i2i-preview"},
+        ],
+        "defaults": {"provider": Q, "model": "qwen-image-3.0-pro"},
+        "execution": "remote", "status": "ready",
+        "impl": "待接（同步接口 ImageGeneration.call 支持 text+image 混排）",
+        "docs": [{"source": Q, "slug": "developer-guides-image-generation-image-editing",
+                  "title": "图像编辑"}],
+    },
+    {
+        "id": "image_inpaint", "name": "局部重绘", "category": "image",
+        "description": "给定原图 + 涂抹掩码 + 描述，只重绘指定区域",
+        "input": {"prompt": "string", "image": "file", "mask": "file"},
+        "output": {"image_path": "string"},
+        "params_schema": {},
+        "providers": [{"id": Q, "name": "千问AI平台·wanx-x-painting", "api_key_ref": "aliyun.api_key",
+                       "model": "wanx-x-painting", "default": True}],
+        "defaults": {"provider": Q}, "execution": "remote", "status": "ready",
+        "impl": "待接",
+        "docs": [{"source": Q, "slug": "developer-guides-image-generation-image-inpainting",
+                  "title": "图像局部重绘"}],
+    },
+    {
+        "id": "image_outpaint", "name": "画面扩展（扩图）", "category": "image",
+        "description": "按宽高比 / 比例 / 方向扩展画面，或旋转扩图",
+        "input": {"image": "file"}, "output": {"image_path": "string"},
+        "params_schema": {"expand_ratio": {"type": "number", "default": 1.5, "label": "扩展比例"}},
+        "providers": [{"id": Q, "name": "千问AI平台·image-out-painting", "api_key_ref": "aliyun.api_key",
+                       "model": "image-out-painting", "default": True}],
+        "defaults": {"provider": Q}, "execution": "remote", "status": "ready", "impl": "待接",
+        "docs": [{"source": Q, "slug": "developer-guides-image-generation-image-out-painting",
+                  "title": "图像画面扩展"}],
+    },
+    {
+        "id": "image_erase", "name": "擦除补全", "category": "image",
+        "description": "保留背景的同时擦除指定区域（去杂物 / 去路人）",
+        "input": {"image": "file", "mask": "file", "keep_mask": "file?"},
+        "output": {"image_path": "string"}, "params_schema": {},
+        "providers": [{"id": Q, "name": "千问AI平台·图像擦除补全", "api_key_ref": "aliyun.api_key",
+                       "model": "image-erase-completion", "default": True}],
+        "defaults": {"provider": Q}, "execution": "remote", "status": "ready", "impl": "待接",
+        "docs": [{"source": Q, "slug": "developer-guides-image-generation-image-erase-completion",
+                  "title": "图像擦除补全"}],
+    },
+    {
+        "id": "image_style_transfer", "name": "图像风格迁移", "category": "image",
+        "description": "把写实图转成指定美术风格（漫画 / 3D / 国风 / 纸艺 …）",
+        "input": {"image": "file", "style": "string"}, "output": {"image_path": "string"},
+        "params_schema": {},
+        "providers": [
+            {"id": Q, "name": "千问AI平台·万相2.1 通用编辑（风格迁移）",
+             "api_key_ref": "aliyun.api_key", "model": "wanx2.1-imageedit", "default": True},
+            {"id": Q, "name": "千问AI平台·Wan（背景生成/风格）", "api_key_ref": "aliyun.api_key",
+             "model": "wan2.7-image-pro"},
+        ],
+        "defaults": {"provider": Q}, "execution": "remote", "status": "ready", "impl": "待接",
+        "docs": [{"source": Q, "slug": "developer-guides-image-generation-wan21-image-editing",
+                  "title": "万相2.1 通用图像编辑（含风格迁移）"}],
+    },
+    {
+        "id": "sketch_to_image", "name": "涂鸦作画", "category": "image",
+        "description": "手绘草图 + 文字描述 → 成品画面",
+        "input": {"image": "file", "prompt": "string"}, "output": {"image_path": "string"},
+        "params_schema": {},
+        "providers": [{"id": Q, "name": "千问AI平台·涂鸦作画", "api_key_ref": "aliyun.api_key",
+                       "model": "wanx-sketch-to-image", "default": True}],
+        "defaults": {"provider": Q}, "execution": "remote", "status": "ready", "impl": "待接",
+        "docs": [{"source": Q, "slug": "developer-guides-image-generation-sketch-to-image",
+                  "title": "涂鸦作画"}],
+    },
+    {
+        "id": "background_generate", "name": "背景生成", "category": "image",
+        "description": "为商品/人物图生成高质量背景（文本引导 / 图像引导 / 边缘引导）",
+        "input": {"image": "file", "prompt": "string"}, "output": {"image_path": "string"},
+        "params_schema": {},
+        "providers": [{"id": Q, "name": "千问AI平台·图像背景生成", "api_key_ref": "aliyun.api_key",
+                       "model": "wanx-background-generation", "default": True}],
+        "defaults": {"provider": Q}, "execution": "remote", "status": "ready", "impl": "待接",
+        "docs": [{"source": Q, "slug": "developer-guides-image-generation-background-generation",
+                  "title": "图像背景生成"}],
+    },
+    {
+        "id": "virtual_model", "name": "虚拟模特 / 换装", "category": "image",
+        "description": "保持真人站姿不变，替换背景与模特（服装试穿 / 模特图批量生成）",
+        "input": {"image": "file", "template": "file"}, "output": {"image_path": "string"},
+        "params_schema": {},
+        "providers": [{"id": Q, "name": "千问AI平台·虚拟模特", "api_key_ref": "aliyun.api_key",
+                       "model": "wanx-virtualmodel", "default": True}],
+        "defaults": {"provider": Q}, "execution": "remote", "status": "ready", "impl": "待接",
+        "docs": [{"source": Q, "slug": "developer-guides-image-generation-virtual-model",
+                  "title": "虚拟模特生成"}],
+    },
+    {
+        "id": "person_segment", "name": "人物分割（抠图）", "category": "image",
+        "description": "识别图中人物并输出像素级掩码（抠人 / 合成前置）",
+        "input": {"image": "file"}, "output": {"mask_path": "string"},
+        "params_schema": {},
+        "providers": [{"id": Q, "name": "千问AI平台·人物实例分割", "api_key_ref": "aliyun.api_key",
+                       "model": "person-instance-segmentation", "default": True}],
+        "defaults": {"provider": Q}, "execution": "remote", "status": "ready", "impl": "待接",
+        "docs": [{"source": Q, "slug": "developer-guides-image-generation-person-instance-segmentation",
+                  "title": "人物实例分割"}],
+    },
+]
+
+# ── 视频类新增 ────────────────────────────────────────────
+VIDEO_CAPS = [
+    {
+        "id": "ref_to_video", "name": "参考驱动生视频（R2V）★", "category": "video",
+        "description": "参考图/视频（最多 5 个混合）+ 文本 → 视频；可同时给角色图 + 场景图 + 首帧，"
+                       "还可绑音色（reference_voice）",
+        "input": {"prompt": "string", "media": "list[{type,url}]"},
+        "output": {"video_path": "string"},
+        "params_schema": {
+            "resolution": {"type": "enum", "options": ["720P", "1080P"], "default": "720P", "label": "清晰度"},
+            "duration": {"type": "number", "min": 2, "max": 10, "default": 5, "label": "时长(秒)"},
+            "ratio": {"type": "enum", "options": ["16:9", "9:16", "1:1"], "default": "9:16", "label": "宽高比"},
+        },
+        "providers": [
+            {"id": Q, "name": "千问AI平台·Wan2.7-R2V（5 参考 + 音色）", "api_key_ref": "aliyun.api_key",
+             "model": "wan2.7-r2v", "default": True},
+            {"id": Q, "name": "千问AI平台·Wan2.6-R2V-Flash（¥0.3/秒）", "api_key_ref": "aliyun.api_key",
+             "model": "wan2.6-r2v-flash"},
+        ],
+        "defaults": {"provider": Q, "model": "wan2.6-r2v-flash", "resolution": "720P", "duration": 5},
+        "execution": "remote", "status": "ready", "impl": "待接",
+        "docs": [{"source": Q, "slug": "api-reference-video-generation-wan27-image-to-video-create-task",
+                  "title": "Wan 2.7 参考生视频"}],
+    },
+    {
+        "id": "keyframe_video", "name": "首尾帧生视频（KF2V）★", "category": "video",
+        "description": "首帧图 + 尾帧图 + 过渡描述 → 平滑过渡视频（固定 5 秒）—— 关键帧 → 视频的主力",
+        "input": {"prompt": "string?", "first_frame_url": "file", "last_frame_url": "file"},
+        "output": {"video_path": "string"},
+        "params_schema": {
+            "resolution": {"type": "enum", "options": ["480P", "720P", "1080P"], "default": "480P", "label": "清晰度"},
+            "duration": {"type": "number", "default": 5, "label": "时长(秒) · 固定 5"},
+            "template": {"type": "string", "default": "", "label": "特效模板（填了则只需首帧）"},
+        },
+        "providers": [
+            {"id": Q, "name": "千问AI平台·Wan2.2-KF2V-Flash（¥0.1/秒·480P）",
+             "api_key_ref": "aliyun.api_key", "model": "wan2.2-kf2v-flash", "default": True},
+            {"id": Q, "name": "千问AI平台·Wan2.1-KF2V-Plus", "api_key_ref": "aliyun.api_key",
+             "model": "wanx2.1-kf2v-plus"},
+        ],
+        "defaults": {"provider": Q, "model": "wan2.2-kf2v-flash", "resolution": "480P"},
+        "execution": "remote", "status": "ready", "impl": "待接",
+        "docs": [{"source": Q, "slug": "api-reference-video-generation-wan-image-to-video-first-last-frames-create-task",
+                  "title": "Wan — 首尾帧图生视频"}],
+    },
+    {
+        "id": "video_style_transform", "name": "视频风格重绘", "category": "video",
+        "description": "把视频转成 8 种艺术风格（日式漫画 / 美式漫画 / 清新漫画 / 3D卡通 / 国风 / 纸艺 / 简易插画 …）",
+        "input": {"video_url": "file"}, "output": {"video_path": "string"},
+        "params_schema": {
+            "style": {"type": "enum",
+                      "options": ["0 日式漫画", "1 美式漫画", "2 清新漫画", "3 3D卡通",
+                                  "4 国风卡通", "5 纸艺风格", "6 简易插画", "7"],
+                      "default": "3 3D卡通", "label": "目标风格"},
+            "animate_emotion": {"type": "bool", "default": False, "label": "情绪表情迁移"},
+            "use_SR": {"type": "bool", "default": False, "label": "超分辨率增强"},
+            "min_len": {"type": "enum", "options": [720, 540], "default": 540, "label": "短边分辨率"},
+        },
+        "providers": [{"id": Q, "name": "千问AI平台·视频风格重绘", "api_key_ref": "aliyun.api_key",
+                       "model": "video-style-transform", "default": True}],
+        "defaults": {"provider": Q}, "execution": "remote", "status": "ready", "impl": "待接",
+        "docs": [{"source": Q, "slug": "api-reference-video-generation-video-style-transform-create-task",
+                  "title": "视频风格重绘"}],
+    },
+    {
+        "id": "video_edit", "name": "统一视频编辑", "category": "video",
+        "description": "一个接口靠 function 切换五大能力：多图参考 / 视频重绘 / 局部编辑 / 视频续写 / 画面扩展",
+        "input": {"function": "enum", "prompt": "string", "ref_images_url": "list[file]",
+                  "video_url": "file?", "mask_*": "any"},
+        "output": {"video_path": "string"},
+        "params_schema": {
+            "function": {"type": "enum",
+                         "options": ["多图参考", "视频重绘", "局部编辑", "视频续写", "画面扩展"],
+                         "default": "局部编辑", "label": "编辑类型"},
+            "strength": {"type": "number", "default": 0.8, "label": "强度"},
+        },
+        "providers": [{"id": Q, "name": "千问AI平台·Wan 统一视频编辑", "api_key_ref": "aliyun.api_key",
+                       "model": "wan-video-editing", "default": True}],
+        "defaults": {"provider": Q}, "execution": "remote", "status": "ready", "impl": "待接",
+        "docs": [{"source": Q, "slug": "openapi-wan-video-editing", "title": "Wan 统一视频编辑"}],
+    },
+    {
+        "id": "character_swap", "name": "视频换人（保留场景光线）", "category": "video",
+        "description": "把我的角色图换到别人的视频里，保留原视频的场景、光线与色调",
+        "input": {"image_url": "file", "video_url": "file"}, "output": {"video_path": "string"},
+        "params_schema": {
+            "mode": {"type": "enum", "options": ["wan-std", "wan-pro"], "default": "wan-std",
+                     "label": "模式（std 快/便宜 · pro 专业）"},
+            "check_image": {"type": "bool", "default": True, "label": "图片合规检测"},
+        },
+        "providers": [{"id": Q, "name": "千问AI平台·视频换脸", "api_key_ref": "aliyun.api_key",
+                       "model": "wan-character-swap", "default": True}],
+        "defaults": {"provider": Q, "mode": "wan-std"}, "execution": "remote", "status": "ready", "impl": "待接",
+        "docs": [{"source": Q, "slug": "openapi-wan-character-swap", "title": "Wan 视频换脸"}],
+    },
+]
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--dry", action="store_true", help="只打印，不写文件")
+    args = ap.parse_args()
+
+    raw = CFG.read_text(encoding="utf-8")
+    d = yaml.safe_load(raw)
+    cats = d.get("categories") or {}
+    caps = d.get("capabilities") or []
+    existing = {c["id"] for c in caps}
+
+    # 1) 分类：新顺序（图像在最前）
+    d["categories"] = NEW_CATEGORIES
+
+    # 2) 已有能力：把图像相关的归到 image
+    MOVE = {"text_to_image", "person_swap"}
+    for c in caps:
+        if c["id"] in MOVE:
+            c["category"] = "image"
+
+    # 3) 追加新能力（已存在的先移除旧的同名，保持新版）
+    add = [c for c in (IMG_CAPS + VIDEO_CAPS) if c["id"] not in existing or c["id"] == "text_to_image"]
+    caps = [c for c in caps if c["id"] not in {x["id"] for x in add}]
+    caps += add
+    d["capabilities"] = caps
+    d["updated"] = datetime.now().strftime("%Y-%m-%d")
+
+    out = yaml.dump(d, allow_unicode=True, sort_keys=False, width=200)
+    n_new = len([x for x in add if x["id"] not in existing])
+    print(f"分类: {len(NEW_CATEGORIES)} · 能力总数: {len(caps)}（新增 {n_new}）")
+    for c in add:
+        print(f"  + [{c['category']}] {c['id']:<24} {c['name']}")
+
+    if args.dry:
+        print("\n(--dry：未写入)")
+        return 0
+
+    bak = CFG.with_suffix(f".yaml.bak_{datetime.now().strftime('%Y%m%d_%H%M%S')}")
+    shutil.copy2(CFG, bak)
+    CFG.write_text(out, encoding="utf-8")
+    print(f"\n✅ 已写入 {CFG.name}（备份 {bak.name}）")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+"""API 文档中心 —— 官方文档抓取脚本（可复用、可增量更新）
+
+用途
+    把各平台官方 API 文档抓到本地 `docs/api/raw/<source>/`，
+    并生成 `docs/api/index.json` 供看板「📚 API 文档中心」页读取。
+    以后需要更新，直接重跑本脚本即可（按 sha256 判断是否变化）。
+
+用法
+    python3 scripts/tools/fetch_api_docs.py              # 增量更新
+    python3 scripts/tools/fetch_api_docs.py --force      # 全量重抓
+    python3 scripts/tools/fetch_api_docs.py --list       # 只列出将抓取的页面，不下载
+    python3 scripts/tools/fetch_api_docs.py --source qwen  # 只抓某个源
+
+设计
+    · 源列表写在 SOURCES（可随时扩平台：可灵 / 火山 / 其他）
+    · 千问AI平台提供 llms.txt 索引 → 自动发现全部页面
+    · 其他平台按显式 URL 列表抓
+    · 全部走 lib.http（禁系统代理 + 重试）
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sys
+import time
+import urllib.request
+from datetime import datetime
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]           # 05_tools/09_ave
+sys.path.insert(0, str(ROOT / "scripts"))
+from lib.http import _NO_PROXY_OPENER  # noqa: E402
+
+OUT_RAW = ROOT / "docs" / "api" / "raw"
+OUT_INDEX = ROOT / "docs" / "api" / "index.json"
+UA = {"User-Agent": "Mozilla/5.0 (compatible; AgentOS-DocFetcher)"}
+
+# ── 抓取源配置 ──────────────────────────────────────────────
+# useful_keywords: 只抓路径/标题含这些关键词的页面（None = 全抓）
+SOURCES = [
+    {
+        "id": "qwen",
+        "name": "千问AI平台（阿里 DashScope）",
+        "index": "https://platform.qianwenai.com/docs/llms.txt",
+        "base": "https://platform.qianwenai.com/docs/",
+        "kind": "llms-txt",
+        # 按【路径】筛选（比标题关键词准）：只保留与图像/视频/语音/多模态相关的目录
+        "include_paths": [
+            "image-generation", "video-generation", "multimodal",
+            "getting-started/image", "getting-started/video", "getting-started/vision",
+            "api-reference/image", "api-reference/video",
+            "speech/", "virtual-model", "background-generation",
+        ],
+        # 明确排除（避免 text-generation / token-plan 之类的噪音）
+        "exclude_paths": [
+            "text-generation", "token-plan", "embeddings", "third-party-models",
+            "run-and-scale", "accuracy-tuning", "tool-calling", "openapi-",
+        ],
+    },
+    {
+        # ⚠️ 可灵文档站是 JS 单页应用：直接抓只会拿到 HTML 骨架（无正文）。
+        #    因此改用 link-only 模式 —— 只登记 URL，供看板跳转到官方在线文档。
+        "id": "kling",
+        "name": "可灵 AI（Kling）",
+        "kind": "link-only",
+        "base": "https://app.klingai.com/cn/dev/document-api/apiReference/model/",
+        "pages": [
+            ("imageGeneration",  "文生图 / 图生图"),
+            ("multiImage2Image", "多图参考生图"),
+            ("videoGeneration",  "视频生成（文生/图生视频）"),
+            ("lipsync",          "对口型"),
+            ("videoExtend",      "视频延长"),
+        ],
+    },
+]
+
+
+def _fetch(url: str, timeout: int = 30) -> str | None:
+    try:
+        r = _NO_PROXY_OPENER.open(urllib.request.Request(url, headers=UA), timeout=timeout)
+        return r.read().decode("utf-8", "ignore")
+    except Exception as e:
+        print(f"    ✗ {url[:90]} → {type(e).__name__}: {str(e)[:70]}")
+        return None
+
+
+def _slug(s: str, maxlen: int = 70) -> str:
+    keep = []
+    for ch in s:
+        if ch.isalnum() or ch in "-_.":
+            keep.append(ch.lower())
+        else:
+            keep.append("-")
+    out = "".join(keep).strip("-")
+    while "--" in out:
+        out = out.replace("--", "-")
+    return out[:maxlen] or "page"
+
+
+def discover_qwen(src: dict) -> list[tuple[str, str, str]]:
+    """从 llms.txt 发现页面 → [(slug, 标题, 完整URL)]"""
+    txt = _fetch(src["index"])
+    if not txt:
+        return []
+    inc = [k.lower() for k in (src.get("include_paths") or [])]
+    exc = [k.lower() for k in (src.get("exclude_paths") or [])]
+    out = []
+    for line in txt.splitlines():
+        if not line.startswith("- ["):
+            continue
+        try:
+            title = line.split("](", 1)[0][3:]
+            url = line.split("](", 1)[1].split(")")[0]
+        except Exception:
+            continue
+        if not url.endswith(".md"):
+            continue
+        low = url.lower()
+        if exc and any(k in low for k in exc):
+            continue
+        if inc and not any(k in low for k in inc):
+            continue
+        slug = _slug(url.split("/docs/", 1)[-1].replace(".md", ""))
+        out.append((slug, title, url))
+    return out
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--force", action="store_true", help="忽略缓存，全量重抓")
+    ap.add_argument("--list", action="store_true", help="只列出，不下载")
+    ap.add_argument("--source", default="", help="只抓某个源 id")
+    args = ap.parse_args()
+
+    OUT_RAW.mkdir(parents=True, exist_ok=True)
+    index: dict = {"generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "sources": []}
+
+    for src in SOURCES:
+        if args.source and src["id"] != args.source:
+            continue
+        print(f"\n=== [{src['id']}] {src['name']} ===")
+        link_only = src["kind"] == "link-only"
+        if src["kind"] == "llms-txt":
+            pages = discover_qwen(src)
+        else:
+            pages = [(s, t, src["base"] + s) for s, t in (src.get("pages") or [])]
+        print(f"  发现 {len(pages)} 个页面" + ("（link-only：只登记 URL，不下载）" if link_only else ""))
+        if link_only:
+            index["sources"].append({
+                "id": src["id"], "name": src["name"], "count": len(pages),
+                "link_only": True,
+                "pages": [{"slug": s, "title": t, "url": u, "file": None, "bytes": 0, "sha256": ""}
+                          for s, t, u in pages],
+            })
+            continue
+        if args.list:
+            for slug, title, url in pages[:80]:
+                print(f"    · {title[:60]}  →  {url[:80]}")
+            continue
+
+        sdir = OUT_RAW / src["id"]
+        sdir.mkdir(parents=True, exist_ok=True)
+        entries, ok, skipped = [], 0, 0
+        for slug, title, url in pages:
+            dest = sdir / f"{slug}.md"
+            if dest.exists() and not args.force:
+                skipped += 1
+                txt = dest.read_text(encoding="utf-8")
+            else:
+                txt = _fetch(url)
+                if txt is None:
+                    continue
+                dest.write_text(txt, encoding="utf-8")
+                ok += 1
+                time.sleep(0.15)          # 温和抓取
+            entries.append({
+                "slug": slug,
+                "title": title,
+                "url": url,
+                "file": str(dest.relative_to(ROOT)),
+                "bytes": len(txt),
+                "sha256": hashlib.sha256(txt.encode()).hexdigest()[:16],
+            })
+        print(f"  下载 {ok} · 跳过(已存在) {skipped} · 合计 {len(entries)}")
+        index["sources"].append({
+            "id": src["id"], "name": src["name"],
+            "count": len(entries), "pages": entries,
+        })
+
+    if not args.list:
+        OUT_INDEX.write_text(json.dumps(index, ensure_ascii=False, indent=2), encoding="utf-8")
+        print(f"\n✅ 索引已写入 {OUT_INDEX.relative_to(ROOT)}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())