feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程，直接发 HTTP
移植自 mac-agent-os 的 mediacrawler_adapter：不起子进程、不开页面，用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin，所以 ingest 那条链路一个字不用改。
**目前能用的（真环境实测，非推断）**：
profile/other : 200, 7075 字节 —— 博主主页指标（粉丝/获赞/作品数/昵称）
aweme/detail : 200, 45425 字节 —— 单条作品详情（含点赞/评论/收藏/分享）
**目前不能用的：作品列表 `aweme/post`。** 两个互相独立的原因：
1. 这个接口被抖音单独升级成了真校验：不带 x-tt-argus 回 403「Uifid Not Found」，
带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过，「真校验」过不了。
同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的，
挑中的恰好是「批量拉作品列表」这个最敏感的动作。
2. 改走页面截获也不行：CDP 浏览器打开博主主页会落到「验证码中间页」（当天大量探测的
代价，过几小时要重测）。
所以现在的边界是：**已知作品的指标刷新能做，自动发现新作品做不了**。
**排查中控住变量后得到的两条事实**（都写进注释了）：
· `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
UA+client hints+Cookie 时是 200 但仅 121 字节的空壳，补上这三个头变 7074 字节。
（我先前猜的 sec-ch-ua 不是关键。）
· 因此 UA 与 client hints 必须**成套地取自同一个浏览器**，所以 BrowserIdentity 一次
从 CDP 取齐 cookie + UA + hints，而不是各自写死。
「200 + 空 body 必须当场报错」也是刻意写死的：放过去它会在下游变成「这个博主没作品」，
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的，还被翻译成「账号被封」。
测试 +11：cookie 解析、请求头成套性（含 uifid 缺失/回退）、产物键名与 store 对齐、
以及 _get 的三条失败路径（空 body / 403 带网关原话 / 正常返回）。
This commit is contained in:
@@ -0,0 +1,461 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
inbox_refine.py - 知识库收件箱提纯脚本
|
||||
|
||||
将 00_inbox/ 中的原始内容按知识属性分类归档到对应目录，
|
||||
更新知识库首页统计和变更日志。
|
||||
|
||||
用法：
|
||||
python3 inbox_refine.py --root ~/workbuddy-agent-os/agent-sync [--dry-run] [--date YYYY-MM-DD]
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
# ─── 角色检查：仅 master 可执行收件箱提纯 ───
|
||||
try:
|
||||
sys.path.insert(0, str(Path.home() / "workbuddy-agent-os" / "agent-sync" / "05_tools" / "01_system"))
|
||||
from role_check import check_role, check_capability
|
||||
if not check_role(["master"]):
|
||||
print("⛔ 非 master 角色，跳过 inbox_refine")
|
||||
sys.exit(0)
|
||||
if not check_capability("inbox_refine"):
|
||||
print("⛔ 能力 inbox_refine 未启用，跳过")
|
||||
sys.exit(0)
|
||||
except ImportError as e:
|
||||
print(f"⚠️ 角色检查模块不可用 ({e})，继续执行")
|
||||
|
||||
# 导入 LLM 分类器
|
||||
try:
|
||||
from llm_classifier import classify_content_enhanced
|
||||
LLM_AVAILABLE = True
|
||||
except ImportError:
|
||||
print("⚠️ LLM 分类器不可用，将使用启发式规则")
|
||||
LLM_AVAILABLE = False
|
||||
|
||||
|
||||
# ── 配置 ──────────────────────────────────────────
|
||||
|
||||
NATURE_TO_DIR = {
|
||||
"fact": "30_facts",
|
||||
"method": "20_methods",
|
||||
"concept": "10_concepts",
|
||||
"axiom": "10_concepts",
|
||||
"regulation": "30_facts",
|
||||
"reference": "40_references",
|
||||
"data": "30_facts",
|
||||
"opinion": "60_opinions",
|
||||
"quote": "60_opinions",
|
||||
}
|
||||
|
||||
SOURCE_TYPE_CONFIDENCE = {
|
||||
"official_doc": 0.9,
|
||||
"literature": 0.8,
|
||||
"experiment": 0.7,
|
||||
"personal_exp": 0.5,
|
||||
"social_media": 0.3,
|
||||
"unknown": 0.4,
|
||||
}
|
||||
|
||||
DIR_CHINESE = {
|
||||
"00_inbox": "📥 收件箱",
|
||||
"01_daily": "📅 日记",
|
||||
"10_concepts": "💡 概念层",
|
||||
"20_methods": "🔧 方法层",
|
||||
"30_facts": "📋 事实层",
|
||||
"40_references": "📎 参考层",
|
||||
"50_resources": "🛠 资源层",
|
||||
"60_opinions": "💭 观点层",
|
||||
"90_archive": "🗄 归档层",
|
||||
}
|
||||
|
||||
TRACKED_DIRS = [
|
||||
"00_inbox", "01_daily", "10_concepts", "20_methods",
|
||||
"30_facts", "40_references", "50_resources", "60_opinions", "90_archive"
|
||||
]
|
||||
|
||||
|
||||
# ── 工具函数 ──────────────────────────────────────
|
||||
|
||||
def count_md_files(directory: Path) -> int:
|
||||
"""统计目录下 .md 文件数量"""
|
||||
if not directory.exists():
|
||||
return 0
|
||||
return sum(1 for f in directory.rglob("*.md") if f.is_file())
|
||||
|
||||
|
||||
def latest_modified(directory: Path) -> str:
|
||||
"""获取目录下最近修改的 .md 文件日期"""
|
||||
if not directory.exists():
|
||||
return "—"
|
||||
md_files = list(directory.rglob("*.md"))
|
||||
if not md_files:
|
||||
return "—"
|
||||
latest = max(f.stat().st_mtime for f in md_files)
|
||||
return datetime.fromtimestamp(latest).strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
def generate_id(date_str: str, knowledge_root: Path) -> str:
|
||||
"""生成唯一知识ID：KB-YYYYMMDD-NNN"""
|
||||
prefix = f"KB-{date_str.replace('-', '')}"
|
||||
# 扫描已有ID确定序号
|
||||
existing = set()
|
||||
for md in knowledge_root.rglob("*.md"):
|
||||
content = md.read_text(encoding="utf-8", errors="ignore")[:500]
|
||||
m = re.search(r'^id:\s*(KB-\d{8}-\d+)', content, re.MULTILINE)
|
||||
if m:
|
||||
existing.add(m.group(1))
|
||||
n = 1
|
||||
while f"{prefix}-{n:03d}" in existing:
|
||||
n += 1
|
||||
return f"{prefix}-{n:03d}"
|
||||
|
||||
|
||||
def parse_frontmatter(text: str) -> dict:
|
||||
"""解析 YAML frontmatter"""
|
||||
if not text.startswith("---"):
|
||||
return {}
|
||||
match = re.match(r'^---\s*\n(.*?)\n---', text, re.DOTALL)
|
||||
if not match:
|
||||
return {}
|
||||
fm = {}
|
||||
for line in match.group(1).split("\n"):
|
||||
if ":" in line:
|
||||
key, val = line.split(":", 1)
|
||||
fm[key.strip()] = val.strip().strip('"').strip("'")
|
||||
return fm
|
||||
|
||||
|
||||
def build_frontmatter(kb_id: str, title: str, nature: str, domain: str,
|
||||
confidence: float, source: str, source_type: str,
|
||||
date_str: str, summary: str = "", tags: list = None) -> str:
|
||||
"""构建知识卡片 frontmatter"""
|
||||
if tags is None:
|
||||
tags = ["待补充"]
|
||||
|
||||
# 确保 domain 是列表格式
|
||||
if isinstance(domain, str):
|
||||
domain_list = [domain]
|
||||
else:
|
||||
domain_list = domain if isinstance(domain, list) else ["other"]
|
||||
|
||||
# 构建 tags 字符串
|
||||
tags_str = ", ".join([f'"{tag}"' for tag in tags])
|
||||
|
||||
return f"""---
|
||||
id: {kb_id}
|
||||
title: "{title}"
|
||||
type: {nature if nature in ('concept', 'fact', 'method', 'opinion', 'reference') else 'concept'}
|
||||
status: published
|
||||
nature: {nature}
|
||||
domain: [{", ".join(domain_list)}]
|
||||
subdomain: []
|
||||
tags: [{tags_str}]
|
||||
confidence: {confidence}
|
||||
source: "{source}"
|
||||
source_type: {source_type}
|
||||
date_created: {date_str}
|
||||
date_modified: {date_str}
|
||||
version: 1
|
||||
previous_version: ""
|
||||
superseded_by: ""
|
||||
summary: "{summary}"
|
||||
---"""
|
||||
|
||||
|
||||
# ── 分类逻辑 ──────────────────────────────────────
|
||||
|
||||
def classify_content(text: str, fm: dict) -> dict:
|
||||
"""
|
||||
增强版分类函数：优先使用 LLM 智能分类，失败时降级到启发式规则。
|
||||
如果 frontmatter 已有 nature/domain，优先使用。
|
||||
"""
|
||||
# 如果 LLM 可用，直接使用增强版分类
|
||||
if LLM_AVAILABLE:
|
||||
try:
|
||||
# 直接使用 LLM 分类，分类器内部会处理降级
|
||||
result = classify_content_enhanced(text, fm, use_llm=True)
|
||||
# 适配现有结构
|
||||
return {
|
||||
"nature": result["nature"],
|
||||
"domain": result["domain"][0] if result["domain"] else "other",
|
||||
"confidence": result["confidence"],
|
||||
"source_type": result.get("source_type", "unknown"),
|
||||
"source": result.get("source", ""),
|
||||
"title": result.get("title", ""),
|
||||
"summary": result.get("summary", ""),
|
||||
"tags": result.get("tags", [])
|
||||
}
|
||||
except Exception as e:
|
||||
print(f"⚠️ LLM 分类失败，回退到启发式规则: {e}")
|
||||
|
||||
# LLM 不可用或失败，使用启发式规则
|
||||
result = {
|
||||
"nature": "concept",
|
||||
"domain": "other",
|
||||
"confidence": 0.4,
|
||||
"source_type": "unknown",
|
||||
"source": "",
|
||||
"title": "",
|
||||
"summary": "",
|
||||
"tags": []
|
||||
}
|
||||
|
||||
# 优先使用已有 frontmatter
|
||||
if fm.get("nature") and fm["nature"] in NATURE_TO_DIR:
|
||||
result["nature"] = fm["nature"]
|
||||
if fm.get("domain"):
|
||||
result["domain"] = fm["domain"]
|
||||
if fm.get("confidence"):
|
||||
try:
|
||||
result["confidence"] = float(fm["confidence"])
|
||||
except ValueError:
|
||||
pass
|
||||
if fm.get("source"):
|
||||
result["source"] = fm["source"]
|
||||
if fm.get("source_type"):
|
||||
result["source_type"] = fm["source_type"]
|
||||
if fm.get("title"):
|
||||
result["title"] = fm["title"]
|
||||
|
||||
# 如果没有 nature，用启发式规则
|
||||
if not fm.get("nature"):
|
||||
lower = text.lower()
|
||||
# 方法类关键词
|
||||
method_kw = ["步骤", "教程", "如何", "how to", "方法", "指南", "最佳实践", "best practice"]
|
||||
fact_kw = ["数据", "统计", "报告", "结果", "事实", "date", "report"]
|
||||
opinion_kw = ["认为", "觉得", "观点", "推测", "假设", "opinion", "think"]
|
||||
reference_kw = ["论文", "文献", "引用", "参考", "paper", "reference"]
|
||||
|
||||
if any(kw in lower for kw in method_kw):
|
||||
result["nature"] = "method"
|
||||
elif any(kw in lower for kw in fact_kw):
|
||||
result["nature"] = "fact"
|
||||
elif any(kw in lower for kw in opinion_kw):
|
||||
result["nature"] = "opinion"
|
||||
elif any(kw in lower for kw in reference_kw):
|
||||
result["nature"] = "reference"
|
||||
|
||||
# confidence from source_type
|
||||
if result["source_type"] in SOURCE_TYPE_CONFIDENCE:
|
||||
result["confidence"] = SOURCE_TYPE_CONFIDENCE[result["source_type"]]
|
||||
|
||||
# 生成简单标题和摘要
|
||||
if not result["title"]:
|
||||
first_line = text.split('\n')[0].strip('# ')
|
||||
if len(first_line) < 50:
|
||||
result["title"] = first_line
|
||||
|
||||
if not result["summary"]:
|
||||
clean_text = re.sub(r'[#*\-`]', '', text)
|
||||
result["summary"] = clean_text[:100] + ("..." if len(clean_text) > 100 else "")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ── 主流程 ─────────────────────────────────────────
|
||||
|
||||
def refine_inbox(root: str, dry_run: bool = False, date_str: str = None):
|
||||
"""执行收件箱提纯"""
|
||||
root = Path(os.path.expanduser(root))
|
||||
kb_root = root / "03_knowledge"
|
||||
inbox = kb_root / "00_inbox"
|
||||
|
||||
if not inbox.exists():
|
||||
print(f"❌ 收件箱不存在: {inbox}")
|
||||
return
|
||||
|
||||
date_str = date_str or datetime.now().strftime("%Y-%m-%d")
|
||||
md_files = sorted(inbox.glob("*.md"))
|
||||
|
||||
if not md_files:
|
||||
print(f"📭 收件箱为空，无需提纯 ({date_str})")
|
||||
# 仍然更新首页统计
|
||||
update_homepage(kb_root)
|
||||
return
|
||||
|
||||
print(f"📬 收件箱有 {len(md_files)} 个文件待提纯")
|
||||
|
||||
archived = []
|
||||
skipped = []
|
||||
errors = []
|
||||
|
||||
for md_file in md_files:
|
||||
try:
|
||||
text = md_file.read_text(encoding="utf-8", errors="ignore")
|
||||
fm = parse_frontmatter(text)
|
||||
title = fm.get("title", md_file.stem)
|
||||
|
||||
# 分类
|
||||
classification = classify_content(text, fm)
|
||||
nature = classification["nature"]
|
||||
target_dir_name = NATURE_TO_DIR.get(nature, "10_concepts")
|
||||
target_dir = kb_root / target_dir_name
|
||||
|
||||
# 去重检查
|
||||
is_dup = False
|
||||
for existing in target_dir.rglob("*.md"):
|
||||
existing_text = existing.read_text(encoding="utf-8", errors="ignore")[:200]
|
||||
existing_fm = parse_frontmatter(existing_text)
|
||||
if existing_fm.get("title") == title:
|
||||
is_dup = True
|
||||
break
|
||||
|
||||
if is_dup:
|
||||
print(f" ⏭ 重复跳过: {title}")
|
||||
skipped.append(title)
|
||||
if not dry_run:
|
||||
md_file.unlink()
|
||||
continue
|
||||
|
||||
# 生成 ID 和新文件
|
||||
kb_id = generate_id(date_str, kb_root)
|
||||
domain = classification["domain"]
|
||||
confidence = classification["confidence"]
|
||||
source = classification["source"]
|
||||
source_type = classification["source_type"]
|
||||
|
||||
# 构建新内容
|
||||
body = text
|
||||
if text.startswith("---"):
|
||||
# 去掉旧 frontmatter
|
||||
body = re.sub(r'^---\s*\n.*?\n---\s*\n', '', text, count=1, flags=re.DOTALL)
|
||||
|
||||
new_content = build_frontmatter(
|
||||
kb_id,
|
||||
classification.get("title", title),
|
||||
nature,
|
||||
domain,
|
||||
confidence,
|
||||
source,
|
||||
source_type,
|
||||
date_str,
|
||||
classification.get("summary", ""),
|
||||
classification.get("tags", [])
|
||||
) + "\n\n" + body.strip() + "\n"
|
||||
|
||||
# 确定目标文件名
|
||||
safe_title = re.sub(r'[^\w\u4e00-\u9fff-]', '_', title)[:50]
|
||||
target_file = target_dir / f"{date_str}_{safe_title}.md"
|
||||
|
||||
if dry_run:
|
||||
print(f" 🔍 [DRY] {title} → {target_dir_name}/ ({nature})")
|
||||
else:
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
target_file.write_text(new_content, encoding="utf-8")
|
||||
md_file.unlink()
|
||||
print(f" ✅ {title} → {target_dir_name}/ ({nature}) [{kb_id}]")
|
||||
|
||||
archived.append({
|
||||
"id": kb_id,
|
||||
"title": title,
|
||||
"nature": nature,
|
||||
"dir": target_dir_name,
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 处理失败: {md_file.name} - {e}")
|
||||
errors.append(md_file.name)
|
||||
|
||||
# 更新首页
|
||||
if not dry_run:
|
||||
update_homepage(kb_root)
|
||||
update_changelog(kb_root, date_str, archived, skipped)
|
||||
|
||||
print(f"\n📊 提纯完成: 归档 {len(archived)}, 跳过 {len(skipped)}, 失败 {len(errors)}")
|
||||
|
||||
|
||||
def update_homepage(kb_root: Path):
|
||||
"""更新知识库首页统计"""
|
||||
readme = kb_root / "README.md"
|
||||
if not readme.exists():
|
||||
print("⚠️ 首页 README.md 不存在，跳过统计更新")
|
||||
return
|
||||
|
||||
# 计算各目录统计
|
||||
stats = {}
|
||||
total = 0
|
||||
for dirname in TRACKED_DIRS:
|
||||
dirpath = kb_root / dirname
|
||||
count = count_md_files(dirpath)
|
||||
latest = latest_modified(dirpath)
|
||||
stats[dirname] = (count, latest)
|
||||
total += count
|
||||
|
||||
# 构建统计表格
|
||||
table_lines = []
|
||||
for dirname in TRACKED_DIRS:
|
||||
count, latest = stats[dirname]
|
||||
emoji_name = DIR_CHINESE.get(dirname, dirname)
|
||||
table_lines.append(f"| {emoji_name} | {count} | {latest} |")
|
||||
table_lines.append(f"| **总计** | **{total}** | — |")
|
||||
|
||||
table_text = "\n".join(table_lines)
|
||||
|
||||
# 替换 README 中的统计部分
|
||||
content = readme.read_text(encoding="utf-8")
|
||||
pattern = r'(\| 📥 收件箱 \|.*?)(\| \*\*总计\*\* \| \*\*\d+\*\* \| — \|)'
|
||||
replacement = table_text
|
||||
content = re.sub(pattern, replacement, content, flags=re.DOTALL)
|
||||
|
||||
# 更新最后更新日期
|
||||
today = datetime.now().strftime("%Y-%m-%d")
|
||||
content = re.sub(
|
||||
r'\*最后更新：.*? by inbox_refine 技能\*',
|
||||
f'*最后更新：{today} by inbox_refine 技能*',
|
||||
content
|
||||
)
|
||||
|
||||
readme.write_text(content, encoding="utf-8")
|
||||
print(f"📄 首页统计已更新 (总计 {total} 个文件)")
|
||||
|
||||
|
||||
def update_changelog(kb_root: Path, date_str: str, archived: list, skipped: list):
|
||||
"""更新变更日志"""
|
||||
changelog = kb_root / "CHANGELOG.md"
|
||||
if not changelog.exists():
|
||||
initial = "# 知识库变更日志\n"
|
||||
changelog.write_text(initial, encoding="utf-8")
|
||||
|
||||
content = changelog.read_text(encoding="utf-8")
|
||||
|
||||
# 构建新条目
|
||||
entry = f"\n## {date_str}\n\n"
|
||||
if archived:
|
||||
entry += "### 归档\n"
|
||||
for item in archived:
|
||||
entry += f"- [{item['nature']}] {item['title']} → {item['dir']}/ ({item['id']})\n"
|
||||
if skipped:
|
||||
entry += f"\n### 跳过（重复）\n"
|
||||
for title in skipped:
|
||||
entry += f"- {title}\n"
|
||||
|
||||
# 在第一个 ## 之前插入（跳过标题行）
|
||||
lines = content.split("\n")
|
||||
insert_idx = 1
|
||||
for i, line in enumerate(lines):
|
||||
if line.startswith("## "):
|
||||
insert_idx = i
|
||||
break
|
||||
|
||||
lines.insert(insert_idx, entry)
|
||||
changelog.write_text("\n".join(lines), encoding="utf-8")
|
||||
print(f"📝 变更日志已更新")
|
||||
|
||||
|
||||
# ── 入口 ──────────────────────────────────────────
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="AgentOS 知识库收件箱提纯")
|
||||
parser.add_argument("--root", default="~/workbuddy-agent-os/agent-sync", help="agent-os 根目录")
|
||||
parser.add_argument("--dry-run", action="store_true", help="试运行，不实际修改文件")
|
||||
parser.add_argument("--date", default=None, help="日期 (YYYY-MM-DD)")
|
||||
args = parser.parse_args()
|
||||
|
||||
refine_inbox(args.root, args.dry_run, args.date)
|
||||
Reference in New Issue
Block a user