butubb/MediaCrawler
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程，直接发 HTTP
Deploy VitePress site to Pages / build (push) Canceled after 0s
Details
Deploy VitePress site to Pages / Deploy (push) Canceled after 0s
Details

Browse Source 
移植自 mac-agent-os 的 mediacrawler_adapter：不起子进程、不开页面，用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin，所以 ingest 那条链路一个字不用改。

**目前能用的（真环境实测，非推断）**：

    profile/other : 200, 7075 字节  —— 博主主页指标（粉丝/获赞/作品数/昵称）
    aweme/detail  : 200, 45425 字节 —— 单条作品详情（含点赞/评论/收藏/分享）

**目前不能用的：作品列表 `aweme/post`。** 两个互相独立的原因：
  1. 这个接口被抖音单独升级成了真校验：不带 x-tt-argus 回 403「Uifid Not Found」，
     带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过，「真校验」过不了。
     同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的，
     挑中的恰好是「批量拉作品列表」这个最敏感的动作。
  2. 改走页面截获也不行：CDP 浏览器打开博主主页会落到「验证码中间页」（当天大量探测的
     代价，过几小时要重测）。

  所以现在的边界是：**已知作品的指标刷新能做，自动发现新作品做不了**。

**排查中控住变量后得到的两条事实**（都写进注释了）：
  · `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
    UA+client hints+Cookie 时是 200 但仅 121 字节的空壳，补上这三个头变 7074 字节。
    （我先前猜的 sec-ch-ua 不是关键。）
  · 因此 UA 与 client hints 必须**成套地取自同一个浏览器**，所以 BrowserIdentity 一次
    从 CDP 取齐 cookie + UA + hints，而不是各自写死。

「200 + 空 body 必须当场报错」也是刻意写死的：放过去它会在下游变成「这个博主没作品」，
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的，还被翻译成「账号被封」。

测试 +11：cookie 解析、请求头成套性（含 uifid 缺失/回退）、产物键名与 store 对齐、
以及 _get 的三条失败路径（空 body / 403 带网关原话 / 正常返回）。
This commit is contained in:
butubb
2026-10-10 17:12:25 +08:00
parent 21ff01b894
commit 2112c1a870
1601 changed files with 367425 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mac-agent-os-main/02_skills/collect_to_inbox/collect_to_inbox.py
+312
View File
@@ -0,0 +1,312 @@
#!/usr/bin/env python3
"""
collect_to_inbox.py —— 扫描知识库各分类目录，提取主要内容生成标准化 MD 放入收件箱
用法：
python3 collect_to_inbox.py --root ~/workbuddy-agent-os/agent-sync [--dry-run] [--force]
参数：
--root agent-os 根目录（默认 ~/workbuddy-agent-os/agent-sync）
--dry-run 只预览不写入
--force 忽略 collected 标记，重新收集
"""
import argparse
import os
import re
import sys
from datetime import datetime
from pathlib import Path
# ─── 角色检查：所有机器均可执行汇聚 ───
try:
sys.path.insert(0, str(Path.home() / "workbuddy-agent-os" / "agent-sync" / "05_tools" / "01_system"))
from role_check import check_capability
if not check_capability("knowledge_collection"):
print("⛔ 能力 knowledge_collection 未启用，跳过 collect_to_inbox")
sys.exit(0)
except ImportError as e:
print(f"⚠️ 角色检查模块不可用 ({e})，继续执行")
# 扫描的源目录配置：(相对路径, 提取策略)
SCAN_DIRS = [
("03_knowledge/50_resources/视频笔记", "extract_summary"),
("03_knowledge/50_resources/字幕存档", "extract_head"),
("03_knowledge/50_resources/阅读笔记", "extract_summary"),
("03_knowledge/50_resources/全文存档", "extract_head"),
("03_knowledge/50_resources/翻译存档", "extract_summary"),
("03_knowledge/50_resources/灵感素材", "extract_summary"),
("03_knowledge/50_resources/语音转写", "extract_key_points"),
("03_knowledge/20_methods", "extract_method"),
("03_knowledge/01_daily/闪念笔记", "extract_key_points"),
("03_knowledge/40_references", "extract_summary"),
]
# 不扫描的目录
SKIP_DIRS = {"00_inbox", "99_system", "90_archive", "10_concepts", "30_facts", "60_opinions", "50_resources/资源索引"}
def parse_frontmatter(content: str) -> tuple[dict, str]:
"""解析 Frontmatter，返回 (metadata_dict, body_text)"""
if not content.startswith("---"):
return {}, content
end = content.find("---", 3)
if end == -1:
return {}, content
fm_text = content[3:end].strip()
body = content[end + 3:].strip()
metadata = {}
for line in fm_text.split("\n"):
line = line.strip()
if ":" in line:
key, _, val = line.partition(":")
key = key.strip()
val = val.strip().strip('"').strip("'")
if val:
metadata[key] = val
return metadata, body
def extract_summary(body: str, max_chars: int = 500) -> str:
"""提取摘要：取前 max_chars 字符"""
if len(body) <= max_chars:
return body
return body[:max_chars] + "\n\n...（内容已截断，完整内容见源文件）"
def extract_head(body: str, max_chars: int = 500) -> str:
"""提取头部内容"""
return extract_summary(body, max_chars)
def extract_key_points(body: str, max_chars: int = 500) -> str:
"""提取关键要点：优先提取列表项"""
lines = body.split("\n")
points = []
for line in lines:
stripped = line.strip()
if stripped.startswith(("- ", "* ", "1.", "2.", "3.", "4.", "5.")):
points.append(stripped)
if len("\n".join(points)) > max_chars:
break
if points:
result = "\n".join(points[:20])
if len(result) > max_chars:
result = result[:max_chars] + "\n\n...（内容已截断）"
return result
return extract_summary(body, max_chars)
def extract_method(body: str, max_chars: int = 500) -> str:
"""提取方法论：方法名 + 步骤"""
lines = body.split("\n")
# 提取标题和编号列表
method_lines = []
for line in lines:
stripped = line.strip()
if stripped.startswith("#") or stripped.startswith(("-", "*", "1.", "2.", "3.", "4.", "5.")):
method_lines.append(stripped)
if len("\n".join(method_lines)) > max_chars:
break
if method_lines:
return "\n".join(method_lines[:30])
return extract_summary(body, max_chars)
EXTRACT_FUNCS = {
"extract_summary": extract_summary,
"extract_head": extract_head,
"extract_key_points": extract_key_points,
"extract_method": extract_method,
}
def infer_nature(source_dir: str, metadata: dict) -> str:
"""根据源目录和元数据推断 nature"""
dir_lower = source_dir.lower()
if "方法" in dir_lower or "method" in dir_lower:
return "method"
if "视频笔记" in dir_lower:
return "resource"
if "字幕" in dir_lower or "转写" in dir_lower:
return "reference"
if "阅读" in dir_lower or "全文" in dir_lower:
return "reference"
if "翻译" in dir_lower:
return "reference"
if "灵感" in dir_lower:
return "opinion"
if "闪念" in dir_lower:
return "opinion"
if "references" in dir_lower:
return "reference"
# 从 tags 推断
tags = metadata.get("tags", "")
if "方法论" in tags or "方法" in tags:
return "method"
if "概念" in tags:
return "concept"
return "resource"
def infer_domain(metadata: dict) -> str:
"""从标签推断领域"""
tags = metadata.get("tags", "").lower()
domain_keywords = {
"技术": ["编程", "代码", "开发", "tech", "python", "javascript", "ai", "llm", "机器学习"],
"金融": ["投资", "股票", "基金", "finance", "交易", "财经"],
"产品": ["产品", "设计", "ux", "product"],
"效率": ["效率", "工具", "workflow", "自动化"],
"阅读": ["读书", "阅读", "书评", "book"],
"生活": ["生活", "旅行", "美食", "健身"],
}
for domain, keywords in domain_keywords.items():
if any(kw in tags for kw in keywords):
return domain
return "general"
def collect_to_inbox(root_dir: str, dry_run: bool = False, force: bool = False):
"""主函数：扫描各目录，提取内容到收件箱"""
root = Path(root_dir).expanduser()
inbox_dir = root / "03_knowledge" / "00_inbox"
stats = {"scanned": 0, "collected": 0, "skipped_collected": 0, "skipped_duplicate": 0, "errors": 0}
# 确保收件箱存在
if not dry_run:
inbox_dir.mkdir(parents=True, exist_ok=True)
# 获取收件箱已有文件名集合（用于去重）
existing_inbox = set()
if inbox_dir.exists():
for f in inbox_dir.glob("*.md"):
existing_inbox.add(f.name)
collected_date = datetime.now().strftime("%Y-%m-%d")
for rel_dir, extract_strategy in SCAN_DIRS:
scan_dir = root / rel_dir
if not scan_dir.exists():
continue
# 递归扫描子目录（灵感素材下有平台子目录）
for md_file in sorted(scan_dir.rglob("*.md")):
stats["scanned"] += 1
try:
content = md_file.read_text(encoding="utf-8")
except Exception as e:
print(f" ❌ 读取失败: {md_file.name} - {e}")
stats["errors"] += 1
continue
metadata, body = parse_frontmatter(content)
# 检查是否已收集
if not force and metadata.get("collected") == "true":
stats["skipped_collected"] += 1
continue
# 去重检查
dest_name = md_file.name
if dest_name in existing_inbox:
stats["skipped_duplicate"] += 1
continue
# 提取内容
extract_func = EXTRACT_FUNCS.get(extract_strategy, extract_summary)
extracted = extract_func(body)
# 推断属性
nature = infer_nature(rel_dir, metadata)
domain = infer_domain(metadata)
title = metadata.get("title", md_file.stem)
source_date = metadata.get("date", collected_date)
tags = metadata.get("tags", "")
source_rel = str(md_file.relative_to(root))
# 生成收件箱文件内容
inbox_content = f"""---
title: "{title}"
source_dir: {rel_dir}
source_file: {md_file.name}
date: {source_date}
collected_date: {collected_date}
tags: {tags}
nature: {nature}
domain: {domain}
status: inbox
---
# {title}
> 来源：{rel_dir}
{extracted}
"""
if dry_run:
print(f" 📥 [预览] {md_file.name} → 00_inbox/{dest_name}")
print(f" nature={nature}, domain={domain}, 策略={extract_strategy}")
else:
# 写入收件箱
dest_path = inbox_dir / dest_name
dest_path.write_text(inbox_content, encoding="utf-8")
print(f" 📥 {md_file.name} → 00_inbox/{dest_name}")
# 标记原文件为已收集
if content.startswith("---"):
# 在 Frontmatter 末尾添加 collected 标记
end = content.find("---", 3)
if end != -1:
fm = content[3:end]
if "collected:" not in fm:
fm += f"\ncollected: true\ncollected_date: {collected_date}"
new_content = "---" + fm + content[end:]
md_file.write_text(new_content, encoding="utf-8")
else:
# 没有 Frontmatter，添加一个简单的
new_content = f"---\ncollected: true\ncollected_date: {collected_date}\n---\n\n{content}"
md_file.write_text(new_content, encoding="utf-8")
existing_inbox.add(dest_name)
stats["collected"] += 1
# 输出统计
print(f"\n{'[预览] ' if dry_run else ''}收集统计：")
print(f" 扫描文件：{stats['scanned']}")
print(f" 提取到收件箱：{stats['collected']}")
print(f" 跳过（已收集）：{stats['skipped_collected']}")
print(f" 跳过（重复）：{stats['skipped_duplicate']}")
print(f" 错误：{stats['errors']}")
return stats
def main():
parser = argparse.ArgumentParser(description="收集各分类目录内容到收件箱")
parser.add_argument("--root", default="~/workbuddy-agent-os/agent-sync", help="agent-os 根目录")
parser.add_argument("--dry-run", action="store_true", help="只预览不写入")
parser.add_argument("--force", action="store_true", help="忽略 collected 标记，重新收集")
args = parser.parse_args()
print(f"=== Collect to Inbox ===")
print(f"根目录：{args.root}")
print(f"模式：{'预览' if args.dry_run else '执行'}")
print()
collect_to_inbox(args.root, dry_run=args.dry_run, force=args.force)
if __name__ == "__main__":
main()