feat(monitor): 抖音接入博主监控

上游爬虫本身不缺抖音能力（三模式、四项指标、二级评论都与小红书对等、指标还是同名同列），缺的全在监控层的适配。这次把「平台之间不一样」的管子集中到一个新模块，再把散落的 xhs 硬编码接上去。 * 新增 api/monitor/adapters.py：产物目录名、jsonl 字段别名、目标链接形态与正则、通知链接模板。不放进 platforms.py 是因为那个模块被 describe_all() 整个序列化进 /api/config/platforms 交给前端，塞进正则和目录名会让爬虫内部细节漏进 API 载荷。代价是两个注册表可能漂移，用一条测试钉住「声明接通就必须有适配器」。 * 两个必须知道的坑，都在这版里处理掉了： 1) 抖音的平台 id 是 dy，而 store 把产物写在 douyin/ 下（store/douyin/_store_impl.py:47）。不改就是 ingest 一个文件都读不到 —— 不报错，只是 0 条，然后被冒充成「疑似登录失效」。 2) 抖音的作品没有 note_id（叫 aweme_id）、评论也用 aweme_id 指作品。ingest 第一步是 `if not note_id: continue`，不映射就逐条全丢。另外抖音顶层评论的 parent_comment_id 是字符串 "0"，归一成空串，免得前端多出悬空的父节点。 * 顺带把「东西抓到了、只是没落在期望目录里」单独识别出来。这类故障的现象和登录失效一模一样，按登录失效报会把人指去查完全错误的方向。 * 修两个既有 bug（今天只有小红书所以无害，加抖音就踩响）： - service.py update_task 换目标时漏传 task.platform，回落到默认小红书 - scheduler.py 取 cookie 没传 platform，抖音任务会读着小红书那份 cookie 不动 * 行为变更（已与用户确认）：cookie 闸门改成「没 cookie 且没开 CDP」才跳过。 CDP 模式下登录态来自被接管的浏览器，粘不粘 cookie 由不得它决定；不放行的话，选了「接管已有 Chrome」却没粘 cookie 的用户会看到任务永远不触发，而且不报错。副作用是开启了 CDP 的小红书任务也不再被该闸门拦住 —— 语义上是对的。 * 目标输入框的示例链接与措辞改由能力矩阵提供（notes_label 抖音说「作品」、小红书说「笔记」；「建议只填纯 ID」是小红书专属劝告，抖音链接不带令牌，不再显示）。测试 +22 条（858 通过），其中最关键的是「抖音作品/评论不被静默丢弃」与「产物目录名不等于平台 id」两条 —— 都是把最难查的失败模式钉死在回归网里。注意：抖音这条路的**端到端尚未验证**，需要一份可用的抖音登录态（CDP 那台 Chrome 里登录，或导出一份 cookie）。单测覆盖的是解析与入库，真实抓取还没跑过。
2026-10-10 14:55:28 +08:00
parent e348de48d3
commit 06718a1351
17 changed files with 863 additions and 105 deletions
@@ -45,6 +45,7 @@ from sqlalchemy.ext.asyncio import AsyncSession

 from tools.time_util import get_current_timestamp

+from . import adapters
 from .platforms import PLATFORM_XHS
 from .models import (
    EVENT_AUTH_FAILURE,
@@ -171,8 +172,11 @@ def find_run_files(
    Glob rather than reconstructing the name: both the crawler type and the date
    are runtime-dependent. Returns lists because a crawl crossing midnight
    produces one file per day.
+
+    ``platform`` 是**监控层的平台 id**，而爬虫落盘的目录名未必同名（抖音的 id 是
+    ``dy``、目录是 ``douyin``），所以这里经 adapters 解析 —— 调用方不必知道这个差异。
    """
-    jsonl_dir = out_dir / platform / "jsonl"
+    jsonl_dir = out_dir / adapters.artifact_dir(platform) / "jsonl"
    if not jsonl_dir.is_dir():
        return [], []

@@ -182,6 +186,30 @@ def find_run_files(
    )


+def _misplaced_output_dirs(out_dir: Path, expected: str) -> List[str]:
+    """在 out_dir 下找「有产物、但目录名不是期望的那个」的目录。
+
+    这是专门为**最难查的那类故障**准备的：产物目录名与平台对不上时，ingest 一个文件
+    都找不到，现象和「登录态失效」一模一样 —— 而实际上登录好好的、数据也抓到了，
+    只是没人去对的地方读。上游哪天改了 store 的目录名，这里能直接把实情说出来。
+    """
+    found = []
+    try:
+        children = list(out_dir.iterdir())
+    except OSError:
+        return found
+
+    for child in children:
+        if child.name == expected or not child.is_dir():
+            continue
+        try:
+            if any(child.glob("jsonl/*_contents_*.jsonl")):
+                found.append(child.name)
+        except OSError:
+            continue
+    return sorted(found)
+
+
 async def _emit(
    session: AsyncSession,
    run: MonitorRun,
@@ -271,13 +299,18 @@ async def _ingest_notes(
    run: MonitorRun,
    records: List[Dict[str, Any]],
    is_baseline: bool,
+    adapter: adapters.PlatformAdapter,
 ) -> int:
-    """Upsert notes, write metric snapshots, and emit new-note/delta events."""
+    """Upsert notes, write metric snapshots, and emit new-note/delta events.
+
+    记录里的字段一律经 ``adapter`` 读。抖音的作品没有 ``note_id``（叫 ``aweme_id``），
+    按名字硬取的话每条记录都会在下面第一行被 continue 掉 —— 一条都不报错地全丢。
+    """
    now = get_current_timestamp()
    new_count = 0

    for record in records:
-        note_id = record.get("note_id")
+        note_id = adapter.note_field(record, "note_id")
        if not note_id:
            continue

@@ -288,21 +321,20 @@ async def _ingest_notes(
            )
        )

-        title = (record.get("title") or "")[:500]
-        raw_images = record.get("image_list") or ""
-        cover = raw_images.split(",")[0] if raw_images else ""
+        title = (adapter.note_field(record, "title") or "")[:500]
+        cover = adapter.cover(record)

        if note is None:
            note = MonitorNote(
                task_id=run.task_id,
                note_id=note_id,
                title=title,
-                note_url=record.get("note_url") or "",
+                note_url=adapter.note_field(record, "note_url") or "",
                cover=cover,
-                creator_hash=record.get("creator_hash") or "",
-                creator_name=record.get("nickname") or "",
-                source_kind=record.get("type") or "",
-                published_at=_as_int(record.get("time")),
+                creator_hash=adapter.note_field(record, "creator_hash") or "",
+                creator_name=adapter.note_field(record, "creator_name") or "",
+                source_kind=adapter.note_field(record, "source_kind") or "",
+                published_at=_as_int(adapter.note_field(record, "published_at")),
                first_seen_run_id=run.id,
                first_seen_at=now,
                last_seen_run_id=run.id,
@@ -330,8 +362,9 @@ async def _ingest_notes(
            if cover:
                note.cover = cover
            # 昵称也要刷：作者改昵称是常事，只在首次入库写一次会一直显示旧的。
-            if record.get("nickname"):
-                note.creator_name = record["nickname"]
+            creator_name = adapter.note_field(record, "creator_name")
+            if creator_name:
+                note.creator_name = creator_name
            note.last_seen_run_id = run.id
            note.last_seen_at = now

@@ -429,14 +462,18 @@ async def _ingest_comments(
    records: List[Dict[str, Any]],
    is_baseline: bool,
    previous_run_started_at: Optional[int],
+    adapter: adapters.PlatformAdapter,
 ) -> int:
-    """Upsert comments and emit events for ones never seen before."""
+    """Upsert comments and emit events for ones never seen before.
+
+    与作品同理，评论记录也要经 ``adapter`` 读：抖音的评论用 ``aweme_id`` 指作品。
+    """
    now = get_current_timestamp()
    new_count = 0

    for record in records:
-        comment_id = record.get("comment_id")
-        note_id = record.get("note_id")
+        comment_id = adapter.comment_field(record, "comment_id")
+        note_id = adapter.comment_field(record, "note_id")
        if not comment_id or not note_id:
            continue

@@ -450,19 +487,22 @@ async def _ingest_comments(
        if exists is not None:
            continue

-        create_time = _as_int(record.get("create_time"))
+        create_time = _as_int(adapter.comment_field(record, "create_time"))
        session.add(
            MonitorComment(
                task_id=run.task_id,
                note_id=note_id,
                comment_id=comment_id,
-                content=(record.get("content") or "")[:2000],
-                nickname=record.get("nickname") or "",
-                creator_hash=record.get("creator_hash") or "",
+                content=(adapter.comment_field(record, "content") or "")[:2000],
+                nickname=adapter.comment_field(record, "creator_name") or "",
+                creator_hash=adapter.comment_field(record, "creator_hash") or "",
                create_time=create_time,
-                like_count=parse_count(record.get("like_count")),
-                sub_comment_count=_as_int(record.get("sub_comment_count")) or 0,
-                parent_comment_id=record.get("parent_comment_id") or "",
+                like_count=parse_count(adapter.comment_field(record, "like_count")),
+                sub_comment_count=_as_int(
+                    adapter.comment_field(record, "sub_comment_count")
+                )
+                or 0,
+                parent_comment_id=adapter.parent_comment_id(record),
                first_seen_run_id=run.id,
                first_seen_at=now,
            )
@@ -523,6 +563,9 @@ async def ingest_run(
        )
        return IngestResult(status=RUN_FAILED, error=run.error_message)

+    adapter = adapters.adapter(task.platform)
+    subdir = adapters.artifact_dir(task.platform)
+
    contents_paths, comment_paths = find_run_files(out_dir, task.platform)
    contents = [record for path in contents_paths for record in _read_jsonl(path)]
    comments = [record for path in comment_paths for record in _read_jsonl(path)]
@@ -538,6 +581,32 @@ async def ingest_run(
    if not contents:
        run.status = RUN_PARTIAL

+        # 先排除「东西抓到了，只是没落在我们找的那个目录里」。这种故障的现象和登录失效
+        # 一模一样，但登录其实是好的 —— 按登录失效报会把人指到完全错的方向去查。
+        misplaced = _misplaced_output_dirs(out_dir, subdir)
+        if misplaced:
+            run.error_message = (
+                f"crawler wrote into {misplaced} but platform {task.platform} "
+                f"expects {subdir}"
+            )
+            await _emit(
+                session,
+                run,
+                EVENT_NO_DATA,
+                f"采集产物目录与平台不匹配（实际 {misplaced}、期望 {subdir}），本次未读到任何作品",
+                severity="error",
+                payload={
+                    "out_dir": str(out_dir),
+                    "expected": subdir,
+                    "found": misplaced,
+                },
+            )
+            return IngestResult(
+                status=RUN_PARTIAL,
+                error=run.error_message,
+                comments_fetched=len(comments),
+            )
+
        # Blaming the cookie is only honest if nothing else is authenticating.
        # A sibling task that just succeeded proves the login works, so the
        # fault is with this target (bad/expired per-creator token, an empty
@@ -587,10 +656,10 @@ async def ingest_run(
        comments_fetched=len(comments),
        is_baseline=is_baseline,
    )
-    result.new_notes = await _ingest_notes(session, run, contents, is_baseline)
+    result.new_notes = await _ingest_notes(session, run, contents, is_baseline, adapter)
    if task.enable_comments:
        result.new_comments = await _ingest_comments(
-            session, run, comments, is_baseline, previous_started_at
+            session, run, comments, is_baseline, previous_started_at, adapter
        )

    run.new_notes = result.new_notes