From 3b6a437e6c7f190cfec45162bfa2de6179642d90 Mon Sep 17 00:00:00 2001 From: butubb <1422726308@qq.com> Date: Sat, 10 Oct 2026 17:21:12 +0800 Subject: [PATCH] =?UTF-8?q?feat(monitor):=20=E6=8A=96=E9=9F=B3=E7=9B=91?= =?UTF-8?q?=E6=8E=A7=E6=94=B9=E8=B5=B0=E6=96=B0=E7=9A=84=20Web=20=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E5=AE=A2=E6=88=B7=E7=AB=AF=EF=BC=88=E6=8E=A5=E4=B8=8A?= =?UTF-8?q?=E4=B8=8A=E4=B8=80=E7=89=88=E7=9A=84=E7=A7=BB=E6=A4=8D=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 上一版只把客户端写出来、验证了它单独可用，**但没有接进任何地方** —— 所以你建的任务跑起来 仍然在调爬虫子进程，报的仍然是那句自己编的「account blocked」。这一步把它接上。 * runner 的 Phase 2 按平台分岔：dy 走进程内 HTTP 客户端（douyin_fetch），其余平台照旧走 爬虫子进程。抖音那条不再起 Playwright、不再构造那串自相矛盾的浏览器指纹参数。 * 新增 douyin_fetch：把采到的东西写成 store/douyin 那套 jsonl 形状 —— **ingest 完全不知道 数据是从哪来的**，重采样/差分/事件/通知/报表全都照旧，一个字没改。 * 失败不再假装：一条都没采到就以非零退出码 + **真实原因**交给 ingest，落成 「采集进程异常退出（code=1）：…」。绝不会再掉进「疑似登录失效」那个分支。 * 已知作品列表接口（aweme/post）被抖音单独加了真校验（200 + 空 body），所以加了退化： 拿不到列表就用库里已知的 aweme_id 逐条走 detail 刷新。**边界是：已知作品的指标能继续 更新，新作品发现不了** —— 这个边界会以一条 warning 日志留下痕迹，不让它看起来一切正常。 * 顺带给客户端补上 video_detail（实测可用：200 / 45425 字节），退化路径靠它。 测试 +6：产物目录与文件名、评论文件即使为空也要建（ingest 靠它区分「没评论」和 「什么都没抓到」）、重复作品只写一次、列表被挡时的退化、彻底失败仍写出产物与原因、 评论失败不连累作品。 --- api/monitor/douyin_api.py | 23 +++++ api/monitor/douyin_fetch.py | 144 +++++++++++++++++++++++++++ api/monitor/runner.py | 148 ++++++++++++++++++---------- tests/test_douyin_fetch.py | 190 ++++++++++++++++++++++++++++++++++++ 4 files changed, 455 insertions(+), 50 deletions(-) create mode 100644 api/monitor/douyin_fetch.py create mode 100644 tests/test_douyin_fetch.py diff --git a/api/monitor/douyin_api.py b/api/monitor/douyin_api.py index 5260c28..76edafe 100644 --- a/api/monitor/douyin_api.py +++ b/api/monitor/douyin_api.py @@ -326,6 +326,29 @@ async def author_videos( return [normalize_aweme(aweme) for aweme in awemes] +async def video_detail(aweme_id: str, *, cookie: str = "") -> Dict[str, Any]: + """单条作品的详情，已翻译成 store 的键名。 + + 这个接口**没有**被那道真校验挡着（实测 200 / 45425 字节），所以在拿不到作品列表时， + 它是「刷新已知作品指标」的唯一途径。 + """ + identity = await browser_identity(cookie) + if not _has_session(identity.cookie): + raise DouyinApiError("抖音登录态不可用：浏览器里没有会话，库里的 cookie 也没有") + + payload = await _get( + DETAIL_PATH, + {"aweme_id": aweme_id, "device_platform": "webapp", "aid": 6383}, + identity, + ) + aweme = payload.get("aweme_detail") or {} + if not aweme: + raise DouyinApiError( + f"接口没返回作品（status_code={payload.get('status_code')}）" + ) + return normalize_aweme(aweme) + + async def author_profile(sec_user_id: str, *, cookie: str = "") -> Dict[str, Any]: """博主主页指标：昵称 / 粉丝数 / 总获赞 / 作品数。""" identity = await browser_identity(cookie) diff --git a/api/monitor/douyin_fetch.py b/api/monitor/douyin_fetch.py new file mode 100644 index 0000000..9dbe3a0 --- /dev/null +++ b/api/monitor/douyin_fetch.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/monitor/douyin_fetch.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守对应平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + +"""抖音采集：直接走 Web 接口，不起爬虫子进程。 + +与 ``media_platform/douyin`` 那条路的分工： + +* 那边起一个 Playwright 子进程、构造一大串**自相矛盾的浏览器指纹参数**（参数说自己是 + Mac + Chrome 125，UA 说自己是 Linux + Chrome 155），网关回一个 200 + 空 body， + 然后被翻译成 ``Exception("account blocked")`` —— 看起来像账号被封，其实什么都不是。 +* 这边不起子进程、只发必要参数，用浏览器里那份登录态。见 ``douyin_api``。 + +采到的东西写成 ``store/douyin`` 那套 jsonl 形状，所以 **ingest 完全不知道数据是怎么来的** +—— 重采样、差分、事件、通知、报表全都照旧。 +""" + +import json +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Iterable, List, Sequence + +from tools import utils + +from . import adapters, douyin_api +from .models import MODE_CREATOR, MonitorTask + +# 拉多少条作品。接口单页上限就是 20，要多了也没用。 +DEFAULT_VIDEO_LIMIT = 20 + + +async def collect( + out_dir: Path, + *, + platform: str, + mode: str, + limit: int, + want_comments: bool, + comment_limit: int, + targets: Sequence[Any], + known_aweme_ids: Iterable[str] = (), + cookie: str = "", +) -> Dict[str, Any]: + """跑一轮抖音采集，把产物写进 ``out_dir`` 下 store 那个目录里。 + + 参数是散的、不收 ORM 对象：调用方那边 ``task`` 在会话关掉之后就 detached 了， + 传对象进来迟早会踩到「属性已过期」。 + + **不抛异常**：失败也把（可能为空的）产物落下去，并把原因放进 ``errors`` 交给调用方。 + 让调用方去决定这是「一轮正常但没数据」还是「一轮失败」—— 这个判断不该藏在这里。 + """ + notes: List[Dict[str, Any]] = [] + comments: List[Dict[str, Any]] = [] + errors: List[str] = [] + + for target in targets: + sec_user_id = target.external_id + seen_aweme: set = set() + + # 作品列表是主路径；它被那道真校验挡着时，退化到「标题/昵称靠主页接口，作品靠 + # 已知 id 逐条刷新」—— 拿不到新作品，但已知作品的指标还能继续更新。 + try: + videos = await douyin_api.author_videos( + sec_user_id, count=limit, cookie=cookie + ) + except douyin_api.DouyinApiError as exc: + errors.append(f"拉取博主 {sec_user_id} 的作品列表失败：{exc}") + videos = [] + for aweme_id in known_aweme_ids: + try: + videos.append(await douyin_api.video_detail(aweme_id, cookie=cookie)) + except douyin_api.DouyinApiError as detail_exc: + errors.append(f"刷新作品 {aweme_id} 失败：{detail_exc}") + + for video in videos: + aweme_id = video.get("aweme_id") + if not aweme_id or aweme_id in seen_aweme: + continue + seen_aweme.add(aweme_id) + notes.append(video) + + if want_comments: + try: + comments.extend( + await douyin_api.video_comments( + aweme_id, count=comment_limit, cookie=cookie + ) + ) + except douyin_api.DouyinApiError as exc: + errors.append(f"拉取作品 {aweme_id} 的评论失败：{exc}") + + jsonl_dir = _write_artifacts(out_dir, platform, mode, notes, comments) + return { + "notes": len(notes), + "comments": len(comments), + "errors": errors, + "jsonl_dir": str(jsonl_dir), + } + + +def _write_artifacts( + out_dir: Path, + platform: str, + mode: str, + notes: List[Dict[str, Any]], + comments: List[Dict[str, Any]], +) -> Path: + """按爬虫那套目录与文件名写 jsonl。 + + 目录名取 ``adapters.artifact_dir``（抖音是 ``douyin``，而不是平台 id ``dy``）—— + 和 ingest 找文件用的是同一个来源，两边不会走散。 + """ + jsonl_dir = out_dir / adapters.artifact_dir(platform) / "jsonl" + jsonl_dir.mkdir(parents=True, exist_ok=True) + + kind = "creator" if mode == MODE_CREATOR else "detail" + date = datetime.now().strftime("%Y-%m-%d") + + _write_jsonl(jsonl_dir / f"{kind}_contents_{date}.jsonl", notes) + # 评论文件即使没有评论也建出来：ingest 靠「文件在不在」区分「这一轮没评论」和 + # 「这一轮什么都没抓到」，两种情况的含义完全不同。 + _write_jsonl(jsonl_dir / f"{kind}_comments_{date}.jsonl", comments) + return jsonl_dir + + +def _write_jsonl(path: Path, records: List[Dict[str, Any]]) -> None: + with path.open("w", encoding="utf-8") as handle: + for record in records: + handle.write(json.dumps(record, ensure_ascii=False) + "\n") + utils.logger.info(f"[douyin_fetch] 写出 {len(records)} 条 -> {path.name}") diff --git a/api/monitor/runner.py b/api/monitor/runner.py index 3e6def1..2d8ae02 100644 --- a/api/monitor/runner.py +++ b/api/monitor/runner.py @@ -28,6 +28,8 @@ import os from pathlib import Path from typing import Iterable, List, Optional +from sqlalchemy import select + from tools.time_util import get_current_timestamp from ..schemas import ( @@ -38,7 +40,7 @@ from ..schemas import ( SaveDataOptionEnum, ) from ..services import crawler_manager -from . import adapters, app_settings, covers, notify +from . import adapters, app_settings, covers, douyin_fetch, notify from .db import get_session from .ingest import IngestResult, diagnose_failure, ingest_run from .models import ( @@ -47,6 +49,7 @@ from .models import ( RUN_PENDING, RUN_RUNNING, RUN_TIMEOUT, + MonitorNote, MonitorRun, MonitorTarget, MonitorTask, @@ -194,57 +197,102 @@ async def execute_task(task_id: int, trigger: str = "manual") -> IngestResult: max_notes_count = task.max_notes_count max_comments_count = task.max_comments_count timeout_seconds = task.run_timeout_seconds + # 抖音的作品列表接口被那道真校验挡着（见 douyin_fetch），拿不到列表时就靠这些 + # 已知的 aweme_id 逐条刷新 —— 新作品发现不了，但已有作品的指标还能继续更新。 + known_aweme_ids = list( + await session.scalars( + select(MonitorNote.note_id).where(MonitorNote.task_id == task.id) + ) + ) - # --- Phase 2: run the crawler outside any transaction --------------------- - cookie_file = out_dir / ".cookies" - _write_cookie_file(cookie_file, cookie) + # --- Phase 2: run the collection outside any transaction ------------------ + # 抖音走**进程内 HTTP 客户端**，不起 Playwright 子进程：那边会构造一大串自相矛盾的 + # 浏览器指纹参数（参数说 Mac + Chrome 125、UA 说 Linux + Chrome 155），网关回一个 + # 200 + 空 body，然后被翻译成「account blocked」—— 看着像账号被封，其实什么都不是。 + # 见 douyin_fetch / douyin_api。 + in_process_tail: List[str] = [] + if platform == adapters.PLATFORM_DY: + fetched = await douyin_fetch.collect( + out_dir, + platform=platform, + mode=mode, + limit=max_notes_count, + want_comments=enable_comments, + comment_limit=max_comments_count, + targets=targets, + known_aweme_ids=known_aweme_ids, + cookie=cookie, + ) + in_process_tail = list(fetched["errors"]) + # 一条都没采到 = 这一轮失败，并把**真因**当作退出诊断传下去。否则它会掉进 + # ingest 的「疑似登录失效」分支 —— 又骗人一次，正是这套东西一直在犯的毛病。 + exit_code = 1 if (fetched["errors"] and not fetched["notes"]) else 0 + if fetched["errors"] and fetched["notes"]: + # 有产物但带着错误，说明走了退化路径（比如作品列表被挡，只刷新了已知作品）。 + # 这一轮状态是成功，但**不是**一切正常 —— 得留下痕迹，否则没人知道新作品 + # 其实没在发现。 + print( + "[monitor.runner] 抖音采集部分失败：" + + "；".join(fetched["errors"])[:300] + ) + else: + cookie_file = out_dir / ".cookies" + _write_cookie_file(cookie_file, cookie) - request = CrawlerStartRequest( - platform=_PLATFORM_ENUM[platform], - login_type=LoginTypeEnum.COOKIE, - crawler_type=CrawlerTypeEnum.CREATOR if mode == MODE_CREATOR else CrawlerTypeEnum.DETAIL, - creator_ids=",".join(urls) if mode == MODE_CREATOR else "", - specified_ids=",".join(urls) if mode != MODE_CREATOR else "", - start_page=1, - enable_comments=enable_comments, - enable_sub_comments=strategy["enable_sub_comments"], - enable_media=False, - save_option=SaveDataOptionEnum.JSONL, - cookies="", - headless=True, - max_notes_count=max_notes_count, - max_comments_count=max_comments_count, - # Isolate this run's output: the crawler names files by date only, so - # otherwise same-day runs would append into one shared file. - save_data_path=str(out_dir), - # Attach to the browser already running on CDP_DEBUG_PORT when the - # operator enabled it; otherwise launch a private, throwaway browser. - enable_cdp_mode=cdp_enabled, - # Only injecting web_session is not enough to sign requests from a cold - # browser profile. - inject_all_cookies=True, - save_login_state=True, - cookies_file=str(cookie_file), - max_concurrency_num=1, - # Strategy + proxy, surfaced on the Settings page. - crawler_max_sleep_sec=strategy["crawl_sleep_sec"], - enable_ip_proxy=strategy["enable_ip_proxy"], - ip_proxy_pool_count=strategy["proxy_pool_count"], - ip_proxy_provider_name=strategy["proxy_provider"], - static_proxy_url=strategy["static_proxy_url"] or None, + request = CrawlerStartRequest( + platform=_PLATFORM_ENUM[platform], + login_type=LoginTypeEnum.COOKIE, + crawler_type=CrawlerTypeEnum.CREATOR if mode == MODE_CREATOR else CrawlerTypeEnum.DETAIL, + creator_ids=",".join(urls) if mode == MODE_CREATOR else "", + specified_ids=",".join(urls) if mode != MODE_CREATOR else "", + start_page=1, + enable_comments=enable_comments, + enable_sub_comments=strategy["enable_sub_comments"], + enable_media=False, + save_option=SaveDataOptionEnum.JSONL, + cookies="", + headless=True, + max_notes_count=max_notes_count, + max_comments_count=max_comments_count, + # Isolate this run's output: the crawler names files by date only, so + # otherwise same-day runs would append into one shared file. + save_data_path=str(out_dir), + # Attach to the browser already running on CDP_DEBUG_PORT when the + # operator enabled it; otherwise launch a private, throwaway browser. + enable_cdp_mode=cdp_enabled, + # Only injecting web_session is not enough to sign requests from a cold + # browser profile. + inject_all_cookies=True, + save_login_state=True, + cookies_file=str(cookie_file), + max_concurrency_num=1, + # Strategy + proxy, surfaced on the Settings page. + crawler_max_sleep_sec=strategy["crawl_sleep_sec"], + enable_ip_proxy=strategy["enable_ip_proxy"], + ip_proxy_pool_count=strategy["proxy_pool_count"], + ip_proxy_provider_name=strategy["proxy_provider"], + static_proxy_url=strategy["static_proxy_url"] or None, + ) + + async with get_session() as session: + run = await session.get(MonitorRun, run_id) + if run is not None: + run.status = RUN_RUNNING + run.started_at = get_current_timestamp() + + try: + exit_code = await crawler_manager.run_and_wait(request, timeout=timeout_seconds) + finally: + _remove_cookie_file(cookie_file) + + # 失败时的诊断来源。抖音那条路没有子进程，尾巴就是它自己报的错 —— **别去读 + # crawler_manager 的尾巴**，那里面是上一轮别的平台留下的东西，会张冠李戴。 + output_tail = ( + in_process_tail + if platform == adapters.PLATFORM_DY + else crawler_manager.get_output_tail() ) - async with get_session() as session: - run = await session.get(MonitorRun, run_id) - if run is not None: - run.status = RUN_RUNNING - run.started_at = get_current_timestamp() - - try: - exit_code = await crawler_manager.run_and_wait(request, timeout=timeout_seconds) - finally: - _remove_cookie_file(cookie_file) - # --- Phase 3: ingest ------------------------------------------------------ async with get_session() as session: run = await session.get(MonitorRun, run_id) @@ -261,7 +309,7 @@ async def execute_task(task_id: int, trigger: str = "manual") -> IngestResult: run.error_message = "Run was killed by timeout or failed to start" # -1 同时代表「超时」和「根本没起来」，两者要查的东西完全不同。输出尾巴里 # 有异常就带上它，否则运行历史里只能看到这句没有信息量的话。 - cause = diagnose_failure(crawler_manager.get_output_tail()) + cause = diagnose_failure(output_tail) if cause: run.error_message = f"{run.error_message}；原因：{cause}" result = IngestResult(status=RUN_TIMEOUT, error=run.error_message) @@ -271,7 +319,7 @@ async def execute_task(task_id: int, trigger: str = "manual") -> IngestResult: # 把爬虫输出的尾巴交给 ingest：退出码本身说明不了问题，运行历史里要显示的 # 是真正的报错（比如抖音的 DataFetchError: account blocked）。 result = await ingest_run( - session, run, task, out_dir, output_tail=crawler_manager.get_output_tail() + session, run, task, out_dir, output_tail=output_tail ) # A run that authenticated fine is the only useful signal that the diff --git a/tests/test_douyin_fetch.py b/tests/test_douyin_fetch.py new file mode 100644 index 0000000..12b324a --- /dev/null +++ b/tests/test_douyin_fetch.py @@ -0,0 +1,190 @@ +# -*- coding: utf-8 -*- +"""抖音采集编排 —— 不碰网络，把 douyin_api 整个换掉。 + +验的是编排本身：产物落在正确的目录、文件名是 store 那套、以及**作品列表被挡时的退化** +（用已知 aweme_id 逐条刷新）—— 那条退化路径决定了今天这个功能是「完全没用」还是 +「已知作品还能看」。 +""" + +import json + +import pytest + +from api.monitor import douyin_api, douyin_fetch + + +def _video(aweme_id: str, likes: str = "1") -> dict: + return { + "aweme_id": aweme_id, + "title": f"title-{aweme_id}", + "aweme_url": f"https://www.douyin.com/video/{aweme_id}", + "cover_url": "", + "aweme_type": "0", + "create_time": 1790574515, + "creator_hash": "hash", + "nickname": "博主", + "liked_count": likes, + "comment_count": "0", + "collected_count": "0", + "share_count": "0", + } + + +def _comment(aweme_id: str, index: int) -> dict: + return { + "comment_id": f"c{index}", + "aweme_id": aweme_id, + "content": f"评论{index}", + "nickname": "路人", + "creator_hash": "h2", + "create_time": 1790574600, + "like_count": "0", + "sub_comment_count": "0", + "parent_comment_id": "0", + } + + +class _Target: + def __init__(self, external_id: str) -> None: + self.external_id = external_id + + +def _read(path): + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line] + + +async def _collect(tmp_path, **overrides): + kwargs = dict( + platform="dy", + mode="creator", + limit=20, + want_comments=True, + comment_limit=20, + targets=[_Target("MS4w-sec")], + cookie="sessionid=x", + ) + kwargs.update(overrides) + return await douyin_fetch.collect(tmp_path, **kwargs) + + +class TestHappyPath: + @pytest.mark.asyncio + async def test_writes_the_layout_ingest_expects(self, monkeypatch, tmp_path): + async def fake_videos(sec_user_id, count=20, *, cookie=""): + return [_video("111"), _video("222")] + + async def fake_comments(aweme_id, count=20, *, cookie=""): + return [_comment(aweme_id, 1)] + + monkeypatch.setattr(douyin_api, "author_videos", fake_videos) + monkeypatch.setattr(douyin_api, "video_comments", fake_comments) + + result = await _collect(tmp_path) + + assert result["notes"] == 2 + assert result["comments"] == 2 + assert result["errors"] == [] + + # 目录名必须是 douyin（不是平台 id dy）—— ingest 找文件用的是同一个来源。 + jsonl_dir = tmp_path / "douyin" / "jsonl" + assert jsonl_dir.is_dir() + + contents = list(jsonl_dir.glob("*_contents_*.jsonl")) + comments = list(jsonl_dir.glob("*_comments_*.jsonl")) + assert len(contents) == 1 + assert len(comments) == 1 + + notes = _read(contents[0]) + assert [n["aweme_id"] for n in notes] == ["111", "222"] + # 键名照抄 store/douyin —— ingest 靠这个读出来。 + assert notes[0]["aweme_url"] == "https://www.douyin.com/video/111" + # 秒，不是毫秒；换算交给 adapters。 + assert notes[0]["create_time"] == 1790574515 + + assert _read(comments[0])[0]["aweme_id"] == "111" + + @pytest.mark.asyncio + async def test_the_comment_file_exists_even_without_comments(self, monkeypatch, tmp_path): + """评论文件必须建出来。 + + ingest 靠「文件在不在」区分「这一轮没评论」和「这一轮什么都没抓到」—— + 两种情况的含义完全不同。 + """ + + async def fake_videos(sec_user_id, count=20, *, cookie=""): + return [_video("111")] + + monkeypatch.setattr(douyin_api, "author_videos", fake_videos) + + await _collect(tmp_path, want_comments=False) + + assert len(list((tmp_path / "douyin" / "jsonl").glob("*_comments_*.jsonl"))) == 1 + + @pytest.mark.asyncio + async def test_duplicate_works_are_written_once(self, monkeypatch, tmp_path): + async def fake_videos(sec_user_id, count=20, *, cookie=""): + return [_video("111"), _video("111")] + + monkeypatch.setattr(douyin_api, "author_videos", fake_videos) + + result = await _collect(tmp_path, want_comments=False) + + assert result["notes"] == 1 + + +class TestDegradation: + """作品列表被挡时的行为 —— 决定了这个功能今天有没有用。""" + + @pytest.mark.asyncio + async def test_falls_back_to_refreshing_known_works(self, monkeypatch, tmp_path): + async def blocked(sec_user_id, count=20, *, cookie=""): + raise douyin_api.DouyinApiError("接口返回了空内容") + + refreshed = [] + + async def fake_detail(aweme_id, *, cookie=""): + refreshed.append(aweme_id) + return _video(aweme_id, likes="9") + + monkeypatch.setattr(douyin_api, "author_videos", blocked) + monkeypatch.setattr(douyin_api, "video_detail", fake_detail) + + result = await _collect( + tmp_path, want_comments=False, known_aweme_ids=["999", "888"] + ) + + assert refreshed == ["999", "888"] + assert result["notes"] == 2 + # 但错误照样报出来 —— 这一轮是「部分可用」，不是「一切正常」，别粉饰。 + assert any("作品列表失败" in error for error in result["errors"]) + + @pytest.mark.asyncio + async def test_nothing_at_all_still_reports_the_reason(self, monkeypatch, tmp_path): + async def blocked(sec_user_id, count=20, *, cookie=""): + raise douyin_api.DouyinApiError("接口返回了空内容") + + monkeypatch.setattr(douyin_api, "author_videos", blocked) + + result = await _collect(tmp_path, want_comments=False) + + assert result["notes"] == 0 + assert result["errors"] + # 产物仍然写出来（空的），让调用方去判断这是失败而不是「这个博主没作品」。 + assert (tmp_path / "douyin" / "jsonl").is_dir() + + @pytest.mark.asyncio + async def test_a_failing_comment_fetch_does_not_lose_the_work(self, monkeypatch, tmp_path): + async def fake_videos(sec_user_id, count=20, *, cookie=""): + return [_video("111")] + + async def broken_comments(aweme_id, count=20, *, cookie=""): + raise douyin_api.DouyinApiError("评论接口抽风") + + monkeypatch.setattr(douyin_api, "author_videos", fake_videos) + monkeypatch.setattr(douyin_api, "video_comments", broken_comments) + + result = await _collect(tmp_path) + + # 评论拿不到是小事，作品不能跟着丢。 + assert result["notes"] == 1 + assert any("评论失败" in error for error in result["errors"])