# -*- coding: utf-8 -*- """抖音采集编排 —— 不碰网络,把 douyin_api 整个换掉。 验的是编排本身:产物落在正确的目录、文件名是 store 那套、以及**作品列表被挡时的退化** (用已知 aweme_id 逐条刷新)—— 那条退化路径决定了今天这个功能是「完全没用」还是 「已知作品还能看」。 """ import json import pytest from api.monitor import douyin_api, douyin_fetch def _video(aweme_id: str, likes: str = "1") -> dict: return { "aweme_id": aweme_id, "title": f"title-{aweme_id}", "aweme_url": f"https://www.douyin.com/video/{aweme_id}", "cover_url": "", "aweme_type": "0", "create_time": 1790574515, "creator_hash": "hash", "nickname": "博主", "liked_count": likes, "comment_count": "0", "collected_count": "0", "share_count": "0", } def _comment(aweme_id: str, index: int) -> dict: return { "comment_id": f"c{index}", "aweme_id": aweme_id, "content": f"评论{index}", "nickname": "路人", "creator_hash": "h2", "create_time": 1790574600, "like_count": "0", "sub_comment_count": "0", "parent_comment_id": "0", } class _Target: def __init__(self, external_id: str) -> None: self.external_id = external_id def _read(path): return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line] async def _collect(tmp_path, **overrides): kwargs = dict( platform="dy", mode="creator", limit=20, want_comments=True, comment_limit=20, targets=[_Target("MS4w-sec")], cookie="sessionid=x", ) kwargs.update(overrides) return await douyin_fetch.collect(tmp_path, **kwargs) class TestHappyPath: @pytest.mark.asyncio async def test_writes_the_layout_ingest_expects(self, monkeypatch, tmp_path): async def fake_videos(sec_user_id, count=20, *, cookie=""): return [_video("111"), _video("222")] async def fake_comments(aweme_id, count=20, *, cookie=""): return [_comment(aweme_id, 1)] monkeypatch.setattr(douyin_api, "author_videos", fake_videos) monkeypatch.setattr(douyin_api, "video_comments", fake_comments) result = await _collect(tmp_path) assert result["notes"] == 2 assert result["comments"] == 2 assert result["errors"] == [] # 目录名必须是 douyin(不是平台 id dy)—— ingest 找文件用的是同一个来源。 jsonl_dir = tmp_path / "douyin" / "jsonl" assert jsonl_dir.is_dir() contents = list(jsonl_dir.glob("*_contents_*.jsonl")) comments = list(jsonl_dir.glob("*_comments_*.jsonl")) assert len(contents) == 1 assert len(comments) == 1 notes = _read(contents[0]) assert [n["aweme_id"] for n in notes] == ["111", "222"] # 键名照抄 store/douyin —— ingest 靠这个读出来。 assert notes[0]["aweme_url"] == "https://www.douyin.com/video/111" # 秒,不是毫秒;换算交给 adapters。 assert notes[0]["create_time"] == 1790574515 assert _read(comments[0])[0]["aweme_id"] == "111" @pytest.mark.asyncio async def test_the_comment_file_exists_even_without_comments(self, monkeypatch, tmp_path): """评论文件必须建出来。 ingest 靠「文件在不在」区分「这一轮没评论」和「这一轮什么都没抓到」—— 两种情况的含义完全不同。 """ async def fake_videos(sec_user_id, count=20, *, cookie=""): return [_video("111")] monkeypatch.setattr(douyin_api, "author_videos", fake_videos) await _collect(tmp_path, want_comments=False) assert len(list((tmp_path / "douyin" / "jsonl").glob("*_comments_*.jsonl"))) == 1 @pytest.mark.asyncio async def test_duplicate_works_are_written_once(self, monkeypatch, tmp_path): async def fake_videos(sec_user_id, count=20, *, cookie=""): return [_video("111"), _video("111")] monkeypatch.setattr(douyin_api, "author_videos", fake_videos) result = await _collect(tmp_path, want_comments=False) assert result["notes"] == 1 class TestDegradation: """作品列表被挡时的行为 —— 决定了这个功能今天有没有用。""" @pytest.mark.asyncio async def test_falls_back_to_refreshing_known_works(self, monkeypatch, tmp_path): async def blocked(sec_user_id, count=20, *, cookie=""): raise douyin_api.DouyinApiError("接口返回了空内容") refreshed = [] async def fake_detail(aweme_id, *, cookie=""): refreshed.append(aweme_id) return _video(aweme_id, likes="9") monkeypatch.setattr(douyin_api, "author_videos", blocked) monkeypatch.setattr(douyin_api, "video_detail", fake_detail) result = await _collect( tmp_path, want_comments=False, known_aweme_ids=["999", "888"] ) assert refreshed == ["999", "888"] assert result["notes"] == 2 # 但错误照样报出来 —— 这一轮是「部分可用」,不是「一切正常」,别粉饰。 assert any("作品列表失败" in error for error in result["errors"]) @pytest.mark.asyncio async def test_nothing_at_all_still_reports_the_reason(self, monkeypatch, tmp_path): async def blocked(sec_user_id, count=20, *, cookie=""): raise douyin_api.DouyinApiError("接口返回了空内容") monkeypatch.setattr(douyin_api, "author_videos", blocked) result = await _collect(tmp_path, want_comments=False) assert result["notes"] == 0 assert result["errors"] # 产物仍然写出来(空的),让调用方去判断这是失败而不是「这个博主没作品」。 assert (tmp_path / "douyin" / "jsonl").is_dir() @pytest.mark.asyncio async def test_a_failing_comment_fetch_does_not_lose_the_work(self, monkeypatch, tmp_path): async def fake_videos(sec_user_id, count=20, *, cookie=""): return [_video("111")] async def broken_comments(aweme_id, count=20, *, cookie=""): raise douyin_api.DouyinApiError("评论接口抽风") monkeypatch.setattr(douyin_api, "author_videos", fake_videos) monkeypatch.setattr(douyin_api, "video_comments", broken_comments) result = await _collect(tmp_path) # 评论拿不到是小事,作品不能跟着丢。 assert result["notes"] == 1 assert any("评论失败" in error for error in result["errors"])