feat(monitor): 抖音监控改走新的 Web 接口客户端(接上上一版的移植)
上一版只把客户端写出来、验证了它单独可用,**但没有接进任何地方** —— 所以你建的任务跑起来 仍然在调爬虫子进程,报的仍然是那句自己编的「account blocked」。这一步把它接上。 * runner 的 Phase 2 按平台分岔:dy 走进程内 HTTP 客户端(douyin_fetch),其余平台照旧走 爬虫子进程。抖音那条不再起 Playwright、不再构造那串自相矛盾的浏览器指纹参数。 * 新增 douyin_fetch:把采到的东西写成 store/douyin 那套 jsonl 形状 —— **ingest 完全不知道 数据是从哪来的**,重采样/差分/事件/通知/报表全都照旧,一个字没改。 * 失败不再假装:一条都没采到就以非零退出码 + **真实原因**交给 ingest,落成 「采集进程异常退出(code=1):…」。绝不会再掉进「疑似登录失效」那个分支。 * 已知作品列表接口(aweme/post)被抖音单独加了真校验(200 + 空 body),所以加了退化: 拿不到列表就用库里已知的 aweme_id 逐条走 detail 刷新。**边界是:已知作品的指标能继续 更新,新作品发现不了** —— 这个边界会以一条 warning 日志留下痕迹,不让它看起来一切正常。 * 顺带给客户端补上 video_detail(实测可用:200 / 45425 字节),退化路径靠它。 测试 +6:产物目录与文件名、评论文件即使为空也要建(ingest 靠它区分「没评论」和 「什么都没抓到」)、重复作品只写一次、列表被挡时的退化、彻底失败仍写出产物与原因、 评论失败不连累作品。
This commit is contained in:
@@ -326,6 +326,29 @@ async def author_videos(
|
||||
return [normalize_aweme(aweme) for aweme in awemes]
|
||||
|
||||
|
||||
async def video_detail(aweme_id: str, *, cookie: str = "") -> Dict[str, Any]:
|
||||
"""单条作品的详情,已翻译成 store 的键名。
|
||||
|
||||
这个接口**没有**被那道真校验挡着(实测 200 / 45425 字节),所以在拿不到作品列表时,
|
||||
它是「刷新已知作品指标」的唯一途径。
|
||||
"""
|
||||
identity = await browser_identity(cookie)
|
||||
if not _has_session(identity.cookie):
|
||||
raise DouyinApiError("抖音登录态不可用:浏览器里没有会话,库里的 cookie 也没有")
|
||||
|
||||
payload = await _get(
|
||||
DETAIL_PATH,
|
||||
{"aweme_id": aweme_id, "device_platform": "webapp", "aid": 6383},
|
||||
identity,
|
||||
)
|
||||
aweme = payload.get("aweme_detail") or {}
|
||||
if not aweme:
|
||||
raise DouyinApiError(
|
||||
f"接口没返回作品(status_code={payload.get('status_code')})"
|
||||
)
|
||||
return normalize_aweme(aweme)
|
||||
|
||||
|
||||
async def author_profile(sec_user_id: str, *, cookie: str = "") -> Dict[str, Any]:
|
||||
"""博主主页指标:昵称 / 粉丝数 / 总获赞 / 作品数。"""
|
||||
identity = await browser_identity(cookie)
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2025 [email protected]
|
||||
#
|
||||
# This file is part of MediaCrawler project.
|
||||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/monitor/douyin_fetch.py
|
||||
# GitHub: https://github.com/NanmiCoder
|
||||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||
#
|
||||
# 声明:本代码仅供学习和研究目的使用。使用者应遵守以下原则:
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守对应平台的使用条款和robots.txt规则。
|
||||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||
# 4. 应合理控制请求频率,避免给目标平台带来不必要的负担。
|
||||
# 5. 不得用于任何非法或不当的用途。
|
||||
#
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
"""抖音采集:直接走 Web 接口,不起爬虫子进程。
|
||||
|
||||
与 ``media_platform/douyin`` 那条路的分工:
|
||||
|
||||
* 那边起一个 Playwright 子进程、构造一大串**自相矛盾的浏览器指纹参数**(参数说自己是
|
||||
Mac + Chrome 125,UA 说自己是 Linux + Chrome 155),网关回一个 200 + 空 body,
|
||||
然后被翻译成 ``Exception("account blocked")`` —— 看起来像账号被封,其实什么都不是。
|
||||
* 这边不起子进程、只发必要参数,用浏览器里那份登录态。见 ``douyin_api``。
|
||||
|
||||
采到的东西写成 ``store/douyin`` 那套 jsonl 形状,所以 **ingest 完全不知道数据是怎么来的**
|
||||
—— 重采样、差分、事件、通知、报表全都照旧。
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Sequence
|
||||
|
||||
from tools import utils
|
||||
|
||||
from . import adapters, douyin_api
|
||||
from .models import MODE_CREATOR, MonitorTask
|
||||
|
||||
# 拉多少条作品。接口单页上限就是 20,要多了也没用。
|
||||
DEFAULT_VIDEO_LIMIT = 20
|
||||
|
||||
|
||||
async def collect(
|
||||
out_dir: Path,
|
||||
*,
|
||||
platform: str,
|
||||
mode: str,
|
||||
limit: int,
|
||||
want_comments: bool,
|
||||
comment_limit: int,
|
||||
targets: Sequence[Any],
|
||||
known_aweme_ids: Iterable[str] = (),
|
||||
cookie: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
"""跑一轮抖音采集,把产物写进 ``out_dir`` 下 store 那个目录里。
|
||||
|
||||
参数是散的、不收 ORM 对象:调用方那边 ``task`` 在会话关掉之后就 detached 了,
|
||||
传对象进来迟早会踩到「属性已过期」。
|
||||
|
||||
**不抛异常**:失败也把(可能为空的)产物落下去,并把原因放进 ``errors`` 交给调用方。
|
||||
让调用方去决定这是「一轮正常但没数据」还是「一轮失败」—— 这个判断不该藏在这里。
|
||||
"""
|
||||
notes: List[Dict[str, Any]] = []
|
||||
comments: List[Dict[str, Any]] = []
|
||||
errors: List[str] = []
|
||||
|
||||
for target in targets:
|
||||
sec_user_id = target.external_id
|
||||
seen_aweme: set = set()
|
||||
|
||||
# 作品列表是主路径;它被那道真校验挡着时,退化到「标题/昵称靠主页接口,作品靠
|
||||
# 已知 id 逐条刷新」—— 拿不到新作品,但已知作品的指标还能继续更新。
|
||||
try:
|
||||
videos = await douyin_api.author_videos(
|
||||
sec_user_id, count=limit, cookie=cookie
|
||||
)
|
||||
except douyin_api.DouyinApiError as exc:
|
||||
errors.append(f"拉取博主 {sec_user_id} 的作品列表失败:{exc}")
|
||||
videos = []
|
||||
for aweme_id in known_aweme_ids:
|
||||
try:
|
||||
videos.append(await douyin_api.video_detail(aweme_id, cookie=cookie))
|
||||
except douyin_api.DouyinApiError as detail_exc:
|
||||
errors.append(f"刷新作品 {aweme_id} 失败:{detail_exc}")
|
||||
|
||||
for video in videos:
|
||||
aweme_id = video.get("aweme_id")
|
||||
if not aweme_id or aweme_id in seen_aweme:
|
||||
continue
|
||||
seen_aweme.add(aweme_id)
|
||||
notes.append(video)
|
||||
|
||||
if want_comments:
|
||||
try:
|
||||
comments.extend(
|
||||
await douyin_api.video_comments(
|
||||
aweme_id, count=comment_limit, cookie=cookie
|
||||
)
|
||||
)
|
||||
except douyin_api.DouyinApiError as exc:
|
||||
errors.append(f"拉取作品 {aweme_id} 的评论失败:{exc}")
|
||||
|
||||
jsonl_dir = _write_artifacts(out_dir, platform, mode, notes, comments)
|
||||
return {
|
||||
"notes": len(notes),
|
||||
"comments": len(comments),
|
||||
"errors": errors,
|
||||
"jsonl_dir": str(jsonl_dir),
|
||||
}
|
||||
|
||||
|
||||
def _write_artifacts(
|
||||
out_dir: Path,
|
||||
platform: str,
|
||||
mode: str,
|
||||
notes: List[Dict[str, Any]],
|
||||
comments: List[Dict[str, Any]],
|
||||
) -> Path:
|
||||
"""按爬虫那套目录与文件名写 jsonl。
|
||||
|
||||
目录名取 ``adapters.artifact_dir``(抖音是 ``douyin``,而不是平台 id ``dy``)——
|
||||
和 ingest 找文件用的是同一个来源,两边不会走散。
|
||||
"""
|
||||
jsonl_dir = out_dir / adapters.artifact_dir(platform) / "jsonl"
|
||||
jsonl_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
kind = "creator" if mode == MODE_CREATOR else "detail"
|
||||
date = datetime.now().strftime("%Y-%m-%d")
|
||||
|
||||
_write_jsonl(jsonl_dir / f"{kind}_contents_{date}.jsonl", notes)
|
||||
# 评论文件即使没有评论也建出来:ingest 靠「文件在不在」区分「这一轮没评论」和
|
||||
# 「这一轮什么都没抓到」,两种情况的含义完全不同。
|
||||
_write_jsonl(jsonl_dir / f"{kind}_comments_{date}.jsonl", comments)
|
||||
return jsonl_dir
|
||||
|
||||
|
||||
def _write_jsonl(path: Path, records: List[Dict[str, Any]]) -> None:
|
||||
with path.open("w", encoding="utf-8") as handle:
|
||||
for record in records:
|
||||
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
utils.logger.info(f"[douyin_fetch] 写出 {len(records)} 条 -> {path.name}")
|
||||
+98
-50
@@ -28,6 +28,8 @@ import os
|
||||
from pathlib import Path
|
||||
from typing import Iterable, List, Optional
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from tools.time_util import get_current_timestamp
|
||||
|
||||
from ..schemas import (
|
||||
@@ -38,7 +40,7 @@ from ..schemas import (
|
||||
SaveDataOptionEnum,
|
||||
)
|
||||
from ..services import crawler_manager
|
||||
from . import adapters, app_settings, covers, notify
|
||||
from . import adapters, app_settings, covers, douyin_fetch, notify
|
||||
from .db import get_session
|
||||
from .ingest import IngestResult, diagnose_failure, ingest_run
|
||||
from .models import (
|
||||
@@ -47,6 +49,7 @@ from .models import (
|
||||
RUN_PENDING,
|
||||
RUN_RUNNING,
|
||||
RUN_TIMEOUT,
|
||||
MonitorNote,
|
||||
MonitorRun,
|
||||
MonitorTarget,
|
||||
MonitorTask,
|
||||
@@ -194,57 +197,102 @@ async def execute_task(task_id: int, trigger: str = "manual") -> IngestResult:
|
||||
max_notes_count = task.max_notes_count
|
||||
max_comments_count = task.max_comments_count
|
||||
timeout_seconds = task.run_timeout_seconds
|
||||
# 抖音的作品列表接口被那道真校验挡着(见 douyin_fetch),拿不到列表时就靠这些
|
||||
# 已知的 aweme_id 逐条刷新 —— 新作品发现不了,但已有作品的指标还能继续更新。
|
||||
known_aweme_ids = list(
|
||||
await session.scalars(
|
||||
select(MonitorNote.note_id).where(MonitorNote.task_id == task.id)
|
||||
)
|
||||
)
|
||||
|
||||
# --- Phase 2: run the crawler outside any transaction ---------------------
|
||||
cookie_file = out_dir / ".cookies"
|
||||
_write_cookie_file(cookie_file, cookie)
|
||||
# --- Phase 2: run the collection outside any transaction ------------------
|
||||
# 抖音走**进程内 HTTP 客户端**,不起 Playwright 子进程:那边会构造一大串自相矛盾的
|
||||
# 浏览器指纹参数(参数说 Mac + Chrome 125、UA 说 Linux + Chrome 155),网关回一个
|
||||
# 200 + 空 body,然后被翻译成「account blocked」—— 看着像账号被封,其实什么都不是。
|
||||
# 见 douyin_fetch / douyin_api。
|
||||
in_process_tail: List[str] = []
|
||||
if platform == adapters.PLATFORM_DY:
|
||||
fetched = await douyin_fetch.collect(
|
||||
out_dir,
|
||||
platform=platform,
|
||||
mode=mode,
|
||||
limit=max_notes_count,
|
||||
want_comments=enable_comments,
|
||||
comment_limit=max_comments_count,
|
||||
targets=targets,
|
||||
known_aweme_ids=known_aweme_ids,
|
||||
cookie=cookie,
|
||||
)
|
||||
in_process_tail = list(fetched["errors"])
|
||||
# 一条都没采到 = 这一轮失败,并把**真因**当作退出诊断传下去。否则它会掉进
|
||||
# ingest 的「疑似登录失效」分支 —— 又骗人一次,正是这套东西一直在犯的毛病。
|
||||
exit_code = 1 if (fetched["errors"] and not fetched["notes"]) else 0
|
||||
if fetched["errors"] and fetched["notes"]:
|
||||
# 有产物但带着错误,说明走了退化路径(比如作品列表被挡,只刷新了已知作品)。
|
||||
# 这一轮状态是成功,但**不是**一切正常 —— 得留下痕迹,否则没人知道新作品
|
||||
# 其实没在发现。
|
||||
print(
|
||||
"[monitor.runner] 抖音采集部分失败:"
|
||||
+ ";".join(fetched["errors"])[:300]
|
||||
)
|
||||
else:
|
||||
cookie_file = out_dir / ".cookies"
|
||||
_write_cookie_file(cookie_file, cookie)
|
||||
|
||||
request = CrawlerStartRequest(
|
||||
platform=_PLATFORM_ENUM[platform],
|
||||
login_type=LoginTypeEnum.COOKIE,
|
||||
crawler_type=CrawlerTypeEnum.CREATOR if mode == MODE_CREATOR else CrawlerTypeEnum.DETAIL,
|
||||
creator_ids=",".join(urls) if mode == MODE_CREATOR else "",
|
||||
specified_ids=",".join(urls) if mode != MODE_CREATOR else "",
|
||||
start_page=1,
|
||||
enable_comments=enable_comments,
|
||||
enable_sub_comments=strategy["enable_sub_comments"],
|
||||
enable_media=False,
|
||||
save_option=SaveDataOptionEnum.JSONL,
|
||||
cookies="",
|
||||
headless=True,
|
||||
max_notes_count=max_notes_count,
|
||||
max_comments_count=max_comments_count,
|
||||
# Isolate this run's output: the crawler names files by date only, so
|
||||
# otherwise same-day runs would append into one shared file.
|
||||
save_data_path=str(out_dir),
|
||||
# Attach to the browser already running on CDP_DEBUG_PORT when the
|
||||
# operator enabled it; otherwise launch a private, throwaway browser.
|
||||
enable_cdp_mode=cdp_enabled,
|
||||
# Only injecting web_session is not enough to sign requests from a cold
|
||||
# browser profile.
|
||||
inject_all_cookies=True,
|
||||
save_login_state=True,
|
||||
cookies_file=str(cookie_file),
|
||||
max_concurrency_num=1,
|
||||
# Strategy + proxy, surfaced on the Settings page.
|
||||
crawler_max_sleep_sec=strategy["crawl_sleep_sec"],
|
||||
enable_ip_proxy=strategy["enable_ip_proxy"],
|
||||
ip_proxy_pool_count=strategy["proxy_pool_count"],
|
||||
ip_proxy_provider_name=strategy["proxy_provider"],
|
||||
static_proxy_url=strategy["static_proxy_url"] or None,
|
||||
request = CrawlerStartRequest(
|
||||
platform=_PLATFORM_ENUM[platform],
|
||||
login_type=LoginTypeEnum.COOKIE,
|
||||
crawler_type=CrawlerTypeEnum.CREATOR if mode == MODE_CREATOR else CrawlerTypeEnum.DETAIL,
|
||||
creator_ids=",".join(urls) if mode == MODE_CREATOR else "",
|
||||
specified_ids=",".join(urls) if mode != MODE_CREATOR else "",
|
||||
start_page=1,
|
||||
enable_comments=enable_comments,
|
||||
enable_sub_comments=strategy["enable_sub_comments"],
|
||||
enable_media=False,
|
||||
save_option=SaveDataOptionEnum.JSONL,
|
||||
cookies="",
|
||||
headless=True,
|
||||
max_notes_count=max_notes_count,
|
||||
max_comments_count=max_comments_count,
|
||||
# Isolate this run's output: the crawler names files by date only, so
|
||||
# otherwise same-day runs would append into one shared file.
|
||||
save_data_path=str(out_dir),
|
||||
# Attach to the browser already running on CDP_DEBUG_PORT when the
|
||||
# operator enabled it; otherwise launch a private, throwaway browser.
|
||||
enable_cdp_mode=cdp_enabled,
|
||||
# Only injecting web_session is not enough to sign requests from a cold
|
||||
# browser profile.
|
||||
inject_all_cookies=True,
|
||||
save_login_state=True,
|
||||
cookies_file=str(cookie_file),
|
||||
max_concurrency_num=1,
|
||||
# Strategy + proxy, surfaced on the Settings page.
|
||||
crawler_max_sleep_sec=strategy["crawl_sleep_sec"],
|
||||
enable_ip_proxy=strategy["enable_ip_proxy"],
|
||||
ip_proxy_pool_count=strategy["proxy_pool_count"],
|
||||
ip_proxy_provider_name=strategy["proxy_provider"],
|
||||
static_proxy_url=strategy["static_proxy_url"] or None,
|
||||
)
|
||||
|
||||
async with get_session() as session:
|
||||
run = await session.get(MonitorRun, run_id)
|
||||
if run is not None:
|
||||
run.status = RUN_RUNNING
|
||||
run.started_at = get_current_timestamp()
|
||||
|
||||
try:
|
||||
exit_code = await crawler_manager.run_and_wait(request, timeout=timeout_seconds)
|
||||
finally:
|
||||
_remove_cookie_file(cookie_file)
|
||||
|
||||
# 失败时的诊断来源。抖音那条路没有子进程,尾巴就是它自己报的错 —— **别去读
|
||||
# crawler_manager 的尾巴**,那里面是上一轮别的平台留下的东西,会张冠李戴。
|
||||
output_tail = (
|
||||
in_process_tail
|
||||
if platform == adapters.PLATFORM_DY
|
||||
else crawler_manager.get_output_tail()
|
||||
)
|
||||
|
||||
async with get_session() as session:
|
||||
run = await session.get(MonitorRun, run_id)
|
||||
if run is not None:
|
||||
run.status = RUN_RUNNING
|
||||
run.started_at = get_current_timestamp()
|
||||
|
||||
try:
|
||||
exit_code = await crawler_manager.run_and_wait(request, timeout=timeout_seconds)
|
||||
finally:
|
||||
_remove_cookie_file(cookie_file)
|
||||
|
||||
# --- Phase 3: ingest ------------------------------------------------------
|
||||
async with get_session() as session:
|
||||
run = await session.get(MonitorRun, run_id)
|
||||
@@ -261,7 +309,7 @@ async def execute_task(task_id: int, trigger: str = "manual") -> IngestResult:
|
||||
run.error_message = "Run was killed by timeout or failed to start"
|
||||
# -1 同时代表「超时」和「根本没起来」,两者要查的东西完全不同。输出尾巴里
|
||||
# 有异常就带上它,否则运行历史里只能看到这句没有信息量的话。
|
||||
cause = diagnose_failure(crawler_manager.get_output_tail())
|
||||
cause = diagnose_failure(output_tail)
|
||||
if cause:
|
||||
run.error_message = f"{run.error_message};原因:{cause}"
|
||||
result = IngestResult(status=RUN_TIMEOUT, error=run.error_message)
|
||||
@@ -271,7 +319,7 @@ async def execute_task(task_id: int, trigger: str = "manual") -> IngestResult:
|
||||
# 把爬虫输出的尾巴交给 ingest:退出码本身说明不了问题,运行历史里要显示的
|
||||
# 是真正的报错(比如抖音的 DataFetchError: account blocked)。
|
||||
result = await ingest_run(
|
||||
session, run, task, out_dir, output_tail=crawler_manager.get_output_tail()
|
||||
session, run, task, out_dir, output_tail=output_tail
|
||||
)
|
||||
|
||||
# A run that authenticated fine is the only useful signal that the
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""抖音采集编排 —— 不碰网络,把 douyin_api 整个换掉。
|
||||
|
||||
验的是编排本身:产物落在正确的目录、文件名是 store 那套、以及**作品列表被挡时的退化**
|
||||
(用已知 aweme_id 逐条刷新)—— 那条退化路径决定了今天这个功能是「完全没用」还是
|
||||
「已知作品还能看」。
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from api.monitor import douyin_api, douyin_fetch
|
||||
|
||||
|
||||
def _video(aweme_id: str, likes: str = "1") -> dict:
|
||||
return {
|
||||
"aweme_id": aweme_id,
|
||||
"title": f"title-{aweme_id}",
|
||||
"aweme_url": f"https://www.douyin.com/video/{aweme_id}",
|
||||
"cover_url": "",
|
||||
"aweme_type": "0",
|
||||
"create_time": 1790574515,
|
||||
"creator_hash": "hash",
|
||||
"nickname": "博主",
|
||||
"liked_count": likes,
|
||||
"comment_count": "0",
|
||||
"collected_count": "0",
|
||||
"share_count": "0",
|
||||
}
|
||||
|
||||
|
||||
def _comment(aweme_id: str, index: int) -> dict:
|
||||
return {
|
||||
"comment_id": f"c{index}",
|
||||
"aweme_id": aweme_id,
|
||||
"content": f"评论{index}",
|
||||
"nickname": "路人",
|
||||
"creator_hash": "h2",
|
||||
"create_time": 1790574600,
|
||||
"like_count": "0",
|
||||
"sub_comment_count": "0",
|
||||
"parent_comment_id": "0",
|
||||
}
|
||||
|
||||
|
||||
class _Target:
|
||||
def __init__(self, external_id: str) -> None:
|
||||
self.external_id = external_id
|
||||
|
||||
|
||||
def _read(path):
|
||||
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line]
|
||||
|
||||
|
||||
async def _collect(tmp_path, **overrides):
|
||||
kwargs = dict(
|
||||
platform="dy",
|
||||
mode="creator",
|
||||
limit=20,
|
||||
want_comments=True,
|
||||
comment_limit=20,
|
||||
targets=[_Target("MS4w-sec")],
|
||||
cookie="sessionid=x",
|
||||
)
|
||||
kwargs.update(overrides)
|
||||
return await douyin_fetch.collect(tmp_path, **kwargs)
|
||||
|
||||
|
||||
class TestHappyPath:
|
||||
@pytest.mark.asyncio
|
||||
async def test_writes_the_layout_ingest_expects(self, monkeypatch, tmp_path):
|
||||
async def fake_videos(sec_user_id, count=20, *, cookie=""):
|
||||
return [_video("111"), _video("222")]
|
||||
|
||||
async def fake_comments(aweme_id, count=20, *, cookie=""):
|
||||
return [_comment(aweme_id, 1)]
|
||||
|
||||
monkeypatch.setattr(douyin_api, "author_videos", fake_videos)
|
||||
monkeypatch.setattr(douyin_api, "video_comments", fake_comments)
|
||||
|
||||
result = await _collect(tmp_path)
|
||||
|
||||
assert result["notes"] == 2
|
||||
assert result["comments"] == 2
|
||||
assert result["errors"] == []
|
||||
|
||||
# 目录名必须是 douyin(不是平台 id dy)—— ingest 找文件用的是同一个来源。
|
||||
jsonl_dir = tmp_path / "douyin" / "jsonl"
|
||||
assert jsonl_dir.is_dir()
|
||||
|
||||
contents = list(jsonl_dir.glob("*_contents_*.jsonl"))
|
||||
comments = list(jsonl_dir.glob("*_comments_*.jsonl"))
|
||||
assert len(contents) == 1
|
||||
assert len(comments) == 1
|
||||
|
||||
notes = _read(contents[0])
|
||||
assert [n["aweme_id"] for n in notes] == ["111", "222"]
|
||||
# 键名照抄 store/douyin —— ingest 靠这个读出来。
|
||||
assert notes[0]["aweme_url"] == "https://www.douyin.com/video/111"
|
||||
# 秒,不是毫秒;换算交给 adapters。
|
||||
assert notes[0]["create_time"] == 1790574515
|
||||
|
||||
assert _read(comments[0])[0]["aweme_id"] == "111"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_comment_file_exists_even_without_comments(self, monkeypatch, tmp_path):
|
||||
"""评论文件必须建出来。
|
||||
|
||||
ingest 靠「文件在不在」区分「这一轮没评论」和「这一轮什么都没抓到」——
|
||||
两种情况的含义完全不同。
|
||||
"""
|
||||
|
||||
async def fake_videos(sec_user_id, count=20, *, cookie=""):
|
||||
return [_video("111")]
|
||||
|
||||
monkeypatch.setattr(douyin_api, "author_videos", fake_videos)
|
||||
|
||||
await _collect(tmp_path, want_comments=False)
|
||||
|
||||
assert len(list((tmp_path / "douyin" / "jsonl").glob("*_comments_*.jsonl"))) == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplicate_works_are_written_once(self, monkeypatch, tmp_path):
|
||||
async def fake_videos(sec_user_id, count=20, *, cookie=""):
|
||||
return [_video("111"), _video("111")]
|
||||
|
||||
monkeypatch.setattr(douyin_api, "author_videos", fake_videos)
|
||||
|
||||
result = await _collect(tmp_path, want_comments=False)
|
||||
|
||||
assert result["notes"] == 1
|
||||
|
||||
|
||||
class TestDegradation:
|
||||
"""作品列表被挡时的行为 —— 决定了这个功能今天有没有用。"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_falls_back_to_refreshing_known_works(self, monkeypatch, tmp_path):
|
||||
async def blocked(sec_user_id, count=20, *, cookie=""):
|
||||
raise douyin_api.DouyinApiError("接口返回了空内容")
|
||||
|
||||
refreshed = []
|
||||
|
||||
async def fake_detail(aweme_id, *, cookie=""):
|
||||
refreshed.append(aweme_id)
|
||||
return _video(aweme_id, likes="9")
|
||||
|
||||
monkeypatch.setattr(douyin_api, "author_videos", blocked)
|
||||
monkeypatch.setattr(douyin_api, "video_detail", fake_detail)
|
||||
|
||||
result = await _collect(
|
||||
tmp_path, want_comments=False, known_aweme_ids=["999", "888"]
|
||||
)
|
||||
|
||||
assert refreshed == ["999", "888"]
|
||||
assert result["notes"] == 2
|
||||
# 但错误照样报出来 —— 这一轮是「部分可用」,不是「一切正常」,别粉饰。
|
||||
assert any("作品列表失败" in error for error in result["errors"])
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_nothing_at_all_still_reports_the_reason(self, monkeypatch, tmp_path):
|
||||
async def blocked(sec_user_id, count=20, *, cookie=""):
|
||||
raise douyin_api.DouyinApiError("接口返回了空内容")
|
||||
|
||||
monkeypatch.setattr(douyin_api, "author_videos", blocked)
|
||||
|
||||
result = await _collect(tmp_path, want_comments=False)
|
||||
|
||||
assert result["notes"] == 0
|
||||
assert result["errors"]
|
||||
# 产物仍然写出来(空的),让调用方去判断这是失败而不是「这个博主没作品」。
|
||||
assert (tmp_path / "douyin" / "jsonl").is_dir()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_failing_comment_fetch_does_not_lose_the_work(self, monkeypatch, tmp_path):
|
||||
async def fake_videos(sec_user_id, count=20, *, cookie=""):
|
||||
return [_video("111")]
|
||||
|
||||
async def broken_comments(aweme_id, count=20, *, cookie=""):
|
||||
raise douyin_api.DouyinApiError("评论接口抽风")
|
||||
|
||||
monkeypatch.setattr(douyin_api, "author_videos", fake_videos)
|
||||
monkeypatch.setattr(douyin_api, "video_comments", broken_comments)
|
||||
|
||||
result = await _collect(tmp_path)
|
||||
|
||||
# 评论拿不到是小事,作品不能跟着丢。
|
||||
assert result["notes"] == 1
|
||||
assert any("评论失败" in error for error in result["errors"])
|
||||
Reference in New Issue
Block a user