feat(monitor): 抖音监控改走新的 Web 接口客户端(接上上一版的移植)
Deploy VitePress site to Pages / build (push) Canceled after 0s
Deploy VitePress site to Pages / Deploy (push) Canceled after 0s

上一版只把客户端写出来、验证了它单独可用,**但没有接进任何地方** —— 所以你建的任务跑起来
仍然在调爬虫子进程,报的仍然是那句自己编的「account blocked」。这一步把它接上。

* runner 的 Phase 2 按平台分岔:dy 走进程内 HTTP 客户端(douyin_fetch),其余平台照旧走
  爬虫子进程。抖音那条不再起 Playwright、不再构造那串自相矛盾的浏览器指纹参数。
* 新增 douyin_fetch:把采到的东西写成 store/douyin 那套 jsonl 形状 —— **ingest 完全不知道
  数据是从哪来的**,重采样/差分/事件/通知/报表全都照旧,一个字没改。
* 失败不再假装:一条都没采到就以非零退出码 + **真实原因**交给 ingest,落成
  「采集进程异常退出(code=1):…」。绝不会再掉进「疑似登录失效」那个分支。
* 已知作品列表接口(aweme/post)被抖音单独加了真校验(200 + 空 body),所以加了退化:
  拿不到列表就用库里已知的 aweme_id 逐条走 detail 刷新。**边界是:已知作品的指标能继续
  更新,新作品发现不了** —— 这个边界会以一条 warning 日志留下痕迹,不让它看起来一切正常。
* 顺带给客户端补上 video_detail(实测可用:200 / 45425 字节),退化路径靠它。

测试 +6:产物目录与文件名、评论文件即使为空也要建(ingest 靠它区分「没评论」和
「什么都没抓到」)、重复作品只写一次、列表被挡时的退化、彻底失败仍写出产物与原因、
评论失败不连累作品。
This commit is contained in:
2026-10-10 17:21:12 +08:00
parent 4f83a075f6
commit 3b6a437e6c
4 changed files with 455 additions and 50 deletions
+23
View File
@@ -326,6 +326,29 @@ async def author_videos(
return [normalize_aweme(aweme) for aweme in awemes]
async def video_detail(aweme_id: str, *, cookie: str = "") -> Dict[str, Any]:
"""单条作品的详情,已翻译成 store 的键名。
这个接口**没有**被那道真校验挡着(实测 200 / 45425 字节),所以在拿不到作品列表时,
它是「刷新已知作品指标」的唯一途径。
"""
identity = await browser_identity(cookie)
if not _has_session(identity.cookie):
raise DouyinApiError("抖音登录态不可用:浏览器里没有会话,库里的 cookie 也没有")
payload = await _get(
DETAIL_PATH,
{"aweme_id": aweme_id, "device_platform": "webapp", "aid": 6383},
identity,
)
aweme = payload.get("aweme_detail") or {}
if not aweme:
raise DouyinApiError(
f"接口没返回作品(status_code={payload.get('status_code')})"
)
return normalize_aweme(aweme)
async def author_profile(sec_user_id: str, *, cookie: str = "") -> Dict[str, Any]:
"""博主主页指标:昵称 / 粉丝数 / 总获赞 / 作品数。"""
identity = await browser_identity(cookie)
+144
View File
@@ -0,0 +1,144 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2025 [email protected]
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/monitor/douyin_fetch.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
# 声明:本代码仅供学习和研究目的使用。使用者应遵守以下原则:
# 1. 不得用于任何商业用途。
# 2. 使用时应遵守对应平台的使用条款和robots.txt规则。
# 3. 不得进行大规模爬取或对平台造成运营干扰。
# 4. 应合理控制请求频率,避免给目标平台带来不必要的负担。
# 5. 不得用于任何非法或不当的用途。
#
# 详细许可条款请参阅项目根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
"""抖音采集:直接走 Web 接口,不起爬虫子进程。
与 ``media_platform/douyin`` 那条路的分工:
* 那边起一个 Playwright 子进程、构造一大串**自相矛盾的浏览器指纹参数**(参数说自己是
Mac + Chrome 125,UA 说自己是 Linux + Chrome 155),网关回一个 200 + 空 body,
然后被翻译成 ``Exception("account blocked")`` —— 看起来像账号被封,其实什么都不是。
* 这边不起子进程、只发必要参数,用浏览器里那份登录态。见 ``douyin_api``。
采到的东西写成 ``store/douyin`` 那套 jsonl 形状,所以 **ingest 完全不知道数据是怎么来的**
—— 重采样、差分、事件、通知、报表全都照旧。
"""
import json
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, Iterable, List, Sequence
from tools import utils
from . import adapters, douyin_api
from .models import MODE_CREATOR, MonitorTask
# 拉多少条作品。接口单页上限就是 20,要多了也没用。
DEFAULT_VIDEO_LIMIT = 20
async def collect(
out_dir: Path,
*,
platform: str,
mode: str,
limit: int,
want_comments: bool,
comment_limit: int,
targets: Sequence[Any],
known_aweme_ids: Iterable[str] = (),
cookie: str = "",
) -> Dict[str, Any]:
"""跑一轮抖音采集,把产物写进 ``out_dir`` 下 store 那个目录里。
参数是散的、不收 ORM 对象:调用方那边 ``task`` 在会话关掉之后就 detached 了,
传对象进来迟早会踩到「属性已过期」。
**不抛异常**:失败也把(可能为空的)产物落下去,并把原因放进 ``errors`` 交给调用方。
让调用方去决定这是「一轮正常但没数据」还是「一轮失败」—— 这个判断不该藏在这里。
"""
notes: List[Dict[str, Any]] = []
comments: List[Dict[str, Any]] = []
errors: List[str] = []
for target in targets:
sec_user_id = target.external_id
seen_aweme: set = set()
# 作品列表是主路径;它被那道真校验挡着时,退化到「标题/昵称靠主页接口,作品靠
# 已知 id 逐条刷新」—— 拿不到新作品,但已知作品的指标还能继续更新。
try:
videos = await douyin_api.author_videos(
sec_user_id, count=limit, cookie=cookie
)
except douyin_api.DouyinApiError as exc:
errors.append(f"拉取博主 {sec_user_id} 的作品列表失败:{exc}")
videos = []
for aweme_id in known_aweme_ids:
try:
videos.append(await douyin_api.video_detail(aweme_id, cookie=cookie))
except douyin_api.DouyinApiError as detail_exc:
errors.append(f"刷新作品 {aweme_id} 失败:{detail_exc}")
for video in videos:
aweme_id = video.get("aweme_id")
if not aweme_id or aweme_id in seen_aweme:
continue
seen_aweme.add(aweme_id)
notes.append(video)
if want_comments:
try:
comments.extend(
await douyin_api.video_comments(
aweme_id, count=comment_limit, cookie=cookie
)
)
except douyin_api.DouyinApiError as exc:
errors.append(f"拉取作品 {aweme_id} 的评论失败:{exc}")
jsonl_dir = _write_artifacts(out_dir, platform, mode, notes, comments)
return {
"notes": len(notes),
"comments": len(comments),
"errors": errors,
"jsonl_dir": str(jsonl_dir),
}
def _write_artifacts(
out_dir: Path,
platform: str,
mode: str,
notes: List[Dict[str, Any]],
comments: List[Dict[str, Any]],
) -> Path:
"""按爬虫那套目录与文件名写 jsonl。
目录名取 ``adapters.artifact_dir``(抖音是 ``douyin``,而不是平台 id ``dy``)——
和 ingest 找文件用的是同一个来源,两边不会走散。
"""
jsonl_dir = out_dir / adapters.artifact_dir(platform) / "jsonl"
jsonl_dir.mkdir(parents=True, exist_ok=True)
kind = "creator" if mode == MODE_CREATOR else "detail"
date = datetime.now().strftime("%Y-%m-%d")
_write_jsonl(jsonl_dir / f"{kind}_contents_{date}.jsonl", notes)
# 评论文件即使没有评论也建出来:ingest 靠「文件在不在」区分「这一轮没评论」和
# 「这一轮什么都没抓到」,两种情况的含义完全不同。
_write_jsonl(jsonl_dir / f"{kind}_comments_{date}.jsonl", comments)
return jsonl_dir
def _write_jsonl(path: Path, records: List[Dict[str, Any]]) -> None:
with path.open("w", encoding="utf-8") as handle:
for record in records:
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
utils.logger.info(f"[douyin_fetch] 写出 {len(records)} 条 -> {path.name}")
+98 -50
View File
@@ -28,6 +28,8 @@ import os
from pathlib import Path
from typing import Iterable, List, Optional
from sqlalchemy import select
from tools.time_util import get_current_timestamp
from ..schemas import (
@@ -38,7 +40,7 @@ from ..schemas import (
SaveDataOptionEnum,
)
from ..services import crawler_manager
from . import adapters, app_settings, covers, notify
from . import adapters, app_settings, covers, douyin_fetch, notify
from .db import get_session
from .ingest import IngestResult, diagnose_failure, ingest_run
from .models import (
@@ -47,6 +49,7 @@ from .models import (
RUN_PENDING,
RUN_RUNNING,
RUN_TIMEOUT,
MonitorNote,
MonitorRun,
MonitorTarget,
MonitorTask,
@@ -194,57 +197,102 @@ async def execute_task(task_id: int, trigger: str = "manual") -> IngestResult:
max_notes_count = task.max_notes_count
max_comments_count = task.max_comments_count
timeout_seconds = task.run_timeout_seconds
# 抖音的作品列表接口被那道真校验挡着(见 douyin_fetch),拿不到列表时就靠这些
# 已知的 aweme_id 逐条刷新 —— 新作品发现不了,但已有作品的指标还能继续更新。
known_aweme_ids = list(
await session.scalars(
select(MonitorNote.note_id).where(MonitorNote.task_id == task.id)
)
)
# --- Phase 2: run the crawler outside any transaction ---------------------
cookie_file = out_dir / ".cookies"
_write_cookie_file(cookie_file, cookie)
# --- Phase 2: run the collection outside any transaction ------------------
# 抖音走**进程内 HTTP 客户端**,不起 Playwright 子进程:那边会构造一大串自相矛盾的
# 浏览器指纹参数(参数说 Mac + Chrome 125、UA 说 Linux + Chrome 155),网关回一个
# 200 + 空 body,然后被翻译成「account blocked」—— 看着像账号被封,其实什么都不是。
# 见 douyin_fetch / douyin_api。
in_process_tail: List[str] = []
if platform == adapters.PLATFORM_DY:
fetched = await douyin_fetch.collect(
out_dir,
platform=platform,
mode=mode,
limit=max_notes_count,
want_comments=enable_comments,
comment_limit=max_comments_count,
targets=targets,
known_aweme_ids=known_aweme_ids,
cookie=cookie,
)
in_process_tail = list(fetched["errors"])
# 一条都没采到 = 这一轮失败,并把**真因**当作退出诊断传下去。否则它会掉进
# ingest 的「疑似登录失效」分支 —— 又骗人一次,正是这套东西一直在犯的毛病。
exit_code = 1 if (fetched["errors"] and not fetched["notes"]) else 0
if fetched["errors"] and fetched["notes"]:
# 有产物但带着错误,说明走了退化路径(比如作品列表被挡,只刷新了已知作品)。
# 这一轮状态是成功,但**不是**一切正常 —— 得留下痕迹,否则没人知道新作品
# 其实没在发现。
print(
"[monitor.runner] 抖音采集部分失败:"
+ ";".join(fetched["errors"])[:300]
)
else:
cookie_file = out_dir / ".cookies"
_write_cookie_file(cookie_file, cookie)
request = CrawlerStartRequest(
platform=_PLATFORM_ENUM[platform],
login_type=LoginTypeEnum.COOKIE,
crawler_type=CrawlerTypeEnum.CREATOR if mode == MODE_CREATOR else CrawlerTypeEnum.DETAIL,
creator_ids=",".join(urls) if mode == MODE_CREATOR else "",
specified_ids=",".join(urls) if mode != MODE_CREATOR else "",
start_page=1,
enable_comments=enable_comments,
enable_sub_comments=strategy["enable_sub_comments"],
enable_media=False,
save_option=SaveDataOptionEnum.JSONL,
cookies="",
headless=True,
max_notes_count=max_notes_count,
max_comments_count=max_comments_count,
# Isolate this run's output: the crawler names files by date only, so
# otherwise same-day runs would append into one shared file.
save_data_path=str(out_dir),
# Attach to the browser already running on CDP_DEBUG_PORT when the
# operator enabled it; otherwise launch a private, throwaway browser.
enable_cdp_mode=cdp_enabled,
# Only injecting web_session is not enough to sign requests from a cold
# browser profile.
inject_all_cookies=True,
save_login_state=True,
cookies_file=str(cookie_file),
max_concurrency_num=1,
# Strategy + proxy, surfaced on the Settings page.
crawler_max_sleep_sec=strategy["crawl_sleep_sec"],
enable_ip_proxy=strategy["enable_ip_proxy"],
ip_proxy_pool_count=strategy["proxy_pool_count"],
ip_proxy_provider_name=strategy["proxy_provider"],
static_proxy_url=strategy["static_proxy_url"] or None,
request = CrawlerStartRequest(
platform=_PLATFORM_ENUM[platform],
login_type=LoginTypeEnum.COOKIE,
crawler_type=CrawlerTypeEnum.CREATOR if mode == MODE_CREATOR else CrawlerTypeEnum.DETAIL,
creator_ids=",".join(urls) if mode == MODE_CREATOR else "",
specified_ids=",".join(urls) if mode != MODE_CREATOR else "",
start_page=1,
enable_comments=enable_comments,
enable_sub_comments=strategy["enable_sub_comments"],
enable_media=False,
save_option=SaveDataOptionEnum.JSONL,
cookies="",
headless=True,
max_notes_count=max_notes_count,
max_comments_count=max_comments_count,
# Isolate this run's output: the crawler names files by date only, so
# otherwise same-day runs would append into one shared file.
save_data_path=str(out_dir),
# Attach to the browser already running on CDP_DEBUG_PORT when the
# operator enabled it; otherwise launch a private, throwaway browser.
enable_cdp_mode=cdp_enabled,
# Only injecting web_session is not enough to sign requests from a cold
# browser profile.
inject_all_cookies=True,
save_login_state=True,
cookies_file=str(cookie_file),
max_concurrency_num=1,
# Strategy + proxy, surfaced on the Settings page.
crawler_max_sleep_sec=strategy["crawl_sleep_sec"],
enable_ip_proxy=strategy["enable_ip_proxy"],
ip_proxy_pool_count=strategy["proxy_pool_count"],
ip_proxy_provider_name=strategy["proxy_provider"],
static_proxy_url=strategy["static_proxy_url"] or None,
)
async with get_session() as session:
run = await session.get(MonitorRun, run_id)
if run is not None:
run.status = RUN_RUNNING
run.started_at = get_current_timestamp()
try:
exit_code = await crawler_manager.run_and_wait(request, timeout=timeout_seconds)
finally:
_remove_cookie_file(cookie_file)
# 失败时的诊断来源。抖音那条路没有子进程,尾巴就是它自己报的错 —— **别去读
# crawler_manager 的尾巴**,那里面是上一轮别的平台留下的东西,会张冠李戴。
output_tail = (
in_process_tail
if platform == adapters.PLATFORM_DY
else crawler_manager.get_output_tail()
)
async with get_session() as session:
run = await session.get(MonitorRun, run_id)
if run is not None:
run.status = RUN_RUNNING
run.started_at = get_current_timestamp()
try:
exit_code = await crawler_manager.run_and_wait(request, timeout=timeout_seconds)
finally:
_remove_cookie_file(cookie_file)
# --- Phase 3: ingest ------------------------------------------------------
async with get_session() as session:
run = await session.get(MonitorRun, run_id)
@@ -261,7 +309,7 @@ async def execute_task(task_id: int, trigger: str = "manual") -> IngestResult:
run.error_message = "Run was killed by timeout or failed to start"
# -1 同时代表「超时」和「根本没起来」,两者要查的东西完全不同。输出尾巴里
# 有异常就带上它,否则运行历史里只能看到这句没有信息量的话。
cause = diagnose_failure(crawler_manager.get_output_tail())
cause = diagnose_failure(output_tail)
if cause:
run.error_message = f"{run.error_message};原因:{cause}"
result = IngestResult(status=RUN_TIMEOUT, error=run.error_message)
@@ -271,7 +319,7 @@ async def execute_task(task_id: int, trigger: str = "manual") -> IngestResult:
# 把爬虫输出的尾巴交给 ingest:退出码本身说明不了问题,运行历史里要显示的
# 是真正的报错(比如抖音的 DataFetchError: account blocked)。
result = await ingest_run(
session, run, task, out_dir, output_tail=crawler_manager.get_output_tail()
session, run, task, out_dir, output_tail=output_tail
)
# A run that authenticated fine is the only useful signal that the
+190
View File
@@ -0,0 +1,190 @@
# -*- coding: utf-8 -*-
"""抖音采集编排 —— 不碰网络,把 douyin_api 整个换掉。
验的是编排本身:产物落在正确的目录、文件名是 store 那套、以及**作品列表被挡时的退化**
(用已知 aweme_id 逐条刷新)—— 那条退化路径决定了今天这个功能是「完全没用」还是
「已知作品还能看」。
"""
import json
import pytest
from api.monitor import douyin_api, douyin_fetch
def _video(aweme_id: str, likes: str = "1") -> dict:
return {
"aweme_id": aweme_id,
"title": f"title-{aweme_id}",
"aweme_url": f"https://www.douyin.com/video/{aweme_id}",
"cover_url": "",
"aweme_type": "0",
"create_time": 1790574515,
"creator_hash": "hash",
"nickname": "博主",
"liked_count": likes,
"comment_count": "0",
"collected_count": "0",
"share_count": "0",
}
def _comment(aweme_id: str, index: int) -> dict:
return {
"comment_id": f"c{index}",
"aweme_id": aweme_id,
"content": f"评论{index}",
"nickname": "路人",
"creator_hash": "h2",
"create_time": 1790574600,
"like_count": "0",
"sub_comment_count": "0",
"parent_comment_id": "0",
}
class _Target:
def __init__(self, external_id: str) -> None:
self.external_id = external_id
def _read(path):
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line]
async def _collect(tmp_path, **overrides):
kwargs = dict(
platform="dy",
mode="creator",
limit=20,
want_comments=True,
comment_limit=20,
targets=[_Target("MS4w-sec")],
cookie="sessionid=x",
)
kwargs.update(overrides)
return await douyin_fetch.collect(tmp_path, **kwargs)
class TestHappyPath:
@pytest.mark.asyncio
async def test_writes_the_layout_ingest_expects(self, monkeypatch, tmp_path):
async def fake_videos(sec_user_id, count=20, *, cookie=""):
return [_video("111"), _video("222")]
async def fake_comments(aweme_id, count=20, *, cookie=""):
return [_comment(aweme_id, 1)]
monkeypatch.setattr(douyin_api, "author_videos", fake_videos)
monkeypatch.setattr(douyin_api, "video_comments", fake_comments)
result = await _collect(tmp_path)
assert result["notes"] == 2
assert result["comments"] == 2
assert result["errors"] == []
# 目录名必须是 douyin(不是平台 id dy)—— ingest 找文件用的是同一个来源。
jsonl_dir = tmp_path / "douyin" / "jsonl"
assert jsonl_dir.is_dir()
contents = list(jsonl_dir.glob("*_contents_*.jsonl"))
comments = list(jsonl_dir.glob("*_comments_*.jsonl"))
assert len(contents) == 1
assert len(comments) == 1
notes = _read(contents[0])
assert [n["aweme_id"] for n in notes] == ["111", "222"]
# 键名照抄 store/douyin —— ingest 靠这个读出来。
assert notes[0]["aweme_url"] == "https://www.douyin.com/video/111"
# 秒,不是毫秒;换算交给 adapters。
assert notes[0]["create_time"] == 1790574515
assert _read(comments[0])[0]["aweme_id"] == "111"
@pytest.mark.asyncio
async def test_the_comment_file_exists_even_without_comments(self, monkeypatch, tmp_path):
"""评论文件必须建出来。
ingest 靠「文件在不在」区分「这一轮没评论」和「这一轮什么都没抓到」——
两种情况的含义完全不同。
"""
async def fake_videos(sec_user_id, count=20, *, cookie=""):
return [_video("111")]
monkeypatch.setattr(douyin_api, "author_videos", fake_videos)
await _collect(tmp_path, want_comments=False)
assert len(list((tmp_path / "douyin" / "jsonl").glob("*_comments_*.jsonl"))) == 1
@pytest.mark.asyncio
async def test_duplicate_works_are_written_once(self, monkeypatch, tmp_path):
async def fake_videos(sec_user_id, count=20, *, cookie=""):
return [_video("111"), _video("111")]
monkeypatch.setattr(douyin_api, "author_videos", fake_videos)
result = await _collect(tmp_path, want_comments=False)
assert result["notes"] == 1
class TestDegradation:
"""作品列表被挡时的行为 —— 决定了这个功能今天有没有用。"""
@pytest.mark.asyncio
async def test_falls_back_to_refreshing_known_works(self, monkeypatch, tmp_path):
async def blocked(sec_user_id, count=20, *, cookie=""):
raise douyin_api.DouyinApiError("接口返回了空内容")
refreshed = []
async def fake_detail(aweme_id, *, cookie=""):
refreshed.append(aweme_id)
return _video(aweme_id, likes="9")
monkeypatch.setattr(douyin_api, "author_videos", blocked)
monkeypatch.setattr(douyin_api, "video_detail", fake_detail)
result = await _collect(
tmp_path, want_comments=False, known_aweme_ids=["999", "888"]
)
assert refreshed == ["999", "888"]
assert result["notes"] == 2
# 但错误照样报出来 —— 这一轮是「部分可用」,不是「一切正常」,别粉饰。
assert any("作品列表失败" in error for error in result["errors"])
@pytest.mark.asyncio
async def test_nothing_at_all_still_reports_the_reason(self, monkeypatch, tmp_path):
async def blocked(sec_user_id, count=20, *, cookie=""):
raise douyin_api.DouyinApiError("接口返回了空内容")
monkeypatch.setattr(douyin_api, "author_videos", blocked)
result = await _collect(tmp_path, want_comments=False)
assert result["notes"] == 0
assert result["errors"]
# 产物仍然写出来(空的),让调用方去判断这是失败而不是「这个博主没作品」。
assert (tmp_path / "douyin" / "jsonl").is_dir()
@pytest.mark.asyncio
async def test_a_failing_comment_fetch_does_not_lose_the_work(self, monkeypatch, tmp_path):
async def fake_videos(sec_user_id, count=20, *, cookie=""):
return [_video("111")]
async def broken_comments(aweme_id, count=20, *, cookie=""):
raise douyin_api.DouyinApiError("评论接口抽风")
monkeypatch.setattr(douyin_api, "author_videos", fake_videos)
monkeypatch.setattr(douyin_api, "video_comments", broken_comments)
result = await _collect(tmp_path)
# 评论拿不到是小事,作品不能跟着丢。
assert result["notes"] == 1
assert any("评论失败" in error for error in result["errors"])