feat(monitor): 抖音监控改走新的 Web 接口客户端(接上上一版的移植)
Deploy VitePress site to Pages / build (push) Canceled after 0s
Deploy VitePress site to Pages / Deploy (push) Canceled after 0s

上一版只把客户端写出来、验证了它单独可用,**但没有接进任何地方** —— 所以你建的任务跑起来
仍然在调爬虫子进程,报的仍然是那句自己编的「account blocked」。这一步把它接上。

* runner 的 Phase 2 按平台分岔:dy 走进程内 HTTP 客户端(douyin_fetch),其余平台照旧走
  爬虫子进程。抖音那条不再起 Playwright、不再构造那串自相矛盾的浏览器指纹参数。
* 新增 douyin_fetch:把采到的东西写成 store/douyin 那套 jsonl 形状 —— **ingest 完全不知道
  数据是从哪来的**,重采样/差分/事件/通知/报表全都照旧,一个字没改。
* 失败不再假装:一条都没采到就以非零退出码 + **真实原因**交给 ingest,落成
  「采集进程异常退出(code=1):…」。绝不会再掉进「疑似登录失效」那个分支。
* 已知作品列表接口(aweme/post)被抖音单独加了真校验(200 + 空 body),所以加了退化:
  拿不到列表就用库里已知的 aweme_id 逐条走 detail 刷新。**边界是:已知作品的指标能继续
  更新,新作品发现不了** —— 这个边界会以一条 warning 日志留下痕迹,不让它看起来一切正常。
* 顺带给客户端补上 video_detail(实测可用:200 / 45425 字节),退化路径靠它。

测试 +6:产物目录与文件名、评论文件即使为空也要建(ingest 靠它区分「没评论」和
「什么都没抓到」)、重复作品只写一次、列表被挡时的退化、彻底失败仍写出产物与原因、
评论失败不连累作品。
This commit is contained in:
2026-10-10 17:21:12 +08:00
parent 4f83a075f6
commit 3b6a437e6c
4 changed files with 455 additions and 50 deletions
+98 -50
View File
@@ -28,6 +28,8 @@ import os
from pathlib import Path
from typing import Iterable, List, Optional
from sqlalchemy import select
from tools.time_util import get_current_timestamp
from ..schemas import (
@@ -38,7 +40,7 @@ from ..schemas import (
SaveDataOptionEnum,
)
from ..services import crawler_manager
from . import adapters, app_settings, covers, notify
from . import adapters, app_settings, covers, douyin_fetch, notify
from .db import get_session
from .ingest import IngestResult, diagnose_failure, ingest_run
from .models import (
@@ -47,6 +49,7 @@ from .models import (
RUN_PENDING,
RUN_RUNNING,
RUN_TIMEOUT,
MonitorNote,
MonitorRun,
MonitorTarget,
MonitorTask,
@@ -194,57 +197,102 @@ async def execute_task(task_id: int, trigger: str = "manual") -> IngestResult:
max_notes_count = task.max_notes_count
max_comments_count = task.max_comments_count
timeout_seconds = task.run_timeout_seconds
# 抖音的作品列表接口被那道真校验挡着(见 douyin_fetch),拿不到列表时就靠这些
# 已知的 aweme_id 逐条刷新 —— 新作品发现不了,但已有作品的指标还能继续更新。
known_aweme_ids = list(
await session.scalars(
select(MonitorNote.note_id).where(MonitorNote.task_id == task.id)
)
)
# --- Phase 2: run the crawler outside any transaction ---------------------
cookie_file = out_dir / ".cookies"
_write_cookie_file(cookie_file, cookie)
# --- Phase 2: run the collection outside any transaction ------------------
# 抖音走**进程内 HTTP 客户端**,不起 Playwright 子进程:那边会构造一大串自相矛盾的
# 浏览器指纹参数(参数说 Mac + Chrome 125、UA 说 Linux + Chrome 155),网关回一个
# 200 + 空 body,然后被翻译成「account blocked」—— 看着像账号被封,其实什么都不是。
# 见 douyin_fetch / douyin_api。
in_process_tail: List[str] = []
if platform == adapters.PLATFORM_DY:
fetched = await douyin_fetch.collect(
out_dir,
platform=platform,
mode=mode,
limit=max_notes_count,
want_comments=enable_comments,
comment_limit=max_comments_count,
targets=targets,
known_aweme_ids=known_aweme_ids,
cookie=cookie,
)
in_process_tail = list(fetched["errors"])
# 一条都没采到 = 这一轮失败,并把**真因**当作退出诊断传下去。否则它会掉进
# ingest 的「疑似登录失效」分支 —— 又骗人一次,正是这套东西一直在犯的毛病。
exit_code = 1 if (fetched["errors"] and not fetched["notes"]) else 0
if fetched["errors"] and fetched["notes"]:
# 有产物但带着错误,说明走了退化路径(比如作品列表被挡,只刷新了已知作品)。
# 这一轮状态是成功,但**不是**一切正常 —— 得留下痕迹,否则没人知道新作品
# 其实没在发现。
print(
"[monitor.runner] 抖音采集部分失败:"
+ ";".join(fetched["errors"])[:300]
)
else:
cookie_file = out_dir / ".cookies"
_write_cookie_file(cookie_file, cookie)
request = CrawlerStartRequest(
platform=_PLATFORM_ENUM[platform],
login_type=LoginTypeEnum.COOKIE,
crawler_type=CrawlerTypeEnum.CREATOR if mode == MODE_CREATOR else CrawlerTypeEnum.DETAIL,
creator_ids=",".join(urls) if mode == MODE_CREATOR else "",
specified_ids=",".join(urls) if mode != MODE_CREATOR else "",
start_page=1,
enable_comments=enable_comments,
enable_sub_comments=strategy["enable_sub_comments"],
enable_media=False,
save_option=SaveDataOptionEnum.JSONL,
cookies="",
headless=True,
max_notes_count=max_notes_count,
max_comments_count=max_comments_count,
# Isolate this run's output: the crawler names files by date only, so
# otherwise same-day runs would append into one shared file.
save_data_path=str(out_dir),
# Attach to the browser already running on CDP_DEBUG_PORT when the
# operator enabled it; otherwise launch a private, throwaway browser.
enable_cdp_mode=cdp_enabled,
# Only injecting web_session is not enough to sign requests from a cold
# browser profile.
inject_all_cookies=True,
save_login_state=True,
cookies_file=str(cookie_file),
max_concurrency_num=1,
# Strategy + proxy, surfaced on the Settings page.
crawler_max_sleep_sec=strategy["crawl_sleep_sec"],
enable_ip_proxy=strategy["enable_ip_proxy"],
ip_proxy_pool_count=strategy["proxy_pool_count"],
ip_proxy_provider_name=strategy["proxy_provider"],
static_proxy_url=strategy["static_proxy_url"] or None,
request = CrawlerStartRequest(
platform=_PLATFORM_ENUM[platform],
login_type=LoginTypeEnum.COOKIE,
crawler_type=CrawlerTypeEnum.CREATOR if mode == MODE_CREATOR else CrawlerTypeEnum.DETAIL,
creator_ids=",".join(urls) if mode == MODE_CREATOR else "",
specified_ids=",".join(urls) if mode != MODE_CREATOR else "",
start_page=1,
enable_comments=enable_comments,
enable_sub_comments=strategy["enable_sub_comments"],
enable_media=False,
save_option=SaveDataOptionEnum.JSONL,
cookies="",
headless=True,
max_notes_count=max_notes_count,
max_comments_count=max_comments_count,
# Isolate this run's output: the crawler names files by date only, so
# otherwise same-day runs would append into one shared file.
save_data_path=str(out_dir),
# Attach to the browser already running on CDP_DEBUG_PORT when the
# operator enabled it; otherwise launch a private, throwaway browser.
enable_cdp_mode=cdp_enabled,
# Only injecting web_session is not enough to sign requests from a cold
# browser profile.
inject_all_cookies=True,
save_login_state=True,
cookies_file=str(cookie_file),
max_concurrency_num=1,
# Strategy + proxy, surfaced on the Settings page.
crawler_max_sleep_sec=strategy["crawl_sleep_sec"],
enable_ip_proxy=strategy["enable_ip_proxy"],
ip_proxy_pool_count=strategy["proxy_pool_count"],
ip_proxy_provider_name=strategy["proxy_provider"],
static_proxy_url=strategy["static_proxy_url"] or None,
)
async with get_session() as session:
run = await session.get(MonitorRun, run_id)
if run is not None:
run.status = RUN_RUNNING
run.started_at = get_current_timestamp()
try:
exit_code = await crawler_manager.run_and_wait(request, timeout=timeout_seconds)
finally:
_remove_cookie_file(cookie_file)
# 失败时的诊断来源。抖音那条路没有子进程,尾巴就是它自己报的错 —— **别去读
# crawler_manager 的尾巴**,那里面是上一轮别的平台留下的东西,会张冠李戴。
output_tail = (
in_process_tail
if platform == adapters.PLATFORM_DY
else crawler_manager.get_output_tail()
)
async with get_session() as session:
run = await session.get(MonitorRun, run_id)
if run is not None:
run.status = RUN_RUNNING
run.started_at = get_current_timestamp()
try:
exit_code = await crawler_manager.run_and_wait(request, timeout=timeout_seconds)
finally:
_remove_cookie_file(cookie_file)
# --- Phase 3: ingest ------------------------------------------------------
async with get_session() as session:
run = await session.get(MonitorRun, run_id)
@@ -261,7 +309,7 @@ async def execute_task(task_id: int, trigger: str = "manual") -> IngestResult:
run.error_message = "Run was killed by timeout or failed to start"
# -1 同时代表「超时」和「根本没起来」,两者要查的东西完全不同。输出尾巴里
# 有异常就带上它,否则运行历史里只能看到这句没有信息量的话。
cause = diagnose_failure(crawler_manager.get_output_tail())
cause = diagnose_failure(output_tail)
if cause:
run.error_message = f"{run.error_message};原因:{cause}"
result = IngestResult(status=RUN_TIMEOUT, error=run.error_message)
@@ -271,7 +319,7 @@ async def execute_task(task_id: int, trigger: str = "manual") -> IngestResult:
# 把爬虫输出的尾巴交给 ingest:退出码本身说明不了问题,运行历史里要显示的
# 是真正的报错(比如抖音的 DataFetchError: account blocked)。
result = await ingest_run(
session, run, task, out_dir, output_tail=crawler_manager.get_output_tail()
session, run, task, out_dir, output_tail=output_tail
)
# A run that authenticated fine is the only useful signal that the