feat(monitor): 抖音监控改走新的 Web 接口客户端(接上上一版的移植)
上一版只把客户端写出来、验证了它单独可用,**但没有接进任何地方** —— 所以你建的任务跑起来 仍然在调爬虫子进程,报的仍然是那句自己编的「account blocked」。这一步把它接上。 * runner 的 Phase 2 按平台分岔:dy 走进程内 HTTP 客户端(douyin_fetch),其余平台照旧走 爬虫子进程。抖音那条不再起 Playwright、不再构造那串自相矛盾的浏览器指纹参数。 * 新增 douyin_fetch:把采到的东西写成 store/douyin 那套 jsonl 形状 —— **ingest 完全不知道 数据是从哪来的**,重采样/差分/事件/通知/报表全都照旧,一个字没改。 * 失败不再假装:一条都没采到就以非零退出码 + **真实原因**交给 ingest,落成 「采集进程异常退出(code=1):…」。绝不会再掉进「疑似登录失效」那个分支。 * 已知作品列表接口(aweme/post)被抖音单独加了真校验(200 + 空 body),所以加了退化: 拿不到列表就用库里已知的 aweme_id 逐条走 detail 刷新。**边界是:已知作品的指标能继续 更新,新作品发现不了** —— 这个边界会以一条 warning 日志留下痕迹,不让它看起来一切正常。 * 顺带给客户端补上 video_detail(实测可用:200 / 45425 字节),退化路径靠它。 测试 +6:产物目录与文件名、评论文件即使为空也要建(ingest 靠它区分「没评论」和 「什么都没抓到」)、重复作品只写一次、列表被挡时的退化、彻底失败仍写出产物与原因、 评论失败不连累作品。
This commit is contained in:
+98
-50
@@ -28,6 +28,8 @@ import os
|
||||
from pathlib import Path
|
||||
from typing import Iterable, List, Optional
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from tools.time_util import get_current_timestamp
|
||||
|
||||
from ..schemas import (
|
||||
@@ -38,7 +40,7 @@ from ..schemas import (
|
||||
SaveDataOptionEnum,
|
||||
)
|
||||
from ..services import crawler_manager
|
||||
from . import adapters, app_settings, covers, notify
|
||||
from . import adapters, app_settings, covers, douyin_fetch, notify
|
||||
from .db import get_session
|
||||
from .ingest import IngestResult, diagnose_failure, ingest_run
|
||||
from .models import (
|
||||
@@ -47,6 +49,7 @@ from .models import (
|
||||
RUN_PENDING,
|
||||
RUN_RUNNING,
|
||||
RUN_TIMEOUT,
|
||||
MonitorNote,
|
||||
MonitorRun,
|
||||
MonitorTarget,
|
||||
MonitorTask,
|
||||
@@ -194,57 +197,102 @@ async def execute_task(task_id: int, trigger: str = "manual") -> IngestResult:
|
||||
max_notes_count = task.max_notes_count
|
||||
max_comments_count = task.max_comments_count
|
||||
timeout_seconds = task.run_timeout_seconds
|
||||
# 抖音的作品列表接口被那道真校验挡着(见 douyin_fetch),拿不到列表时就靠这些
|
||||
# 已知的 aweme_id 逐条刷新 —— 新作品发现不了,但已有作品的指标还能继续更新。
|
||||
known_aweme_ids = list(
|
||||
await session.scalars(
|
||||
select(MonitorNote.note_id).where(MonitorNote.task_id == task.id)
|
||||
)
|
||||
)
|
||||
|
||||
# --- Phase 2: run the crawler outside any transaction ---------------------
|
||||
cookie_file = out_dir / ".cookies"
|
||||
_write_cookie_file(cookie_file, cookie)
|
||||
# --- Phase 2: run the collection outside any transaction ------------------
|
||||
# 抖音走**进程内 HTTP 客户端**,不起 Playwright 子进程:那边会构造一大串自相矛盾的
|
||||
# 浏览器指纹参数(参数说 Mac + Chrome 125、UA 说 Linux + Chrome 155),网关回一个
|
||||
# 200 + 空 body,然后被翻译成「account blocked」—— 看着像账号被封,其实什么都不是。
|
||||
# 见 douyin_fetch / douyin_api。
|
||||
in_process_tail: List[str] = []
|
||||
if platform == adapters.PLATFORM_DY:
|
||||
fetched = await douyin_fetch.collect(
|
||||
out_dir,
|
||||
platform=platform,
|
||||
mode=mode,
|
||||
limit=max_notes_count,
|
||||
want_comments=enable_comments,
|
||||
comment_limit=max_comments_count,
|
||||
targets=targets,
|
||||
known_aweme_ids=known_aweme_ids,
|
||||
cookie=cookie,
|
||||
)
|
||||
in_process_tail = list(fetched["errors"])
|
||||
# 一条都没采到 = 这一轮失败,并把**真因**当作退出诊断传下去。否则它会掉进
|
||||
# ingest 的「疑似登录失效」分支 —— 又骗人一次,正是这套东西一直在犯的毛病。
|
||||
exit_code = 1 if (fetched["errors"] and not fetched["notes"]) else 0
|
||||
if fetched["errors"] and fetched["notes"]:
|
||||
# 有产物但带着错误,说明走了退化路径(比如作品列表被挡,只刷新了已知作品)。
|
||||
# 这一轮状态是成功,但**不是**一切正常 —— 得留下痕迹,否则没人知道新作品
|
||||
# 其实没在发现。
|
||||
print(
|
||||
"[monitor.runner] 抖音采集部分失败:"
|
||||
+ ";".join(fetched["errors"])[:300]
|
||||
)
|
||||
else:
|
||||
cookie_file = out_dir / ".cookies"
|
||||
_write_cookie_file(cookie_file, cookie)
|
||||
|
||||
request = CrawlerStartRequest(
|
||||
platform=_PLATFORM_ENUM[platform],
|
||||
login_type=LoginTypeEnum.COOKIE,
|
||||
crawler_type=CrawlerTypeEnum.CREATOR if mode == MODE_CREATOR else CrawlerTypeEnum.DETAIL,
|
||||
creator_ids=",".join(urls) if mode == MODE_CREATOR else "",
|
||||
specified_ids=",".join(urls) if mode != MODE_CREATOR else "",
|
||||
start_page=1,
|
||||
enable_comments=enable_comments,
|
||||
enable_sub_comments=strategy["enable_sub_comments"],
|
||||
enable_media=False,
|
||||
save_option=SaveDataOptionEnum.JSONL,
|
||||
cookies="",
|
||||
headless=True,
|
||||
max_notes_count=max_notes_count,
|
||||
max_comments_count=max_comments_count,
|
||||
# Isolate this run's output: the crawler names files by date only, so
|
||||
# otherwise same-day runs would append into one shared file.
|
||||
save_data_path=str(out_dir),
|
||||
# Attach to the browser already running on CDP_DEBUG_PORT when the
|
||||
# operator enabled it; otherwise launch a private, throwaway browser.
|
||||
enable_cdp_mode=cdp_enabled,
|
||||
# Only injecting web_session is not enough to sign requests from a cold
|
||||
# browser profile.
|
||||
inject_all_cookies=True,
|
||||
save_login_state=True,
|
||||
cookies_file=str(cookie_file),
|
||||
max_concurrency_num=1,
|
||||
# Strategy + proxy, surfaced on the Settings page.
|
||||
crawler_max_sleep_sec=strategy["crawl_sleep_sec"],
|
||||
enable_ip_proxy=strategy["enable_ip_proxy"],
|
||||
ip_proxy_pool_count=strategy["proxy_pool_count"],
|
||||
ip_proxy_provider_name=strategy["proxy_provider"],
|
||||
static_proxy_url=strategy["static_proxy_url"] or None,
|
||||
request = CrawlerStartRequest(
|
||||
platform=_PLATFORM_ENUM[platform],
|
||||
login_type=LoginTypeEnum.COOKIE,
|
||||
crawler_type=CrawlerTypeEnum.CREATOR if mode == MODE_CREATOR else CrawlerTypeEnum.DETAIL,
|
||||
creator_ids=",".join(urls) if mode == MODE_CREATOR else "",
|
||||
specified_ids=",".join(urls) if mode != MODE_CREATOR else "",
|
||||
start_page=1,
|
||||
enable_comments=enable_comments,
|
||||
enable_sub_comments=strategy["enable_sub_comments"],
|
||||
enable_media=False,
|
||||
save_option=SaveDataOptionEnum.JSONL,
|
||||
cookies="",
|
||||
headless=True,
|
||||
max_notes_count=max_notes_count,
|
||||
max_comments_count=max_comments_count,
|
||||
# Isolate this run's output: the crawler names files by date only, so
|
||||
# otherwise same-day runs would append into one shared file.
|
||||
save_data_path=str(out_dir),
|
||||
# Attach to the browser already running on CDP_DEBUG_PORT when the
|
||||
# operator enabled it; otherwise launch a private, throwaway browser.
|
||||
enable_cdp_mode=cdp_enabled,
|
||||
# Only injecting web_session is not enough to sign requests from a cold
|
||||
# browser profile.
|
||||
inject_all_cookies=True,
|
||||
save_login_state=True,
|
||||
cookies_file=str(cookie_file),
|
||||
max_concurrency_num=1,
|
||||
# Strategy + proxy, surfaced on the Settings page.
|
||||
crawler_max_sleep_sec=strategy["crawl_sleep_sec"],
|
||||
enable_ip_proxy=strategy["enable_ip_proxy"],
|
||||
ip_proxy_pool_count=strategy["proxy_pool_count"],
|
||||
ip_proxy_provider_name=strategy["proxy_provider"],
|
||||
static_proxy_url=strategy["static_proxy_url"] or None,
|
||||
)
|
||||
|
||||
async with get_session() as session:
|
||||
run = await session.get(MonitorRun, run_id)
|
||||
if run is not None:
|
||||
run.status = RUN_RUNNING
|
||||
run.started_at = get_current_timestamp()
|
||||
|
||||
try:
|
||||
exit_code = await crawler_manager.run_and_wait(request, timeout=timeout_seconds)
|
||||
finally:
|
||||
_remove_cookie_file(cookie_file)
|
||||
|
||||
# 失败时的诊断来源。抖音那条路没有子进程,尾巴就是它自己报的错 —— **别去读
|
||||
# crawler_manager 的尾巴**,那里面是上一轮别的平台留下的东西,会张冠李戴。
|
||||
output_tail = (
|
||||
in_process_tail
|
||||
if platform == adapters.PLATFORM_DY
|
||||
else crawler_manager.get_output_tail()
|
||||
)
|
||||
|
||||
async with get_session() as session:
|
||||
run = await session.get(MonitorRun, run_id)
|
||||
if run is not None:
|
||||
run.status = RUN_RUNNING
|
||||
run.started_at = get_current_timestamp()
|
||||
|
||||
try:
|
||||
exit_code = await crawler_manager.run_and_wait(request, timeout=timeout_seconds)
|
||||
finally:
|
||||
_remove_cookie_file(cookie_file)
|
||||
|
||||
# --- Phase 3: ingest ------------------------------------------------------
|
||||
async with get_session() as session:
|
||||
run = await session.get(MonitorRun, run_id)
|
||||
@@ -261,7 +309,7 @@ async def execute_task(task_id: int, trigger: str = "manual") -> IngestResult:
|
||||
run.error_message = "Run was killed by timeout or failed to start"
|
||||
# -1 同时代表「超时」和「根本没起来」,两者要查的东西完全不同。输出尾巴里
|
||||
# 有异常就带上它,否则运行历史里只能看到这句没有信息量的话。
|
||||
cause = diagnose_failure(crawler_manager.get_output_tail())
|
||||
cause = diagnose_failure(output_tail)
|
||||
if cause:
|
||||
run.error_message = f"{run.error_message};原因:{cause}"
|
||||
result = IngestResult(status=RUN_TIMEOUT, error=run.error_message)
|
||||
@@ -271,7 +319,7 @@ async def execute_task(task_id: int, trigger: str = "manual") -> IngestResult:
|
||||
# 把爬虫输出的尾巴交给 ingest:退出码本身说明不了问题,运行历史里要显示的
|
||||
# 是真正的报错(比如抖音的 DataFetchError: account blocked)。
|
||||
result = await ingest_run(
|
||||
session, run, task, out_dir, output_tail=crawler_manager.get_output_tail()
|
||||
session, run, task, out_dir, output_tail=output_tail
|
||||
)
|
||||
|
||||
# A run that authenticated fine is the only useful signal that the
|
||||
|
||||
Reference in New Issue
Block a user