feat(monitor): 运行历史要写清楚失败原因,不能只写「退出码 1」
用户的要求:运行历史的说明要写清楚。现在确实写不清楚 —— 抖音那次失败,运行历史里 只有一句 `Crawler exited with code 1`,而真正的报错 `DataFetchError: account blocked` 埋在子进程的 stderr 里,谁也看不到。 那两者本来是断开的两条路:子进程的输出只流向日志 WebSocket(前端 Terminal 看得到), 而监控层调 run_and_wait() 只拿得到一个退出码。 * crawler_manager 在 _push_log() 里留一份输出尾巴(80 行,每次 start 清空)—— 那是所有输出的唯一出口,挂这儿不会漏。新增 get_output_tail()。 * ingest 新增 diagnose_failure():倒着找第一行像异常的行(traceback 的末行), 认不出就退回最后一行;管理器自己补的「Crawler exited with code」不是原因,排除掉。 * describe_exit_code() 接受这个原因并附在消息里;失败事件的标题也带上,这样企业微信 通知和事件流不用翻日志就能看懂。 * runner 把尾巴交给 ingest;「超时/没起来」那条分支同样带上原因 —— -1 同时代表两种 情况,而要查的东西完全不同。 * 运行历史那一格是截断的(240px),而失败原因现在有一整行 —— 补上 title 悬停显示, 并放宽到 320px。没有悬停提示等于把最要紧的半句藏起来。 测试 +5:能挑出异常行、不会把管理器自己的话当成原因、没有输出时不报错、认不出时退回 最后一行、以及失败运行同时记下退出码与真因(含事件标题)。
This commit is contained in:
+51
-8
@@ -38,7 +38,7 @@ import json
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Dict, List, Optional, Sequence
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
@@ -122,13 +122,46 @@ _WINDOWS_EXIT_REASONS = {
|
||||
}
|
||||
|
||||
|
||||
def describe_exit_code(code: int) -> str:
|
||||
"""Render an exit code so a human can act on it."""
|
||||
def describe_exit_code(code: int, cause: Optional[str] = None) -> str:
|
||||
"""Render an exit code so a human can act on it.
|
||||
|
||||
只有退出码时信息量约等于零(`code 1` 什么都能是),所以把从输出尾巴里认出来的
|
||||
异常一并附上 —— 运行历史里那一格显示的正是这句话。
|
||||
"""
|
||||
unsigned = code & 0xFFFFFFFF if code < 0 else code
|
||||
reason = _WINDOWS_EXIT_REASONS.get(unsigned)
|
||||
base = f"Crawler exited with code {code}"
|
||||
if reason:
|
||||
return f"Crawler exited with code {code} (0x{unsigned:08X}): {reason}"
|
||||
return f"Crawler exited with code {code}"
|
||||
base = f"{base} (0x{unsigned:08X}): {reason}"
|
||||
return f"{base};原因:{cause}" if cause else base
|
||||
|
||||
|
||||
# 从爬虫输出里认出一行「异常」。Python 的 traceback 末行形如
|
||||
# ``media_platform.douyin.exception.DataFetchError: account blocked``。
|
||||
_EXCEPTION_LINE_RE = re.compile(r"^[\w.]*[A-Za-z](?:Error|Exception|Timeout)\b")
|
||||
|
||||
|
||||
def diagnose_failure(output_tail: Optional[Sequence[str]]) -> Optional[str]:
|
||||
"""从爬虫输出的末尾挑出最能说明问题的一行。
|
||||
|
||||
「退出码 1」等于什么都没说:真正的报错埋在子进程的 stderr 里。倒着找第一行看起来
|
||||
像异常的行(traceback 的末行),找不到就退回最后一行有效输出。
|
||||
"""
|
||||
if not output_tail:
|
||||
return None
|
||||
|
||||
lines = [line.strip() for line in output_tail if line and line.strip()]
|
||||
# 管理器自己补的那两句不是爬虫的报错,别被当成失败原因。
|
||||
noise = ("Crawler exited with code", "Crawler completed successfully")
|
||||
lines = [line for line in lines if not line.startswith(noise)]
|
||||
if not lines:
|
||||
return None
|
||||
|
||||
for line in reversed(lines):
|
||||
if _EXCEPTION_LINE_RE.match(line):
|
||||
return line[:300]
|
||||
|
||||
return lines[-1][:300]
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -558,6 +591,7 @@ async def ingest_run(
|
||||
run: MonitorRun,
|
||||
task: MonitorTask,
|
||||
out_dir: Path,
|
||||
output_tail: Optional[Sequence[str]] = None,
|
||||
) -> IngestResult:
|
||||
"""Ingest one finished run and return what changed.
|
||||
|
||||
@@ -568,14 +602,23 @@ async def ingest_run(
|
||||
# A non-zero exit is a genuine crash: trust nothing this run produced.
|
||||
if run.exit_code not in (0, None):
|
||||
run.status = RUN_FAILED
|
||||
run.error_message = describe_exit_code(run.exit_code)
|
||||
cause = diagnose_failure(output_tail)
|
||||
run.error_message = describe_exit_code(run.exit_code, cause)
|
||||
title = f"采集进程异常退出(code={run.exit_code})"
|
||||
if cause:
|
||||
# 标题里也带上真因:企业微信通知和事件流都只看这一行,不写就还得去翻日志。
|
||||
title = f"{title}:{cause}"
|
||||
await _emit(
|
||||
session,
|
||||
run,
|
||||
EVENT_RUN_FAILED,
|
||||
f"采集进程异常退出(code={run.exit_code})",
|
||||
title,
|
||||
severity="error",
|
||||
payload={"exit_code": run.exit_code, "detail": run.error_message},
|
||||
payload={
|
||||
"exit_code": run.exit_code,
|
||||
"detail": run.error_message,
|
||||
"cause": cause,
|
||||
},
|
||||
)
|
||||
return IngestResult(status=RUN_FAILED, error=run.error_message)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user