butubb
9e13a7f686
用户报的现象：任务一直显示「排队中」。查库确认有两批 run 卡在 pending（任务 7 的 44/45、 任务 15 的 62/63）。两个原因，一个是我上一版改坏的： 1) **`RUN_RUNNING` 被我缩进进了爬虫那条分支。** 抖音走的是另一条路，于是它**从不标记 「运行中」** —— 建完 pending 那一行就直接进采集，中途一旦出事（异常、进程被重启）， 状态就永远停在 pending。这是我加平台分岔时把原本在两条路公共位置的一行挪进去了。 2) **`recover()` 只收 `running`，够不着 `pending`。** 那行是上一轮建的、后面的采集却 根本没机会开始（进程重启），它永远不会自己往前走。于是重启也救不回来，界面上就是 一个永远「排队中」的幽灵。现在 pending 一起收。 两处都补了测试：抖音路的 run 必须在**采集开始之前**就已经是 running（这条改回去就会 失败）；recover 要把 pending 也标成 interrupted。
281 lines
12 KiB
Python
281 lines
12 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright (c) 2025 [email protected]
|
|
#
|
|
# This file is part of MediaCrawler project.
|
|
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/monitor/scheduler.py
|
|
# GitHub: https://github.com/NanmiCoder
|
|
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
|
#
|
|
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
|
# 1. 不得用于任何商业用途。
|
|
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
|
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
|
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
|
# 5. 不得用于任何非法或不当的用途。
|
|
#
|
|
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
|
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
|
|
|
"""Background scheduler for monitor tasks.
|
|
|
|
One asyncio loop polls for due tasks and hands them to the runner. A plain loop
|
|
is enough here: there is exactly one process, one global crawler subprocess, and
|
|
therefore no concurrency to coordinate -- a cron-style library would add a
|
|
dependency without adding a capability.
|
|
|
|
Two families of schedule, and the difference matters:
|
|
|
|
* ``interval`` is **fixed-delay**, not fixed-rate -- ``next_run_at`` is measured
|
|
from the moment a run starts, so a slow run cannot make its task fire
|
|
back-to-back.
|
|
* the clock modes (``daily``/``weekly``) are **fixed-time** -- recomputed from the
|
|
calendar, so a run that starts late does not drag every later run with it.
|
|
|
|
The arithmetic for both lives in schedule.py.
|
|
|
|
The loop also carries the 上游更新检查: it is not a crawl, so it shares none of
|
|
the rules above (no subprocess, no active-hours gate) -- see
|
|
``_maybe_check_upstream``. It rides this loop rather than getting a thread of its
|
|
own because it is one HTTP-shaped fetch per day.
|
|
"""
|
|
|
|
import asyncio
|
|
import random
|
|
from datetime import datetime
|
|
from typing import Optional
|
|
|
|
from sqlalchemy import select
|
|
|
|
from tools.time_util import get_current_timestamp
|
|
|
|
from ..services import crawler_manager
|
|
from . import app_settings, schedule, upstream
|
|
from .db import get_session
|
|
from .models import (
|
|
RUN_INTERRUPTED,
|
|
RUN_PENDING,
|
|
RUN_RUNNING,
|
|
MonitorRun,
|
|
MonitorTask,
|
|
)
|
|
from .runner import execute_task
|
|
from .settings import get_cookie
|
|
|
|
POLL_INTERVAL_SECONDS = 20
|
|
# Spread tasks sharing an interval so they do not all come due on the same tick.
|
|
# Applied to interval mode only -- see the advance step below.
|
|
JITTER_SECONDS = 60
|
|
|
|
|
|
class MonitorScheduler:
|
|
"""Polls the task table and runs whatever is due."""
|
|
|
|
def __init__(self) -> None:
|
|
self._loop_task: Optional[asyncio.Task] = None
|
|
self._stopping = asyncio.Event()
|
|
# Avoids logging "no cookie" on every single tick. Per platform, because
|
|
# warning once for Xiaohongshu must not silence the warning for Douyin.
|
|
self._warned_no_cookie: set = set()
|
|
|
|
async def start(self) -> None:
|
|
if self._loop_task is not None and not self._loop_task.done():
|
|
return
|
|
self._stopping.clear()
|
|
self._loop_task = asyncio.create_task(self._run_loop())
|
|
|
|
async def stop(self) -> None:
|
|
self._stopping.set()
|
|
if self._loop_task is not None:
|
|
self._loop_task.cancel()
|
|
try:
|
|
await self._loop_task
|
|
except asyncio.CancelledError:
|
|
pass
|
|
self._loop_task = None
|
|
|
|
async def _run_loop(self) -> None:
|
|
try:
|
|
await self.recover()
|
|
except Exception as exc: # pragma: no cover - defensive
|
|
print(f"[monitor.scheduler] recovery failed: {exc}")
|
|
|
|
while not self._stopping.is_set():
|
|
try:
|
|
await self.tick()
|
|
except Exception as exc: # pragma: no cover - keep the loop alive
|
|
print(f"[monitor.scheduler] tick failed: {exc}")
|
|
# 独立于采集任务，因此单独一段 try：上游检查失败不该影响采集调度，
|
|
# 反过来也一样。
|
|
try:
|
|
await self._maybe_check_upstream()
|
|
except Exception as exc: # pragma: no cover - keep the loop alive
|
|
print(f"[monitor.scheduler] upstream check failed: {exc}")
|
|
await asyncio.sleep(POLL_INTERVAL_SECONDS)
|
|
|
|
async def _maybe_check_upstream(self) -> None:
|
|
"""到点就 fetch 一次上游仓库，看它有没有新提交。
|
|
|
|
与采集任务的三条规则都不同，各有理由：它不碰浏览器、也不占采集子进程，
|
|
所以不看 ``is_busy``；它只发一个 git 请求，没有被平台风控的风险，所以也不
|
|
受活跃时段限制 —— 定时检查放在半夜反而是最合适的。
|
|
"""
|
|
async with get_session() as session:
|
|
if not await app_settings.get_value(
|
|
session, "upstream_check_enabled", fallback=False
|
|
):
|
|
return
|
|
interval_minutes = int(
|
|
await app_settings.get_value(
|
|
session, "upstream_check_interval_minutes", fallback=1440
|
|
)
|
|
)
|
|
state = await upstream.load_state(session)
|
|
|
|
checked_at = int(state.get("checked_at") or 0)
|
|
now = get_current_timestamp()
|
|
# 失败也会写 checked_at，所以不通的时候同样是每个间隔重试一次，
|
|
# 而不是每个 tick（20 秒）都去撞一次墙。
|
|
if checked_at and now - checked_at < max(1, interval_minutes) * 60_000:
|
|
return
|
|
|
|
result = await upstream.run_check()
|
|
if result.get("behind"):
|
|
print(
|
|
f"[monitor.scheduler] 上游 {result.get('branch')} 领先 "
|
|
f"{result['behind']} 个提交"
|
|
)
|
|
elif not result.get("ok"):
|
|
print(f"[monitor.scheduler] 上游检查失败：{result.get('error')}")
|
|
|
|
async def recover(self) -> None:
|
|
"""Clean up state left behind by a server restart.
|
|
|
|
A run still marked ``running`` cannot be running -- its subprocess died
|
|
with the previous process. Marking it interrupted stops it from blocking
|
|
the UI as a phantom in-flight run.
|
|
|
|
**``pending`` 同样是残留**：那一行是上一轮建的，可它后面的采集根本没机会开始
|
|
（进程被重启，或者采集那条路抛了异常），所以它永远不会自己往前走。只清 running
|
|
的话，它会永远挂在界面上显示「排队中」—— 用户看到的就是任务卡住了。
|
|
"""
|
|
async with get_session() as session:
|
|
stale = (
|
|
await session.scalars(
|
|
select(MonitorRun).where(
|
|
MonitorRun.status.in_((RUN_RUNNING, RUN_PENDING))
|
|
)
|
|
)
|
|
).all()
|
|
for run in stale:
|
|
run.status = RUN_INTERRUPTED
|
|
run.finished_at = get_current_timestamp()
|
|
if stale:
|
|
print(
|
|
f"[monitor.scheduler] marked {len(stale)} interrupted run(s) "
|
|
f"left over from a previous process"
|
|
)
|
|
|
|
async def tick(self) -> None:
|
|
"""Run one due task, if the crawler is free and we are in the active window."""
|
|
# The crawler subprocess is a global singleton, so a manual crawl and a
|
|
# monitor run cannot overlap. Returning without advancing next_run_at
|
|
# leaves the task due, and it is picked up on a later tick.
|
|
if crawler_manager.is_busy():
|
|
return
|
|
|
|
async with get_session() as session:
|
|
if not await self._within_active_hours(session):
|
|
# Deliberately does not advance next_run_at: the task simply runs
|
|
# when the window next opens, rather than being skipped for a day.
|
|
return
|
|
|
|
await self._run_due_task()
|
|
|
|
async def _within_active_hours(self, session) -> bool:
|
|
"""Whether scheduled runs are allowed right now (local time)."""
|
|
start, end = await app_settings.active_hours(session)
|
|
hour = datetime.now().hour
|
|
if start <= end:
|
|
return start <= hour <= end
|
|
# Window wraps past midnight, e.g. 22 -> 6.
|
|
return hour >= start or hour <= end
|
|
|
|
async def _run_due_task(self) -> None:
|
|
async with get_session() as session:
|
|
task = await session.scalar(
|
|
select(MonitorTask)
|
|
.where(
|
|
MonitorTask.enabled.is_(True),
|
|
MonitorTask.next_run_at.is_not(None),
|
|
MonitorTask.next_run_at <= get_current_timestamp(),
|
|
)
|
|
.order_by(MonitorTask.next_run_at)
|
|
.limit(1)
|
|
)
|
|
|
|
if task is None:
|
|
return
|
|
|
|
# 没有 cookie 就跳过，是为了不让任务每轮白跑一趟出个认证失败。任务留在
|
|
# due 状态而不推进 —— 用户一粘上 cookie 它就能自己跑起来。
|
|
#
|
|
# **但开着 CDP 时必须放行**：那种模式下登录态来自被接管的那个浏览器，
|
|
# 粘不粘 cookie 根本轮不到它决定成败。不放行的话，选了「接管已有 Chrome」
|
|
# 却没粘 cookie 的用户会发现任务永远不被触发，而且什么错都不报。
|
|
cookie = await get_cookie(session, task.platform)
|
|
if not cookie:
|
|
cdp_enabled = await app_settings.get_value(
|
|
session, "cdp_enabled", fallback=False
|
|
)
|
|
if not cdp_enabled:
|
|
if task.platform not in self._warned_no_cookie:
|
|
print(
|
|
f"[monitor.scheduler] no {task.platform} cookie configured; "
|
|
"scheduled tasks will not run until one is set or CDP is enabled"
|
|
)
|
|
self._warned_no_cookie.add(task.platform)
|
|
return
|
|
self._warned_no_cookie.discard(task.platform)
|
|
|
|
# Advance before running so a crash mid-run cannot cause an immediate
|
|
# re-fire, and so a long outage coalesces into a single run instead
|
|
# of one run per missed interval.
|
|
now = get_current_timestamp()
|
|
following = schedule.next_occurrence(
|
|
mode=task.schedule_mode,
|
|
interval_minutes=task.interval_minutes,
|
|
hours=schedule.parse_hours(task.schedule_hours),
|
|
days=schedule.parse_days(task.schedule_days),
|
|
minute=task.schedule_minute,
|
|
after_ms=now,
|
|
)
|
|
|
|
if following is None:
|
|
# A clock schedule with no times can never fire. The API rejects
|
|
# that shape, so this guards against a hand-edited row: park the
|
|
# task with no next run rather than leaving it permanently due and
|
|
# re-running it on every tick.
|
|
task.next_run_at = None
|
|
print(
|
|
f"[monitor.scheduler] task {task.id} has no usable schedule "
|
|
f"and will not run until one is set"
|
|
)
|
|
elif task.schedule_mode == schedule.MODE_INTERVAL:
|
|
# Jitter belongs to the interval mode only. Spreading identical
|
|
# intervals apart is the point; nudging a time the operator
|
|
# explicitly picked is not -- it just looks like a broken clock.
|
|
task.next_run_at = following + random.randint(0, JITTER_SECONDS) * 1000
|
|
else:
|
|
task.next_run_at = following
|
|
|
|
task_id = task.id
|
|
|
|
try:
|
|
await execute_task(task_id, trigger="scheduled")
|
|
except Exception as exc:
|
|
print(f"[monitor.scheduler] task {task_id} failed: {exc}")
|
|
|
|
|
|
# Global singleton, mirroring the crawler_manager pattern.
|
|
monitor_scheduler = MonitorScheduler()
|