MediaCrawler/api/monitor/scheduler.py

# -*- coding: utf-8 -*-
# Copyright (c) 2025 [email protected]
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/monitor/scheduler.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
# 1. 不得用于任何商业用途。
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
# 3. 不得进行大规模爬取或对平台造成运营干扰。
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
# 5. 不得用于任何非法或不当的用途。
#
# 详细许可条款请参阅项目根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。

"""Background scheduler for monitor tasks.

One asyncio loop polls for due tasks and hands them to the runner. A plain loop
is enough here: there is exactly one process, one global crawler subprocess, and
therefore no concurrency to coordinate -- a cron-style library would add a
dependency without adding a capability.

Two families of schedule, and the difference matters:

* ``interval`` is **fixed-delay**, not fixed-rate -- ``next_run_at`` is measured
  from the moment a run starts, so a slow run cannot make its task fire
  back-to-back.
* the clock modes (``daily``/``weekly``) are **fixed-time** -- recomputed from the
  calendar, so a run that starts late does not drag every later run with it.

The arithmetic for both lives in schedule.py.
"""

import asyncio
import random
from datetime import datetime
from typing import Optional

from sqlalchemy import select

from tools.time_util import get_current_timestamp

from ..services import crawler_manager
from . import app_settings, schedule
from .db import get_session
from .models import MonitorRun, MonitorTask, RUN_INTERRUPTED, RUN_RUNNING
from .runner import execute_task
from .settings import get_cookie

POLL_INTERVAL_SECONDS = 20
# Spread tasks sharing an interval so they do not all come due on the same tick.
# Applied to interval mode only -- see the advance step below.
JITTER_SECONDS = 60


class MonitorScheduler:
    """Polls the task table and runs whatever is due."""

    def __init__(self) -> None:
        self._loop_task: Optional[asyncio.Task] = None
        self._stopping = asyncio.Event()
        # Avoids logging "no cookie" on every single tick.
        self._warned_no_cookie = False

    async def start(self) -> None:
        if self._loop_task is not None and not self._loop_task.done():
            return
        self._stopping.clear()
        self._loop_task = asyncio.create_task(self._run_loop())

    async def stop(self) -> None:
        self._stopping.set()
        if self._loop_task is not None:
            self._loop_task.cancel()
            try:
                await self._loop_task
            except asyncio.CancelledError:
                pass
            self._loop_task = None

    async def _run_loop(self) -> None:
        try:
            await self.recover()
        except Exception as exc:  # pragma: no cover - defensive
            print(f"[monitor.scheduler] recovery failed: {exc}")

        while not self._stopping.is_set():
            try:
                await self.tick()
            except Exception as exc:  # pragma: no cover - keep the loop alive
                print(f"[monitor.scheduler] tick failed: {exc}")
            await asyncio.sleep(POLL_INTERVAL_SECONDS)

    async def recover(self) -> None:
        """Clean up state left behind by a server restart.

        A run still marked ``running`` cannot be running -- its subprocess died
        with the previous process. Marking it interrupted stops it from blocking
        the UI as a phantom in-flight run.
        """
        async with get_session() as session:
            stale = (
                await session.scalars(
                    select(MonitorRun).where(MonitorRun.status == RUN_RUNNING)
                )
            ).all()
            for run in stale:
                run.status = RUN_INTERRUPTED
                run.finished_at = get_current_timestamp()
            if stale:
                print(
                    f"[monitor.scheduler] marked {len(stale)} interrupted run(s) "
                    f"left over from a previous process"
                )

    async def tick(self) -> None:
        """Run one due task, if the crawler is free and we are in the active window."""
        # The crawler subprocess is a global singleton, so a manual crawl and a
        # monitor run cannot overlap. Returning without advancing next_run_at
        # leaves the task due, and it is picked up on a later tick.
        if crawler_manager.is_busy():
            return

        async with get_session() as session:
            if not await self._within_active_hours(session):
                # Deliberately does not advance next_run_at: the task simply runs
                # when the window next opens, rather than being skipped for a day.
                return

        await self._run_due_task()

    async def _within_active_hours(self, session) -> bool:
        """Whether scheduled runs are allowed right now (local time)."""
        start, end = await app_settings.active_hours(session)
        hour = datetime.now().hour
        if start <= end:
            return start <= hour <= end
        # Window wraps past midnight, e.g. 22 -> 6.
        return hour >= start or hour <= end

    async def _run_due_task(self) -> None:
        async with get_session() as session:
            task = await session.scalar(
                select(MonitorTask)
                .where(
                    MonitorTask.enabled.is_(True),
                    MonitorTask.next_run_at.is_not(None),
                    MonitorTask.next_run_at <= get_current_timestamp(),
                )
                .order_by(MonitorTask.next_run_at)
                .limit(1)
            )

            if task is None:
                return

            # No cookie means every run would report an auth failure. Leave the
            # task due rather than advancing: it starts working the moment the
            # user pastes one.
            cookie = await get_cookie(session)
            if not cookie:
                if not self._warned_no_cookie:
                    print(
                        "[monitor.scheduler] no XHS cookie configured; "
                        "scheduled tasks will not run until one is set"
                    )
                    self._warned_no_cookie = True
                return
            self._warned_no_cookie = False

            # Advance before running so a crash mid-run cannot cause an immediate
            # re-fire, and so a long outage coalesces into a single run instead
            # of one run per missed interval.
            now = get_current_timestamp()
            following = schedule.next_occurrence(
                mode=task.schedule_mode,
                interval_minutes=task.interval_minutes,
                hours=schedule.parse_hours(task.schedule_hours),
                days=schedule.parse_days(task.schedule_days),
                minute=task.schedule_minute,
                after_ms=now,
            )

            if following is None:
                # A clock schedule with no times can never fire. The API rejects
                # that shape, so this guards against a hand-edited row: park the
                # task with no next run rather than leaving it permanently due and
                # re-running it on every tick.
                task.next_run_at = None
                print(
                    f"[monitor.scheduler] task {task.id} has no usable schedule "
                    f"and will not run until one is set"
                )
            elif task.schedule_mode == schedule.MODE_INTERVAL:
                # Jitter belongs to the interval mode only. Spreading identical
                # intervals apart is the point; nudging a time the operator
                # explicitly picked is not -- it just looks like a broken clock.
                task.next_run_at = following + random.randint(0, JITTER_SECONDS) * 1000
            else:
                task.next_run_at = following

            task_id = task.id

        try:
            await execute_task(task_id, trigger="scheduled")
        except Exception as exc:
            print(f"[monitor.scheduler] task {task_id} failed: {exc}")


# Global singleton, mirroring the crawler_manager pattern.
monitor_scheduler = MonitorScheduler()