MediaCrawler/api/monitor/scheduler.py

# -*- coding: utf-8 -*-
# Copyright (c) 2025 [email protected]
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/monitor/scheduler.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
# 1. 不得用于任何商业用途。
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
# 3. 不得进行大规模爬取或对平台造成运营干扰。
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
# 5. 不得用于任何非法或不当的用途。
#
# 详细许可条款请参阅项目根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。

"""Background scheduler for monitor tasks.

One asyncio loop polls for due tasks and hands them to the runner. A plain loop
is enough here: there is exactly one process, one global crawler subprocess, and
therefore no concurrency to coordinate -- a cron-style library would add a
dependency without adding a capability.

Two families of schedule, and the difference matters:

* ``interval`` is **fixed-delay**, not fixed-rate -- ``next_run_at`` is measured
  from the moment a run starts, so a slow run cannot make its task fire
  back-to-back.
* the clock modes (``daily``/``weekly``) are **fixed-time** -- recomputed from the
  calendar, so a run that starts late does not drag every later run with it.

The arithmetic for both lives in schedule.py.

The loop also carries the 上游更新检查: it is not a crawl, so it shares none of
the rules above (no subprocess, no active-hours gate) -- see
``_maybe_check_upstream``. It rides this loop rather than getting a thread of its
own because it is one HTTP-shaped fetch per day.
"""

import asyncio
import random
from datetime import datetime
from typing import Optional

from sqlalchemy import select

from tools.time_util import get_current_timestamp

from ..services import crawler_manager
from . import app_settings, schedule, upstream
from .db import get_session
from .models import MonitorRun, MonitorTask, RUN_INTERRUPTED, RUN_RUNNING
from .runner import execute_task
from .settings import get_cookie

POLL_INTERVAL_SECONDS = 20
# Spread tasks sharing an interval so they do not all come due on the same tick.
# Applied to interval mode only -- see the advance step below.
JITTER_SECONDS = 60


class MonitorScheduler:
    """Polls the task table and runs whatever is due."""

    def __init__(self) -> None:
        self._loop_task: Optional[asyncio.Task] = None
        self._stopping = asyncio.Event()
        # Avoids logging "no cookie" on every single tick. Per platform, because
        # warning once for Xiaohongshu must not silence the warning for Douyin.
        self._warned_no_cookie: set = set()

    async def start(self) -> None:
        if self._loop_task is not None and not self._loop_task.done():
            return
        self._stopping.clear()
        self._loop_task = asyncio.create_task(self._run_loop())

    async def stop(self) -> None:
        self._stopping.set()
        if self._loop_task is not None:
            self._loop_task.cancel()
            try:
                await self._loop_task
            except asyncio.CancelledError:
                pass
            self._loop_task = None

    async def _run_loop(self) -> None:
        try:
            await self.recover()
        except Exception as exc:  # pragma: no cover - defensive
            print(f"[monitor.scheduler] recovery failed: {exc}")

        while not self._stopping.is_set():
            try:
                await self.tick()
            except Exception as exc:  # pragma: no cover - keep the loop alive
                print(f"[monitor.scheduler] tick failed: {exc}")
            # 独立于采集任务，因此单独一段 try：上游检查失败不该影响采集调度，
            # 反过来也一样。
            try:
                await self._maybe_check_upstream()
            except Exception as exc:  # pragma: no cover - keep the loop alive
                print(f"[monitor.scheduler] upstream check failed: {exc}")
            await asyncio.sleep(POLL_INTERVAL_SECONDS)

    async def _maybe_check_upstream(self) -> None:
        """到点就 fetch 一次上游仓库，看它有没有新提交。

        与采集任务的三条规则都不同，各有理由：它不碰浏览器、也不占采集子进程，
        所以不看 ``is_busy``；它只发一个 git 请求，没有被平台风控的风险，所以也不
        受活跃时段限制 —— 定时检查放在半夜反而是最合适的。
        """
        async with get_session() as session:
            if not await app_settings.get_value(
                session, "upstream_check_enabled", fallback=False
            ):
                return
            interval_minutes = int(
                await app_settings.get_value(
                    session, "upstream_check_interval_minutes", fallback=1440
                )
            )
            state = await upstream.load_state(session)

        checked_at = int(state.get("checked_at") or 0)
        now = get_current_timestamp()
        # 失败也会写 checked_at，所以不通的时候同样是每个间隔重试一次，
        # 而不是每个 tick（20 秒）都去撞一次墙。
        if checked_at and now - checked_at < max(1, interval_minutes) * 60_000:
            return

        result = await upstream.run_check()
        if result.get("behind"):
            print(
                f"[monitor.scheduler] 上游 {result.get('branch')} 领先 "
                f"{result['behind']} 个提交"
            )
        elif not result.get("ok"):
            print(f"[monitor.scheduler] 上游检查失败：{result.get('error')}")

    async def recover(self) -> None:
        """Clean up state left behind by a server restart.

        A run still marked ``running`` cannot be running -- its subprocess died
        with the previous process. Marking it interrupted stops it from blocking
        the UI as a phantom in-flight run.
        """
        async with get_session() as session:
            stale = (
                await session.scalars(
                    select(MonitorRun).where(MonitorRun.status == RUN_RUNNING)
                )
            ).all()
            for run in stale:
                run.status = RUN_INTERRUPTED
                run.finished_at = get_current_timestamp()
            if stale:
                print(
                    f"[monitor.scheduler] marked {len(stale)} interrupted run(s) "
                    f"left over from a previous process"
                )

    async def tick(self) -> None:
        """Run one due task, if the crawler is free and we are in the active window."""
        # The crawler subprocess is a global singleton, so a manual crawl and a
        # monitor run cannot overlap. Returning without advancing next_run_at
        # leaves the task due, and it is picked up on a later tick.
        if crawler_manager.is_busy():
            return

        async with get_session() as session:
            if not await self._within_active_hours(session):
                # Deliberately does not advance next_run_at: the task simply runs
                # when the window next opens, rather than being skipped for a day.
                return

        await self._run_due_task()

    async def _within_active_hours(self, session) -> bool:
        """Whether scheduled runs are allowed right now (local time)."""
        start, end = await app_settings.active_hours(session)
        hour = datetime.now().hour
        if start <= end:
            return start <= hour <= end
        # Window wraps past midnight, e.g. 22 -> 6.
        return hour >= start or hour <= end

    async def _run_due_task(self) -> None:
        async with get_session() as session:
            task = await session.scalar(
                select(MonitorTask)
                .where(
                    MonitorTask.enabled.is_(True),
                    MonitorTask.next_run_at.is_not(None),
                    MonitorTask.next_run_at <= get_current_timestamp(),
                )
                .order_by(MonitorTask.next_run_at)
                .limit(1)
            )

            if task is None:
                return

            # 没有 cookie 就跳过，是为了不让任务每轮白跑一趟出个认证失败。任务留在
            # due 状态而不推进 —— 用户一粘上 cookie 它就能自己跑起来。
            #
            # **但开着 CDP 时必须放行**：那种模式下登录态来自被接管的那个浏览器，
            # 粘不粘 cookie 根本轮不到它决定成败。不放行的话，选了「接管已有 Chrome」
            # 却没粘 cookie 的用户会发现任务永远不被触发，而且什么错都不报。
            cookie = await get_cookie(session, task.platform)
            if not cookie:
                cdp_enabled = await app_settings.get_value(
                    session, "cdp_enabled", fallback=False
                )
                if not cdp_enabled:
                    if task.platform not in self._warned_no_cookie:
                        print(
                            f"[monitor.scheduler] no {task.platform} cookie configured; "
                            "scheduled tasks will not run until one is set or CDP is enabled"
                        )
                        self._warned_no_cookie.add(task.platform)
                    return
            self._warned_no_cookie.discard(task.platform)

            # Advance before running so a crash mid-run cannot cause an immediate
            # re-fire, and so a long outage coalesces into a single run instead
            # of one run per missed interval.
            now = get_current_timestamp()
            following = schedule.next_occurrence(
                mode=task.schedule_mode,
                interval_minutes=task.interval_minutes,
                hours=schedule.parse_hours(task.schedule_hours),
                days=schedule.parse_days(task.schedule_days),
                minute=task.schedule_minute,
                after_ms=now,
            )

            if following is None:
                # A clock schedule with no times can never fire. The API rejects
                # that shape, so this guards against a hand-edited row: park the
                # task with no next run rather than leaving it permanently due and
                # re-running it on every tick.
                task.next_run_at = None
                print(
                    f"[monitor.scheduler] task {task.id} has no usable schedule "
                    f"and will not run until one is set"
                )
            elif task.schedule_mode == schedule.MODE_INTERVAL:
                # Jitter belongs to the interval mode only. Spreading identical
                # intervals apart is the point; nudging a time the operator
                # explicitly picked is not -- it just looks like a broken clock.
                task.next_run_at = following + random.randint(0, JITTER_SECONDS) * 1000
            else:
                task.next_run_at = following

            task_id = task.id

        try:
            await execute_task(task_id, trigger="scheduled")
        except Exception as exc:
            print(f"[monitor.scheduler] task {task_id} failed: {exc}")


# Global singleton, mirroring the crawler_manager pattern.
monitor_scheduler = MonitorScheduler()