# -*- coding: utf-8 -*- # Copyright (c) 2025 relakkes@gmail.com # # This file is part of MediaCrawler project. # Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/api/monitor/scheduler.py # GitHub: https://github.com/NanmiCoder # Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 # # 声明:本代码仅供学习和研究目的使用。使用者应遵守以下原则: # 1. 不得用于任何商业用途。 # 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 # 3. 不得进行大规模爬取或对平台造成运营干扰。 # 4. 应合理控制请求频率,避免给目标平台带来不必要的负担。 # 5. 不得用于任何非法或不当的用途。 # # 详细许可条款请参阅项目根目录下的LICENSE文件。 # 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 """Background scheduler for monitor tasks. One asyncio loop polls for due tasks and hands them to the runner. A plain loop is enough here: there is exactly one process, one global crawler subprocess, and therefore no concurrency to coordinate -- a cron-style library would add a dependency without adding a capability. Two families of schedule, and the difference matters: * ``interval`` is **fixed-delay**, not fixed-rate -- ``next_run_at`` is measured from the moment a run starts, so a slow run cannot make its task fire back-to-back. * the clock modes (``daily``/``weekly``) are **fixed-time** -- recomputed from the calendar, so a run that starts late does not drag every later run with it. The arithmetic for both lives in schedule.py. The loop also carries the 上游更新检查: it is not a crawl, so it shares none of the rules above (no subprocess, no active-hours gate) -- see ``_maybe_check_upstream``. It rides this loop rather than getting a thread of its own because it is one HTTP-shaped fetch per day. """ import asyncio import random from datetime import datetime from typing import Optional from sqlalchemy import select from tools.time_util import get_current_timestamp from ..services import crawler_manager from . import app_settings, schedule, upstream from .db import get_session from .models import MonitorRun, MonitorTask, RUN_INTERRUPTED, RUN_RUNNING from .runner import execute_task from .settings import get_cookie POLL_INTERVAL_SECONDS = 20 # Spread tasks sharing an interval so they do not all come due on the same tick. # Applied to interval mode only -- see the advance step below. JITTER_SECONDS = 60 class MonitorScheduler: """Polls the task table and runs whatever is due.""" def __init__(self) -> None: self._loop_task: Optional[asyncio.Task] = None self._stopping = asyncio.Event() # Avoids logging "no cookie" on every single tick. Per platform, because # warning once for Xiaohongshu must not silence the warning for Douyin. self._warned_no_cookie: set = set() async def start(self) -> None: if self._loop_task is not None and not self._loop_task.done(): return self._stopping.clear() self._loop_task = asyncio.create_task(self._run_loop()) async def stop(self) -> None: self._stopping.set() if self._loop_task is not None: self._loop_task.cancel() try: await self._loop_task except asyncio.CancelledError: pass self._loop_task = None async def _run_loop(self) -> None: try: await self.recover() except Exception as exc: # pragma: no cover - defensive print(f"[monitor.scheduler] recovery failed: {exc}") while not self._stopping.is_set(): try: await self.tick() except Exception as exc: # pragma: no cover - keep the loop alive print(f"[monitor.scheduler] tick failed: {exc}") # 独立于采集任务,因此单独一段 try:上游检查失败不该影响采集调度, # 反过来也一样。 try: await self._maybe_check_upstream() except Exception as exc: # pragma: no cover - keep the loop alive print(f"[monitor.scheduler] upstream check failed: {exc}") await asyncio.sleep(POLL_INTERVAL_SECONDS) async def _maybe_check_upstream(self) -> None: """到点就 fetch 一次上游仓库,看它有没有新提交。 与采集任务的三条规则都不同,各有理由:它不碰浏览器、也不占采集子进程, 所以不看 ``is_busy``;它只发一个 git 请求,没有被平台风控的风险,所以也不 受活跃时段限制 —— 定时检查放在半夜反而是最合适的。 """ async with get_session() as session: if not await app_settings.get_value( session, "upstream_check_enabled", fallback=False ): return interval_minutes = int( await app_settings.get_value( session, "upstream_check_interval_minutes", fallback=1440 ) ) state = await upstream.load_state(session) checked_at = int(state.get("checked_at") or 0) now = get_current_timestamp() # 失败也会写 checked_at,所以不通的时候同样是每个间隔重试一次, # 而不是每个 tick(20 秒)都去撞一次墙。 if checked_at and now - checked_at < max(1, interval_minutes) * 60_000: return result = await upstream.run_check() if result.get("behind"): print( f"[monitor.scheduler] 上游 {result.get('branch')} 领先 " f"{result['behind']} 个提交" ) elif not result.get("ok"): print(f"[monitor.scheduler] 上游检查失败:{result.get('error')}") async def recover(self) -> None: """Clean up state left behind by a server restart. A run still marked ``running`` cannot be running -- its subprocess died with the previous process. Marking it interrupted stops it from blocking the UI as a phantom in-flight run. """ async with get_session() as session: stale = ( await session.scalars( select(MonitorRun).where(MonitorRun.status == RUN_RUNNING) ) ).all() for run in stale: run.status = RUN_INTERRUPTED run.finished_at = get_current_timestamp() if stale: print( f"[monitor.scheduler] marked {len(stale)} interrupted run(s) " f"left over from a previous process" ) async def tick(self) -> None: """Run one due task, if the crawler is free and we are in the active window.""" # The crawler subprocess is a global singleton, so a manual crawl and a # monitor run cannot overlap. Returning without advancing next_run_at # leaves the task due, and it is picked up on a later tick. if crawler_manager.is_busy(): return async with get_session() as session: if not await self._within_active_hours(session): # Deliberately does not advance next_run_at: the task simply runs # when the window next opens, rather than being skipped for a day. return await self._run_due_task() async def _within_active_hours(self, session) -> bool: """Whether scheduled runs are allowed right now (local time).""" start, end = await app_settings.active_hours(session) hour = datetime.now().hour if start <= end: return start <= hour <= end # Window wraps past midnight, e.g. 22 -> 6. return hour >= start or hour <= end async def _run_due_task(self) -> None: async with get_session() as session: task = await session.scalar( select(MonitorTask) .where( MonitorTask.enabled.is_(True), MonitorTask.next_run_at.is_not(None), MonitorTask.next_run_at <= get_current_timestamp(), ) .order_by(MonitorTask.next_run_at) .limit(1) ) if task is None: return # 没有 cookie 就跳过,是为了不让任务每轮白跑一趟出个认证失败。任务留在 # due 状态而不推进 —— 用户一粘上 cookie 它就能自己跑起来。 # # **但开着 CDP 时必须放行**:那种模式下登录态来自被接管的那个浏览器, # 粘不粘 cookie 根本轮不到它决定成败。不放行的话,选了「接管已有 Chrome」 # 却没粘 cookie 的用户会发现任务永远不被触发,而且什么错都不报。 cookie = await get_cookie(session, task.platform) if not cookie: cdp_enabled = await app_settings.get_value( session, "cdp_enabled", fallback=False ) if not cdp_enabled: if task.platform not in self._warned_no_cookie: print( f"[monitor.scheduler] no {task.platform} cookie configured; " "scheduled tasks will not run until one is set or CDP is enabled" ) self._warned_no_cookie.add(task.platform) return self._warned_no_cookie.discard(task.platform) # Advance before running so a crash mid-run cannot cause an immediate # re-fire, and so a long outage coalesces into a single run instead # of one run per missed interval. now = get_current_timestamp() following = schedule.next_occurrence( mode=task.schedule_mode, interval_minutes=task.interval_minutes, hours=schedule.parse_hours(task.schedule_hours), days=schedule.parse_days(task.schedule_days), minute=task.schedule_minute, after_ms=now, ) if following is None: # A clock schedule with no times can never fire. The API rejects # that shape, so this guards against a hand-edited row: park the # task with no next run rather than leaving it permanently due and # re-running it on every tick. task.next_run_at = None print( f"[monitor.scheduler] task {task.id} has no usable schedule " f"and will not run until one is set" ) elif task.schedule_mode == schedule.MODE_INTERVAL: # Jitter belongs to the interval mode only. Spreading identical # intervals apart is the point; nudging a time the operator # explicitly picked is not -- it just looks like a broken clock. task.next_run_at = following + random.randint(0, JITTER_SECONDS) * 1000 else: task.next_run_at = following task_id = task.id try: await execute_task(task_id, trigger="scheduled") except Exception as exc: print(f"[monitor.scheduler] task {task_id} failed: {exc}") # Global singleton, mirroring the crawler_manager pattern. monitor_scheduler = MonitorScheduler()