mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-16 18:31:59 +00:00
a2a19df9df
- 表:schedules(Webhook 地址和签名密钥加密存放、next_run_at 存 UTC、version 乐观锁)、schedule_fires((任务,
fire_at, kind) 唯一)、webhook_deliveries;新表由 create_all 建,不用迁移
- 输入模板:{{ schedule.yesterday }} 这类变量按计划触发时间在任务时区算(不按「现在」:补发晚了、零点前后立即
运行,「昨天」都是那次触发的昨天),上一次计划触发时间按 cron 往前找,同一次触发永远渲染出同一份输入;未知
变量、未知过滤器、{{ }} 不成对都拦下,声明了入口字段的工作流还对照字段名和必填项
- 调度循环:每 15 秒一拍,条件 UPDATE 认领到点的任务并推进到下一次,两拍并发只有一拍认领得到,触发记录的唯一
约束是第二道保险;上一次运行没结束(含停在审批、服务重启挂起的)记跳过;错过的只看最近一次,在宽限内补一次、
超出记跳过,积压不补;发起走 launch_run,拒绝的原话记成未能发起,别的异常记通用说明、详情进日志;stop 等手上
这一拍收尾,开关关着时 start 不起循环
- Webhook:通用 JSON(可选 HMAC 签名头)、飞书文本、企业微信 markdown、钉钉 markdown,按各平台算法签名;运行
结束、失败、确有待审批时按订阅推一次,结束的等封存好再推,没订阅、没配 Webhook 的也记下已处理,之后配上不会
补推旧的;10 秒超时、失败重试 3 次(4xx 不重试),每次推送记一条记录;通知里没有报告数字和原始报错,日志和
错误里只有主机名(httpx 自己的请求日志也换成主机名)
- 设置 notify.public_base_url:通知里的运行链接 = 它 + /runs/{id},只收 http(s) 地址,末尾的 / 去掉
175 lines
8.5 KiB
Python
175 lines
8.5 KiB
Python
"""调度循环:每一拍认领到点的定时任务、发起运行,再顺便查已发起的运行要不要推送通知。
|
||
|
||
认领靠条件 UPDATE:UPDATE schedules SET next_run_at = 下一次 WHERE id = … AND next_run_at = 旧值。两拍撞在
|
||
一起(或者以后多开了一个进程),只有一拍改得动这一行,另一拍拿到 0 行就知道这一次已经有人处理了。触发记录
|
||
(任务, fire_at, kind) 唯一是第二道保险。
|
||
|
||
错过的触发(服务停机、进程卡住):只看最近应触发的那一次——在宽限内补一次(kind=misfire),超出就记一条
|
||
skipped_misfire;更早的积压只记一条跳过,不补。下一次一律推进到现在之后,不会醒来就把积压的一口气全发起。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import contextlib
|
||
import logging
|
||
from datetime import datetime
|
||
|
||
from sqlalchemy import select, update
|
||
|
||
from app.db.base import SessionLocal, utcnow
|
||
from app.db.models import Schedule, ScheduleFire
|
||
from app.scheduler import cron, service
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
#: 一拍的间隔(秒)。测试里 start(interval=…) 调小
|
||
TICK_SECONDS = 15.0
|
||
#: 晚了这么多秒以内都算准点(kind=scheduled),和补发宽限无关:拍与拍之间的十几秒不该算错过
|
||
ON_TIME_S = 60
|
||
#: stop 等手上那一拍收尾最多等多久,超过就取消
|
||
STOP_GRACE_S = 10.0
|
||
|
||
|
||
async def fire_due(now: datetime | None = None) -> list[ScheduleFire]:
|
||
"""处理所有到点的任务(enabled 且 next_run_at <= now),返回这一拍记下的触发。一个任务出错不影响别的。"""
|
||
now = now or utcnow()
|
||
async with SessionLocal() as session:
|
||
due = (await session.execute(
|
||
select(Schedule.id, Schedule.next_run_at)
|
||
.where(Schedule.enabled.is_(True), Schedule.next_run_at.is_not(None), Schedule.next_run_at <= now)
|
||
.order_by(Schedule.next_run_at)
|
||
)).all()
|
||
out: list[ScheduleFire] = []
|
||
for schedule_id, claimed_at in due:
|
||
try:
|
||
out.extend(await _due_one(schedule_id, claimed_at, now))
|
||
except Exception: # noqa: BLE001 - 一个任务出错不能拖住这一拍的其他任务,也不能让循环停下
|
||
logger.exception("定时任务 %s 这一拍处理失败", schedule_id)
|
||
return out
|
||
|
||
|
||
async def _due_one(schedule_id: str, claimed_at: datetime, now: datetime) -> list[ScheduleFire]:
|
||
async with SessionLocal() as session:
|
||
row = await session.get(Schedule, schedule_id)
|
||
if row is None or not row.enabled or row.next_run_at != claimed_at:
|
||
return []
|
||
cron_expr, tz, grace = row.cron, row.timezone, max(int(row.misfire_grace_s or 0), 0)
|
||
try:
|
||
spec, zone = cron.parse(cron_expr), cron.zone(tz)
|
||
# 最近应触发的那一次。库里的 next_run_at 理应就是一个触发时刻,万一不是(手改过库)就按它算
|
||
latest = max(cron.last_due(spec, zone, now) or claimed_at, claimed_at)
|
||
upcoming = cron.next_fire(spec, zone, now)
|
||
except (cron.CronError, cron.TimezoneError) as e:
|
||
# 存进库里之后才坏掉的表达式(手改过库、时区库升级删了时区):停止排期,在触发历史里照实说
|
||
if not await _claim(schedule_id, claimed_at, None):
|
||
return []
|
||
logger.warning("定时任务 %s 的 cron 或时区无法解析,已停止排期:%s", schedule_id, e)
|
||
fire = await service._record(schedule_id, claimed_at, "scheduled", "failed_to_start",
|
||
f"cron 表达式或时区无法解析,已停止排期:{e}", {})
|
||
return [fire] if fire else []
|
||
if not await _claim(schedule_id, claimed_at, upcoming):
|
||
return []
|
||
|
||
out: list[ScheduleFire | None] = []
|
||
lateness = (now - latest).total_seconds()
|
||
if lateness <= max(ON_TIME_S, grace):
|
||
if latest > claimed_at:
|
||
# 最近这一次照常(或补一次),更早的积压只记一条,不补
|
||
last_missed = cron.prev_fire(spec, zone, latest) or claimed_at
|
||
reason = f"服务未运行,错过了 {_span(claimed_at, last_missed, tz)} 的触发;积压的触发不补,只发起最近一次"
|
||
out.append(await service.record_skip(schedule_id, claimed_at, reason))
|
||
kind = "scheduled" if lateness <= ON_TIME_S else "misfire"
|
||
out.append(await service.fire_once(schedule_id, latest, kind))
|
||
else:
|
||
reason = (f"服务未运行,错过了 {_span(claimed_at, latest, tz)} 的触发,"
|
||
f"超出补发宽限({service.duration_text(grace)}),没有发起")
|
||
out.append(await service.record_skip(schedule_id, latest, reason))
|
||
return [fire for fire in out if fire is not None]
|
||
|
||
|
||
async def _claim(schedule_id: str, claimed_at: datetime, upcoming: datetime | None) -> bool:
|
||
"""把 next_run_at 从 claimed_at 推进到 upcoming。改动了一行才算认领到。
|
||
|
||
updated_at 照旧:它说的是任务最近一次被人修改,推进排期不算。version 也不动——乐观锁防的是两个人同时改设置,
|
||
调度推进 next_run_at 不该让正在编辑的人保存时撞上 409。"""
|
||
async with SessionLocal() as session:
|
||
result = await session.execute(
|
||
update(Schedule)
|
||
.where(Schedule.id == schedule_id, Schedule.enabled.is_(True), Schedule.next_run_at == claimed_at)
|
||
.values(next_run_at=upcoming, updated_at=Schedule.updated_at)
|
||
.execution_options(synchronize_session=False)
|
||
)
|
||
await session.commit()
|
||
return result.rowcount == 1
|
||
|
||
|
||
def _span(first: datetime, last: datetime, tz: str) -> str:
|
||
a, b = service.local_text(first, tz), service.local_text(last, tz)
|
||
return a if a == b else f"{a} 至 {b}"
|
||
|
||
|
||
async def tick(now: datetime | None = None) -> None:
|
||
"""完整的一拍:发起到点的,再推送该推送的通知。测试和手动调用用;循环里两件事分开跑(见 _Runner._loop)。"""
|
||
from app.scheduler import webhook
|
||
|
||
await fire_due(now)
|
||
await webhook.notify_due(now)
|
||
|
||
|
||
class _Runner:
|
||
"""调度循环的任务。照 sandbox/manager.py 回收闲置会话的写法:一个 asyncio 任务,出错记日志接着下一拍。"""
|
||
|
||
def __init__(self) -> None:
|
||
self.task: asyncio.Task[None] | None = None
|
||
#: 通知推送另起任务:Webhook 超时加重试最长要几十秒,不能拖着下一拍的发起。同一时刻只跑一份
|
||
self.notify: asyncio.Task[None] | None = None
|
||
self._stopping: asyncio.Event | None = None
|
||
|
||
def running(self) -> bool:
|
||
return self.task is not None and not self.task.done()
|
||
|
||
async def start(self, interval: float = TICK_SECONDS) -> None:
|
||
if self.running():
|
||
return
|
||
self._stopping = asyncio.Event()
|
||
self.task = asyncio.create_task(self._loop(interval, self._stopping), name="agentlab-scheduler")
|
||
logger.info("定时任务调度已启动,每 %s 秒检查一次", interval)
|
||
|
||
async def _loop(self, interval: float, stopping: asyncio.Event) -> None:
|
||
from app.scheduler import webhook
|
||
|
||
while not stopping.is_set():
|
||
try:
|
||
await fire_due()
|
||
if self.notify is None or self.notify.done():
|
||
self.notify = asyncio.create_task(webhook.notify_due(), name="agentlab-scheduler-notify")
|
||
except asyncio.CancelledError:
|
||
raise
|
||
except Exception: # noqa: BLE001 - 这一拍出错不该让循环停下
|
||
logger.exception("定时任务调度这一拍出错,下一拍照常进行")
|
||
with contextlib.suppress(TimeoutError):
|
||
await asyncio.wait_for(stopping.wait(), timeout=interval)
|
||
|
||
async def stop(self) -> None:
|
||
"""让循环收完手上这一拍再停;等不到(STOP_GRACE_S)就取消。没 start 过直接返回。"""
|
||
tasks = [t for t in (self.task, self.notify) if t is not None]
|
||
self.task = self.notify = None
|
||
if self._stopping is not None:
|
||
self._stopping.set()
|
||
for task in tasks:
|
||
if task.done():
|
||
continue
|
||
try:
|
||
await asyncio.wait_for(asyncio.shield(task), timeout=STOP_GRACE_S)
|
||
except TimeoutError:
|
||
task.cancel()
|
||
with contextlib.suppress(asyncio.CancelledError, Exception):
|
||
await task
|
||
except Exception: # noqa: BLE001 - 停的时候不再追究这一拍的错
|
||
pass
|
||
if tasks:
|
||
logger.info("定时任务调度已停止")
|
||
|
||
|
||
runner = _Runner()
|