diff --git a/backend/app/api/schedules.py b/backend/app/api/schedules.py index 728cac71..9664d6c4 100644 --- a/backend/app/api/schedules.py +++ b/backend/app/api/schedules.py @@ -6,7 +6,7 @@ """ from __future__ import annotations -from datetime import datetime +from datetime import datetime, timedelta from typing import Any from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, Response @@ -87,7 +87,7 @@ def _timezone(value: Any) -> str: def _run_class(value: Any) -> str: if value not in service.RUN_CLASSES: - raise _invalid(f"运行类别需要是 formal(正式运行)或 exploratory(探索运行),当前为「{value}」", "run_class") + raise _invalid(f"运行类别需要选择正式运行或探索运行{_current(value)}", "run_class") return value @@ -98,35 +98,45 @@ def _template(value: Any) -> dict[str, str]: raise _invalid(str(e), "input_template") from e +def _current(value: Any) -> str: + return service.current_value(value) + + def _enabled(value: Any) -> bool: if not isinstance(value, bool): - raise _invalid(f"启用状态需要是 true 或 false,当前为「{value}」", "enabled") + raise _invalid(f"启用状态需要是 true 或 false{_current(value)}", "enabled") return value def _overlap(value: Any) -> str: if value != "skip": - raise _invalid(f"重叠处理目前只支持 skip(上一次运行还没结束时跳过这次触发),当前为「{value}」", "overlap") + raise _invalid(f"重叠处理目前只支持「跳过」:上一次运行还没结束时跳过这次触发{_current(value)}", "overlap") return value def _grace(value: Any) -> int: if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= MAX_GRACE_S: - raise _invalid(f"补发宽限需要是 0 到 {MAX_GRACE_S} 之间的整数秒数(最长 7 天),当前为「{value}」", + raise _invalid(f"补发宽限需要是 0 到 {MAX_GRACE_S} 之间的整数秒数(最长 7 天){_current(value)}", "misfire_grace_s") return value def _format(value: Any) -> str: if value not in webhook.FORMATS: - raise _invalid(f"不支持的 Webhook 格式「{value}」,可选的格式:{'、'.join(webhook.FORMATS)}", "webhook") + labels = "、".join(webhook.FORMAT_LABEL[f] for f in webhook.FORMATS) + raise _invalid(f"Webhook 格式需要是{labels}之一{_current(value)}", "webhook") return value def _events(value: Any) -> list[str]: - if not isinstance(value, list) or any(e not in webhook.EVENTS for e in value): - raise _invalid("Webhook 的通知事件只能从 finished(运行完成)、failed(运行失败)、awaiting_approval(等待审批)" - f"中选择,当前为「{value}」", "webhook") + labels = "、".join(webhook.EVENT_LABEL[e] for e in webhook.EVENTS) + if not isinstance(value, list): + raise _invalid(f"Webhook 的通知事件需要从{labels}中选择", "webhook") + unknown = [e for e in value if e not in webhook.EVENTS] + if unknown: + first = unknown[0] + named = f"「{first}」" if isinstance(first, str) and first.strip() else "中有一项" + raise _invalid(f"Webhook 的通知事件{named}不可选,只能从{labels}中选择", "webhook") return [e for e in webhook.EVENTS if e in value] @@ -149,16 +159,28 @@ def _blank(value: Any) -> bool: return value is None or (isinstance(value, str) and not value.strip()) +def _matched_format(value: Any, url: str) -> str: + """校验格式,并要求和地址对得上:群机器人的地址只认自己平台的格式(见 webhook.bot_format)。""" + fmt = _format(value) + if problem := webhook.format_problem(fmt, url): + raise _invalid(problem, "webhook") + return fmt + + def _new_webhook(value: Any) -> dict[str, Any] | None: - """创建时的 webhook:null 或 url 为空表示不配;否则校验格式、地址、密钥、事件(事件省略表示三种都要)。""" + """创建时的 webhook:null 或 url 为空表示不配;否则校验格式、地址、密钥、事件(事件省略表示三种都要)。 + 格式省略时按地址认:群机器人的地址用它平台的格式,别的用通用格式。""" if value is None: return None if not isinstance(value, dict): raise _invalid("Webhook 设置需要是一个对象,或者为空表示不推送", "webhook") if _blank(value.get("url")): return None - return {"format": _format(value.get("format", "generic")), "url": _url(value.get("url")), - "secret": _secret(value.get("secret")), "events": _events(value.get("events", list(webhook.EVENTS)))} + url = _url(value.get("url")) + given = value.get("format") + fmt = _matched_format(given, url) if given is not None else webhook.bot_format(url) or "generic" + return {"format": fmt, "url": url, "secret": _secret(value.get("secret")), + "events": _events(value.get("events", list(webhook.EVENTS)))} def _webhook_columns(hook: dict[str, Any] | None) -> dict[str, Any]: @@ -170,7 +192,10 @@ def _webhook_columns(hook: dict[str, Any] | None) -> dict[str, Any]: def _patched_webhook(row: Schedule, value: Any) -> dict[str, Any]: """PATCH 的 webhook → 要写的列。null 或 url 写空串:删除整个 Webhook;url 省略:地址不变;secret 省略:密钥不变、 - 写空串:去掉密钥。""" + 写空串:去掉密钥。format 省略:换了地址的按新地址认(群机器人用它平台的格式),没换的不变。 + + 地址或格式真的变了才查两者对不对得上:库里已有对不上的设置(这条校验之前存的)时,整份表单回写只为停用、 + 改名,不能因为 Webhook 被拦下。""" if value is None: return _webhook_columns(None) if not isinstance(value, dict): @@ -180,13 +205,24 @@ def _patched_webhook(row: Schedule, value: Any) -> dict[str, Any]: has_url = bool(row.webhook_url) if "url" not in value and not has_url: raise _invalid("请填写 Webhook 地址", "webhook") + url = _url(value.get("url")) if "url" in value else None + if value.get("format") is not None: + fmt = _format(value["format"]) + elif url is not None: + fmt = webhook.bot_format(url) or (row.webhook_format if has_url else "generic") + else: + fmt = row.webhook_format + if url is not None or fmt != row.webhook_format: + target = url or decrypt(row.webhook_url) + if target and (problem := webhook.format_problem(fmt, target)): + raise _invalid(problem, "webhook") out: dict[str, Any] = { - "webhook_format": _format(value.get("format", row.webhook_format if has_url else "generic")), + "webhook_format": _format(fmt), "webhook_events": _events(value.get("events", list(row.webhook_events or []) if has_url else list(webhook.EVENTS))), } - if "url" in value: - out["webhook_url"] = encrypt(_url(value.get("url"))) + if url is not None: + out["webhook_url"] = encrypt(url) if "secret" in value: secret = _secret(value.get("secret")) out["webhook_secret"] = encrypt(secret) if secret else None @@ -254,14 +290,16 @@ async def _fire_outs(session: AsyncSession, fires: list[ScheduleFire]) -> list[d )).scalars() for d in rows: notified.setdefault(d.fire_id, []).append({"event": d.event, "status": d.status, "at": _iso(d.created_at)}) + # 正在发起的触发:库里是占位的「未能发起」(为了进程被强杀时照实说),进程还在、只是还没发起完,按已发起给 + launching = {f.id for f in fires if service.launching(f.id)} return [{ "id": f.id, "fire_at": _iso(f.fire_at), "kind": f.kind, - "status": f.status, + "status": "started" if f.id in launching else f.status, "run_id": f.run_id, "run_status": statuses.get(f.run_id) if f.run_id else None, - "reason": f.reason, + "reason": None if f.id in launching else f.reason, "notified": notified.get(f.id, []), "created_at": _iso(f.created_at), } for f in fires] @@ -403,8 +441,11 @@ async def update_schedule(schedule_id: str, request: Request, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """改任意可改的项,带上读到的 version;对不上回 409。只读的项(id、next_run_at、last_fire……)带了也不理。 - 改了 cron、时区或启用状态就重算下一次触发(从现在算,不补停用期间错过的)。改了工作流、运行类别或输入模板 - 就按改后的组合重新校验;只停用、改名不查工作流——工作流后来撤了发布,也要能先把任务停下来。 + 看的是值变没变,不是键在不在:编辑表单把整份字段带回来是正常用法。 + - 工作流、运行类别、输入模板真改了才按改后的组合重新校验。只停用、改名不查工作流——工作流删了、撤了发布, + 也要能先把任务停下来。 + - cron、时区、启用状态真改了才重算下一次触发(见 _next_after_edit)。没改的不动它:到点了、这一拍还没来时 + 保存一次,这一次照常发起,不会被推到下一次、连记录都没有就丢了。 """ body = await _body(request) row = await _get(session, schedule_id) @@ -439,11 +480,18 @@ async def update_schedule(schedule_id: str, request: Request, def merged(key: str) -> Any: return values.get(key, getattr(row, key)) - if {"workflow_id", "run_class", "input_template"} & values.keys(): + def changed(key: str) -> bool: + if key not in values: + return False + stored = getattr(row, key) + return values[key] != (dict(stored or {}) if key == "input_template" else stored) + + if changed("workflow_id") or changed("run_class") or changed("input_template"): workflow = await _check_workflow(session, merged("workflow_id"), merged("run_class"), merged("input_template")) values["workflow_name"] = workflow.name - if {"cron", "timezone", "enabled"} & values.keys(): - values["next_run_at"] = service.next_run(merged("cron"), merged("timezone"), merged("enabled")) + # cron、时区在上面已经规范化(字段之间一个空格、时区的标准写法),和库里存的同一种写法,直接比 + if changed("cron") or changed("timezone") or changed("enabled"): + values["next_run_at"] = _next_after_edit(row, merged("cron"), merged("timezone"), merged("enabled")) # 条件 UPDATE 才是真的乐观锁:两个人同时读到第 3 版、同时保存,只有先到的那个改得动 result = await session.execute( update(Schedule).where(Schedule.id == schedule_id, Schedule.version == version) @@ -456,6 +504,21 @@ async def update_schedule(schedule_id: str, request: Request, return await _schedule_out(session, await _get(session, schedule_id)) +def _next_after_edit(row: Schedule, cron_expr: str, tz: str, enabled: bool) -> datetime | None: + """改了 cron、时区或启用状态之后的下一次触发。 + + 停用:没有下一次。刚启用:从现在算,不补停用期间错过的。启用着改了 cron、时区:原来那一次已经到点、这一拍还没来 + 的,从那一刻接着按新的表达式算——新表达式也覆盖那一刻(每天 → 工作日)就还发起这一次,只属于旧表达式的不再发起。 + """ + if not enabled: + return None + now = utcnow() + due = service.as_utc(row.next_run_at) if row.enabled and row.next_run_at is not None else None + if due is not None and due <= now: + return cron.next_fire(cron_expr, tz, due - timedelta(microseconds=1)) + return service.next_run(cron_expr, tz, True, now) + + @router.delete("/{schedule_id}", status_code=204) async def delete_schedule(schedule_id: str, session: AsyncSession = Depends(get_session)) -> Response: """删除任务,连同它的触发历史和推送记录。它发起过的运行留着(运行上的来源 id 也留着):那是追溯凭证。""" diff --git a/backend/app/db/models.py b/backend/app/db/models.py index 0a250142..db3e6dac 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -815,8 +815,9 @@ class ScheduleFire(Base): reason: Mapped[str | None] = mapped_column(Text, default=None) # 渲染后的输入,就是交给运行的那一份 inputs: Mapped[dict[str, Any]] = mapped_column(default=dict) - # 已经按哪个运行状态处理过通知(succeeded / failed / cancelled / interrupted / failed_to_start)。 - # 运行状态变了、和它不同才再看要不要推送,同一状态只处理一次 + # 已经按哪个运行状态处理过通知:succeeded / failed / cancelled / failed_to_start;停在断点上的另带第几段执行, + # 停在审批是 interrupted:approval:<段>、挂起是 interrupted:held:<段>(见 scheduler/webhook.py 的 _state_of)。 + # 和现在算出来的不同才再看要不要推送,同一状态只处理一次 notified_status: Mapped[str | None] = mapped_column(String(30), default=None) # 和同一任务上一次运行的比较结果(工件 id),后续的运行对比工作包写 compare_artifact: Mapped[str | None] = mapped_column(String(64), default=None) diff --git a/backend/app/scheduler/loop.py b/backend/app/scheduler/loop.py index d3620f0a..7a6ed3ee 100644 --- a/backend/app/scheduler/loop.py +++ b/backend/app/scheduler/loop.py @@ -26,8 +26,11 @@ logger = logging.getLogger(__name__) TICK_SECONDS = 15.0 #: 晚了这么多秒以内都算准点(kind=scheduled),和补发宽限无关:拍与拍之间的十几秒不该算错过 ON_TIME_S = 60 -#: stop 等手上那一拍收尾最多等多久,超过就取消 -STOP_GRACE_S = 10.0 +#: stop 等手上那一拍的发起收尾最多等多久,超过就取消。关停时 main.py 先停调度、再让 run_manager 平稳挂起在执行 +#: 的运行,docker stop 默认只给 10 秒:这里要留出大半 +STOP_GRACE_S = 3.0 +#: stop 等正在推送的通知多久。通知本来就是至多一次,接收端挂住时等满超时加重试(几十秒)换不来什么,宽限一过就取消 +NOTIFY_GRACE_S = 1.5 async def fire_due(now: datetime | None = None) -> list[ScheduleFire]: @@ -149,7 +152,8 @@ class _Runner: while not stopping.is_set(): try: await fire_due() - if self.notify is None or self.notify.done(): + # stop 撞在这一拍的发起当中时,收尾不再新起通知扫描:stop 只等它动手时手上的那一份 + if not stopping.is_set() and (self.notify is None or self.notify.done()): self.notify = asyncio.create_task(_notify(webhook), name="agentlab-scheduler-notify") except asyncio.CancelledError: raise @@ -159,24 +163,34 @@ class _Runner: await asyncio.wait_for(stopping.wait(), timeout=interval) async def stop(self) -> None: - """让循环收完手上这一拍再停;等不到(STOP_GRACE_S)就取消。没 start 过直接返回。""" - tasks = [t for t in (self.task, self.notify) if t is not None] - self.task = self.notify = None + """让循环收完手上这一拍再停、正在推送的通知推完;各等一个宽限(STOP_GRACE_S、NOTIFY_GRACE_S),两件事并行 + 等,过了就取消。没 start 过直接返回。 + + 先置停止标志、再取两个任务:标志置上之后循环不会再起新的通知扫描,取到的就是最后一份。""" if self._stopping is not None: self._stopping.set() - for task in tasks: - if task.done(): - continue - try: - await asyncio.wait_for(asyncio.shield(task), timeout=STOP_GRACE_S) - except TimeoutError: - task.cancel() - with contextlib.suppress(asyncio.CancelledError, Exception): - await task - except Exception: # noqa: BLE001 - 停的时候不再追究这一拍的错 - pass - if tasks: + tick, notify = self.task, self.notify + await asyncio.gather(_settle(tick, STOP_GRACE_S), _settle(notify, NOTIFY_GRACE_S)) + if self.notify is not None and self.notify is not notify: + # 兜底:万一还是落下了一份(以后改循环时漏掉上面的条件),不让它在关停期间接着推 + await _settle(self.notify, 0) + self.task = self.notify = None + if tick is not None or notify is not None: logger.info("定时任务调度已停止") +async def _settle(task: asyncio.Task[None] | None, grace: float) -> None: + """等 task 结束,最多等 grace 秒,过了就取消。task 自己的异常不再追究:停的时候只管它停下。""" + if task is None or task.done(): + return + try: + await asyncio.wait_for(asyncio.shield(task), timeout=grace) + except TimeoutError: + task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + except Exception: # noqa: BLE001 + pass + + runner = _Runner() diff --git a/backend/app/scheduler/service.py b/backend/app/scheduler/service.py index 932c439a..8eab64e3 100644 --- a/backend/app/scheduler/service.py +++ b/backend/app/scheduler/service.py @@ -13,11 +13,11 @@ import weakref from datetime import date, datetime, timedelta, timezone from typing import Any -from sqlalchemy import select, update +from sqlalchemy import func, select, update from sqlalchemy.exc import IntegrityError from app.db.base import SessionLocal, utcnow -from app.db.models import Run, Schedule, ScheduleFire +from app.db.models import Approval, Run, Schedule, ScheduleFire from app.engine.expressions import _FILTERS, render_template from app.scheduler import cron @@ -26,7 +26,9 @@ logger = logging.getLogger(__name__) #: 变量清单的示例按这个时区算(GET /api/schedules/variables 没带时区时) DEFAULT_TIMEZONE = "Asia/Shanghai" RUN_CLASSES = ("formal", "exploratory") -#: 这几个状态的运行算「还没结束」:停在审批上、服务重启挂起的(interrupted)也算,它们随时会接着执行 +#: 这几个状态的运行算「还没结束」,挡住这个任务之后的触发。interrupted 也算(规格如此):停在审批上的,批完会 +#: 接着执行;服务重启挂起的却不会自己接着执行,要有人去继续或放弃——所以跳过的原因分开写(见 overlap_reason), +#: 挂起的另外按失败推一次通知(webhook._state_of),不然任务就此静默停摆 ACTIVE_RUN = ("queued", "running", "interrupted") #: 模板里能用的变量:(变量名, 说明)。说明写在变量清单接口里,和示例一起给界面 @@ -45,7 +47,8 @@ VARIABLES: tuple[tuple[str, str], ...] = ( _NAMES = tuple(f"schedule.{key}" for key, _ in VARIABLES) _TEMPLATE_RE = re.compile(r"\{\{(.*?)\}\}", re.DOTALL) -#: 触发记下来、运行还没发起时先写成这样:发起成功再改成 started。进程在这中间被强杀,留下的记录照实说没有发起 +#: 触发记下来、运行还没发起时先写成这样:发起成功再改成 started。进程在这中间被强杀,留下的记录照实说没有发起。 +#: 进程还在、只是还没发起完的,接口按「已发起」给(见 launching) INTERRUPTED_REASON = "服务在发起运行前停止,这次触发没有发起运行" #: launch_run 抛了 LaunchRefused 以外的异常(库出错之类)。详细信息只进日志 GENERIC_REASON = "发起运行时出错,详细信息见服务日志" @@ -55,6 +58,18 @@ class TemplateError(ValueError): """输入模板写错了。str() 是给人看的中文说明,指出是哪个输入。""" +def current_value(value: Any) -> str: + """报错末尾回显当前值的那半句:没填写说「当前未填写」,文本、数字照原样,true / false 按 JSON 的写法;列表、对象 + 不回显——给人看的说明里不出现 Python 的写法(None、['a', 'b'])。""" + if value is None or (isinstance(value, str) and not value.strip()): + return ",当前未填写" + if isinstance(value, bool): + return f",当前为「{'true' if value else 'false'}」" + if isinstance(value, (str, int, float)): + return f",当前为「{value}」" + return "" + + # -------------------------------------------------------------------------- # 模板变量 # -------------------------------------------------------------------------- @@ -116,7 +131,8 @@ def check_template(template: Any) -> dict[str, str]: if name in out: raise TemplateError(f"输入模板里字段「{name}」写了两次,请只保留一个") if not isinstance(text, str): - raise TemplateError(f"输入「{name}」的值需要是文本(可以带 {{{{ schedule.yesterday }}}} 这样的变量),当前为「{text}」") + raise TemplateError(f"输入「{name}」的值需要是文本(可以带 {{{{ schedule.yesterday }}}} 这样的变量)" + f"{current_value(text)}") _check_text(name, text) out[name] = text return out @@ -210,6 +226,15 @@ def launch_lock() -> asyncio.Lock: return lock +#: 正在发起的触发:占位记录已写进库、launch_run 还没返回。通知扫描有 launch_lock 挡着,接口的读(任务列表的 +#: last_fire、触发历史)没有:它们据此按「已发起」给,不把还在发起的说成「未能发起」(单进程部署,进程内的集合就够) +_LAUNCHING: set[str] = set() + + +def launching(fire_id: str) -> bool: + return fire_id in _LAUNCHING + + def actor_label(name: str) -> str: """定时任务发起的运行记在谁名下。""" return f"定时任务:{name}"[:100] @@ -244,23 +269,27 @@ async def fire_once(schedule_id: str, fire_at: datetime, kind: str, *, actor: st return await _record(schedule_id, fire_at, kind, "failed_to_start", f"无法生成这次运行的输入:{e}", {}) busy = await _active_run(schedule_id) if busy is not None: - reason = f"上一次运行({busy[:8]})还没有结束,按设置跳过这次触发" - return await _record(schedule_id, fire_at, kind, "skipped_overlap", reason, inputs) + return await _record(schedule_id, fire_at, kind, "skipped_overlap", overlap_reason(*busy), inputs) fire = await _record(schedule_id, fire_at, kind, "failed_to_start", INTERRUPTED_REASON, inputs, fresh=True) if fire is None: return await _existing(schedule_id, fire_at, kind) from app.engine.launch import LaunchRefused, launch_run + _LAUNCHING.add(fire.id) try: - async with SessionLocal() as session: - run = await launch_run(session, workflow_id=workflow_id, run_class=run_class, input_payload=inputs, - actor=actor or actor_label(name), schedule_id=schedule_id) - except LaunchRefused as e: - return await _finish(fire.id, status="failed_to_start", reason=e.message) - except Exception: # noqa: BLE001 - 发起失败不能让调度循环停下;详细信息进日志,记录上写通用说明 - logger.exception("定时任务「%s」(%s)发起运行失败", name, schedule_id) - return await _finish(fire.id, status="failed_to_start", reason=GENERIC_REASON) - return await _finish(fire.id, status="started", reason=None, run_id=run.id) + try: + async with SessionLocal() as session: + run = await launch_run(session, workflow_id=workflow_id, run_class=run_class, + input_payload=inputs, actor=actor or actor_label(name), + schedule_id=schedule_id) + except LaunchRefused as e: + return await _finish(fire.id, status="failed_to_start", reason=e.message) + except Exception: # noqa: BLE001 - 发起失败不能让调度循环停下;详细信息进日志,记录上写通用说明 + logger.exception("定时任务「%s」(%s)发起运行失败", name, schedule_id) + return await _finish(fire.id, status="failed_to_start", reason=GENERIC_REASON) + return await _finish(fire.id, status="started", reason=None, run_id=run.id) + finally: + _LAUNCHING.discard(fire.id) async def record_skip(schedule_id: str, fire_at: datetime, reason: str) -> ScheduleFire | None: @@ -268,12 +297,32 @@ async def record_skip(schedule_id: str, fire_at: datetime, reason: str) -> Sched return await _record(schedule_id, fire_at, "misfire", "skipped_misfire", reason, {}) -async def _active_run(schedule_id: str) -> str | None: - """这个任务发起的、还没结束的运行 id(最近的那一个),没有返回 None。""" +async def _active_run(schedule_id: str) -> tuple[str, str, bool] | None: + """这个任务发起的、还没结束的运行(最近的那一个):(运行 id, 状态, 是否停在审批),没有返回 None。 + + 停在审批:有 pending 的审批,或者刚批完(answered)、还没接着执行。interrupted 而两样都没有的是挂起的。""" async with SessionLocal() as session: - return await session.scalar( - select(Run.id).where(Run.schedule_id == schedule_id, Run.status.in_(ACTIVE_RUN)) - .order_by(Run.created_at.desc()).limit(1)) + row = (await session.execute( + select(Run.id, Run.status).where(Run.schedule_id == schedule_id, Run.status.in_(ACTIVE_RUN)) + .order_by(Run.created_at.desc()).limit(1))).first() + if row is None: + return None + run_id, status = row + waiting = status == "interrupted" and bool(await session.scalar( + select(func.count()).select_from(Approval).where( + Approval.run_id == run_id, Approval.status.in_(("pending", "answered"))))) + return run_id, status, waiting + + +def overlap_reason(run_id: str, status: str, waiting: bool) -> str: + """跳过的原因按挡路的那次运行说清楚该等还是该处理。运行编号和界面上的短编号一样写(#前 6 位)。""" + tag = f"#{run_id[:6]}" + if status != "interrupted": + return f"上一次运行 {tag} 仍在进行中,本次触发已跳过" + if waiting: + return f"上一次运行 {tag} 正在等待审批,处理审批、运行结束后才会按时发起新的运行;本次触发已跳过" + return (f"上一次运行 {tag} 已挂起(通常是服务重启所致),请在「记录」中继续运行或放弃这次运行," + "它结束后才会按时发起新的运行;本次触发已跳过") async def _record(schedule_id: str, fire_at: datetime, kind: str, status: str, reason: str | None, diff --git a/backend/app/scheduler/webhook.py b/backend/app/scheduler/webhook.py index 390fb0b4..85907b66 100644 --- a/backend/app/scheduler/webhook.py +++ b/backend/app/scheduler/webhook.py @@ -1,4 +1,4 @@ -"""定时任务的通知:运行结束、失败、停在审批时按任务的设置推送 Webhook。 +"""定时任务的通知:运行结束、失败、停在审批(或因服务重启挂起)时按任务的设置推送 Webhook。 四种格式:通用 JSON、飞书(文本消息)、企业微信(markdown)、钉钉(markdown)。配了签名密钥时:通用格式在请求头 里带 HMAC-SHA256 签名,飞书、钉钉按各自平台的算法签(飞书放在请求体,钉钉放在地址上);企业微信的 key 本来就在 @@ -8,8 +8,13 @@ Webhook 地址本身就是凭证(企业微信的 key、钉钉的 access_token 加密落库,日志、推送记录、接口返回里只出现主机名。httpx 每发一个请求都会在 INFO 级别记一行完整地址,这里装了 一个过滤器把推送期间的那几行换成主机名。 -通知里只有任务名、状态、出具档位、缺口条数、运行链接、封存哈希前 12 位和起止时间,不放报告里的具体数字,也不放 -失败时的原始报错——报错里可能带着查询到的数据,Webhook 推去的群不一定有权看。 +通知里只有任务名、状态、出具档位、缺口条数、运行链接、封存哈希(界面上叫清单哈希)前 12 位和起止时间,失败时 +另写失败节点的名字。不放报告里的具体数字,也不放失败时的原始报错——报错里可能带着查询到的数据,Webhook 推去的群 +不一定有权看。 + +三个平台出错时都回 HTTP 200、错误码放在响应体里,所以送没送达要看响应体:按地址的主机认平台,不看选的格式—— +格式选错了(飞书的地址选了通用格式)正是最需要认出来的情形。群机器人的地址只认自己平台的格式,保存时就拦下 +对不上的(见 bot_format)。 """ from __future__ import annotations @@ -20,6 +25,7 @@ import hashlib import hmac import json import logging +import unicodedata from dataclasses import dataclass from datetime import datetime, timedelta, timezone from typing import Any @@ -29,6 +35,7 @@ import httpx from sqlalchemy import and_, func, or_, select, update from app.core.crypto import decrypt +from app.core.events import EventType from app.db.base import SessionLocal, utcnow from app.db.models import Approval, Run, RunEvent, Schedule, ScheduleFire, WebhookDelivery from app.engine.labels import ISSUANCE_LABEL @@ -38,6 +45,9 @@ logger = logging.getLogger(__name__) FORMATS = ("generic", "feishu", "wecom", "dingtalk") EVENTS = ("finished", "failed", "awaiting_approval") +#: 格式、事件在报错里的叫法,和界面上的选项一致 +FORMAT_LABEL = {"generic": "通用", "feishu": "飞书", "wecom": "企业微信", "dingtalk": "钉钉"} +EVENT_LABEL = {"finished": "运行结束", "failed": "运行失败", "awaiting_approval": "等待审批"} #: 单次请求的超时(秒) TIMEOUT_S = 10.0 #: 失败后再试 3 次,每次之前等这么久(秒)。4xx(429 除外)不重试:地址或参数错了,重试也一样 @@ -49,10 +59,12 @@ SEAL_WAIT_S = 60 #: 测试用:换成 httpx.MockTransport 接住请求 _transport: httpx.AsyncBaseTransport | None = None -_HEADLINE = {"finished": "运行完成", "failed": "运行失败", "awaiting_approval": "等待审批", "test": "Webhook 测试"} -#: 运行状态在通知里的叫法 -_STATUS_LABEL = {"succeeded": "已完成", "failed": "失败", "interrupted": "停在人工审批", "cancelled": "已取消", - "failed_to_start": "未能发起"} +_HEADLINE = {"finished": "运行完成", "failed": "运行失败", "awaiting_approval": "等待审批"} +#: 运行状态在通知里的叫法,和界面一致。interrupted 分两种,见 _status_label +_STATUS_LABEL = {"succeeded": "已完成", "failed": "失败", "cancelled": "已取消", "failed_to_start": "未能发起"} +#: 挂起的运行要人去处理:不处理,这个任务之后的触发都按「上一次还没结束」跳过 +SUSPENDED_REASON = "运行停在断点上、没有待审批,通常是服务重启所致" +SUSPENDED_ACTION = "请在 AgentLab 的「记录」中继续运行或放弃这次运行;它结束之前,这个定时任务之后的触发都会跳过" # -------------------------------------------------------------------------- @@ -60,30 +72,86 @@ _STATUS_LABEL = {"succeeded": "已完成", "failed": "失败", "interrupted": " # -------------------------------------------------------------------------- +_URL_FORMAT = ("Webhook 地址需要是 http:// 或 https:// 开头的完整地址,例如飞书机器人的 " + "https://open.feishu.cn/open-apis/bot/v2/hook/…") + + def check_url(url: Any) -> str: - """校验 Webhook 地址,返回去掉首尾空白的地址。不对抛 ValueError(中文说明,不回显地址)。""" + """校验 Webhook 地址,返回去掉首尾空白的地址。不对抛 ValueError(中文说明,不回显地址)。 + + 除了 http(s)、有主机名,还要 httpx 真发得出去:端口写错、夹着不可见的字符(从聊天软件复制时常带上零宽空格)、 + 主机名不合 IDNA 规则的,保存时就拦下。以前这些照样存进去,到「测试 Webhook」才在 httpx 里抛出 500 或英文报错。 + """ text = url.strip() if isinstance(url, str) else "" + if not text: + raise ValueError(_URL_FORMAT) + if any(c.isspace() or unicodedata.category(c) in ("Cc", "Cf") for c in text): + raise ValueError("Webhook 地址中有空格或不可见的字符(从聊天软件复制时常会带上),请删除后重新填写") + if len(text) > 2000: + raise ValueError("Webhook 地址过长(超过 2000 个字符),请检查是否粘贴了多余的内容") try: parts = urlsplit(text) except ValueError: - parts = None - if (not text or parts is None or parts.scheme not in ("http", "https") or not parts.hostname - or any(c.isspace() for c in text)): - raise ValueError("Webhook 地址需要是 http:// 或 https:// 开头的完整地址,例如飞书机器人的 " - "https://open.feishu.cn/open-apis/bot/v2/hook/…") - if len(text) > 2000: - raise ValueError("Webhook 地址过长(超过 2000 个字符),请检查是否粘贴了多余的内容") + raise ValueError(_URL_FORMAT) from None + if parts.scheme not in ("http", "https") or not parts.hostname: + raise ValueError(_URL_FORMAT) + try: + port = parts.port + except ValueError: # 不是数字,或者超出 65535 + port = -1 + if port is not None and not 1 <= port <= 65535: + raise ValueError("Webhook 地址中的端口需要是 1 到 65535 之间的数字") + try: + # 照发送时的样子建一个请求(不发出去):httpx 到这一步才按 IDNA 解码主机名 + httpx.Request("POST", text) + except (httpx.InvalidURL, ValueError): # IDNA 的错误是 UnicodeError,也是 ValueError + raise ValueError("Webhook 地址的主机名不符合规范,无法发送,请检查后重新填写") from None return text +#: 群机器人的地址(主机,路径前缀)→ 格式。这些地址只认自己平台的消息格式:格式选错了,平台回错误码,群里收不到 +_BOTS = ( + ("open.feishu.cn", "/open-apis/bot/", "feishu"), + ("open.larksuite.com", "/open-apis/bot/", "feishu"), + ("qyapi.weixin.qq.com", "/cgi-bin/webhook/", "wecom"), + ("oapi.dingtalk.com", "/robot/", "dingtalk"), +) +#: 平台的主机 → 按哪个平台读 HTTP 200 里的错误码。同一主机上不是群机器人的接口(飞书集成平台的 Webhook 触发器 +#: 收任意 JSON)也是同一套错误码约定 +_PLATFORM_HOSTS = {host: fmt for host, _, fmt in _BOTS} + + +def bot_format(url: str) -> str | None: + """这条地址是哪个平台的群机器人(feishu / wecom / dingtalk),不是群机器人的返回 None。""" + try: + parts = urlsplit(url) + except ValueError: + return None + host = (parts.hostname or "").lower() + return next((fmt for bot_host, prefix, fmt in _BOTS if host == bot_host and parts.path.startswith(prefix)), None) + + +def format_problem(fmt: str, url: str) -> str | None: + """格式和地址对不上时的说明(群机器人的地址选了别的格式),对得上返回 None。""" + expected = bot_format(url) + if expected is None or expected == fmt: + return None + label = FORMAT_LABEL[expected] + return f"这是{label}群机器人的地址,只接收{label}的消息格式,Webhook 格式请选择「{label}」" + + def host_of(url: str) -> str: - """主机名(带非默认端口)。日志、报错里只写它。""" + """主机名(带非默认端口)。日志、报错里只写它。端口写错的地址也照样给出主机名。""" try: parts = urlsplit(url) host = parts.hostname or "?" - return f"{host}:{parts.port}" if parts.port else host except ValueError: return "?" + try: + port = parts.port + except ValueError: + return host + return f"{host}:{port}" if port else host def url_hint(url: str) -> str: @@ -146,7 +214,10 @@ _install_filter() @dataclass class Notice: """一条通知要说的事。event 是 finished / failed / awaiting_approval / test;status 是运行状态,没能发起的是 - failed_to_start。没有的项为 None。""" + failed_to_start。没有的项为 None。 + + suspended:运行停在断点上、没有待审批(通常是服务重启挂起的),按 failed 事件推。error_node:失败节点的名字。 + fmt、signed 只给测试消息用:说明里只说这条消息证明得了的事。""" event: str schedule_id: str @@ -165,6 +236,10 @@ class Notice: finished_at: datetime | None = None reason: str | None = None pending_approvals: int | None = None + suspended: bool = False + error_node: str | None = None + fmt: str | None = None + signed: bool = False def _local(moment: datetime | None, tz: str, *, seconds: bool = True) -> str | None: @@ -181,39 +256,70 @@ def _local(moment: datetime | None, tz: str, *, seconds: bool = True) -> str | N def message(notice: Notice) -> tuple[str, list[tuple[str, str]], str | None]: """(标题, [(项, 内容)…], 运行链接)。三种文本格式都从这一份排出来。""" - if notice.event == "failed" and notice.status == "failed_to_start": + if notice.event == "test": + return f"定时任务「{notice.schedule_name}」Webhook 测试", [("说明", _test_text(notice))], None + if notice.status == "failed_to_start": headline = "未能发起运行" + elif notice.suspended: + headline = "运行已挂起" else: headline = _HEADLINE.get(notice.event, notice.event) title = f"定时任务「{notice.schedule_name}」{headline}" lines: list[tuple[str, str]] = [] - if notice.event == "test": - lines.append(("说明", "这是一条测试消息,收到说明 Webhook 地址和签名配置正确")) - return title, lines, None if notice.status: - lines.append(("状态", _STATUS_LABEL.get(notice.status, notice.status))) + lines.append(("状态", _status_label(notice))) if notice.tier: gaps = f"(缺口 {notice.gaps} 条)" if notice.gaps is not None else "" lines.append(("出具档位", f"{ISSUANCE_LABEL.get(notice.tier, notice.tier)}{gaps}")) if notice.pending_approvals: - lines.append(("待审批", f"{notice.pending_approvals} 项,请到 AgentLab 处理")) + lines.append(("待审批", f"{notice.pending_approvals} 项,请到 AgentLab 处理;运行结束之前," + "这个定时任务之后的触发都会跳过")) if notice.status == "failed_to_start" and notice.reason: lines.append(("原因", notice.reason)) + elif notice.suspended: + lines += [("原因", SUSPENDED_REASON), ("处理", SUSPENDED_ACTION)] elif notice.status == "failed": - lines.append(("原因", "运行失败,详情请查看运行记录")) + # 不写原始报错(可能带着查询到的数据),写失败节点的名字和去哪里处理 + if notice.error_node: + lines.append(("失败节点", f"「{notice.error_node}」")) + lines.append(("处理", "请在 AgentLab 的运行记录中查看报错,修正后可继续运行")) if notice.fire_at is not None: - lines.append(("计划触发", f"{_local(notice.fire_at, notice.timezone, seconds=False)}({notice.timezone})")) + when = f"{_local(notice.fire_at, notice.timezone, seconds=False)}({notice.timezone})" + # 立即运行不写「计划触发」:群里的人会以为报告按时出了 + if notice.fire_kind == "manual": + lines.append(("手动触发", when)) + elif notice.fire_kind == "misfire": + lines.append(("计划触发", f"{when},错过后补发")) + else: + lines.append(("计划触发", when)) if notice.started_at is not None: lines.append(("开始", _local(notice.started_at, notice.timezone) or "")) if notice.finished_at is not None: lines.append(("结束", _local(notice.finished_at, notice.timezone) or "")) if notice.manifest_hash: - lines.append(("封存哈希", notice.manifest_hash[:12])) + lines.append(("清单哈希", notice.manifest_hash[:12])) if notice.run_id: - lines.append(("运行编号", notice.run_id)) + # 和界面上的短编号一样写(#前 6 位);完整 id 在运行链接里 + lines.append(("运行", f"#{notice.run_id[:6]}")) return title, lines, notice.run_url +def _status_label(notice: Notice) -> str: + if notice.status == "interrupted": + return "等待审批" if notice.pending_approvals else "已挂起,可继续运行" + return _STATUS_LABEL.get(notice.status or "", notice.status or "") + + +def _test_text(notice: Notice) -> str: + """测试消息的说明只说收到它证明得了的事:飞书、钉钉的机器人开了签名校验时,验签不过就不收;通用格式的接收端验没 + 验签这边不知道;企业微信不签名,填了密钥也不提。""" + if notice.signed and notice.fmt in ("feishu", "dingtalk"): + return "这是一条测试消息。收到即表示 Webhook 地址可用;机器人开启了签名校验的,签名密钥也配置正确" + if notice.signed: + return "这是一条测试消息。收到即表示 Webhook 地址可用;请求头带有签名,接收端可据此核对签名密钥" + return "这是一条测试消息。收到即表示 Webhook 地址配置正确" + + def _text(notice: Notice) -> str: title, lines, link = message(notice) out = [f"【AgentLab】{title}", *(f"{k}:{v}" for k, v in lines)] @@ -268,7 +374,9 @@ def _generic(notice: Notice, now: datetime) -> dict[str, Any]: "manifest_hash": notice.manifest_hash[:12] if notice.manifest_hash else None, "started_at": _iso(notice.started_at), "finished_at": _iso(notice.finished_at), - "reason": notice.reason if notice.status == "failed_to_start" else None, + "reason": (notice.reason if notice.status == "failed_to_start" + else f"{SUSPENDED_REASON}。{SUSPENDED_ACTION}" if notice.suspended else None), + "error_node": notice.error_node, "sent_at": _iso(now), } @@ -367,6 +475,11 @@ async def _post_once(client: httpx.AsyncClient, fmt: str, target: str, headers: return Outcome(False, None, 1, f"无法连接 {host}", retryable=True) except httpx.HTTPError as e: return Outcome(False, None, 1, f"请求 {host} 失败({type(e).__name__})", retryable=True) + except httpx.InvalidURL: + # 保存时的地址校验加严之前存进库的地址:httpx 抛的不是 HTTPError。重试也一样,请人改地址 + return Outcome(False, None, 1, f"发往 {host} 的 Webhook 地址无效,请重新填写地址") + except Exception as e: # noqa: BLE001 - 端口越界之类在传输层才抛(不是 HTTPError),不能让测试接口回 500 + return Outcome(False, None, 1, f"无法向 {host} 发送请求({type(e).__name__}),请检查 Webhook 地址") finally: _SENDING.reset(token) status = response.status_code @@ -375,7 +488,7 @@ async def _post_once(client: httpx.AsyncClient, fmt: str, target: str, headers: detail = _platform_message(reply) text = f"{host} 返回 HTTP {status}" + (f":{detail}" if detail else "") return Outcome(False, status, 1, _scrub(text, url, secret)[:300], retryable=status == 429 or status >= 500) - if problem := _platform_error(fmt, reply): + if problem := _platform_error(fmt, url, reply): return Outcome(False, status, 1, _scrub(f"{host} {problem}", url, secret)[:300], retryable=True) return Outcome(True, status, 1) @@ -395,13 +508,20 @@ def _platform_message(reply: dict[str, Any] | None) -> str: return str(text)[:200] -def _platform_error(fmt: str, reply: dict[str, Any] | None) -> str | None: +def _platform_error(fmt: str, url: str, reply: dict[str, Any] | None) -> str | None: """HTTP 200 里的平台错误码:飞书 code(老接口 StatusCode),企业微信、钉钉 errcode。0 是成功。 - 说明里写主机名(调用方拼在前面),不写平台名:主机名已经说明是哪个平台,配错了格式时也不会张冠李戴。""" - if not reply or fmt == "generic": + 按地址的主机认平台,不看选的格式:以前只看所选格式自己的字段(通用格式完全不看),飞书的地址选了通用格式、 + 平台回 code 19002,照样记「已送达」。主机不是这几个平台的(自建接收端、代理转发)才按所选格式读,通用格式 + 不解读——自建接收端的 code 字段是什么意思由它自己定。说明里写主机名(调用方拼在前面),不写平台名。""" + try: + host = (urlsplit(url).hostname or "").lower() + except ValueError: + host = "" + platform = _PLATFORM_HOSTS.get(host) or (fmt if fmt != "generic" else None) + if not reply or platform is None: return None - code = reply.get("code", reply.get("StatusCode")) if fmt == "feishu" else reply.get("errcode") + code = reply.get("code", reply.get("StatusCode")) if platform == "feishu" else reply.get("errcode") if code in (None, 0, "0"): return None detail = _platform_message(reply) @@ -425,8 +545,8 @@ async def _record(schedule_id: str, fire_id: str | None, run_id: str | None, eve #: 运行到了这些状态才可能要推送 _WATCHED = ("succeeded", "failed", "cancelled", "interrupted") -#: 运行状态 → 事件。cancelled 是有人手动停的,不推 -_EVENT_OF = {"succeeded": "finished", "failed": "failed", "cancelled": None, "interrupted": "awaiting_approval"} +#: 结束的运行 → 事件。cancelled 是有人手动停的,不推。interrupted 分停在审批和挂起两种,见 _state_of +_EVENT_OF = {"succeeded": "finished", "failed": "failed", "cancelled": None} @dataclass @@ -442,6 +562,7 @@ async def notify_due(now: datetime | None = None) -> int: 处理过的状态记在 notified_status 上,用条件 UPDATE 认领(WHERE notified_status = 旧值):两拍撞在一起也只 推一次。没订阅这个事件、任务没配 Webhook 的也记下已处理——之后再配上 Webhook,不会把旧的补推一遍。 + 停在断点上的运行每一拍都要重新看(见 _state_of):它的已处理状态带着第几段执行,和运行状态本身对不上。 """ now = now or utcnow() jobs: list[_Job] = [] @@ -465,7 +586,8 @@ async def notify_due(now: datetime | None = None) -> int: base_url = (await notify_settings(session))["public_base_url"] for fire, run in rows: state, event, pending = await _state_of(session, fire, run, now) - if state is None or not await _mark(fire.id, fire.notified_status, state): + if state is None or state == fire.notified_status \ + or not await _mark(fire.id, fire.notified_status, state): continue schedule = await session.get(Schedule, fire.schedule_id) url = decrypt(schedule.webhook_url) if schedule and schedule.webhook_url else None @@ -481,22 +603,39 @@ async def notify_due(now: datetime | None = None) -> int: async def _state_of(session: Any, fire: ScheduleFire, run: Run | None, now: datetime) -> tuple[str | None, str | None, int]: - """(要记下的已处理状态, 要推送的事件, 待审批数)。还不能下结论(没封存、挂起但没有待审批)时状态为 None。""" + """(要记下的已处理状态, 要推送的事件, 待审批数)。还不能下结论(没封存、审批刚批完还没接着执行)时状态为 None。 + + 停在断点上(interrupted)的分两种,各推一次: + - 有待审批:停在审批,推 awaiting_approval。批完接着执行、又停在下一个审批点时要再推,所以已处理状态里带着这是 + 第几段执行(每段执行从一条 run.started 开始),同一个审批点不重复推。 + - 没有待审批:通常是服务重启挂起的。它不会自己接着执行,这个任务之后的触发都会按「上一次还没结束」跳过, + 不说一声任务就此静默停摆——按 failed 推一次,写明要去继续运行或放弃。 + 审批批完到运行改回 running 之间,审批是 answered、还没有 resolved:那不是挂起,等下一拍。 + """ if fire.status == "failed_to_start": return "failed_to_start", "failed", 0 if run is None: return None, None, 0 if run.status in ("succeeded", "failed", "cancelled"): - finished = run.finished_at or now - if not run.manifest_hash and (now - service.as_utc(finished)).total_seconds() < SEAL_WAIT_S: + # runner 先写终态和结束时间、再封存,两步之间被这一拍撞上就等。没有结束时间的(服务重启时还在排队、被改成 + # failed 的)永远不会封存:按它最后一次更新的时间等,以前按「现在」等,每一拍都等、永远不推 + since = run.finished_at or run.updated_at or fire.created_at + if not run.manifest_hash and (now - service.as_utc(since)).total_seconds() < SEAL_WAIT_S: return None, None, 0 return run.status, _EVENT_OF[run.status], 0 - if run.status == "interrupted": - # 服务重启挂起的运行也是 interrupted,但没人要审批:确有待审批才算停在审批 - pending = await session.scalar(select(func.count()).select_from(Approval).where( - Approval.run_id == run.id, Approval.status == "pending")) or 0 - return ("interrupted", "awaiting_approval", pending) if pending else (None, None, 0) - return None, None, 0 + if run.status != "interrupted": + return None, None, 0 + counts = dict((await session.execute( + select(Approval.status, func.count()).where( + Approval.run_id == run.id, Approval.status.in_(("pending", "answered"))) + .group_by(Approval.status))).all()) + segment = await session.scalar(select(func.count()).select_from(RunEvent).where( + RunEvent.run_id == run.id, RunEvent.type == EventType.RUN_STARTED.value)) or 0 + if counts.get("pending"): + return f"interrupted:approval:{segment}", "awaiting_approval", counts["pending"] + if counts.get("answered"): + return None, None, 0 + return f"interrupted:held:{segment}", "failed", 0 async def _mark(fire_id: str, old: str | None, state: str) -> bool: @@ -523,9 +662,22 @@ async def _notice(session: Any, schedule: Schedule, fire: ScheduleFire, run: Run finished_at=run.finished_at if run else None, reason=fire.reason if fire.status == "failed_to_start" else None, pending_approvals=pending or None, + suspended=run is not None and run.status == "interrupted" and not pending, + error_node=_failed_node(run) if run is not None and run.status == "failed" else None, ) +def _failed_node(run: Run) -> str | None: + """失败节点在图上的名字(没有名字的给节点 id)。节点名是搭图时起的,不带查询到的数据。""" + if not run.error_node_id: + return None + for node in (run.graph or {}).get("nodes") or []: + if isinstance(node, dict) and node.get("id") == run.error_node_id: + label = (node.get("data") or {}).get("label") if isinstance(node.get("data"), dict) else None + return str(label or run.error_node_id)[:80] + return None + + async def _issuance_of(session: Any, run: Run) -> tuple[str | None, int | None]: """出具档位和缺口条数:成果上的 _issuance,取不到时退回最后一条 issuance 事件(同 api/evidence.py 的读法)。""" issuance = (run.output or {}).get("_issuance") if isinstance(run.output, dict) else None @@ -549,19 +701,24 @@ async def _deliver(job: _Job) -> None: async def send_test(schedule_id: str) -> dict[str, Any]: """按任务存的 Webhook 发一条测试消息(不重试),记一条 event=test 的推送记录。任务不存在抛 KeyError, - 没配 Webhook 抛 ValueError。返回 {ok, http_status, message}。""" + 没配 Webhook、地址解不开抛 ValueError。 + + 返回 {ok, http_status, message}。message 是 ok、http_status 之外的补充说明:失败时是原因(只有主机名),送达时 + 提醒去接收端看一眼。界面在「测试消息已送达(HTTP 200)」「测试消息发送失败」后面接它,这里不再重复那半句。""" async with SessionLocal() as session: schedule = await session.get(Schedule, schedule_id) if schedule is None: raise KeyError(schedule_id) - url = decrypt(schedule.webhook_url) if schedule.webhook_url else None - if not url or schedule.webhook_format not in FORMATS: + if not schedule.webhook_url or schedule.webhook_format not in FORMATS: raise ValueError("这个定时任务没有配置 Webhook,请先填写地址并保存") + url = decrypt(schedule.webhook_url) + if not url: + raise ValueError("Webhook 地址无法解密(加密密钥可能已更换),请重新填写地址并保存") fmt, secret = schedule.webhook_format, decrypt(schedule.webhook_secret) notice = Notice(event="test", schedule_id=schedule.id, schedule_name=schedule.name, - timezone=schedule.timezone) + timezone=schedule.timezone, fmt=fmt, signed=bool(secret) and fmt != "wecom") outcome = await send(fmt, url, secret, notice, retry=False) await _record(schedule_id, None, None, "test", outcome) if outcome.ok: - return {"ok": True, "http_status": outcome.http_status, "message": f"测试消息已送达(HTTP {outcome.http_status})"} - return {"ok": False, "http_status": outcome.http_status, "message": f"推送失败:{outcome.error}"} + return {"ok": True, "http_status": outcome.http_status, "message": "请到接收端确认已收到"} + return {"ok": False, "http_status": outcome.http_status, "message": outcome.error or "发送失败"} diff --git a/backend/tests/test_scheduler_loop.py b/backend/tests/test_scheduler_loop.py index 60e8c947..94dab830 100644 --- a/backend/tests/test_scheduler_loop.py +++ b/backend/tests/test_scheduler_loop.py @@ -4,29 +4,35 @@ 触发记录 (任务, fire_at, kind) 唯一,撞上约束的不再发起 - 发起走 engine/launch.py 的 launch_run:记在「定时任务:<名字>」名下、带 schedule_id,输入按 fire_at 渲染; 正式运行用当前发布版本,没有发布版本被拒,拒绝的原话记成 failed_to_start;别的异常记通用说明、详情进日志 -- 上一次的运行还没结束(排队、运行中、停在审批 / 挂起都算)记 skipped_overlap +- 上一次的运行还没结束(排队、运行中、停在审批 / 挂起都算)记 skipped_overlap,原因按挡路的运行写清楚:仍在 + 进行中、正在等待审批、已挂起(要去「记录」里继续运行或放弃);挂起的那次按失败推一次通知,不是每次触发都推 - 错过的触发:最近一次在宽限内补一次(kind=misfire),更早的积压只记一条跳过;最近一次也超出宽限就记一条 skipped_misfire,推进到未来的下一次,不补积压 -- start / stop:起一个 asyncio 任务按拍执行,stop 干净地停下;AGENTLAB_SCHEDULER 关着时 start 不起循环 +- start / stop:起一个 asyncio 任务按拍执行,stop 干净地停下(不落下新起的通知扫描,接收端挂住时宽限一过就取消); + AGENTLAB_SCHEDULER 关着时 start 不起循环 工作流、字段、任务名都是虚构的。 """ from __future__ import annotations import asyncio +import json import logging +import time import uuid from datetime import datetime, timedelta, timezone +import httpx import pytest from sqlalchemy import delete, func, select from app import scheduler from app.core.config import Settings +from app.core.crypto import encrypt from app.db.base import SessionLocal -from app.db.models import Run, Schedule, ScheduleFire, Workflow, WorkflowVersion +from app.db.models import Approval, Run, Schedule, ScheduleFire, Workflow, WorkflowVersion from app.engine.runner import run_manager -from app.scheduler import loop, service +from app.scheduler import cron, loop, service, webhook UTC = timezone.utc #: 上海 2026-10-05(周一)09:00 @@ -171,22 +177,79 @@ async def test_disabled_schedule_is_not_fired(): assert await fires(sid) == [] and await runs_of(sid) == [] -@pytest.mark.parametrize("status", ["queued", "running", "interrupted"]) -async def test_previous_run_still_going_means_skip(status): +@pytest.mark.parametrize("status, approval, says", [ + ("queued", None, "仍在进行中"), + ("running", None, "仍在进行中"), + ("interrupted", "pending", "正在等待审批"), + # 批了、还没接着执行(审批记成 answered 到运行改回 running 之间):仍是停在审批,不是挂起 + ("interrupted", "answered", "正在等待审批"), + # 服务重启挂起的:没有待审批,也不会自己接着执行,原因要说清楚该去做什么 + ("interrupted", None, "已挂起(通常是服务重启所致),请在「记录」中继续运行或放弃这次运行"), +]) +async def test_previous_run_still_going_means_skip(status, approval, says): wf = await workflow() sid = await schedule(wf) async with SessionLocal() as session: busy = Run(workflow_id=wf, workflow_name="门店周报", status=status, schedule_id=sid, run_class="formal") session.add(busy) + await session.flush() + if approval: + session.add(Approval(run_id=busy.id, node_id="gate", title="审核周报", status=approval)) await session.commit() await loop.fire_due(NINE + timedelta(seconds=5)) [fire] = await fires(sid) assert (fire.status, fire.run_id) == ("skipped_overlap", None) - assert busy.id[:8] in fire.reason and "跳过" in fire.reason + # 运行编号和界面一样写成 #前 6 位;不再说「按设置」——重叠处理只有跳过一档,用户没选过什么 + assert f"上一次运行 #{busy.id[:6]} " in fire.reason and says in fire.reason, fire.reason + assert fire.reason.endswith("本次触发已跳过") and "按设置" not in fire.reason assert [r.id for r in await runs_of(sid)] == [busy.id] assert await next_of(sid) == NINE + timedelta(days=1) +async def test_suspended_run_is_reported_once_and_later_fires_say_what_to_do(monkeypatch): + """上一次运行在服务重启时被挂起:它不会自己接着执行,按规则之后的触发都跳过。跳过的原因写明要去继续或放弃, + 订阅了失败通知的推一次(不是每次触发都推);有人放弃了它之后,下一次照常发起""" + requests: list[httpx.Request] = [] + + def receive(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json={}) + + monkeypatch.setattr(webhook, "_transport", httpx.MockTransport(receive)) + wf = await workflow() + sid = await schedule(wf) + async with SessionLocal() as session: + await session.execute(Schedule.__table__.update().where(Schedule.id == sid).values( + webhook_format="generic", webhook_url=encrypt("https://hooks.example.com/agentlab/inbox-5c1d"), + webhook_events=["finished", "failed", "awaiting_approval"])) + held = Run(workflow_id=wf, workflow_name="门店周报", status="interrupted", schedule_id=sid, run_class="formal", + error="服务重启,运行已挂起,可从断点继续运行") + session.add(held) + await session.flush() + session.add(ScheduleFire(schedule_id=sid, fire_at=NINE - timedelta(days=1), kind="scheduled", + status="started", run_id=held.id)) + await session.commit() + for day in range(3): + now = NINE + timedelta(days=day, seconds=5) + await loop.fire_due(now) + await webhook.notify_due(now) + skipped = [f for f in await fires(sid) if f.status == "skipped_overlap"] + assert len(skipped) == 3 + assert all(f"#{held.id[:6]} 已挂起" in f.reason and "继续运行" in f.reason for f in skipped) + [pushed] = requests + body = json.loads(pushed.content) + assert (body["event"], body["status"], body["run_id"]) == ("failed", "interrupted", held.id) + assert "挂起" in body["title"] and "继续运行" in body["text"] + # 放弃之后,下一次按时发起 + async with SessionLocal() as session: + await session.execute(Run.__table__.update().where(Run.id == held.id).values(status="cancelled")) + await session.commit() + await loop.fire_due(NINE + timedelta(days=3, seconds=5)) + newest = (await fires(sid))[-1] + assert (newest.status, newest.fire_at) == ("started", NINE + timedelta(days=3)) + await settle(newest.run_id) + + # -------------------------------------------------------------------------- # 错过的触发 # -------------------------------------------------------------------------- @@ -307,18 +370,20 @@ async def test_start_does_nothing_when_switched_off(): async def test_loop_ticks_until_stopped(): - sid = await schedule(await workflow(), next_run_at=datetime.now(UTC) - timedelta(seconds=1), cron="* * * * *") + # 下一次放在整分上(和 cron 对齐):放在「现在 - 1 秒」的话,这一秒里恰好跨过整分时会多记一条错过的跳过 + due = cron.last_due("* * * * *", "UTC", datetime.now(UTC)) + sid = await schedule(await workflow(), next_run_at=due, cron="* * * * *") on = Settings(_env_file=None, scheduler=True) assert await scheduler.start(interval=0.05, config=on) is True assert scheduler.running() for _ in range(200): - if await fires(sid): + if any(f.status == "started" for f in await fires(sid)): break await asyncio.sleep(0.02) await scheduler.stop() assert scheduler.running() is False - [fire] = await fires(sid) - assert fire.status == "started" + # 只认发起的那一条:建任务和第一拍之间跨过整分时,前面还会有一条错过的跳过,那不是这里要测的 + [fire] = [f for f in await fires(sid) if f.status == "started"] await settle(fire.run_id) # 停下之后不再有拍:把下一次拨回过去也不会再发起 async with SessionLocal() as session: @@ -339,3 +404,61 @@ async def test_start_twice_keeps_one_loop(): assert loop.runner.task is first await scheduler.stop() assert first.done() + + +def _notify_tasks() -> list[asyncio.Task]: + return [t for t in asyncio.all_tasks() if t.get_name() == "agentlab-scheduler-notify" and not t.done()] + + +async def test_stop_in_the_middle_of_a_tick_leaves_no_notify_task_behind(monkeypatch): + """stop 撞在一拍的发起当中:这一拍收尾时不再新起通知扫描。以前 stop 先把手上的通知任务清掉再等,循环收尾时 + 看见没有就又起了一个,它不在 stop 等的名单里,日志说「已停止」之后还在推送""" + started = asyncio.Event() + + async def slow_fire_due(now=None): + started.set() + await asyncio.sleep(0.2) + return [] + + async def notify_due(now=None): + await asyncio.sleep(5) + return 0 + + monkeypatch.setattr(loop, "fire_due", slow_fire_due) + monkeypatch.setattr(webhook, "notify_due", notify_due) + runner = loop._Runner() + await runner.start(interval=0.05) + await started.wait() + await runner.stop() + assert runner.task is None and runner.notify is None + assert _notify_tasks() == [] + + +async def test_stop_does_not_wait_out_a_hanging_webhook(monkeypatch): + """接收端挂住、通知扫描还在重试:stop 不等满超时加重试,宽限一过就取消。关停时后面还有 run_manager 要平稳挂起 + 运行,docker stop 默认只给 10 秒""" + hanging = asyncio.Event() + + async def fire_due(now=None): + return [] + + async def notify_due(now=None): + hanging.set() + await asyncio.sleep(30) + return 0 + + monkeypatch.setattr(loop, "fire_due", fire_due) + monkeypatch.setattr(webhook, "notify_due", notify_due) + monkeypatch.setattr(loop, "NOTIFY_GRACE_S", 0.1) + runner = loop._Runner() + await runner.start(interval=0.05) + await asyncio.wait_for(hanging.wait(), timeout=5) + began = time.monotonic() + await runner.stop() + assert time.monotonic() - began < 2 + assert _notify_tasks() == [] + + +def test_stop_fits_in_the_shutdown_budget(): + """两件事并行等,最坏也就是较长的那个宽限:给关停时的 run_manager.shutdown 留出大半的 docker stop 时间""" + assert max(loop.STOP_GRACE_S, loop.NOTIFY_GRACE_S) <= 3 diff --git a/backend/tests/test_scheduler_webhook.py b/backend/tests/test_scheduler_webhook.py index 2a90f8b4..98c41404 100644 --- a/backend/tests/test_scheduler_webhook.py +++ b/backend/tests/test_scheduler_webhook.py @@ -3,12 +3,14 @@ - 四种格式的请求体和签名:通用 JSON(配了密钥时 X-AgentLab-Signature = HMAC-SHA256(密钥, 时间戳.请求体))、 飞书文本(timestamp + sign,sign 是以「时间戳\\n密钥」为键的 HMAC-SHA256)、企业微信 markdown(key 在地址里)、 钉钉 markdown(地址上加 timestamp 和 sign) -- 通知里有任务名、状态、出具档位、缺口条数、运行链接(对外访问地址 + /runs/{id})、封存哈希前 12 位、开始结束 - 时间;不放报告里的具体数字,不放失败的原始报错 -- 每一拍查已发起的触发:运行状态变了、任务订阅了这个事件、这个状态还没处理过才推送;停在审批只在确有待审批时 - 算;结束的运行等封存好再推;没订阅、没配 Webhook 的也记下已处理,之后再配上不会补发旧的 -- 失败重试 3 次(4xx 不重试),每次推送记一条 WebhookDelivery;日志和错误里只有主机名,没有地址、key、 - access_token、密钥(httpx 自己的请求日志也一样) +- 通知里有任务名、状态、出具档位、缺口条数、运行链接(对外访问地址 + /runs/{id})、清单哈希前 12 位、开始结束 + 时间,失败时写失败节点的名字;不放报告里的具体数字,不放失败的原始报错。立即运行写「手动触发」,补发标明补发 +- 每一拍查已发起的触发:运行状态变了、任务订阅了这个事件、这个状态还没处理过才推送。停在审批只在确有待审批时 + 算,每停一个审批点推一次;挂起(没有待审批)按失败推一次;结束的运行等封存好再推(没有结束时间、不会再封存的 + 按最后更新时间等);没订阅、没配 Webhook 的也记下已处理,之后再配上不会补发旧的 +- 失败重试 3 次(4xx 不重试),每次推送记一条 WebhookDelivery;平台的错误码按地址的主机认,不看选的格式; + 日志和错误里只有主机名,没有地址、key、access_token、密钥(httpx 自己的请求日志也一样) +- 地址校验:httpx 发不出去的(端口写错、不可见字符、主机名不合 IDNA 规则)保存时就拦下 用 httpx.MockTransport 接住请求,不连外网。任务名、地址、密钥、数字都是虚构的。 """ @@ -29,7 +31,7 @@ from sqlalchemy import delete, select from app.core.crypto import encrypt from app.db.base import SessionLocal -from app.db.models import Approval, Run, Schedule, ScheduleFire, Setting, WebhookDelivery +from app.db.models import Approval, Run, RunEvent, Schedule, ScheduleFire, Setting, WebhookDelivery from app.scheduler import webhook UTC = timezone.utc @@ -151,15 +153,42 @@ def test_dingtalk_markdown_and_signed_address(): def test_awaiting_approval_and_failed_to_start_texts(): - waiting = webhook.message(notice(event="awaiting_approval", status="interrupted", tier=None, gaps=None, - pending_approvals=2, finished_at=None)) - assert "等待审批" in waiting[0] and any("2 项" in value for _, value in waiting[1]) + title, lines, _ = webhook.message(notice(event="awaiting_approval", status="interrupted", tier=None, gaps=None, + pending_approvals=2, finished_at=None)) + # 状态的叫法和界面一致:「等待审批」,审批也可能来自 Agent、工具的审批策略,不一定是人工审批节点 + assert "等待审批" in title and dict(lines)["状态"] == "等待审批" + assert dict(lines)["待审批"].startswith("2 项") refused = webhook.message(notice(event="failed", status="failed_to_start", run_id=None, run_url=None, tier=None, gaps=None, manifest_hash=None, started_at=None, finished_at=None, reason="「门店周报」还没有发布版本,请先在编排页发布")) assert "未能发起" in refused[0] and any("还没有发布版本" in value for _, value in refused[1]) +def test_trigger_line_follows_the_kind_of_fire(): + """立即运行不写「计划触发」(群里的人会以为报告按时出了),补发标明是补发""" + manual = webhook._text(notice(fire_kind="manual")) + assert "手动触发:2026-10-05 09:00(Asia/Shanghai)" in manual and "计划触发" not in manual + caught_up = webhook._text(notice(fire_kind="misfire")) + assert "计划触发:2026-10-05 09:00(Asia/Shanghai),错过后补发" in caught_up + assert "计划触发:2026-10-05 09:00(Asia/Shanghai)\n" in webhook._text(notice()) + + +def test_failed_run_text_points_to_the_node_without_repeating_itself(): + title, lines, _ = webhook.message(notice(event="failed", status="failed", tier=None, gaps=None, + error_node="查询门店销售")) + shown = dict(lines) + assert title.endswith("运行失败") and shown["状态"] == "失败" + assert shown["失败节点"] == "「查询门店销售」" and "继续运行" in shown["处理"] + # 不再写「原因:运行失败,详情请查看运行记录」这种同义反复 + assert "原因" not in shown + + +def test_hash_and_run_id_are_named_like_the_run_page(): + shown = dict(webhook.message(notice())[1]) + assert shown["清单哈希"] == "a1b2c3d4e5f6" and shown["运行"] == "#r01234" + assert "封存哈希" not in shown and "运行编号" not in shown + + def test_url_hint_shows_only_host_and_last_four(): assert webhook.url_hint(FEISHU) == "open.feishu.cn/…5b4c" assert webhook.url_hint(WECOM) == "qyapi.weixin.qq.com/…6666" @@ -173,6 +202,32 @@ def test_bad_addresses_are_refused(url): assert "Webhook 地址" in str(caught.value) +@pytest.mark.parametrize("url", [ + "https://hooks.example.com:abc/inbox-7f3a", # 端口不是数字 + "https://hooks.example.com:99999/inbox-7f3a", # 端口越界 + "https://hooks.example.com:0/inbox-7f3a", + "https://hooks.exa​mple.com/inbox-7f3a", # 从聊天软件粘贴带上的零宽空格 + "https://hooks.example.com/inbox?key=ab\x7fcd", # 控制字符 + "https://xn--bad.example.com/inbox-7f3a", # 不合 IDNA 规则的主机名 +]) +def test_addresses_httpx_cannot_send_are_refused_in_chinese(url): + """校验以前只看 scheme 和主机名:这些地址照样存进去,到「测试 Webhook」才在 httpx 里炸成 500 或英文报错""" + with pytest.raises(ValueError) as caught: + webhook.check_url(url) + text = str(caught.value) + assert text.startswith("Webhook 地址") and "Invalid" not in text and "Codepoint" not in text + assert "7f3a" not in text and "abcd" not in text # 不回显地址 + + +def test_platform_hosts_need_their_own_format(): + assert webhook.bot_format(FEISHU) == "feishu" and webhook.bot_format(WECOM) == "wecom" + assert webhook.bot_format(DINGTALK) == "dingtalk" + assert webhook.bot_format("https://open.larksuite.com/open-apis/bot/v2/hook/0f1e2d3c") == "feishu" + # 同一主机上不是群机器人的地址(飞书集成平台的 Webhook 触发器收任意 JSON)、自建接收端:不限格式 + assert webhook.bot_format("https://open.feishu.cn/anycross/trigger/callback/MDk2NTc0") is None + assert webhook.bot_format(GENERIC) is None + + # -------------------------------------------------------------------------- # 推送与重试 # -------------------------------------------------------------------------- @@ -211,6 +266,36 @@ async def test_platform_error_code_counts_as_failure(recorder): assert "310000" in (await webhook.send("dingtalk", DINGTALK, SECRET, notice())).error +@pytest.mark.parametrize("fmt, url, reply", [ + ("generic", FEISHU, {"code": 19002, "msg": "params error, msg_type need"}), + ("wecom", FEISHU, {"code": 19002, "msg": "params error, msg_type need"}), + ("dingtalk", FEISHU, {"code": 19002, "msg": "params error, msg_type need"}), + ("generic", WECOM, {"errcode": 40008, "errmsg": "invalid message type"}), + ("feishu", WECOM, {"errcode": 40008, "errmsg": "invalid message type"}), + ("generic", DINGTALK, {"errcode": 40035, "errmsg": "missing parameter"}), + ("feishu", DINGTALK, {"errcode": 40035, "errmsg": "missing parameter"}), +]) +async def test_platform_error_codes_are_read_by_host_whatever_the_format(recorder, fmt, url, reply): + """三个平台出错都回 HTTP 200、错误码在响应体里。以前只看所选格式自己的字段(通用格式完全不看):格式和地址 + 对不上时测试显示「已送达」,正式推送也记成功,群里却什么都没收到""" + recorder.replies = [httpx.Response(200, json=reply)] + outcome = await webhook.send(fmt, url, None, notice(), retry=False) + assert not outcome.ok and "返回错误码" in outcome.error, outcome + + +async def test_custom_receivers_reply_is_not_second_guessed(recorder): + """自建接收端回什么由它自己定:HTTP 2xx 就算送达,不按平台的错误码去解读它的 code 字段""" + recorder.replies = [httpx.Response(200, json={"code": 7, "msg": "queued"})] + assert (await webhook.send("generic", GENERIC, None, notice(), retry=False)).ok + + +async def test_an_unsendable_saved_address_fails_cleanly(recorder): + """校验加严之前存进库的地址(端口写错之类):httpx 抛的不是 HTTPError,记一次失败、写主机名,不往外抛""" + outcome = await webhook.send("generic", "https://hooks.example.com:abc/inbox-7f3a", None, notice()) + assert (outcome.ok, outcome.attempts) == (False, 1) + assert "hooks.example.com" in outcome.error and "地址" in outcome.error and "7f3a" not in outcome.error + + async def test_connection_errors_name_only_the_host(recorder): recorder.replies = [httpx.ConnectError(f"cannot reach {WECOM}")] outcome = await webhook.send("wecom", WECOM, None, notice()) @@ -353,21 +438,89 @@ async def test_waits_for_the_seal_before_pushing(recorder): assert json.loads(recorder.requests[0].content)["manifest_hash"] == "beef" * 3 -async def test_awaiting_approval_only_with_a_real_pending_approval(recorder): - """服务重启挂起的运行也是 interrupted,但没有待审批:不推。有了待审批推一次,批完跑完再推完成""" - _, fire_id, run_id = await make(run_status="interrupted") - assert await webhook.notify_due() == 0 and (await fire_row(fire_id)).notified_status is None +async def add_approval(run_id: str, status: str = "pending") -> None: async with SessionLocal() as session: - session.add(Approval(run_id=run_id, node_id="gate", title="审核周报", status="pending")) + session.add(Approval(run_id=run_id, node_id="gate", title="审核周报", status=status)) await session.commit() + + +async def new_segment(run_id: str, seq: int) -> None: + """运行又执行了一段(审批后恢复、继续运行都从一条 run.started 开始)""" + async with SessionLocal() as session: + session.add(RunEvent(run_id=run_id, seq=seq, type="run.started", ts=float(seq), data={})) + await session.commit() + + +async def test_awaiting_approval_is_pushed_once_per_stop(recorder): + """停在审批推一次;批完接着执行、又停在下一个审批点,再推一次;跑完推完成。同一个审批点不重复推""" + _, fire_id, run_id = await make(run_status="interrupted") + await new_segment(run_id, 1) + await add_approval(run_id) assert await webhook.notify_due() == 1 data = json.loads(recorder.requests[0].content) assert (data["event"], data["status"], data["pending_approvals"]) == ("awaiting_approval", "interrupted", 1) assert await webhook.notify_due() == 0 + # 批了第一个,执行一段之后停在第二个审批点 + async with SessionLocal() as session: + await session.execute(Approval.__table__.update().where(Approval.run_id == run_id).values(status="resolved")) + await session.commit() + await new_segment(run_id, 2) + await add_approval(run_id) + assert await webhook.notify_due() == 1 + assert await webhook.notify_due() == 0 await set_run(run_id, status="succeeded", finished_at=datetime.now(UTC), manifest_hash="ab" * 32) assert await webhook.notify_due() == 1 - assert [json.loads(r.content)["event"] for r in recorder.requests] == ["awaiting_approval", "finished"] - assert [d.event for d in await deliveries(fire_id)] == ["awaiting_approval", "finished"] + events = ["awaiting_approval", "awaiting_approval", "finished"] + assert [json.loads(r.content)["event"] for r in recorder.requests] == events + assert [d.event for d in await deliveries(fire_id)] == events + + +async def test_suspended_run_is_pushed_once_as_a_failure(recorder): + """服务重启挂起的运行(interrupted、没有待审批)不会自己接着执行,这个任务之后的触发都会跳过:按失败事件推 + 一次,写明要去继续运行或放弃。以前它既不算停在审批、也不算结束,一条都不推,任务就此静默停摆""" + _, fire_id, run_id = await make(run_status="interrupted", fmt="feishu", url=FEISHU) + assert await webhook.notify_due() == 1 + text = json.loads(recorder.requests[0].content)["content"]["text"] + assert "运行已挂起" in text and "状态:已挂起,可继续运行" in text and "通常是服务重启所致" in text + assert "处理:请在 AgentLab 的「记录」中继续运行或放弃这次运行" in text + assert await webhook.notify_due() == 0 + [delivery] = await deliveries(fire_id) + assert (delivery.event, delivery.status) == ("failed", "ok") + # 继续运行之后又挂起了一次(又一次服务重启):再推一次 + await new_segment(run_id, 1) + assert await webhook.notify_due() == 1 + + +async def test_answered_but_not_yet_resumed_is_not_a_suspension(recorder): + """审批刚批完、运行还没改回 running 的那一瞬间:没有 pending 的审批,但也不是挂起,不推挂起""" + _, fire_id, run_id = await make(run_status="interrupted") + await add_approval(run_id, status="answered") + assert await webhook.notify_due() == 0 and recorder.requests == [] + assert (await fire_row(fire_id)).notified_status is None + + +async def test_failed_run_without_finish_time_is_still_notified(recorder): + """服务重启时还在排队的运行被改成 failed:不写结束时间,也永远不会封存。以前按「结束时间 = 现在」等封存, + 每一拍都等、永远不推;现在按它最后一次更新的时间等,过了等封存的时限就推""" + _, fire_id, run_id = await make(run_status="failed", sealed=False, output={}) + await set_run(run_id, finished_at=None) + assert await webhook.notify_due() == 0 + later = datetime.now(UTC) + timedelta(seconds=webhook.SEAL_WAIT_S + 5) + assert await webhook.notify_due(later) == 1 + data = json.loads(recorder.requests[0].content) + assert (data["event"], data["status"], data["manifest_hash"]) == ("failed", "failed", None) + assert (await fire_row(fire_id)).notified_status == "failed" + + +async def test_failed_notice_names_the_failed_node(recorder): + _, _, run_id = await make(run_status="failed", output={}, fmt="wecom", url=WECOM) + await set_run(run_id, error_node_id="q1", error="查询超时:门店 0042 的数据……", graph={"nodes": [ + {"id": "q1", "type": "sql", "data": {"label": "查询门店销售", "config": {}}}], "edges": []}) + await webhook.notify_due() + content = json.loads(recorder.requests[0].content)["markdown"]["content"] + assert "失败节点:「查询门店销售」" in content + # 原始报错不进通知:里面可能带着查询到的数据 + assert "0042" not in content and "查询超时" not in content async def test_failed_delivery_is_recorded_without_secrets(recorder): @@ -386,14 +539,31 @@ async def test_failed_delivery_is_recorded_without_secrets(recorder): async def test_test_message(recorder): sid, _, _ = await make(fmt="feishu", url=FEISHU, secret=SECRET, run_status=None, fire_status="skipped_overlap") result = await webhook.send_test(sid) - assert result == {"ok": True, "http_status": 200, "message": "测试消息已送达(HTTP 200)"} + # message 是 ok、http_status 之外的补充说明:界面在「测试消息已送达(HTTP 200)」后面接它,不再重复那半句 + assert result == {"ok": True, "http_status": 200, "message": "请到接收端确认已收到"} data = json.loads(recorder.requests[0].content) - assert "测试" in data["content"]["text"] and "门店周报" in data["content"]["text"] and data["sign"] + text = data["content"]["text"] + assert "测试" in text and "门店周报" in text and data["sign"] + assert "机器人开启了签名校验的,签名密钥也配置正确" in text and text.count("说明") == 1 async with SessionLocal() as session: [row] = (await session.execute(select(WebhookDelivery).where(WebhookDelivery.schedule_id == sid))).scalars() assert (row.event, row.status, row.fire_id, row.attempts) == ("test", "ok", None, 1) recorder.replies = [httpx.ConnectError("refused")] failed = await webhook.send_test(sid) - assert failed["ok"] is False and failed["http_status"] is None and "open.feishu.cn" in failed["message"] + assert failed == {"ok": False, "http_status": None, "message": "无法连接 open.feishu.cn"} + + +@pytest.mark.parametrize("fmt, url, secret, says, not_says", [ + ("feishu", FEISHU, None, "Webhook 地址配置正确", "签名"), + # 企业微信不签名:填了密钥也不说签名配置正确 + ("wecom", WECOM, SECRET, "Webhook 地址配置正确", "签名"), + # 通用格式带签名,但接收端验没验签这边不知道:只说收到即地址可用、可据此核对 + ("generic", GENERIC, SECRET, "接收端可据此核对签名密钥", "签名密钥也配置正确"), +]) +async def test_test_message_only_claims_what_it_proves(recorder, fmt, url, secret, says, not_says): + sid, _, _ = await make(fmt=fmt, url=url, secret=secret, run_status=None, fire_status="skipped_overlap") + await webhook.send_test(sid) + body = recorder.requests[0].content.decode() + assert says in body and not_says not in body.split("说明", 1)[1] diff --git a/backend/tests/test_schedules_api.py b/backend/tests/test_schedules_api.py index cc44788c..fe4eb26d 100644 --- a/backend/tests/test_schedules_api.py +++ b/backend/tests/test_schedules_api.py @@ -4,8 +4,10 @@ misfire_grace_s, webhook: {format, url_hint, has_secret, events} | null, next_run_at, last_fire, description_text, created_by, created_at, updated_at, version};Webhook 地址和密钥加密存放,接口只给主机和末 4 位 - Fire:{id, fire_at, kind, status, run_id, run_status, reason, notified: [{event, status, at}], created_at} -- 创建、修改时校验 cron、时区、工作流存在、正式运行要有发布版本、输入模板;不对回 422 {detail(中文), field} -- PATCH 带 version,对不上回 409;webhook.url 省略表示不改,空串表示删除 Webhook +- 创建、修改时校验 cron、时区、工作流存在、正式运行要有发布版本、输入模板;不对回 422 {detail(中文), field}, + 说明里不回显 Python 的值;群机器人的地址要配它平台的格式 +- PATCH 带 version,对不上回 409(两人同时保存时靠条件 UPDATE);webhook.url 省略表示不改,空串表示删除 + Webhook;看值变没变:整份表单回写不重新校验没改的工作流,也不动到点了的下一次触发 - 预览下几次触发和渲染后的输入;变量清单;立即运行(同一套重叠规则);触发历史翻页;测试 Webhook 工作流、任务名、地址、密钥都是虚构的。 @@ -24,13 +26,13 @@ import pytest from httpx import ASGITransport, AsyncClient from sqlalchemy import delete, func, select -from app.core.crypto import decrypt +from app.core.crypto import decrypt, encrypt from app.db.base import SessionLocal from app.db.models import Run, Schedule, ScheduleFire, WebhookDelivery, Workflow, WorkflowVersion from app.api import schedules as schedules_api from app.engine.runner import run_manager from app.main import app -from app.scheduler import webhook +from app.scheduler import loop, webhook UTC = timezone.utc HOOK = "https://open.feishu.cn/open-apis/bot/v2/hook/7c6b5a49-dddd-eeee-ffff-1234567890ab" @@ -173,15 +175,100 @@ async def test_stale_version_is_a_conflict(client): assert (await client.get(f"/api/schedules/{made['id']}")).json()["name"] == "甲" +async def test_concurrent_saves_of_the_same_version_conflict(client, monkeypatch): + """两个人同时读到第 1 版、同时保存:读后比对都过了,只有先写进去的那份生效,后到的回 409、不覆盖先到的。 + 这一条靠 UPDATE … WHERE version = 读到的版本 守住,串行的两次请求测不到它""" + wf, _ = await workflow(fields=(DATE_FIELD, {"name": "region"})) + made = await create(client, wf) + sid = made["id"] + real = schedules_api._check_workflow + + async def other_save_lands_first(*args, **kwargs): + # 本次请求读完、比对完版本之后,另一个人的保存先提交了 + async with SessionLocal() as session: + await session.execute(Schedule.__table__.update().where(Schedule.id == sid) + .values(name="先到的", version=Schedule.version + 1)) + await session.commit() + return await real(*args, **kwargs) + + monkeypatch.setattr(schedules_api, "_check_workflow", other_save_lands_first) + template = {"stat_date": "{{ schedule.yesterday }}", "region": "华东"} + res = await client.patch(f"/api/schedules/{sid}", json={"name": "后到的", "input_template": template, "version": 1}) + assert res.status_code == 409 and "刷新" in res.json()["detail"] + got = (await client.get(f"/api/schedules/{sid}")).json() + assert (got["name"], got["input_template"], got["version"]) == ("先到的", made["input_template"], 2) + + +EDITABLE = ("name", "workflow_id", "cron", "timezone", "run_class", "input_template", "enabled", "overlap", + "misfire_grace_s") + + +def full_form(made: dict, **over) -> dict: + """编辑表单整份回写:没改的字段也原样带上""" + return {**{k: made[k] for k in EDITABLE}, "version": made["version"], **over} + + +async def test_full_form_save_can_pause_after_the_workflow_is_gone(client): + """工作流删掉了、或撤了发布:整份带回来、只把启用关掉,也要能停下来。没改的工作流、运行类别、输入不重新校验""" + wf, _ = await workflow() + made = await create(client, wf, webhook=None) + async with SessionLocal() as session: + await session.execute(delete(WorkflowVersion).where(WorkflowVersion.workflow_id == wf)) + await session.execute(delete(Workflow).where(Workflow.id == wf)) + await session.commit() + res = await client.patch(f"/api/schedules/{made['id']}", json=full_form(made, enabled=False)) + assert res.status_code == 200, res.text + assert res.json()["enabled"] is False and res.json()["next_run_at"] is None + # 真改了工作流相关的项,照样校验 + res = await client.patch(f"/api/schedules/{made['id']}", + json={"input_template": {"stat_date": "{{ schedule.today }}"}, "version": 2}) + assert res.status_code == 422 and res.json()["field"] == "workflow_id" + + +async def test_full_form_save_keeps_a_due_fire(client): + """到点了、这一拍还没来:整份带回、只改名字,下一次触发时间不动,这一拍照常发起。以前只要带了 cron、时区、 + 启用就从现在重算,这一次连记录都没有就丢了""" + wf, _ = await workflow() + made = await create(client, wf, webhook=None, cron="* * * * *") + due = datetime.now(UTC).replace(second=0, microsecond=0) + async with SessionLocal() as session: + await session.execute(Schedule.__table__.update().where(Schedule.id == made["id"]).values(next_run_at=due)) + await session.commit() + res = await client.patch(f"/api/schedules/{made['id']}", json=full_form(made, name="每分钟早报")) + assert res.status_code == 200, res.text + assert datetime.fromisoformat(res.json()["next_run_at"]) == due + [fire] = await loop.fire_due() + assert (fire.status, fire.fire_at) == ("started", due) + await settle(fire.run_id) + + +async def test_changed_cron_keeps_a_due_fire_it_still_covers(client): + """改了 cron:从改之前那个到点的时刻接着算。新表达式也覆盖那一刻(每分钟 → 工作日每分钟)就还发起这一次; + 只属于旧表达式的到点时刻不再发起""" + wf, _ = await workflow() + made = await create(client, wf, webhook=None, cron="* * * * *") + due = datetime.now(UTC).replace(second=0, microsecond=0) + async with SessionLocal() as session: + await session.execute(Schedule.__table__.update().where(Schedule.id == made["id"]).values(next_run_at=due)) + await session.commit() + weekday = (due.weekday() + 1) % 7 # cron 的周:0 = 周日 + covered = await client.patch(f"/api/schedules/{made['id']}", + json={"cron": f"* * * * {weekday}", "timezone": "UTC", "version": 1}) + assert datetime.fromisoformat(covered.json()["next_run_at"]) == due + other = (weekday + 1) % 7 + moved = await client.patch(f"/api/schedules/{made['id']}", json={"cron": f"* * * * {other}", "version": 2}) + assert datetime.fromisoformat(moved.json()["next_run_at"]) > datetime.now(UTC) + + async def test_patch_webhook_keeps_or_removes_the_address(client): wf, _ = await workflow() made = await create(client, wf) sid = made["id"] - # 不带 url:地址不变,只改事件和格式 + # 不带 url:地址不变,只改事件(飞书群机器人的地址只能用飞书格式,格式对不上的见 test_known_bot_address_…) res = await client.patch(f"/api/schedules/{sid}", json={ - "webhook": {"format": "dingtalk", "events": ["awaiting_approval"]}, "version": 1}) + "webhook": {"format": "feishu", "events": ["awaiting_approval"]}, "version": 1}) assert res.status_code == 200, res.text - assert res.json()["webhook"] == {"format": "dingtalk", "url_hint": "open.feishu.cn/…90ab", "has_secret": True, + assert res.json()["webhook"] == {"format": "feishu", "url_hint": "open.feishu.cn/…90ab", "has_secret": True, "events": ["awaiting_approval"]} async with SessionLocal() as session: assert decrypt((await session.get(Schedule, sid)).webhook_url) == HOOK @@ -253,7 +340,7 @@ async def test_unknown_schedule_is_404_in_chinese(client, method, path): ({"input_template": {}}, "input_template", "必填"), ({"misfire_grace_s": -1}, "misfire_grace_s", "宽限"), ({"misfire_grace_s": True}, "misfire_grace_s", "宽限"), - ({"overlap": "queue"}, "overlap", "skip"), + ({"overlap": "queue"}, "overlap", "跳过"), ({"enabled": "yes"}, "enabled", "启用"), ({"webhook": {"format": "slack", "url": HOOK}}, "webhook", "格式"), ({"webhook": {"format": "feishu", "url": "open.feishu.cn/hook"}}, "webhook", "Webhook 地址"), @@ -268,6 +355,97 @@ async def test_invalid_create_is_422_with_field(client, over, field, fragment): assert body["field"] == field and fragment in body["detail"], body +@pytest.mark.parametrize("over, field, says, never", [ + # 表单清空、没选时传 null:说「未填写」,不回显 Python 的 None + ({"run_class": None}, "run_class", "未填写", "None"), + ({"misfire_grace_s": None}, "misfire_grace_s", "未填写", "None"), + ({"enabled": None}, "enabled", "未填写", "None"), + ({"misfire_grace_s": True}, "misfire_grace_s", "宽限", "True"), + # 列表不原样回显,点出不认识的那一项 + ({"webhook": {"format": "feishu", "url": HOOK, "events": ["finished", "done"]}}, "webhook", "「done」", "['"), + # 格式写中文名,不列英文枚举 + ({"webhook": {"format": "slack", "url": HOOK}}, "webhook", "通用、飞书、企业微信、钉钉", "generic"), +]) +async def test_invalid_values_are_described_without_python_reprs(client, over, field, says, never): + wf, _ = await workflow() + res = await client.post("/api/schedules", json=payload(wf, **over)) + assert res.status_code == 422 and res.json()["field"] == field + detail = res.json()["detail"] + assert says in detail and never not in detail, detail + + +FEISHU_BOT = HOOK +WECOM_BOT = "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=0a1b2c3d-4444-5555-6666-777788889999" +DINGTALK_BOT = "https://oapi.dingtalk.com/robot/send?access_token=0123456789abcdef0123456789abcdef" + + +@pytest.mark.parametrize("fmt, url, platform", [ + ("generic", FEISHU_BOT, "飞书"), + ("dingtalk", FEISHU_BOT, "飞书"), + ("feishu", WECOM_BOT, "企业微信"), + ("generic", DINGTALK_BOT, "钉钉"), +]) +async def test_known_bot_address_needs_its_own_format(client, fmt, url, platform): + """群机器人只认自己平台的消息格式:格式选错了,平台回错误码、群里收不到。保存时就说出来""" + wf, _ = await workflow() + hook = {"format": fmt, "url": url, "events": ["failed"]} + res = await client.post("/api/schedules", json=payload(wf, webhook=hook)) + assert res.status_code == 422 and res.json()["field"] == "webhook" + assert f"{platform}群机器人" in res.json()["detail"] and f"「{platform}」" in res.json()["detail"] + assert "0123456789abcdef" not in res.text and "0a1b2c3d" not in res.text + + +async def test_format_follows_the_address_when_left_out(client): + wf, _ = await workflow() + made = await create(client, wf, webhook={"url": WECOM_BOT, "events": ["failed"]}) + assert made["webhook"]["format"] == "wecom" + custom = await create(client, wf, webhook={"url": "https://hooks.example.com/agentlab/inbox-7f3a"}) + assert custom["webhook"]["format"] == "generic" + # 改格式(地址不变)也要和地址对得上;换了地址、没给格式,按新地址认 + res = await client.patch(f"/api/schedules/{made['id']}", json={"webhook": {"format": "feishu"}, "version": 1}) + assert res.status_code == 422 and "企业微信群机器人" in res.json()["detail"] + res = await client.patch(f"/api/schedules/{made['id']}", json={"webhook": {"url": DINGTALK_BOT}, "version": 1}) + assert res.status_code == 200 and res.json()["webhook"]["format"] == "dingtalk" + + +async def test_old_mismatch_does_not_block_unrelated_edits(client): + """库里已有对不上的设置(这条校验之前存的):只改名字、停用,不因为 Webhook 被拦""" + wf, _ = await workflow() + made = await create(client, wf) + async with SessionLocal() as session: + await session.execute(Schedule.__table__.update().where(Schedule.id == made["id"]) + .values(webhook_format="generic")) + await session.commit() + res = await client.patch(f"/api/schedules/{made['id']}", json={ + "enabled": False, "webhook": {"format": "generic", "events": ["finished", "failed"]}, "version": 1}) + assert res.status_code == 200, res.text + + +@pytest.mark.parametrize("url", [ + "https://hooks.example.com:abc/inbox-7f3a", "https://hooks.exa\u200bmple.com/inbox-7f3a"]) +async def test_unsendable_address_is_refused_on_save(client, url): + wf, _ = await workflow() + res = await client.post("/api/schedules", json=payload(wf, webhook={"format": "generic", "url": url})) + assert res.status_code == 422 and res.json()["field"] == "webhook" + assert res.json()["detail"].startswith("Webhook 地址") + + +async def test_webhook_test_with_a_saved_unsendable_address_answers_in_chinese(client): + """校验加严之前存进库的地址:测试接口回 200 和中文说明,不是 500""" + wf, _ = await workflow() + made = await create(client, wf, webhook=None) + async with SessionLocal() as session: + await session.execute(Schedule.__table__.update().where(Schedule.id == made["id"]).values( + webhook_format="generic", webhook_url=encrypt("https://hooks.example.com:abc/inbox-7f3a"), + webhook_events=["failed"])) + await session.commit() + res = await client.post(f"/api/schedules/{made['id']}/webhook/test") + assert res.status_code == 200, res.text + body = res.json() + assert body["ok"] is False and body["http_status"] is None + assert "hooks.example.com" in body["message"] and "7f3a" not in body["message"] + + async def test_missing_workflow_and_unpublished_formal(client): res = await client.post("/api/schedules", json=payload(f"gone{uuid.uuid4().hex[:6]}")) assert res.status_code == 422 and res.json()["field"] == "workflow_id" and "不存在" in res.json()["detail"] @@ -378,7 +556,7 @@ async def test_run_now_and_overlap(client, monkeypatch): monkeypatch.setattr(schedules_api, "utcnow", lambda: datetime.now(UTC) + timedelta(seconds=2)) second = (await client.post(f"/api/schedules/{made['id']}/run")).json() assert (second["fire"]["status"], second["run_id"]) == ("skipped_overlap", None) - assert body["run_id"][:8] in second["fire"]["reason"] + assert f"上一次运行 #{body['run_id'][:6]} " in second["fire"]["reason"] # 运行列表能按定时任务筛 listed = (await client.get("/api/runs", params={"schedule_id": made["id"]})).json() assert [r["id"] for r in listed] == [body["run_id"]] @@ -396,6 +574,35 @@ async def test_double_click_in_the_same_second_returns_the_same_fire(client, mon await settle(first["run_id"]) +async def test_a_fire_being_launched_is_not_shown_as_failed(client, monkeypatch): + """发起要先记一条占位再调 launch_run。以前占位写成「未能发起:服务在发起运行前停止」,这期间来读列表、触发 + 历史的会闪出一次假的失败;进程还在、只是还没发起完的,按已发起给(run_id 还是空的)""" + from app.engine import launch + + real = launch.launch_run + halfway = asyncio.Event() + go_on = asyncio.Event() + + async def slow(*args, **kwargs): + halfway.set() + await go_on.wait() + return await real(*args, **kwargs) + + monkeypatch.setattr(launch, "launch_run", slow) + wf, _ = await workflow() + made = await create(client, wf, webhook=None) + pressing = asyncio.create_task(client.post(f"/api/schedules/{made['id']}/run")) + await asyncio.wait_for(halfway.wait(), timeout=5) + last = (await client.get("/api/schedules")).json()["items"][0]["last_fire"] + assert (last["status"], last["reason"], last["run_id"]) == ("started", None, None) + [item] = (await client.get(f"/api/schedules/{made['id']}/fires")).json()["items"] + assert (item["status"], item["reason"]) == ("started", None) + go_on.set() + done = (await pressing).json() + assert done["fire"]["status"] == "started" and done["run_id"] + await settle(done["run_id"]) + + async def test_run_now_refused_for_unpublished_formal(client): wf, name = await workflow() made = await create(client, wf, webhook=None) @@ -459,7 +666,7 @@ async def test_webhook_test_endpoint(client, monkeypatch): made = await create(client, wf) res = await client.post(f"/api/schedules/{made['id']}/webhook/test") assert res.status_code == 200 - assert res.json() == {"ok": True, "http_status": 200, "message": "测试消息已送达(HTTP 200)"} + assert res.json() == {"ok": True, "http_status": 200, "message": "请到接收端确认已收到"} assert str(seen[0].url) == HOOK and "sign" in json.loads(seen[0].content) bare = await create(client, wf, webhook=None) res = await client.post(f"/api/schedules/{bare['id']}/webhook/test") diff --git a/scripts/copy-allow.json b/scripts/copy-allow.json index 4c31bf7a..2a9609ec 100644 --- a/scripts/copy-allow.json +++ b/scripts/copy-allow.json @@ -43,6 +43,12 @@ "rule": "供应商", "reason": "按用户数据源描述里的业务词(采购、供应商)造的示例问法,这里的「供应商」是采购供应商,不是模型接入" }, + { + "file": "backend/app/scheduler/webhook.py", + "text": "钉钉", + "rule": "钉", + "reason": "「钉钉」是 Webhook 通知的平台名(定时任务的消息格式:通用 / 飞书 / 企业微信 / 钉钉),用在格式不可选、格式和群机器人地址对不上的报错里,不是「钉住」的意思" + }, { "file": "backend/app/seed.py", "text": "print(json.dumps(...))",