From d62f921ed4df4460eb11261017bc515bc82b0a30 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 18:39:10 +0800 Subject: [PATCH 1/6] =?UTF-8?q?=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E9=81=97=E7=95=99=E9=97=AE=E9=A2=98=EF=BC=9A=E5=A4=B1=E8=B4=A5?= =?UTF-8?q?=E9=80=9A=E7=9F=A5=E4=B8=8D=E5=86=8D=E8=AE=B8=E8=AF=BA=E8=83=BD?= =?UTF-8?q?=E7=BB=A7=E7=BB=AD=E8=BF=90=E8=A1=8C=E3=80=8130=20=E5=A4=A9?= =?UTF-8?q?=E5=89=8D=E7=9A=84=E8=A7=A6=E5=8F=91=E5=81=9C=E5=9C=A8=E5=AE=A1?= =?UTF-8?q?=E6=89=B9=E4=B8=8A=E7=85=A7=E6=A0=B7=E6=8E=A8=E9=80=81=E3=80=81?= =?UTF-8?q?=E8=B0=83=E5=BA=A6=E5=BE=AA=E7=8E=AF=E6=B5=8B=E8=AF=95=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E4=BE=9D=E8=B5=96=E6=95=B4=E5=88=86=E8=BE=B9=E7=95=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 失败通知的「处理」行改成中性的「请在 AgentLab 的运行记录中查看原因」:服务重启时还在排队、从没开始执行的运行 没有断点,「修正后可继续运行」是错的;开始执行过的也可能续不了(执行第一个节点前就失败、系统升级后断点不兼容), 能不能继续以记录页为准。没有开始时间的失败另写一行原因(没有可继续的进度),通用格式的 reason 字段同写;任务 还在排期上时补「下一次触发会照常发起」,和挂起(之后的触发都跳过)说清区别 - 通知扫描的 30 天窗口加两种例外:停在审批、有待审批的运行不受限制(创建 30 天以后才停到审批上的以前收不到 「等待审批」);运行记录 30 天内有过变化的照扫(等了一个多月的审批批完、跑完,以前收不到「运行结束」)。早就 结束、多日没有变化的旧触发仍不扫 - test_loop_ticks_until_stopped 改成只看「发起了」和停下前后的条数差:建任务和第一拍之间跨过整分时会先记一条 错过的跳过,以前按总条数断言偶发失败。加一组把下一次放在上一个整分的用例,必然走到跨整分这一支 --- backend/app/scheduler/webhook.py | 37 +++- backend/tests/test_scheduler_loop.py | 31 ++- backend/tests/test_scheduler_notice_rules.py | 188 +++++++++++++++++++ backend/tests/test_scheduler_webhook.py | 3 +- 4 files changed, 243 insertions(+), 16 deletions(-) create mode 100644 backend/tests/test_scheduler_notice_rules.py diff --git a/backend/app/scheduler/webhook.py b/backend/app/scheduler/webhook.py index 85907b66..c7b9f2b1 100644 --- a/backend/app/scheduler/webhook.py +++ b/backend/app/scheduler/webhook.py @@ -9,8 +9,8 @@ Webhook 地址本身就是凭证(企业微信的 key、钉钉的 access_token 一个过滤器把推送期间的那几行换成主机名。 通知里只有任务名、状态、出具档位、缺口条数、运行链接、封存哈希(界面上叫清单哈希)前 12 位和起止时间,失败时 -另写失败节点的名字。不放报告里的具体数字,也不放失败时的原始报错——报错里可能带着查询到的数据,Webhook 推去的群 -不一定有权看。 +另写失败节点的名字和去哪里查看(不许诺能继续运行)。不放报告里的具体数字,也不放失败时的原始报错——报错里可能带着 +查询到的数据,Webhook 推去的群不一定有权看。 三个平台出错时都回 HTTP 200、错误码放在响应体里,所以送没送达要看响应体:按地址的主机认平台,不看选的格式—— 格式选错了(飞书的地址选了通用格式)正是最需要认出来的情形。群机器人的地址只认自己平台的格式,保存时就拦下 @@ -52,7 +52,8 @@ EVENT_LABEL = {"finished": "运行结束", "failed": "运行失败", "awaiting_a TIMEOUT_S = 10.0 #: 失败后再试 3 次,每次之前等这么久(秒)。4xx(429 除外)不重试:地址或参数错了,重试也一样 RETRY_DELAYS: tuple[float, ...] = (1.0, 2.0, 4.0) -#: 只看这么多天内的触发:更早的运行状态不会再变,不用每一拍都扫 +#: 只看这么多天内的触发:更早结束的运行状态不会再变,不用每一拍都扫。例外见 notify_due:停在审批上、有待审批的不受 +#: 这个限制,运行记录这些天内有过变化的也照扫——审批可以等上一个多月,批完接着执行、停到下一个审批点或跑完都要推 SCAN_DAYS = 30 #: 运行结束后等封存(清单哈希)最多等多久。runner 先写终态再封存,两步之间被这一拍撞上就等下一拍 SEAL_WAIT_S = 60 @@ -65,6 +66,14 @@ _STATUS_LABEL = {"succeeded": "已完成", "failed": "失败", "cancelled": "已 #: 挂起的运行要人去处理:不处理,这个任务之后的触发都按「上一次还没结束」跳过 SUSPENDED_REASON = "运行停在断点上、没有待审批,通常是服务重启所致" SUSPENDED_ACTION = "请在 AgentLab 的「记录」中继续运行或放弃这次运行;它结束之前,这个定时任务之后的触发都会跳过" +#: 失败的运行怎么处理:不许诺「修正后可继续运行」。没开始执行的没有断点;开始执行过的也可能没有(执行第一个节点前就 +#: 失败)或已不兼容(系统升级),能不能继续以记录页的提示为准 +FAILED_ACTION = "请在 AgentLab 的运行记录中查看原因" +#: 从没开始执行就失败的(服务重启时还在排队,被改成失败、没有开始时间):没有可继续的进度。不照抄运行记录上的报错—— +#: 通知一律不放原始报错,这一句是按「没有开始时间」认出来的,自己写 +NOT_STARTED_REASON = "运行在开始执行前就已中止(通常是服务重启时仍在排队),没有可继续的进度" +#: 失败的运行不挡之后的触发(挂起的才挡):任务还在排期上时补这一句,免得人以为这个定时任务就此停了 +NEXT_AS_USUAL = "下一次触发会照常发起" # -------------------------------------------------------------------------- @@ -217,6 +226,7 @@ class Notice: failed_to_start。没有的项为 None。 suspended:运行停在断点上、没有待审批(通常是服务重启挂起的),按 failed 事件推。error_node:失败节点的名字。 + next_fire:任务启用着、往后还会触发(失败通知据此补「下一次触发会照常发起」)。 fmt、signed 只给测试消息用:说明里只说这条消息证明得了的事。""" event: str @@ -238,9 +248,15 @@ class Notice: pending_approvals: int | None = None suspended: bool = False error_node: str | None = None + next_fire: bool = False fmt: str | None = None signed: bool = False + @property + def never_started(self) -> bool: + """失败的运行从没开始执行(没有开始时间):服务重启时还在排队、被改成失败的就是这种,没有断点可续。""" + return self.status == "failed" and self.run_id is not None and self.started_at is None + def _local(moment: datetime | None, tz: str, *, seconds: bool = True) -> str | None: if moment is None: @@ -282,7 +298,9 @@ def message(notice: Notice) -> tuple[str, list[tuple[str, str]], str | None]: # 不写原始报错(可能带着查询到的数据),写失败节点的名字和去哪里处理 if notice.error_node: lines.append(("失败节点", f"「{notice.error_node}」")) - lines.append(("处理", "请在 AgentLab 的运行记录中查看报错,修正后可继续运行")) + if notice.never_started: + lines.append(("原因", NOT_STARTED_REASON)) + lines.append(("处理", f"{FAILED_ACTION};{NEXT_AS_USUAL}" if notice.next_fire else FAILED_ACTION)) if notice.fire_at is not None: when = f"{_local(notice.fire_at, notice.timezone, seconds=False)}({notice.timezone})" # 立即运行不写「计划触发」:群里的人会以为报告按时出了 @@ -375,7 +393,8 @@ def _generic(notice: Notice, now: datetime) -> dict[str, Any]: "started_at": _iso(notice.started_at), "finished_at": _iso(notice.finished_at), "reason": (notice.reason if notice.status == "failed_to_start" - else f"{SUSPENDED_REASON}。{SUSPENDED_ACTION}" if notice.suspended else None), + else f"{SUSPENDED_REASON}。{SUSPENDED_ACTION}" if notice.suspended + else NOT_STARTED_REASON if notice.never_started else None), "error_node": notice.error_node, "sent_at": _iso(now), } @@ -566,13 +585,18 @@ async def notify_due(now: datetime | None = None) -> int: """ now = now or utcnow() jobs: list[_Job] = [] + cutoff = now - timedelta(days=SCAN_DAYS) + # 扫描窗口按触发记录的创建时间算,两种例外:运行记录窗口内有过变化(运行状态一变 updated_at 就跟着变),以及 + # 停在审批上、有待审批的——以前只看触发记录,创建 30 天以后才停到审批上的运行收不到「等待审批」 + waiting = select(Approval.id).where(Approval.run_id == Run.id, Approval.status == "pending").exists() # 和发起共用一把锁:发起时先记一条「未能发起」再发起运行、成功了才改成 started,扫描不能撞在这中间 async with service.launch_lock(): async with SessionLocal() as session: rows = (await session.execute( select(ScheduleFire, Run) .outerjoin(Run, Run.id == ScheduleFire.run_id) - .where(ScheduleFire.created_at >= now - timedelta(days=SCAN_DAYS), or_( + .where(or_(ScheduleFire.created_at >= cutoff, Run.updated_at >= cutoff, + and_(Run.status == "interrupted", waiting)), or_( and_(ScheduleFire.status == "failed_to_start", ScheduleFire.notified_status.is_(None)), and_(ScheduleFire.status == "started", Run.status.in_(_WATCHED), or_(ScheduleFire.notified_status.is_(None), ScheduleFire.notified_status != Run.status)), @@ -664,6 +688,7 @@ async def _notice(session: Any, schedule: Schedule, fire: ScheduleFire, run: Run pending_approvals=pending or None, suspended=run is not None and run.status == "interrupted" and not pending, error_node=_failed_node(run) if run is not None and run.status == "failed" else None, + next_fire=bool(schedule.enabled and schedule.next_run_at is not None), ) diff --git a/backend/tests/test_scheduler_loop.py b/backend/tests/test_scheduler_loop.py index 94dab830..91ee6d90 100644 --- a/backend/tests/test_scheduler_loop.py +++ b/backend/tests/test_scheduler_loop.py @@ -369,9 +369,20 @@ async def test_start_does_nothing_when_switched_off(): await scheduler.stop() -async def test_loop_ticks_until_stopped(): - # 下一次放在整分上(和 cron 对齐):放在「现在 - 1 秒」的话,这一秒里恰好跨过整分时会多记一条错过的跳过 +async def fire_count(schedule_id: str) -> int: + async with SessionLocal() as session: + return await session.scalar(select(func.count()).select_from(ScheduleFire) + .where(ScheduleFire.schedule_id == schedule_id)) + + +@pytest.mark.parametrize("crossed", [False, True], ids=["on_the_minute", "minute_crossed"]) +async def test_loop_ticks_until_stopped(crossed): + """循环按拍执行、stop 之后不再有拍。只看「发起了」和停下前后的条数差,不看总条数:建任务和第一拍之间恰好跨过 + 整分时,第一拍会先记一条错过的跳过、再发起最近这一次——以前按总条数断言,跨整分时偶发失败。crossed 把下一次 + 放在上一个整分,必然走到这一种""" due = cron.last_due("* * * * *", "UTC", datetime.now(UTC)) + if crossed: + due -= timedelta(minutes=1) sid = await schedule(await workflow(), next_run_at=due, cron="* * * * *") on = Settings(_env_file=None, scheduler=True) assert await scheduler.start(interval=0.05, config=on) is True @@ -382,18 +393,20 @@ async def test_loop_ticks_until_stopped(): await asyncio.sleep(0.02) await scheduler.stop() assert scheduler.running() is False - # 只认发起的那一条:建任务和第一拍之间跨过整分时,前面还会有一条错过的跳过,那不是这里要测的 - [fire] = [f for f in await fires(sid) if f.status == "started"] - await settle(fire.run_id) - # 停下之后不再有拍:把下一次拨回过去也不会再发起 + started = [f for f in await fires(sid) if f.status == "started"] + assert started + if crossed: + assert [f.status for f in await fires(sid)][0] == "skipped_misfire" + for fire in started: + await settle(fire.run_id) + # 停下之后不再有拍:把下一次拨回过去,过一会儿条数一条不多 + before = await fire_count(sid) async with SessionLocal() as session: await session.execute(Schedule.__table__.update().where(Schedule.id == sid) .values(next_run_at=datetime.now(UTC) - timedelta(seconds=1))) await session.commit() await asyncio.sleep(0.2) - async with SessionLocal() as session: - assert await session.scalar(select(func.count()).select_from(ScheduleFire) - .where(ScheduleFire.schedule_id == sid)) == 1 + assert await fire_count(sid) == before async def test_start_twice_keeps_one_loop(): diff --git a/backend/tests/test_scheduler_notice_rules.py b/backend/tests/test_scheduler_notice_rules.py new file mode 100644 index 00000000..dfc11af7 --- /dev/null +++ b/backend/tests/test_scheduler_notice_rules.py @@ -0,0 +1,188 @@ +"""定时任务通知(app/scheduler/webhook.py)的两条规则: + +- 失败通知的「处理」行不许诺能继续运行:服务重启时还在排队、从没开始执行的运行(被改成失败、没有开始时间)没有 + 断点,「修正后可继续运行」是错的。按有没有开始执行分开写——没开始的另写一行原因(没有可继续的进度),处理一律是 + 中性的「请在运行记录中查看原因」;任务还在排期上时补一句「下一次触发会照常发起」:失败的运行不挡之后的触发, + 和挂起的那种(之后的触发都会跳过)说清区别 +- 扫描窗口(SCAN_DAYS)之外的触发:停在审批、有待审批的运行不受窗口限制——创建 30 天以后才停到审批上的照样推送 + 「等待审批」;运行记录在窗口内有变化的(等了一个多月的审批批完、跑完)照样推送结束。早就结束、记录多日没有变化 + 的旧触发仍不扫 + +用 httpx.MockTransport 接住请求,不连外网。任务名、地址、数字都是虚构的。 +""" +from __future__ import annotations + +import json +from datetime import datetime, timedelta, timezone + +import httpx +import pytest +from sqlalchemy import delete + +from app.core.crypto import encrypt +from app.db.base import SessionLocal +from app.db.models import Approval, Run, RunEvent, Schedule, ScheduleFire, Setting +from app.scheduler import webhook + +UTC = timezone.utc +GENERIC = "https://hooks.example.com/agentlab/inbox-3e9d" +NOT_STARTED = "运行在开始执行前就已中止(通常是服务重启时仍在排队),没有可继续的进度" + + +class Recorder: + def __init__(self) -> None: + self.requests: list[httpx.Request] = [] + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.requests.append(request) + return httpx.Response(200, json={}) + + def bodies(self) -> list[dict]: + return [json.loads(r.content) for r in self.requests] + + +@pytest.fixture +def recorder(monkeypatch): + rec = Recorder() + monkeypatch.setattr(webhook, "_transport", httpx.MockTransport(rec)) + monkeypatch.setattr(webhook, "RETRY_DELAYS", (0, 0, 0)) + return rec + + +@pytest.fixture(autouse=True) +async def clean(): + async with SessionLocal() as session: + await session.execute(delete(Schedule)) + await session.execute(delete(Setting).where(Setting.key == "notify")) + await session.commit() + yield + + +def notice(**over) -> webhook.Notice: + base = dict(event="failed", schedule_id="s1", schedule_name="仓库日报", timezone="Asia/Shanghai", + fire_id="f1", fire_at=datetime(2026, 10, 5, 1, 0, tzinfo=UTC), fire_kind="scheduled", + run_id="r9876543210fedcba", status="failed", run_url=None, manifest_hash=None, + started_at=datetime(2026, 10, 5, 1, 0, 2, tzinfo=UTC), finished_at=None) + base.update(over) + return webhook.Notice(**base) + + +async def make(*, run_status: str, started: bool = True, fire_age: timedelta = timedelta(0), + run_age: timedelta = timedelta(0), enabled: bool = True, sealed: bool = True) -> tuple[str, str, str]: + """一个配了通用 Webhook 的任务、一次触发和它发起的运行(不真的执行,状态直接写库)。fire_age、run_age 是触发 + 记录、运行记录最后一次更新离现在多久;started=False 是从没开始执行的(服务重启时还在排队)。""" + now = datetime.now(UTC) + async with SessionLocal() as session: + sched = Schedule(name="仓库日报", workflow_id="wf-demo", cron="0 9 * * *", timezone="Asia/Shanghai", + enabled=enabled, next_run_at=now + timedelta(hours=3) if enabled else None, + webhook_format="generic", webhook_url=encrypt(GENERIC), + webhook_events=["finished", "failed", "awaiting_approval"]) + session.add(sched) + await session.flush() + ended = run_status in ("succeeded", "failed", "cancelled") and started + touched = now - run_age + run = Run(workflow_id=None, workflow_name="仓库日报", status=run_status, schedule_id=sched.id, + run_class="formal", started_at=touched - timedelta(minutes=1) if started else None, + finished_at=touched if ended else None, manifest_hash=("d00d" * 16) if sealed and ended else None, + created_at=now - fire_age, updated_at=touched, + error="服务重启时这次运行仍在排队,没有可恢复的进度,请重新发起运行" if not started else None) + session.add(run) + await session.flush() + fire = ScheduleFire(schedule_id=sched.id, fire_at=(now - fire_age).replace(microsecond=0), kind="scheduled", + status="started", run_id=run.id, created_at=now - fire_age) + session.add(fire) + await session.commit() + return sched.id, fire.id, run.id + + +async def notified(fire_id: str) -> str | None: + async with SessionLocal() as session: + return (await session.get(ScheduleFire, fire_id)).notified_status + + +# -------------------------------------------------------------------------- +# 失败通知的「处理」行 +# -------------------------------------------------------------------------- + + +def test_a_run_that_never_started_is_not_promised_a_continue(): + """服务重启时还在排队的运行没有断点:不写「修正后可继续运行」,另写一行原因说没有可继续的进度""" + _, lines, _ = webhook.message(notice(started_at=None, next_fire=True)) + shown = dict(lines) + assert shown["原因"] == NOT_STARTED + assert shown["处理"] == "请在 AgentLab 的运行记录中查看原因;下一次触发会照常发起" + assert not any("继续运行" in value for _, value in lines), lines + + +def test_a_failed_run_gets_a_neutral_action_without_promising_a_continue(): + """开始执行过的运行也不许诺能继续:断点可能没有(执行第一个节点前就失败)、可能已不兼容(系统升级),能不能继续 + 以记录页为准。失败节点照写,不另写原因(原始报错可能带着查询到的数据)""" + _, lines, _ = webhook.message(notice(error_node="汇总库存", next_fire=True)) + shown = dict(lines) + assert shown["失败节点"] == "「汇总库存」" and "原因" not in shown + assert shown["处理"] == "请在 AgentLab 的运行记录中查看原因;下一次触发会照常发起" + assert "修正后可继续运行" not in webhook._text(notice(error_node="汇总库存")) + + +def test_next_fire_is_only_promised_while_the_schedule_is_on(): + """任务已暂停(或往后不再触发)时不说「下一次触发会照常发起」""" + _, lines, _ = webhook.message(notice(next_fire=False)) + assert dict(lines)["处理"] == "请在 AgentLab 的运行记录中查看原因" + + +async def test_restart_failure_notice_end_to_end(recorder): + """服务重启时还在排队、被改成失败的运行:不会封存,等过封存时限后推一次失败,原因、处理按上面写;通用格式的 + reason 字段也写这句原因。任务停用时不补「下一次触发会照常发起」""" + _, on_fire, _ = await make(run_status="failed", started=False, sealed=False) + _, off_fire, _ = await make(run_status="failed", started=False, sealed=False, enabled=False) + later = datetime.now(UTC) + timedelta(seconds=webhook.SEAL_WAIT_S + 5) + assert await webhook.notify_due(later) == 2 + by_fire = {b["fire_id"]: b for b in recorder.bodies()} + on, off = by_fire[on_fire], by_fire[off_fire] + assert (on["event"], on["status"], on["reason"]) == ("failed", "failed", NOT_STARTED) + assert "处理:请在 AgentLab 的运行记录中查看原因;下一次触发会照常发起" in on["text"] + assert "处理:请在 AgentLab 的运行记录中查看原因\n" in off["text"] + "\n" and "照常发起" not in off["text"] + for body in (on, off): + assert "继续运行" not in body["text"], body["text"] + # 运行记录上的原始报错不进通知 + assert "请重新发起运行" not in body["text"] + + +# -------------------------------------------------------------------------- +# 扫描窗口之外 +# -------------------------------------------------------------------------- + + +async def test_old_fire_stopped_at_an_approval_is_still_pushed(recorder): + """触发是 40 天前的,运行刚停到审批上(或者停下时服务没在推送):有待审批的运行不受 30 天的窗口限制。 + 运行记录本身也很久没有变化,照样推「等待审批」""" + _, fire_id, run_id = await make(run_status="interrupted", fire_age=timedelta(days=40), run_age=timedelta(days=35)) + async with SessionLocal() as session: + session.add(RunEvent(run_id=run_id, seq=1, type="run.started", ts=1.0, data={})) + session.add(Approval(run_id=run_id, node_id="gate", title="审核日报", status="pending")) + await session.commit() + assert await webhook.notify_due() == 1 + [body] = recorder.bodies() + assert (body["event"], body["pending_approvals"]) == ("awaiting_approval", 1) + assert await notified(fire_id) == "interrupted:approval:1" + assert await webhook.notify_due() == 0 + + +async def test_old_fire_whose_run_just_finished_is_pushed(recorder): + """等了一个多月的审批批完、刚跑完:运行记录在窗口内有变化,推「运行结束」""" + _, fire_id, _ = await make(run_status="succeeded", fire_age=timedelta(days=45)) + assert await webhook.notify_due() == 1 + assert recorder.bodies()[0]["event"] == "finished" and await notified(fire_id) == "succeeded" + + +async def test_old_fire_whose_run_ended_long_ago_is_not_scanned(recorder): + """早就结束、多日没有变化的旧触发仍不扫:窗口本来就是为了不每一拍把全部历史翻一遍""" + _, fire_id, _ = await make(run_status="succeeded", fire_age=timedelta(days=45), run_age=timedelta(days=40)) + assert await webhook.notify_due() == 0 + assert recorder.requests == [] and await notified(fire_id) is None + + +async def test_old_suspended_run_without_approval_is_not_rescanned_forever(recorder): + """挂起(没有待审批)、多日没有变化的旧触发不在例外里:例外只给停在审批上的""" + _, fire_id, _ = await make(run_status="interrupted", fire_age=timedelta(days=40), run_age=timedelta(days=40)) + assert await webhook.notify_due() == 0 and await notified(fire_id) is None diff --git a/backend/tests/test_scheduler_webhook.py b/backend/tests/test_scheduler_webhook.py index 98c41404..ca32dca3 100644 --- a/backend/tests/test_scheduler_webhook.py +++ b/backend/tests/test_scheduler_webhook.py @@ -178,7 +178,8 @@ def test_failed_run_text_points_to_the_node_without_repeating_itself(): error_node="查询门店销售")) shown = dict(lines) assert title.endswith("运行失败") and shown["状态"] == "失败" - assert shown["失败节点"] == "「查询门店销售」" and "继续运行" in shown["处理"] + # 处理只说去哪里看,不许诺能继续运行(没有断点的运行续不了,见 test_scheduler_notice_rules.py) + assert shown["失败节点"] == "「查询门店销售」" and shown["处理"] == "请在 AgentLab 的运行记录中查看原因" # 不再写「原因:运行失败,详情请查看运行记录」这种同义反复 assert "原因" not in shown From f0f1b99ead921f65022ab78ba0e17a9cd2f2882b Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 18:45:59 +0800 Subject: [PATCH 2/6] =?UTF-8?q?=E5=AE=9A=E6=97=B6=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E7=BB=93=E6=9D=9F=E5=90=8E=E8=87=AA=E5=8A=A8=E5=92=8C=E5=90=8C?= =?UTF-8?q?=E4=B8=80=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1=E7=9A=84=E4=B8=8A?= =?UTF-8?q?=E4=B8=80=E6=AC=A1=E5=AF=B9=E6=AF=94=EF=BC=9A=E7=BB=93=E6=9E=9C?= =?UTF-8?q?=E5=AD=98=E6=88=90=E5=B7=A5=E4=BB=B6=E8=AE=B0=E5=9C=A8=E8=A7=A6?= =?UTF-8?q?=E5=8F=91=E8=AE=B0=E5=BD=95=E4=B8=8A=EF=BC=8C=E8=A7=A6=E5=8F=91?= =?UTF-8?q?=E5=8E=86=E5=8F=B2=E5=92=8C=E9=80=9A=E7=9F=A5=E5=8F=AA=E7=BB=99?= =?UTF-8?q?=E8=AE=A1=E6=95=B0=E6=91=98=E8=A6=81=E4=B8=8E=E5=AF=B9=E6=AF=94?= =?UTF-8?q?=E9=A1=B5=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 app/scheduler/auto_compare.py:通知扫描认领到运行进入终态(结束、失败、取消)的那一次之后、推送之前, 调运行对比接口的现成函数(default_base、compare_runs,不改它们)比一次。基准只取 default_base 的「同一定时 任务」那一档,退回「同一工作流」的不比——那不是这个任务的上一次。没封存的不比;比不了(基准封存核对不一致等) 记警告、意外异常记堆栈,都不影响通知。整份结果存成 run_compare 工件,不挂在运行名下(不进运行的工件列表), 工件 id 写进 ScheduleFire.compare_artifact;再次进入终态(失败后继续执行)时按新的封存重新比、覆盖旧结果 - 接口:Fire 多一个 compare: {base_run_id, summary}(summary 同对比接口的计数)或 null。运行现在不在终态 (被继续执行)、工件读不出来时为 null;工件摘要按 id 缓存(内容寻址,不会过期),读盘在线程里 - 通知:结束、失败的通知多一行「与上一次相比:只数据变 3 个、口径与数据都变 1 个、SQL 结构变 0 个」(三类总写, 其余各类有才写,两边都没有指标时不写一串 0 个),配了对外访问地址时附对比页链接(/runs/compare?target=…&base=…), 四种格式都带;通用格式多一个 compare 字段。只给个数,不放任何具体数字 - 没配 Webhook 的任务也比:触发历史里要用 --- backend/app/api/schedules.py | 14 +- backend/app/scheduler/__init__.py | 1 + backend/app/scheduler/auto_compare.py | 178 +++++++++++ backend/app/scheduler/webhook.py | 85 +++-- backend/tests/test_scheduler_compare.py | 395 ++++++++++++++++++++++++ backend/tests/test_schedules_api.py | 2 +- 6 files changed, 649 insertions(+), 26 deletions(-) create mode 100644 backend/app/scheduler/auto_compare.py create mode 100644 backend/tests/test_scheduler_compare.py diff --git a/backend/app/api/schedules.py b/backend/app/api/schedules.py index 9664d6c4..b96d57b8 100644 --- a/backend/app/api/schedules.py +++ b/backend/app/api/schedules.py @@ -1,4 +1,4 @@ -"""定时任务:按周期自动发起工作流运行,结束、失败、停在审批时推送 Webhook。 +"""定时任务:按周期自动发起工作流运行,结束、失败、停在审批时推送 Webhook,运行结束后和同一任务的上一次自动对比。 形状按前后端约定(Schedule、Fire 两种对象,见 _schedule_out、_fire_outs)。Webhook 地址和签名密钥加密存放,接口 只给主机和末 4 位(url_hint)、有没有密钥(has_secret)。请求体自己解析、自己校验:错误一律回中文 detail,表单 @@ -18,7 +18,7 @@ from app.api.runs import actor_of from app.core.crypto import decrypt, encrypt from app.db.base import get_session, utcnow from app.db.models import Run, Schedule, ScheduleFire, WebhookDelivery, Workflow, WorkflowVersion -from app.scheduler import cron, service, webhook +from app.scheduler import auto_compare, cron, service, webhook router = APIRouter(prefix="/api/schedules", tags=["schedules"]) @@ -278,7 +278,11 @@ def _webhook_out(row: Schedule) -> dict[str, Any] | None: async def _fire_outs(session: AsyncSession, fires: list[ScheduleFire]) -> list[dict[str, Any]]: - """Fire 对象:run_status 是运行现在的状态,notified 是这次触发的推送记录 [{event, status, at}](按时间先后)。""" + """Fire 对象:run_status 是运行现在的状态,notified 是这次触发的推送记录 [{event, status, at}](按时间先后), + compare 是运行结束后和同一任务上一次的自动对比 {base_run_id, summary}(summary 同对比接口的计数),没有为 null。 + + 运行又被继续执行、现在不在终态时 compare 也给 null:对比页只认结束并封存的运行,链过去只会被拒;再结束时 + 通知扫描会按新的封存重新比。工件读不出来、被改过的同样给 null。""" run_ids = [f.run_id for f in fires if f.run_id] statuses = dict((await session.execute(select(Run.id, Run.status).where(Run.id.in_(run_ids)))).all()) \ if run_ids else {} @@ -292,6 +296,9 @@ async def _fire_outs(session: AsyncSession, fires: list[ScheduleFire]) -> list[d notified.setdefault(d.fire_id, []).append({"event": d.event, "status": d.status, "at": _iso(d.created_at)}) # 正在发起的触发:库里是占位的「未能发起」(为了进程被强杀时照实说),进程还在、只是还没发起完,按已发起给 launching = {f.id for f in fires if service.launching(f.id)} + compared = {f.id: f.compare_artifact for f in fires + if f.compare_artifact and statuses.get(f.run_id) in auto_compare.TERMINAL} + compares = await auto_compare.briefs(list(compared.values())) return [{ "id": f.id, "fire_at": _iso(f.fire_at), @@ -301,6 +308,7 @@ async def _fire_outs(session: AsyncSession, fires: list[ScheduleFire]) -> list[d "run_status": statuses.get(f.run_id) if f.run_id else None, "reason": None if f.id in launching else f.reason, "notified": notified.get(f.id, []), + "compare": compares.get(compared[f.id]) if f.id in compared else None, "created_at": _iso(f.created_at), } for f in fires] diff --git a/backend/app/scheduler/__init__.py b/backend/app/scheduler/__init__.py index aa1ef8b6..beac3ead 100644 --- a/backend/app/scheduler/__init__.py +++ b/backend/app/scheduler/__init__.py @@ -4,6 +4,7 @@ - service.py:输入模板、和工作流入口字段的对照、发起一次触发(调度循环和「立即运行」共用) - loop.py:调度循环,条件 UPDATE 认领到点的任务 - webhook.py:通知推送(通用 / 飞书 / 企业微信 / 钉钉) +- auto_compare.py:运行结束后和同一定时任务的上一次自动对比,结果记在触发记录上,通知里只带计数摘要和对比页链接 服务启动时(main.py 的 lifespan)只在 scheduler_enabled() 时调 start,关闭时总是调 stop——stop 要能在没 start 过时安全返回。start 自己也看一次开关:拷来的数据目录(检查脚本的 --stacks、验收副本)不管从哪条路径起服务, diff --git a/backend/app/scheduler/auto_compare.py b/backend/app/scheduler/auto_compare.py new file mode 100644 index 00000000..dc464b3f --- /dev/null +++ b/backend/app/scheduler/auto_compare.py @@ -0,0 +1,178 @@ +"""定时运行结束后的自动对比:和同一定时任务的上一次运行比一次,结果存成工件,工件 id 记在触发记录上 +(ScheduleFire.compare_artifact)。 + +对比本身调运行对比接口的现成函数(api/compare.py 的 default_base、compare_runs),这里只管什么时候比、跟谁比、结果 +放哪、怎么说给触发历史和通知听: + +- 什么时候:通知扫描(webhook.notify_due)认领到运行进入终态(结束、失败、取消)的那一次之后、推送之前。和推送一样, + 同一个终态只处理一次;失败后被继续执行、又结束的,按新的封存重新比,覆盖旧的结果 +- 跟谁比:default_base 的「同一定时任务」那一档,即这个任务里更早结束、封存过的终态运行中最近的一次。它退回「同一 + 工作流、同一运行类别」时不比:那不是这个定时任务的上一次,别人手动改的输入、另一个任务的参数都会被算成变化 +- 结果:整份对比结果存成工件(kind=run_compare)。不挂在任何运行名下:它不是运行产出的,不该出现在运行的工件列表里。 + 没封存的不比(服务重启时还在排队、被改成失败的,永远不会封存);比不了、出错只记日志,通知照常推送 +- 对外:触发历史(Fire.compare)和通知只给基准是哪一次、各类指标的个数,不给具体数字 +""" +from __future__ import annotations + +import asyncio +import logging +import threading +from typing import Any +from urllib.parse import urlencode + +from fastapi import HTTPException +from sqlalchemy import update + +from app.core import artifact_store +from app.db.base import SessionLocal +from app.db.models import Run, ScheduleFire + +logger = logging.getLogger(__name__) + +KIND = "run_compare" +#: 终态,同对比接口(api/evidence._TERMINAL):只有这几种状态、封存过的运行能参与对比 +TERMINAL = ("succeeded", "failed", "cancelled") +#: 各类指标的叫法,和对比页一致(frontend/src/lib/terms.ts 的 RUN_COMPARE_TEXT.classLabel) +CLASS_LABEL = { + "data_only": "只数据变", + "caliber_and_data": "口径与数据都变", + "sql_changed": "SQL 结构变", + "params_only": "只参数变", + "added": "新增", + "removed": "移除", + "incomparable": "无法比较", + "unchanged": "未变", +} +#: 变化三分法的三类,0 个也写:「SQL 结构变 0 个」本身就是要告诉人的事 +CORE = ("data_only", "caliber_and_data", "sql_changed") +#: 其余各类有才写,次序同对比页的摘要 +OTHERS = ("params_only", "added", "removed", "incomparable", "unchanged") +#: 两边都没有指标时的说法:不写一串「0 个」,免得读成「什么都没变」 +NO_METRICS = "两次运行都没有指标" + +#: 读过的对比工件:工件 id → 摘要。工件按内容寻址、内容不会变,记下来不会过期;触发历史、任务列表每次打开都要读, +#: 不必每次都去盘上读一份几十 KB 的 JSON。只记读成功的,满了丢最早的。读盘在线程里、记下新算的在事件循环里, +#: 两边都动这个字典,加一把线程锁 +_BRIEFS: dict[str, dict[str, Any]] = {} +_BRIEFS_MAX = 512 +_BRIEFS_LOCK = threading.Lock() + + +def summary_text(summary: dict[str, Any]) -> str: + """计数摘要:「只数据变 3 个、口径与数据都变 1 个、SQL 结构变 0 个」,其余各类有才接在后面。只有个数。""" + counts = {key: value for key, value in summary.items() if isinstance(value, int) and not isinstance(value, bool)} + if not any(counts.values()): + return NO_METRICS + parts = [f"{CLASS_LABEL[key]} {counts.get(key, 0)} 个" for key in CORE] + parts += [f"{CLASS_LABEL[key]} {counts[key]} 个" for key in OTHERS if counts.get(key)] + return "、".join(parts) + + +def page_url(base_url: str, target: str, base: str) -> str: + """对比页的地址(对外访问地址 + /runs/compare?target=…&base=…),和界面上运行详情跳过去的是同一个页面。""" + return f"{base_url}/runs/compare?{urlencode({'target': target, 'base': base})}" + + +async def compare_fire(fire_id: str, run_id: str) -> dict[str, Any] | None: + """这次触发发起的运行刚进入终态:和同一定时任务的上一次比一次,结果记进触发记录。返回给通知用的摘要 + {base_run_id, summary};没有可比的、比不了时返回 None,触发记录上之前的结果一并清掉(那是上一次封存的)。 + + 不抛异常:比不了(对比接口拒绝,如基准的封存核对不一致)记一条警告,意外的异常记堆栈,都不影响通知。""" + artifact: str | None = None + brief: dict[str, Any] | None = None + try: + artifact, brief = await _compare(run_id) + except HTTPException as e: + logger.warning("定时任务的运行 %s 自动对比未完成:%s", run_id, e.detail) + except Exception: # noqa: BLE001 - 对比出错不能拖住通知,也不能让通知扫描停下 + logger.exception("定时任务的运行 %s 自动对比出错", run_id) + await _store(fire_id, artifact) + return brief + + +async def _compare(run_id: str) -> tuple[str | None, dict[str, Any] | None]: + # 现成的对比函数在用到时才导入:api/compare.py 连带证据、推演接口,调度模块被导入时不必一起加载 + from app.api import compare as compare_api + + async with SessionLocal() as session: + run = await session.get(Run, run_id) + if run is None or run.status not in TERMINAL or not run.manifest_hash: + return None, None + found = await compare_api.default_base(run_id) + if found is None or found[1] != "same_schedule": + return None, None + base = found[0] + result = await compare_api.compare_runs(run_id, base.id) + brief = _brief(result) + if brief is None: + return None, None + artifact = await artifact_store.put_json(result, kind=KIND, meta={"target_run_id": run_id, "base_run_id": base.id}) + _remember(artifact, brief) + return artifact, _copy(brief) + + +async def _store(fire_id: str, artifact: str | None) -> None: + try: + async with SessionLocal() as session: + await session.execute(update(ScheduleFire).where(ScheduleFire.id == fire_id) + .values(compare_artifact=artifact)) + await session.commit() + except Exception: # noqa: BLE001 - 任务刚被删掉之类:记不下不影响通知 + logger.warning("触发记录 %s 的自动对比结果没能保存", fire_id) + + +def _brief(result: Any) -> dict[str, Any] | None: + """对比结果里给触发历史、通知的那一点:基准是哪一次、各类指标的个数(同对比接口的 summary)。""" + if not isinstance(result, dict): + return None + base, summary = result.get("base"), result.get("summary") + if not isinstance(base, dict) or not isinstance(base.get("id"), str) or not isinstance(summary, dict): + return None + return {"base_run_id": base["id"], + "summary": {key: value for key, value in summary.items() + if isinstance(value, int) and not isinstance(value, bool)}} + + +def _copy(brief: dict[str, Any]) -> dict[str, Any]: + return {"base_run_id": brief["base_run_id"], "summary": dict(brief["summary"])} + + +def _remember(artifact: str, brief: dict[str, Any]) -> None: + with _BRIEFS_LOCK: + if artifact not in _BRIEFS and len(_BRIEFS) >= _BRIEFS_MAX: + _BRIEFS.pop(next(iter(_BRIEFS))) + _BRIEFS[artifact] = brief + + +def brief_of(artifact: str) -> dict[str, Any] | None: + """读一份对比工件的摘要。文件不在、内容和哈希对不上(另记一条警告)、形状不对的返回 None,不抛异常。""" + with _BRIEFS_LOCK: + known = _BRIEFS.get(artifact) + if known is not None: + return _copy(known) + try: + brief = _brief(artifact_store.load(artifact)) + except ValueError: + logger.warning("对比工件 %s… 内容与哈希不符,触发历史中不显示这次对比", artifact[:12]) + return None + if brief is None: + return None + _remember(artifact, brief) + return _copy(brief) + + +async def briefs(artifacts: list[str]) -> dict[str, dict[str, Any]]: + """几份对比工件的摘要:工件 id → 摘要,读不出来的不在里面。读盘在线程里做。""" + wanted = list(dict.fromkeys(a for a in artifacts if a)) + if not wanted: + return {} + + def read() -> dict[str, dict[str, Any]]: + out = {} + for artifact in wanted: + brief = brief_of(artifact) + if brief is not None: + out[artifact] = brief + return out + + return await asyncio.to_thread(read) diff --git a/backend/app/scheduler/webhook.py b/backend/app/scheduler/webhook.py index c7b9f2b1..e74e12b7 100644 --- a/backend/app/scheduler/webhook.py +++ b/backend/app/scheduler/webhook.py @@ -9,8 +9,9 @@ Webhook 地址本身就是凭证(企业微信的 key、钉钉的 access_token 一个过滤器把推送期间的那几行换成主机名。 通知里只有任务名、状态、出具档位、缺口条数、运行链接、封存哈希(界面上叫清单哈希)前 12 位和起止时间,失败时 -另写失败节点的名字和去哪里查看(不许诺能继续运行)。不放报告里的具体数字,也不放失败时的原始报错——报错里可能带着 -查询到的数据,Webhook 推去的群不一定有权看。 +另写失败节点的名字和去哪里查看(不许诺能继续运行);运行结束后和同一任务上一次的对比(auto_compare)只写各类 +指标的个数和对比页链接。不放报告里的具体数字,也不放失败时的原始报错——报错里可能带着查询到的数据,Webhook +推去的群不一定有权看。 三个平台出错时都回 HTTP 200、错误码放在响应体里,所以送没送达要看响应体:按地址的主机认平台,不看选的格式—— 格式选错了(飞书的地址选了通用格式)正是最需要认出来的情形。群机器人的地址只认自己平台的格式,保存时就拦下 @@ -39,7 +40,7 @@ from app.core.events import EventType from app.db.base import SessionLocal, utcnow from app.db.models import Approval, Run, RunEvent, Schedule, ScheduleFire, WebhookDelivery from app.engine.labels import ISSUANCE_LABEL -from app.scheduler import service +from app.scheduler import auto_compare, service logger = logging.getLogger(__name__) @@ -227,6 +228,7 @@ class Notice: suspended:运行停在断点上、没有待审批(通常是服务重启挂起的),按 failed 事件推。error_node:失败节点的名字。 next_fire:任务启用着、往后还会触发(失败通知据此补「下一次触发会照常发起」)。 + compare:和同一定时任务上一次的对比摘要 {base_run_id, summary}(只有个数),compare_url:对比页的链接。 fmt、signed 只给测试消息用:说明里只说这条消息证明得了的事。""" event: str @@ -249,6 +251,8 @@ class Notice: suspended: bool = False error_node: str | None = None next_fire: bool = False + compare: dict[str, Any] | None = None + compare_url: str | None = None fmt: str | None = None signed: bool = False @@ -287,6 +291,9 @@ def message(notice: Notice) -> tuple[str, list[tuple[str, str]], str | None]: if notice.tier: gaps = f"(缺口 {notice.gaps} 条)" if notice.gaps is not None else "" lines.append(("出具档位", f"{ISSUANCE_LABEL.get(notice.tier, notice.tier)}{gaps}")) + if notice.compare: + # 只给各类指标的个数:具体的数在对比页上,跟着运行的权限走,不随消息推进群里 + lines.append(("与上一次相比", auto_compare.summary_text(notice.compare.get("summary") or {}))) if notice.pending_approvals: lines.append(("待审批", f"{notice.pending_approvals} 项,请到 AgentLab 处理;运行结束之前," "这个定时任务之后的触发都会跳过")) @@ -338,31 +345,34 @@ def _test_text(notice: Notice) -> str: return "这是一条测试消息。收到即表示 Webhook 地址配置正确" +def _links(notice: Notice) -> list[tuple[str, str]]: + """消息末尾的链接:运行,有对比时再加对比页。没设对外访问地址时都没有。""" + return [(label, url) for label, url in (("查看运行", notice.run_url), ("查看对比", notice.compare_url)) if url] + + def _text(notice: Notice) -> str: - title, lines, link = message(notice) + title, lines, _ = message(notice) out = [f"【AgentLab】{title}", *(f"{k}:{v}" for k, v in lines)] - if link: - out.append(f"查看运行:{link}") + out += [f"{label}:{url}" for label, url in _links(notice)] return "\n".join(out) def _wecom_markdown(notice: Notice) -> str: - title, lines, link = message(notice) + title, lines, _ = message(notice) color = "info" if notice.event in ("finished", "test") else "warning" out = [f"**【AgentLab】{title}**"] for key, value in lines: shown = f'{value}' if key == "状态" else value out.append(f"> {key}:{shown}") - if link: - out.append(f"[查看运行]({link})") + out += [f"[{label}]({url})" for label, url in _links(notice)] return "\n".join(out) def _dingtalk_markdown(notice: Notice) -> tuple[str, str]: - title, lines, link = message(notice) + title, lines, _ = message(notice) out = [f"#### 【AgentLab】{title}", "", *(f"- {k}:{v}" for k, v in lines)] - if link: - out += ["", f"[查看运行]({link})"] + for label, url in _links(notice): + out += ["", f"[{label}]({url})"] return title, "\n".join(out) @@ -396,6 +406,10 @@ def _generic(notice: Notice, now: datetime) -> dict[str, Any]: else f"{SUSPENDED_REASON}。{SUSPENDED_ACTION}" if notice.suspended else NOT_STARTED_REASON if notice.never_started else None), "error_node": notice.error_node, + # 对比只给基准是哪一次、各类指标的个数和对比页链接 + "compare": ({"base_run_id": notice.compare.get("base_run_id"), + "summary": dict(notice.compare.get("summary") or {}), "url": notice.compare_url} + if notice.compare else None), "sent_at": _iso(now), } @@ -576,15 +590,29 @@ class _Job: notice: Notice +@dataclass +class _Due: + """认领到的一次状态变化接下来要做的事:运行进入终态时和上一次比一次(compare_run),订阅了这个事件时推一条 + (job)。两样都在锁外做:对比要读快照、重算,推送要等接收端,都不能挡着发起。""" + + fire_id: str + compare_run: str | None + job: _Job | None + + async def notify_due(now: datetime | None = None) -> int: """查已发起的触发,运行状态有变化的按订阅推送一次。返回这一拍推送了几条(含失败的)。 处理过的状态记在 notified_status 上,用条件 UPDATE 认领(WHERE notified_status = 旧值):两拍撞在一起也只 推一次。没订阅这个事件、任务没配 Webhook 的也记下已处理——之后再配上 Webhook,不会把旧的补推一遍。 停在断点上的运行每一拍都要重新看(见 _state_of):它的已处理状态带着第几段执行,和运行状态本身对不上。 + + 运行进入终态的那一次,推送之前先和同一定时任务的上一次比一次(auto_compare),结束、失败的通知带上计数摘要和 + 对比页链接。没配 Webhook 的也比:触发历史里要用。比不了、出错不影响推送。 """ now = now or utcnow() - jobs: list[_Job] = [] + dues: list[_Due] = [] + base_url: str | None = None cutoff = now - timedelta(days=SCAN_DAYS) # 扫描窗口按触发记录的创建时间算,两种例外:运行记录窗口内有过变化(运行状态一变 updated_at 就跟着变),以及 # 停在审批上、有待审批的——以前只看触发记录,创建 30 天以后才停到审批上的运行收不到「等待审批」 @@ -603,7 +631,6 @@ async def notify_due(now: datetime | None = None) -> int: )) .order_by(ScheduleFire.created_at) )).all() - base_url: str | None = None if rows: from app.api.settings import notify_settings @@ -613,16 +640,30 @@ async def notify_due(now: datetime | None = None) -> int: if state is None or state == fire.notified_status \ or not await _mark(fire.id, fire.notified_status, state): continue + compare_run = run.id if run is not None and run.status in auto_compare.TERMINAL else None schedule = await session.get(Schedule, fire.schedule_id) url = decrypt(schedule.webhook_url) if schedule and schedule.webhook_url else None - if not (event and url and schedule.webhook_format in FORMATS - and event in (schedule.webhook_events or [])): - continue - notice = await _notice(session, schedule, fire, run, event, base_url, pending) - jobs.append(_Job(schedule.webhook_format, url, decrypt(schedule.webhook_secret), notice)) - if jobs: - await asyncio.gather(*(_deliver(job) for job in jobs)) - return len(jobs) + job = None + if event and url and schedule.webhook_format in FORMATS and event in (schedule.webhook_events or []): + notice = await _notice(session, schedule, fire, run, event, base_url, pending) + job = _Job(schedule.webhook_format, url, decrypt(schedule.webhook_secret), notice) + if compare_run or job: + dues.append(_Due(fire.id, compare_run, job)) + if dues: + await asyncio.gather(*(_follow_up(due, base_url) for due in dues)) + return sum(1 for due in dues if due.job is not None) + + +async def _follow_up(due: _Due, base_url: str | None) -> None: + """先比(需要的话),再推:对比摘要要写进这条通知。对比自己兜住了异常,出什么事都不挡推送。""" + if due.compare_run: + brief = await auto_compare.compare_fire(due.fire_id, due.compare_run) + if brief is not None and due.job is not None: + due.job.notice.compare = brief + if base_url: + due.job.notice.compare_url = auto_compare.page_url(base_url, due.compare_run, brief["base_run_id"]) + if due.job is not None: + await _deliver(due.job) async def _state_of(session: Any, fire: ScheduleFire, run: Run | None, diff --git a/backend/tests/test_scheduler_compare.py b/backend/tests/test_scheduler_compare.py new file mode 100644 index 00000000..480bfb53 --- /dev/null +++ b/backend/tests/test_scheduler_compare.py @@ -0,0 +1,395 @@ +"""定时运行结束后的自动对比(app/scheduler/auto_compare.py,由通知扫描 webhook.notify_due 在推送之前触发): + +- 运行进入终态且已封存、同一定时任务有更早的已封存运行时,用运行对比的现成函数(api/compare.py 的 default_base、 + compare_runs)和它比一次,整份结果存成工件(kind=run_compare,不挂在运行名下、不进运行的工件列表),工件 id 记进 + 触发记录的 compare_artifact。基准是同一定时任务的上一次:同一工作流更近的手动运行、别的定时任务的运行都不算 +- 这个任务的第一次(只有同一工作流别的运行可比)、还没封存时不比;对比出错(基准的封存核对不一致、意外的异常)只记 + 日志,通知照常推送、不带对比 +- 接口:Fire 多一个 compare: {base_run_id, summary} 或 null,summary 同对比接口的计数;运行又被继续执行、不在终态 + 时为 null(对比页只认封存的运行) +- 通知:结束、失败的通知多一行「与上一次相比:只数据变 2 个、口径与数据都变 0 个、SQL 结构变 0 个」,配了对外访问 + 地址时附对比页链接(/runs/compare?target=…&base=…);只给个数,报告、指标、查询里的具体数字一个不出现 + +两次周报运行用 tests/fixtures/compare_runs.py 的虚构运行(区域、金额、日期都是编的),按 runner 的算法封存,不真的 +执行;Webhook 用 httpx.MockTransport 接住,不连外网。 +""" +from __future__ import annotations + +import json +import logging +from datetime import datetime, timedelta, timezone +from urllib.parse import parse_qs, urlsplit + +import httpx +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import delete, select + +from app.core.artifact_store import canonical_json, load, manifest_hash, put_json +from app.core.crypto import encrypt +from app.db.base import SessionLocal +from app.db.models import Artifact, Run, RunEvent, Schedule, ScheduleFire, Setting, Workflow +from app.main import app +from app.scheduler import auto_compare, webhook +from tests.fixtures.compare_runs import NEXT_ROWS, Fake, week + +UTC = timezone.utc +T0 = datetime(2026, 9, 7, 1, 0, tzinfo=UTC) +WEEK = timedelta(days=7) +GENERIC = "https://hooks.example.com/agentlab/inbox-5a7c" +FEISHU = "https://open.feishu.cn/open-apis/bot/v2/hook/1a2b3c4d-aaaa-bbbb-cccc-0d9e8f7a6b5c" +BASE_URL = "https://agentlab.example.com" +#: 两次虚构周报(BASE_ROWS → NEXT_ROWS)之间两个指标都只是数据变了 +SUMMARY = {"unchanged": 0, "data_only": 2, "caliber_and_data": 0, "sql_changed": 0, "params_only": 0, "added": 0, + "removed": 0, "incomparable": 0} +LINE = "与上一次相比:只数据变 2 个、口径与数据都变 0 个、SQL 结构变 0 个" +#: 两次运行查到、算出的数(金额、退款、合计、变化量),通知里一个都不能有 +NUMBERS = ("1200.5", "1,200.5", "1500.5", "1,500.5", "2600.5", "2,600.5", "2800.5", "2,800.5", "300.0", "800.0", + "700.0", "600.0") + + +class Recorder: + def __init__(self) -> None: + self.requests: list[httpx.Request] = [] + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.requests.append(request) + return httpx.Response(200, json={"code": 0, "errcode": 0}) + + def bodies(self) -> list[dict]: + return [json.loads(r.content) for r in self.requests] + + +@pytest.fixture +def recorder(monkeypatch): + rec = Recorder() + monkeypatch.setattr(webhook, "_transport", httpx.MockTransport(rec)) + monkeypatch.setattr(webhook, "RETRY_DELAYS", (0, 0, 0)) + return rec + + +@pytest.fixture(autouse=True) +async def clean(): + async with SessionLocal() as session: + await session.execute(delete(Schedule)) + await session.execute(delete(Setting).where(Setting.key == "notify")) + await session.commit() + yield + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +async def public_address() -> None: + async with SessionLocal() as session: + session.add(Setting(key="notify", value={"public_base_url": BASE_URL})) + await session.commit() + + +async def new_workflow(name: str = "区域周报") -> str: + async with SessionLocal() as session: + workflow = Workflow(name=name) + session.add(workflow) + await session.commit() + return workflow.id + + +async def new_schedule(workflow_id: str, *, fmt: str = "generic", url: str | None = GENERIC, + name: str = "区域周报 · 每周") -> str: + async with SessionLocal() as session: + row = Schedule(name=name, workflow_id=workflow_id, cron="0 9 * * 1", timezone="Asia/Shanghai", + enabled=True, next_run_at=datetime.now(UTC) + timedelta(days=3), + webhook_format=fmt if url else None, webhook_url=encrypt(url) if url else None, + webhook_events=["finished", "failed", "awaiting_approval"]) + session.add(row) + await session.commit() + return row.id + + +async def persist(fake: Fake, workflow_id: str, *, finished: datetime, schedule_id: str | None = None, + status: str = "succeeded", seal: bool = True) -> str: + """把一次虚构运行落库:工件进工件库,事件进运行记录,按 runner 的算法封存(同 test_compare_api.py)。""" + for content in fake.store.values(): + await put_json(content, kind="compare_fixture") + async with SessionLocal() as session: + run = Run(workflow_id=workflow_id, workflow_name="区域周报", status=status, graph=fake.graph, + input=dict(fake.inputs), run_class="formal", version=fake.version, schedule_id=schedule_id, + started_at=finished - timedelta(minutes=2), finished_at=finished, agent_limits={}, + started_by="定时任务:区域周报 · 每周" if schedule_id else "周报编辑") + session.add(run) + await session.flush() + rows = [] + for seq, etype, node_id, data in fake.events: + session.add(RunEvent(run_id=run.id, seq=seq, type=etype, node_id=node_id, ts=float(seq), data=data)) + rows.append((seq, etype, node_id, data)) + run.last_seq = len(rows) + if seal: + run.manifest_hash, run.manifest_seq = manifest_hash(rows), rows[-1][0] + await session.commit() + return run.id + + +async def seal(run_id: str) -> None: + async with SessionLocal() as session: + run = await session.get(Run, run_id) + rows = [tuple(r) for r in await session.execute( + select(RunEvent.seq, RunEvent.type, RunEvent.node_id, RunEvent.data) + .where(RunEvent.run_id == run_id).order_by(RunEvent.seq))] + run.manifest_hash, run.manifest_seq = manifest_hash(rows), rows[-1][0] + await session.commit() + + +async def fire_for(schedule_id: str, run_id: str, at: datetime, *, notified: str | None = None) -> str: + async with SessionLocal() as session: + fire = ScheduleFire(schedule_id=schedule_id, fire_at=at, kind="scheduled", status="started", run_id=run_id, + notified_status=notified) + session.add(fire) + await session.commit() + return fire.id + + +async def fire_row(fire_id: str) -> ScheduleFire: + async with SessionLocal() as session: + return await session.get(ScheduleFire, fire_id) + + +async def set_run(run_id: str, **values) -> None: + async with SessionLocal() as session: + await session.execute(Run.__table__.update().where(Run.id == run_id).values(**values)) + await session.commit() + + +async def weekly_pair(*, target_status: str = "succeeded", target_seal: bool = True, + target_finished: datetime | None = None, fmt: str = "generic", url: str | None = GENERIC): + """同一个定时任务的两次周报:上一次(已推送过)和这一次(还没处理),这次的数变了。返回 (任务, 上一次, 这一次, + 这一次的触发)。""" + workflow = await new_workflow() + sid = await new_schedule(workflow, fmt=fmt, url=url) + base_id = await persist(week("r1"), workflow, finished=T0, schedule_id=sid) + await fire_for(sid, base_id, T0 - timedelta(minutes=2), notified="succeeded") + target_id = await persist(week("r2", NEXT_ROWS), workflow, finished=target_finished or T0 + WEEK, + schedule_id=sid, status=target_status, seal=target_seal) + fire_id = await fire_for(sid, target_id, T0 + WEEK - timedelta(minutes=2)) + return sid, base_id, target_id, fire_id + + +def no_numbers(raw: str) -> list[str]: + return [n for n in NUMBERS if n in raw] + + +# -------------------------------------------------------------------------- +# 什么时候比、跟谁比、存在哪 +# -------------------------------------------------------------------------- + + +async def test_finished_run_is_compared_with_the_previous_run_of_its_schedule(recorder, client): + await public_address() + workflow = await new_workflow() + sid = await new_schedule(workflow) + other = await new_schedule(workflow, url=None, name="区域周报 · 补跑") + base_id = await persist(week("r1"), workflow, finished=T0, schedule_id=sid) + base_fire = await fire_for(sid, base_id, T0 - timedelta(minutes=2), notified="succeeded") + # 同一工作流、比上一次更近的两次:手动发起的、别的定时任务发起的。都不是「同一定时任务的上一次」 + await persist(week("m1"), workflow, finished=T0 + timedelta(days=3)) + await persist(week("o1"), workflow, finished=T0 + timedelta(days=5), schedule_id=other) + target_id = await persist(week("r2", NEXT_ROWS), workflow, finished=T0 + WEEK, schedule_id=sid) + fire_id = await fire_for(sid, target_id, T0 + WEEK - timedelta(minutes=2)) + + assert await webhook.notify_due() == 1 + + artifact = (await fire_row(fire_id)).compare_artifact + assert artifact + from app.api.compare import compare_runs + + stored = load(artifact) + assert stored == json.loads(canonical_json(await compare_runs(target_id, base_id))) + assert stored["base"]["id"] == base_id and stored["summary"] == SUMMARY + async with SessionLocal() as session: + rows = list((await session.execute(select(Artifact).where(Artifact.id == artifact))).scalars()) + # 对比结果不是运行产出的:不挂在运行名下,不进这次运行的工件列表 + assert [(a.kind, a.run_id) for a in rows] == [("run_compare", "")] + assert artifact not in [a["id"] for a in (await client.get(f"/api/runs/{target_id}/artifacts")).json()] + + items = {f["id"]: f for f in (await client.get(f"/api/schedules/{sid}/fires")).json()["items"]} + assert items[fire_id]["compare"] == {"base_run_id": base_id, "summary": SUMMARY} + assert items[base_fire]["compare"] is None + [listed] = [s for s in (await client.get("/api/schedules")).json()["items"] if s["id"] == sid] + assert listed["last_fire"]["id"] == fire_id and listed["last_fire"]["compare"]["base_run_id"] == base_id + + [body] = recorder.bodies() + link = f"{BASE_URL}/runs/compare?target={target_id}&base={base_id}" + assert body["event"] == "finished" and LINE in body["text"] and f"查看对比:{link}" in body["text"] + assert body["compare"] == {"base_run_id": base_id, "summary": SUMMARY, "url": link} + query = parse_qs(urlsplit(body["compare"]["url"]).query) + assert query == {"target": [target_id], "base": [base_id]} + assert no_numbers(recorder.requests[0].content.decode()) == [] + # 下一拍:同一状态不再比、不再推 + assert await webhook.notify_due() == 0 and (await fire_row(fire_id)).compare_artifact == artifact + + +@pytest.mark.parametrize("fmt, url", [("feishu", FEISHU), ("generic", GENERIC)]) +async def test_failed_run_notice_carries_the_counts_too(recorder, fmt, url): + """失败的运行也封存:和上一次比一次,失败通知同样带计数摘要""" + await public_address() + _, base_id, target_id, fire_id = await weekly_pair(target_status="failed", fmt=fmt, url=url) + assert await webhook.notify_due() == 1 + assert (await fire_row(fire_id)).compare_artifact + raw = recorder.requests[0].content.decode() + text = json.loads(raw)["content"]["text"] if fmt == "feishu" else json.loads(raw)["text"] + assert "运行失败" in text and LINE in text and f"base={base_id}" in text and f"target={target_id}" in text + assert no_numbers(raw) == [] + + +async def test_cancelled_run_is_compared_but_not_pushed(recorder, client): + """取消的运行不推送(有人手动停的),对比照样算:触发历史里一样能点「对比上一次」""" + sid, base_id, _, fire_id = await weekly_pair(target_status="cancelled") + assert await webhook.notify_due() == 0 and recorder.requests == [] + items = {f["id"]: f for f in (await client.get(f"/api/schedules/{sid}/fires")).json()["items"]} + assert items[fire_id]["compare"]["base_run_id"] == base_id + + +async def test_first_run_of_a_schedule_has_no_compare(recorder, client): + """这个任务的第一次:同一工作流有更早的手动运行,对比接口会退回「同一工作流」那一档,那不是「上一次」,不比""" + await public_address() + workflow = await new_workflow() + sid = await new_schedule(workflow) + await persist(week("m1"), workflow, finished=T0) + target_id = await persist(week("r2", NEXT_ROWS), workflow, finished=T0 + WEEK, schedule_id=sid) + fire_id = await fire_for(sid, target_id, T0 + WEEK - timedelta(minutes=2)) + assert await webhook.notify_due() == 1 + assert (await fire_row(fire_id)).compare_artifact is None + [item] = (await client.get(f"/api/schedules/{sid}/fires")).json()["items"] + assert item["compare"] is None + [body] = recorder.bodies() + assert body["compare"] is None and "与上一次相比" not in body["text"] and "查看对比" not in body["text"] + assert f"查看运行:{BASE_URL}/runs/{target_id}" in body["text"] + + +async def test_waits_for_the_seal_before_comparing(recorder): + """runner 先写终态、再封存:两步之间被这一拍撞上就不比、不推,封存好了下一拍再比""" + _, base_id, target_id, fire_id = await weekly_pair(target_seal=False, target_finished=datetime.now(UTC)) + assert await webhook.notify_due() == 0 + assert (await fire_row(fire_id)).compare_artifact is None and recorder.requests == [] + await seal(target_id) + assert await webhook.notify_due() == 1 + assert load((await fire_row(fire_id)).compare_artifact)["base"]["id"] == base_id + assert LINE in recorder.bodies()[0]["text"] + + +async def test_no_public_address_means_counts_without_a_link(recorder): + _, _, _, fire_id = await weekly_pair() + assert await webhook.notify_due() == 1 + [body] = recorder.bodies() + assert LINE in body["text"] and "查看对比" not in body["text"] and body["compare"]["url"] is None + assert (await fire_row(fire_id)).compare_artifact + + +# -------------------------------------------------------------------------- +# 比不了、出错 +# -------------------------------------------------------------------------- + + +async def break_seal(run_id: str) -> None: + async with SessionLocal() as session: + event = (await session.execute(select(RunEvent).where( + RunEvent.run_id == run_id, RunEvent.type == "tool.end"))).scalars().first() + event.data = {**event.data, "duration_ms": 999} + await session.commit() + + +async def explode(*args, **kwargs): + raise RuntimeError("快照读取失败:磁盘错误") + + +@pytest.mark.parametrize("failure", ["broken_seal", "unexpected"]) +async def test_compare_failure_only_logs_and_the_notice_still_goes_out(recorder, caplog, monkeypatch, failure): + """基准的封存核对不一致(对比接口回 409)、对比中途抛了意外的异常:只记日志,通知照常推送、不带对比""" + await public_address() + _, base_id, target_id, fire_id = await weekly_pair() + if failure == "broken_seal": + await break_seal(base_id) + else: + from app.api import compare as compare_api + + monkeypatch.setattr(compare_api, "compare_runs", explode) + with caplog.at_level(logging.INFO, logger="app.scheduler"): + assert await webhook.notify_due() == 1 + assert (await fire_row(fire_id)).compare_artifact is None + [body] = recorder.bodies() + assert (body["event"], body["run_id"], body["compare"]) == ("finished", target_id, None) + assert "与上一次相比" not in body["text"] and f"{BASE_URL}/runs/{target_id}" in body["text"] + assert "自动对比" in caplog.text and target_id in caplog.text + if failure == "broken_seal": + assert "封存核对不一致" in caplog.text + + +async def test_compare_is_hidden_once_the_run_is_running_again(client): + """失败后被继续执行(不在终态):对比页只认封存的运行,接口不给 compare;再次结束时重新比""" + sid, base_id, target_id, fire_id = await weekly_pair(target_status="failed", url=None) + await webhook.notify_due() + assert (await fire_row(fire_id)).compare_artifact + await set_run(target_id, status="running") + [item] = [f for f in (await client.get(f"/api/schedules/{sid}/fires")).json()["items"] if f["id"] == fire_id] + assert item["run_status"] == "running" and item["compare"] is None + await set_run(target_id, status="succeeded", finished_at=T0 + WEEK + timedelta(hours=1)) + await webhook.notify_due() + [item] = [f for f in (await client.get(f"/api/schedules/{sid}/fires")).json()["items"] if f["id"] == fire_id] + assert item["compare"] == {"base_run_id": base_id, "summary": SUMMARY} + + +async def test_a_missing_artifact_reads_as_no_compare(client): + """工件文件不在了、被改过:接口给 null,不报错""" + sid, _, _, fire_id = await weekly_pair(url=None) + async with SessionLocal() as session: + await session.execute(ScheduleFire.__table__.update().where(ScheduleFire.id == fire_id) + .values(compare_artifact="0" * 64)) + await session.commit() + resp = await client.get(f"/api/schedules/{sid}/fires") + assert resp.status_code == 200 + assert {f["id"]: f for f in resp.json()["items"]}[fire_id]["compare"] is None + + +# -------------------------------------------------------------------------- +# 计数摘要的写法 +# -------------------------------------------------------------------------- + + +def counts(**over) -> dict[str, int]: + return {**{k: 0 for k in SUMMARY}, **over} + + +def test_summary_text_always_names_the_three_kinds_of_change(): + assert auto_compare.summary_text(counts(data_only=3, caliber_and_data=1)) == \ + "只数据变 3 个、口径与数据都变 1 个、SQL 结构变 0 个" + # 其余各类有才写,次序同对比页 + assert auto_compare.summary_text(counts(sql_changed=1, params_only=4, added=1, removed=2, incomparable=1, + unchanged=5)) == \ + "只数据变 0 个、口径与数据都变 0 个、SQL 结构变 1 个、只参数变 4 个、新增 1 个、移除 2 个、无法比较 1 个、未变 5 个" + # 两边都没有指标:不写一串 0 个,免得读成「什么都没变」 + assert auto_compare.summary_text(counts()) == "两次运行都没有指标" + + +@pytest.mark.parametrize("fmt, url, link_text", [ + ("generic", GENERIC, "查看对比:{link}"), + ("feishu", FEISHU, "查看对比:{link}"), + ("wecom", "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=0a1b2c3d-1111-2222-3333-444455556666", + "[查看对比]({link})"), + ("dingtalk", "https://oapi.dingtalk.com/robot/send?access_token=0123456789abcdef0123456789abcdef", + "[查看对比]({link})"), +]) +def test_every_format_carries_the_counts_and_the_compare_link(fmt, url, link_text): + link = f"{BASE_URL}/runs/compare?target=r2&base=r1" + notice = webhook.Notice(event="finished", schedule_id="s1", schedule_name="区域周报", timezone="Asia/Shanghai", + run_id="r2", status="succeeded", run_url=f"{BASE_URL}/runs/r2", + compare={"base_run_id": "r1", "summary": counts(data_only=3, caliber_and_data=1)}, + compare_url=link) + _, _, body = webhook.build_request(fmt, url, None, notice) + # 请求体是 ensure_ascii=False 的 JSON:中文、链接原样出现在里面 + text = body.decode() + assert "与上一次相比:只数据变 3 个、口径与数据都变 1 个、SQL 结构变 0 个" in text + assert link_text.format(link=link) in text + # 运行链接照旧在前 + assert text.index("/runs/r2") < text.index("/runs/compare") diff --git a/backend/tests/test_schedules_api.py b/backend/tests/test_schedules_api.py index fe4eb26d..cec1eeca 100644 --- a/backend/tests/test_schedules_api.py +++ b/backend/tests/test_schedules_api.py @@ -40,7 +40,7 @@ SECRET = "SECdemo-api-not-real-7" SCHEDULE_KEYS = {"id", "name", "workflow_id", "workflow_name", "cron", "timezone", "run_class", "input_template", "enabled", "overlap", "misfire_grace_s", "webhook", "next_run_at", "last_fire", "description_text", "created_by", "created_at", "updated_at", "version"} -FIRE_KEYS = {"id", "fire_at", "kind", "status", "run_id", "run_status", "reason", "notified", "created_at"} +FIRE_KEYS = {"id", "fire_at", "kind", "status", "run_id", "run_status", "reason", "notified", "compare", "created_at"} @pytest.fixture(autouse=True) From 863b0b289425481352ded36bce7f85a5051a20ab Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 18:58:25 +0800 Subject: [PATCH 3/6] =?UTF-8?q?=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E5=89=8D=E7=AB=AF=EF=BC=9A=E8=A7=A6=E5=8F=91=E5=8E=86=E5=8F=B2?= =?UTF-8?q?=E5=8A=A0=E3=80=8C=E5=AF=B9=E6=AF=94=E4=B8=8A=E4=B8=80=E6=AC=A1?= =?UTF-8?q?=E3=80=8D=EF=BC=8C=E8=BF=90=E8=A1=8C=E8=AF=A6=E6=83=85=E9=A1=B5?= =?UTF-8?q?=E5=A4=B4=E7=9A=84=E3=80=8C=E7=94=B1=E5=AE=9A=E6=97=B6=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E3=80=8CX=E3=80=8D=E5=8F=91=E8=B5=B7=E3=80=8D?= =?UTF-8?q?=E6=94=B9=E6=88=90=E9=94=AE=E7=9B=98=E5=8F=AF=E8=BE=BE=E7=9A=84?= =?UTF-8?q?=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 触发历史:服务端给了 compare 的那次触发,在运行链接前加「对比上一次」,链到对比页(本次是这次触发的运行、 基准是服务端自动对比时取的同一任务上一次),带上来历让对比页写「默认基准:同一定时任务的上一次运行」。悬停写 各类指标的个数,和 Webhook 通知那一行同一个写法(三类总写、其余有才写、两边都没有指标时不写一串 0 个),读屏 当说明念;只有个数。compare 为 null、老服务端不给时不显示 - 运行详情页头:定时任务发起的运行,「谁发起的」那一段换成到 /schedules/:id 的真链接(Tab 到得了、回车能开, 有焦点环)。到点发起的(署名是「定时任务:X」)整句换成「由定时任务「X」发起」,不再写两遍;有人署名点了 立即运行的,署名照写、后面接「来自定时任务「X」」。名字取定时任务现在的名字,取不到用署名里记的;长名字截断, 全名在悬停里。记录页「定时」标记的说明同步改成指向这里 - types.ts 定时任务那一段加 ScheduleFireCompare 和 ScheduleFire.compare;文案进 SCHEDULE_TEXT - check-schedules:触发历史的对比链接(地址、悬停的计数、只在有 compare 的行出现、Tab 到得了、回车打开对比页)、 运行详情页头的链接(现在的名字、真链接、不重复署名、Tab 到得了且有焦点环、回车去定时任务、署名的立即运行、 手动运行没有、长名字截断不撑宽、浅色可读),390px 深浅色下对比链接在屏幕里、看得清 --- frontend/src/lib/terms.ts | 31 ++++ frontend/src/pages/runs/RunDetailView.tsx | 10 +- frontend/src/pages/schedules/FireHistory.tsx | 29 +++- frontend/src/pages/schedules/parts.tsx | 41 ++++- frontend/src/types.ts | 13 ++ scripts/check-schedules.mjs | 167 ++++++++++++++++++- 6 files changed, 280 insertions(+), 11 deletions(-) diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index af39ded1..c5645c78 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -3050,6 +3050,37 @@ export const SCHEDULE_TEXT = { allLoaded: '已全部加载', shown: (n: number) => `已显示 ${n} 条`, + // ---- 触发历史里的对比:运行结束后服务端自动和同一定时任务的上一次比一次 + compareLink: '对比上一次', + /** + * 悬停写各类指标的个数,和 Webhook 通知里那一行同一个写法(服务端 scheduler/auto_compare.py 的 summary_text): + * 变化三分法的三类总写上,0 个也写;其余各类有才写,次序同对比页;两边都没有指标时不写一串 0 个。只有个数, + * 不写数值。各类的叫法用对比页那一套(RUN_COMPARE_TEXT.classLabel) + */ + compareTitle: (summary: Partial>) => { + const n = (key: string) => (typeof summary[key] === 'number' ? summary[key] as number : 0) + const core = ['data_only', 'caliber_and_data', 'sql_changed'] + const rest = ['params_only', 'added', 'removed', 'incomparable', 'unchanged'] + if (![...core, ...rest].some((key) => n(key) > 0)) return '与上一次相比:两次运行都没有指标' + const parts = [...core, ...rest.filter((key) => n(key) > 0)] + .map((key) => `${RUN_COMPARE_TEXT.classLabel[key] ?? key} ${n(key)} 个`) + return `与上一次相比:${parts.join('、')}` + }, + + // ---- 运行详情页头的来源 + /** + * 到点发起(以及没有署名的立即运行)的运行记在「定时任务:名字」名下,前缀和服务端 scheduler/service.py 的 + * actor_label 一致。页头据此不再写「定时任务:X 发起」,换成下面那条链接 + */ + actorPrefix: '定时任务:', + runSource: (name?: string | null) => (name ? `由定时任务「${name}」发起` : '由定时任务发起'), + /** 有人署名点「立即运行」发起的:署名照写(runStarter),后面接这一句 */ + runSourceManual: (name?: string | null) => (name ? `来自定时任务「${name}」` : '来自定时任务'), + runStarter: (who: string) => `${who} 发起`, + /** 悬停写全名(页头窄时链接会截断)和点开去哪 */ + runSourceTitle: (name?: string | null) => + (name ? `定时任务「${name}」:查看设置和触发历史` : '查看这个定时任务的设置和触发历史'), + // ---- 记录页的定时标记 runTag: '定时', runTagTitle: (name?: string | null) => (name ? `由定时任务「${name}」发起,点击查看` : '由定时任务发起,点击查看'), diff --git a/frontend/src/pages/runs/RunDetailView.tsx b/frontend/src/pages/runs/RunDetailView.tsx index 636d005e..3a32e12d 100644 --- a/frontend/src/pages/runs/RunDetailView.tsx +++ b/frontend/src/pages/runs/RunDetailView.tsx @@ -31,6 +31,7 @@ import { ProvenanceBar, Telemetry, runClocks } from './Telemetry' import { TracePane } from './TracePane' import { useRunDetail } from './useRunDetail' import { useCompareEntry } from './compare/entry' +import { RunScheduleSource } from '../schedules/parts' /** * 一次运行的详情:头部读数、凭证、排错路径,下面分三个视图—— @@ -561,9 +562,12 @@ export function RunDetailView({ runId, onChange, onDeleted }: { {formatTime(run.created_at ?? null, new Date(now))} · - - {run.started_by || '未署名'} 发起 - + {/* 定时任务发起的:写成到那个定时任务的链接(键盘可达;记录页的「定时」标记只能鼠标点) */} + {run.schedule_id ? : ( + + {run.started_by || '未署名'} 发起 + + )} · {formatNumber(d.events.length)} 条事件 {workflowGone && · 工作流已删除} diff --git a/frontend/src/pages/schedules/FireHistory.tsx b/frontend/src/pages/schedules/FireHistory.tsx index 3e008910..e1bd9547 100644 --- a/frontend/src/pages/schedules/FireHistory.tsx +++ b/frontend/src/pages/schedules/FireHistory.tsx @@ -1,18 +1,20 @@ import { useCallback, useEffect, useRef, useState } from 'react' import { Link } from 'react-router-dom' -import { ChevronRight, History, RotateCw } from 'lucide-react' +import { ChevronRight, GitCompareArrows, History, RotateCw } from 'lucide-react' import { api } from '../../api/client' import { EmptyState, ErrorState, Skeleton, Spinner, StatusPill, toast, useTicker } from '../../components/ui' import { formatDateTime, parseServerTime } from '../../lib/format' import { SCHEDULE_TEXT as T } from '../../lib/terms' -import type { Schedule, ScheduleFire } from '../../types' +import type { Schedule, ScheduleFire, ScheduleFireCompare } from '../../types' +import { comparePath, type CompareVia } from '../runs/compare/entry' import { FireKindChip, FireStatusChip, ZonedTime } from './parts' const PAGE = 20 /** * 触发历史:每次触发一行,新的在前,20 条一页、「加载更多」往后接。计划时间按定时任务的时区写;跳过、发起失败 - * 写原因;发起了的链到那次运行、写运行现在的状态;发出去的通知逐条写结果。refreshKey 变了(立即运行之后)从头重取 + * 写原因;发起了的链到那次运行、写运行现在的状态;运行结束后服务端和同一任务的上一次比过的,另给「对比上一次」; + * 发出去的通知逐条写结果。refreshKey 变了(立即运行之后)从头重取 */ export function FireHistory({ schedule, refreshKey }: { schedule: Schedule; refreshKey: number }) { const [items, setItems] = useState(null) @@ -110,6 +112,7 @@ function FireRow({ fire: f, tz, now }: { fire: ScheduleFire; tz: string; now: nu + {f.run_id && f.compare && } {f.run_id ? ( {f.run_status && } @@ -137,3 +140,23 @@ function FireRow({ fire: f, tz, now }: { fire: ScheduleFire; tz: string; now: nu ) } + +/** + * 「对比上一次」:去对比页,本次是这次触发的运行、基准是服务端自动对比时取的同一任务的上一次。悬停写各类指标的个数 + * (和通知里那一行同一个写法),读屏把它当说明念。带上来历,对比页据此写「默认基准:同一定时任务的上一次运行」 + */ +function CompareLink({ runId, compare }: { runId: string; compare: ScheduleFireCompare }) { + const via: CompareVia = { base: compare.base_run_id, reason: 'same_schedule' } + return ( + + + {T.compareLink} + + ) +} diff --git a/frontend/src/pages/schedules/parts.tsx b/frontend/src/pages/schedules/parts.tsx index bd0e14f0..193dfe53 100644 --- a/frontend/src/pages/schedules/parts.tsx +++ b/frontend/src/pages/schedules/parts.tsx @@ -269,8 +269,8 @@ export function useScheduleNames(ids: (string | null | undefined)[]): ReadonlyMa * 这里拦住冒泡,只去定时任务、不打开运行。 * * 按钮里放不了可聚焦的链接(读屏会把按钮里的内容压平,Tab 也到不了),所以键盘到不了这个标记:读屏改念 - * 「由定时任务「X」发起」,至少在整行的名字里听到来源。键盘可达的入口要放到运行详情的页头,或者把整行改成 - * 「打开运行」按钮和这个链接并排,都不在本页的文件里,见交付说明 + * 「由定时任务「X」发起」,至少在整行的名字里听到来源。键盘可达的入口在运行详情的页头(RunScheduleSource): + * 打开这次运行之后,Tab 就能到那条链接 */ export function ScheduleTag({ scheduleId, name }: { scheduleId: string; name?: string | null }) { const navigate = useNavigate() @@ -291,3 +291,40 @@ export function ScheduleTag({ scheduleId, name }: { scheduleId: string; name?: s ) } + +// ------------------------------------------------------------------------- +// 运行详情页头的来源 +// ------------------------------------------------------------------------- + +/** + * 运行详情页头里「谁发起的」那一段,定时任务发起的运行用它:「由定时任务「X」发起」是真的链接,Tab 到得了、回车能 + * 打开,去那个定时任务的设置和触发历史。记录页的「定时」标记在整行按钮里,只能用鼠标点,键盘的入口在这里。 + * + * 到点发起的运行(以及没有署名的立即运行)记在「定时任务:X」名下:原来那句「定时任务:X 发起」整句换成这条链接, + * 不写两遍。有人署名点「立即运行」的,署名照写,后面接「来自定时任务「X」」。名字取定时任务现在的名字(改过名的写 + * 新名字),取不到时用署名里记的那个,再没有就不写名字。页头窄时链接截断,全名在悬停里 + */ +export function RunScheduleSource({ scheduleId, startedBy }: { scheduleId: string; startedBy?: string | null }) { + const names = useScheduleNames([scheduleId]) + const recorded = startedBy?.startsWith(T.actorPrefix) ? startedBy.slice(T.actorPrefix.length).trim() : null + const auto = !startedBy || recorded != null + const name = names.get(scheduleId) || recorded || null + const link = ( + + {auto ? T.runSource(name) : T.runSourceManual(name)} + + ) + if (auto || !startedBy) return link + return ( + <> + {T.runStarter(startedBy)} + · + {link} + + ) +} diff --git a/frontend/src/types.ts b/frontend/src/types.ts index cd09e0ba..ed565315 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -3207,6 +3207,15 @@ export interface ScheduleFireNotice { at: string } +/** + * 运行结束后和同一定时任务上一次运行的自动对比:基准是哪一次、各类指标的个数(同对比接口的 summary)。只有个数, + * 具体的数在对比页上 + */ +export interface ScheduleFireCompare { + base_run_id: string + summary: RunCompareSummary +} + /** 一次触发(触发历史的一行) */ export interface ScheduleFire { id: string @@ -3220,6 +3229,10 @@ export interface ScheduleFire { /** 跳过、发起失败的原因(中文) */ reason: string | null notified: ScheduleFireNotice[] + /** + * 和同一定时任务上一次的对比。没有上一次、还没算、比不了、运行又在执行(不在终态)时为 null;老服务端不给 + */ + compare?: ScheduleFireCompare | null created_at: string } diff --git a/scripts/check-schedules.mjs b/scripts/check-schedules.mjs index b125f897..d0ffbc2e 100644 --- a/scripts/check-schedules.mjs +++ b/scripts/check-schedules.mjs @@ -3,7 +3,9 @@ // 守的是这些「看着没坏、其实不能用」的地方:预设生成的 cron 不对(选了每周一 09:00,交上去的是每天)、时区按本机写 // 而不是按定时任务的时区、预览不防抖、非法 cron 只弹一句不知道指哪一项、启用开关不带版本号(别人改过也照样覆盖)、 // 立即运行之后没有下文、触发历史只有一页、保存过的 Webhook 地址又被原样显示出来、改了别的字段却把地址清掉、 -// 记录页看不出哪些运行来自定时任务、390px 窄屏被表单撑出横向滚动。 +// 记录页看不出哪些运行来自定时任务、390px 窄屏被表单撑出横向滚动。运行结束后的自动对比:触发历史里「对比上一次」 +// 链到对比页、悬停只写各类指标的个数;运行详情页头的「由定时任务「X」发起」是键盘到得了的真链接(记录页的「定时」 +// 标记在整行按钮里,只能鼠标点)。 // 还有几处时序和口径:晚到的刷新把刚切换的开关盖回去、先开触发历史再开设置时带着旧版本撞假的 409、保存途中又改了 // 别的字段时替换过的地址明文留在框里、服务端会拒的组合(未发布 + 正式运行、必填没填、已发布版本里没有的字段) // 照样放行、测试消息的结果把服务端的整句又套一层、错过已跳过的那条悬停另说一个原因(和本条记的原因对不上)。 @@ -375,6 +377,13 @@ const S2 = { * 原因却有两种,各放一条、原因照服务端的原话写:f03 是停机一周多、在宽限内恢复时积压的那一次(最近一次就是补发的 * f02),f05 是超出补发宽限的。f06、f07 没有记原因(约定里 reason 可以是 null),看兜底的说明 */ +/** 服务端自动对比给的计数(只有个数):f00 和同一任务的上一次(run-f02)比过 */ +const COMPARE_F00 = { base_run_id: 'run-f02', summary: { unchanged: 4, data_only: 3, caliber_and_data: 1, sql_changed: 0, + params_only: 2, added: 0, removed: 0, incomparable: 0 } } +const COMPARE_F00_TITLE = '与上一次相比:只数据变 3 个、口径与数据都变 1 个、SQL 结构变 0 个、只参数变 2 个、未变 4 个' +const NO_METRICS = { base_run_id: 'run-f09', summary: { unchanged: 0, data_only: 0, caliber_and_data: 0, sql_changed: 0, + params_only: 0, added: 0, removed: 0, incomparable: 0 } } + function makeFires() { const out = [] const local = (at) => { @@ -385,14 +394,15 @@ function makeFires() { for (let i = 0; i < 25; i++) { const at = iso(nextMonday9(-(i + 1))) let f = fire(`f${String(i).padStart(2, '0')}`, { fire_at: at, created_at: at }) - if (i === 0) f = { ...f, kind: 'manual', notified: [{ event: 'finished', status: 'ok', at }, { event: 'failed', status: 'failed', at }] } + if (i === 0) f = { ...f, kind: 'manual', compare: COMPARE_F00, notified: [{ event: 'finished', status: 'ok', at }, { event: 'failed', status: 'failed', at }] } if (i === 1) f = { ...f, status: 'skipped_overlap', run_id: null, run_status: null, reason: '上一次运行(#9f8e7d)尚未结束' } - if (i === 2) f = { ...f, kind: 'misfire', run_status: 'failed' } + if (i === 2) f = { ...f, kind: 'misfire', run_status: 'failed', compare: null } if (i === 3) f = { ...f, ...missed, reason: `服务未运行,错过了 ${local(at)} 的触发;积压的触发不补,只发起最近一次` } if (i === 4) f = { ...f, status: 'failed_to_start', run_id: null, run_status: null, reason: '工作流「门店周报」还没有发布:正式运行需要已发布的版本' } if (i === 5) f = { ...f, ...missed, reason: `服务未运行,错过了 ${local(at)} 的触发,超出补发宽限(1 小时),没有发起` } if (i === 6) f = { ...f, ...missed, reason: null } if (i === 7) f = { ...f, status: 'failed_to_start', run_id: null, run_status: null, reason: null } + if (i === 8) f = { ...f, compare: NO_METRICS } out.push(f) } return out @@ -401,12 +411,27 @@ function makeFires() { const RUNS = [ { id: 'run-sched-1', workflow_id: 'wf-weekly', workflow_name: '门店周报', status: 'succeeded', input: { date: '2026-10-04' }, output: { 周报: '上周销售额 1,234 万元' }, error: null, usage: { duration_ms: 4200 }, run_class: 'formal', version: 3, + started_by: '定时任务:门店周报(旧名)', schedule_id: 's-weekly', created_at: iso(NOW - 2 * H), started_at: iso(NOW - 2 * H), finished_at: iso(NOW - 2 * H + 4200) }, { id: 'run-manual-1', workflow_id: 'wf-weekly', workflow_name: '门店周报', status: 'succeeded', input: { date: '2026-10-03' }, output: { 周报: '手动补一份' }, error: null, usage: { duration_ms: 3900 }, run_class: 'exploratory', version: null, schedule_id: null, created_at: iso(NOW - 3 * H), started_at: iso(NOW - 3 * H), finished_at: iso(NOW - 3 * H + 3900) }, ] +/** 有人署名点了「立即运行」:署名照写,页头另接到定时任务的链接 */ +const RUN_MANUAL_FIRE = { ...RUNS[0], id: 'run-sched-2', started_by: '检查员甲', created_at: iso(NOW - H), started_at: iso(NOW - H), + finished_at: iso(NOW - H + 4200) } +/** 运行详情要的图和事件:一个节点、开始到结束两条,够页头和时间线画出来 */ +const RUN_GRAPH = { nodes: [{ id: 'out', type: 'output', position: { x: 0, y: 0 }, data: { label: '成果', config: {} } }], edges: [] } +function runEvents(run) { + const t0 = Date.parse(run.started_at) / 1000 + return [ + { seq: 1, type: 'run.started', node_id: null, ts: t0, data: { nodes: 1 } }, + { seq: 2, type: 'run.finished', node_id: null, ts: t0 + 4.2, + data: { output: run.output, usage: {}, timing: { wall_ms: 4200, active_ms: 4200, wait_ms: 0 } } }, + ] +} + function freshState(patch = {}) { return { schedules: [structuredClone(S1), structuredClone(S2)], @@ -538,7 +563,13 @@ async function reply(state, method, path, url, body) { if (state.runs && method === 'GET' && path.startsWith('/runs/')) { const run = state.runs.find((r) => path === `/runs/${r.id}`) if (run) return { json: run } + const m = path.match(/^\/runs\/([^/]+)\/(events|graph|artifacts)$/) + const of = m && state.runs.find((r) => r.id === m[1]) + if (of && m[2] === 'events') return { json: runEvents(of) } + if (of && m[2] === 'graph') return { json: { graph: RUN_GRAPH, workflow_id: of.workflow_id, version: of.version } } + if (of) return { json: [] } } + if (state.runs && method === 'GET' && path === '/approvals' && url.searchParams.get('run_id')) return { json: [] } return null } @@ -621,6 +652,18 @@ const saveDone = (page) => until(async () => { /** 等预览算完、说的是这个频率 */ const previewSays = (page, words) => until(async () => (await page.locator('[data-schedule-preview][data-status="ok"]').count()) === 1 && (await page.locator('[data-schedule-preview]').innerText()).includes(words)) +/** + * 只按 Tab 键把焦点移到 sel 上:从页面开头往后按,最多按 max 次。按到了返回按了几次,按不到返回 0。键盘可达就是 + * 这个意思——鼠标能点、程序能 focus() 的不算 + */ +const tabTo = async (page, sel, max = 160) => { + await page.evaluate(() => document.activeElement?.blur?.()) + for (let i = 1; i <= max; i++) { + await page.keyboard.press('Tab') + if (await page.evaluate((s) => !!document.activeElement?.matches?.(s), sel)) return i + } + return 0 +} /** 元素整个在视口的横向范围里 */ const inViewX = (page, sel) => page.evaluate((s) => { const el = document.querySelector(s) @@ -1335,6 +1378,117 @@ await section('详情:触发历史分页', async () => { && await gone.page.locator('[data-schedule-missing] a[href="/schedules"]').count() === 1) }) +await section('详情:触发历史里的「对比上一次」', async () => { + const { page, sent, errors } = await open('/schedules/s-weekly?tab=history') + await page.locator('[data-fire-id]').first().waitFor() + const row = (id) => page.locator(`[data-fire-id="${id}"]`) + const link = row('f00').locator('[data-fire-compare]') + check('有对比的那次:给「对比上一次」', (await link.count()) === 1 && (await link.innerText()).trim() === '对比上一次', + await link.count() ? await link.innerText() : '没有链接') + check('……链到对比页,本次是这次的运行、基准是服务端比过的那一次', + await link.getAttribute('href') === '/runs/compare?target=run-f00&base=run-f02', (await link.getAttribute('href')) ?? '') + const title = (await link.getAttribute('title')) ?? '' + check('……悬停写各类指标的个数:三类总写,其余有才写,没有数值', title === COMPARE_F00_TITLE, title) + check('两边都没有指标:悬停不写一串 0 个', ((await row('f08').locator('[data-fire-compare]').getAttribute('title')) ?? '') + === '与上一次相比:两次运行都没有指标') + const linked = await page.locator('[data-fire-compare]').evaluateAll((els) => els.map((e) => e.closest('[data-fire-id]')?.getAttribute('data-fire-id'))) + check('没有对比的(compare 为 null、老服务端不给、没有运行)不给链接', JSON.stringify(linked) === '["f00","f08"]', JSON.stringify(linked)) + check('运行链接照旧在', await row('f00').locator('a[href="/runs/run-f00"]').count() === 1) + const presses = await tabTo(page, '[data-fire-id="f00"] [data-fire-compare]') + check('键盘:Tab 到得了「对比上一次」', presses > 0, `按了 ${presses} 次`) + await shot(page, 'history-compare-focus-dark') + await page.keyboard.press('Enter') + const url = await until(() => { + const u = new URL(page.url()) + return u.pathname === '/runs/compare' ? u : null + }) + check('……回车打开对比页,带着本次和基准', !!url && url.searchParams.get('target') === 'run-f00' + && url.searchParams.get('base') === 'run-f02', page.url()) + const real = writes(sent, /./).filter((x) => x.key !== 'POST /schedules/preview') + check('只读:没有写请求', real.length === 0, real.map((x) => x.key).join('、')) + check('没有页面报错', !errors.length, errors.join(' | ')) +}) + +// ---------------------------------------------------------------- 运行详情 + +await section('运行详情:页头的定时任务链接', async () => { + const state = freshState({ runs: [...RUNS, RUN_MANUAL_FIRE] }) + const { page, sent, errors } = await open('/runs/run-sched-1', { state }) + await page.locator('[data-run-detail="run-sched-1"]').waitFor({ timeout: 10000 }) + const head = page.locator('[data-run-detail] header').first() + const link = head.locator('[data-run-schedule-link]') + // 署名里记的是旧名字:取到定时任务列表之后写现在的名字 + const said = await until(async () => { + const t = (await link.innerText().catch(() => '')).trim() + return t === '由定时任务「门店周报 · 每周」发起' ? t : null + }) + check('页头写「由定时任务「X」发起」,X 是定时任务现在的名字', !!said, (await link.innerText().catch(() => '没有链接')).trim()) + check('……是到那个定时任务的真链接', (await link.evaluate((el) => el.tagName)) === 'A' + && await link.getAttribute('href') === '/schedules/s-weekly', (await link.getAttribute('href')) ?? '') + const headText = await head.innerText() + check('……不再另写「定时任务:X 发起」', !headText.includes('定时任务:') && !headText.includes('旧名'), oneLine([headText]).slice(0, 160)) + check('……悬停写全名和点开去哪', ((await link.getAttribute('title')) ?? '').includes('门店周报 · 每周')) + const presses = await tabTo(page, '[data-run-detail] header [data-run-schedule-link]') + check('键盘:Tab 到得了页头的定时任务链接', presses > 0, `按了 ${presses} 次`) + const ring = await page.evaluate(() => getComputedStyle(document.activeElement).boxShadow) + check('……聚焦时有焦点环', !!ring && ring !== 'none', ring) + await shot(page, 'run-detail-schedule-link-dark') + await page.keyboard.press('Enter') + check('……回车去那个定时任务', await until(() => new URL(page.url()).pathname === '/schedules/s-weekly'), page.url()) + await page.locator('[data-schedule-form], [data-schedules-page] header').first().waitFor() + + const manual = await open('/runs/run-sched-2', { state: freshState({ runs: [...RUNS, RUN_MANUAL_FIRE] }) }) + await manual.page.locator('[data-run-detail="run-sched-2"]').waitFor({ timeout: 10000 }) + const mHead = manual.page.locator('[data-run-detail] header').first() + const mLink = mHead.locator('[data-run-schedule-link]') + await until(async () => (await mLink.innerText().catch(() => '')).includes('门店周报 · 每周')) + const mText = oneLine([await mHead.innerText()]) + check('有人署名点了立即运行:署名照写,后面接「来自定时任务「X」」的链接', mText.includes('检查员甲 发起') + && (await mLink.innerText()).trim() === '来自定时任务「门店周报 · 每周」' && await mLink.getAttribute('href') === '/schedules/s-weekly', + mText.slice(0, 160)) + + await shot(manual.page, 'run-detail-manual-fire-dark') + const light = await open('/runs/run-sched-1', { state: freshState({ runs: [...RUNS, RUN_MANUAL_FIRE] }), theme: 'light' }) + await light.page.locator('[data-run-detail="run-sched-1"] header [data-run-schedule-link]').waitFor() + const lightTone = await light.page.evaluate(() => { + const link = document.querySelector('[data-run-detail] header [data-run-schedule-link]') + const head = document.querySelector('[data-run-detail] header') + return { theme: document.documentElement.dataset.theme, fg: getComputedStyle(link).color, + bg: getComputedStyle(head).backgroundColor } + }) + check('浅色:页头的定时任务链接字和底色分得开', lightTone.theme === 'light' && lightTone.fg !== lightTone.bg, JSON.stringify(lightTone)) + await shot(light.page, 'run-detail-schedule-link-light') + const plain = await open('/runs/run-manual-1', { state: freshState({ runs: [...RUNS, RUN_MANUAL_FIRE] }) }) + await plain.page.locator('[data-run-detail="run-manual-1"]').waitFor({ timeout: 10000 }) + check('手动发起的运行:页头没有定时任务链接', await plain.page.locator('[data-run-detail] header [data-run-schedule-link]').count() === 0) + + // 名字很长、详情区不宽时链接截断,不把页头撑宽(全名在悬停里) + const longName = freshState({ runs: [...RUNS, RUN_MANUAL_FIRE] }) + longName.schedules = longName.schedules.map((x) => (x.id === 's-weekly' ? { ...x, name: '华东区全部门店按品类与渠道拆分的周度销售与退款复盘' } : x)) + const narrow = await open('/runs/run-sched-1', { state: longName, width: 900 }) + await narrow.page.locator('[data-run-detail="run-sched-1"]').waitFor({ timeout: 10000 }) + const nLink = narrow.page.locator('[data-run-detail] header [data-run-schedule-link]') + await until(async () => (await nLink.innerText().catch(() => '')).includes('复盘')) + const fit = await narrow.page.evaluate(() => { + const head = document.querySelector('[data-run-detail] header') + const link = head?.querySelector('[data-run-schedule-link]') + if (!head || !link) return null + const h = head.getBoundingClientRect() + const l = link.getBoundingClientRect() + return { headRight: Math.round(h.right), linkRight: Math.round(l.right), scroll: head.scrollWidth - head.clientWidth, + clipped: link.scrollWidth > link.clientWidth } + }) + check('长名字:链接截断,不超出页头、页头不横向溢出', !!fit && fit.linkRight <= fit.headRight && fit.scroll <= 1 && fit.clipped, + JSON.stringify(fit)) + // 回车打开的定时任务页会请求预览(POST,但只读),不算写 + const writesSeen = [...writes(sent, /./), ...writes(manual.sent, /./), ...writes(light.sent, /./), ...writes(plain.sent, /./), + ...writes(narrow.sent, /./)] + .filter((x) => x.key !== 'POST /schedules/preview') + check('只读:没有写请求', writesSeen.length === 0, writesSeen.map((x) => x.key).join('、')) + const all = [...errors, ...manual.errors, ...light.errors, ...plain.errors, ...narrow.errors] + check('没有页面报错', !all.length, all.join(' | ')) +}) + // ---------------------------------------------------------------- 记录页 await section('记录页:定时标记', async () => { @@ -1416,6 +1570,13 @@ await section('窄屏 390px 与浅色', async () => { const hist = await open('/schedules/s-weekly?tab=history', { ...narrow, theme }) await hist.page.locator('[data-fire-id]').first().waitFor() check(`${theme}:触发历史不横向溢出`, await overflow(hist.page) <= 1, `${await overflow(hist.page)}px`) + check(`${theme}:「对比上一次」在屏幕里`, await inViewX(hist.page, '[data-fire-id="f00"] [data-fire-compare]')) + const cmp = await hist.page.evaluate(() => { + const link = document.querySelector('[data-fire-id="f00"] [data-fire-compare]') + const list = document.querySelector('[data-fire-list] ul') + return link && list && { fg: getComputedStyle(link).color, bg: getComputedStyle(list).backgroundColor } + }) + check(`${theme}:「对比上一次」的字和底色分得开`, !!cmp && cmp.fg !== cmp.bg, JSON.stringify(cmp)) check(`${theme}:页头的开关、立即运行、删除在屏幕里`, await inViewX(hist.page, 'header [data-schedule-run]') && await inViewX(hist.page, 'header [data-schedule-toggle]') && await inViewX(hist.page, 'header [data-schedule-delete]')) await shot(hist.page, `history-390-${theme}`) From f8ac8dc1d682208dc558a23a6041cce74718fa78 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 07:43:01 +0800 Subject: [PATCH 4/6] =?UTF-8?q?=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E8=87=AA=E5=8A=A8=E5=AF=B9=E6=AF=94=E6=94=B6=E5=B0=BE=EF=BC=9A?= =?UTF-8?q?=E5=9F=BA=E5=87=86=E9=82=A3=E4=B8=80=E6=AC=A1=E4=B8=8D=E5=9C=A8?= =?UTF-8?q?=E7=BB=88=E6=80=81=E6=88=96=E8=A2=AB=E5=88=A0=E6=97=B6=E4=B8=8D?= =?UTF-8?q?=E7=BB=99=E5=AF=B9=E6=AF=94=E3=80=81=E5=AF=B9=E6=AF=94=E5=B7=A5?= =?UTF-8?q?=E4=BB=B6=E8=AF=BB=E4=B8=8D=E5=87=BA=E6=9D=A5=E6=97=B6=E8=A7=A6?= =?UTF-8?q?=E5=8F=91=E5=8E=86=E5=8F=B2=E7=85=A7=E5=B8=B8=E6=89=93=E5=BC=80?= =?UTF-8?q?=EF=BC=9B=E6=A3=80=E6=9F=A5=E8=84=9A=E6=9C=AC=E6=8C=89=E5=AE=9E?= =?UTF-8?q?=E9=99=85=E5=BA=95=E8=89=B2=E9=87=8F=E5=AF=B9=E6=AF=94=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 触发历史的 compare:以前只看这一次的运行在不在终态。基准那一次被强制删除、失败后被继续执行时,对比页比不了, 「对比上一次」链过去只会报错——基准一侧同样要在终态,否则给 null - 对比工件读不出来(权限、磁盘出错)时记一条警告、给 null:以前 OSError 直接抛出,整页触发历史回 500。被改过 (内容与哈希不符)的以前就给 null,这次补上用例 - check-schedules:页头的定时任务链接、触发历史的「对比上一次」改成量 WCAG 对比度(≥ 4.5),底色顺着祖先叠到 不透明为止。以前比的是「字色不等于页头的底色」,页头本身透明,那样比永远成立;深色也补上同样的检查。另加一项 署名的立即运行配长名字:署名照写、链接截断、页头不横向溢出 --- backend/app/api/schedules.py | 8 ++- backend/app/scheduler/auto_compare.py | 6 +- backend/tests/test_scheduler_compare.py | 55 +++++++++++++++--- scripts/check-schedules.mjs | 77 ++++++++++++++++++++----- 4 files changed, 122 insertions(+), 24 deletions(-) diff --git a/backend/app/api/schedules.py b/backend/app/api/schedules.py index b96d57b8..7931aaf3 100644 --- a/backend/app/api/schedules.py +++ b/backend/app/api/schedules.py @@ -282,7 +282,7 @@ async def _fire_outs(session: AsyncSession, fires: list[ScheduleFire]) -> list[d compare 是运行结束后和同一任务上一次的自动对比 {base_run_id, summary}(summary 同对比接口的计数),没有为 null。 运行又被继续执行、现在不在终态时 compare 也给 null:对比页只认结束并封存的运行,链过去只会被拒;再结束时 - 通知扫描会按新的封存重新比。工件读不出来、被改过的同样给 null。""" + 通知扫描会按新的封存重新比。基准那一次被删掉、被继续执行的,工件读不出来、被改过的,同样给 null。""" run_ids = [f.run_id for f in fires if f.run_id] statuses = dict((await session.execute(select(Run.id, Run.status).where(Run.id.in_(run_ids)))).all()) \ if run_ids else {} @@ -299,6 +299,12 @@ async def _fire_outs(session: AsyncSession, fires: list[ScheduleFire]) -> list[d compared = {f.id: f.compare_artifact for f in fires if f.compare_artifact and statuses.get(f.run_id) in auto_compare.TERMINAL} compares = await auto_compare.briefs(list(compared.values())) + # 基准一侧同样要在终态:被删掉(强制删除封存的正式运行)、失败后被继续执行的,对比页比不了 + base_ids = list({brief["base_run_id"] for brief in compares.values()}) + if base_ids: + bases = dict((await session.execute(select(Run.id, Run.status).where(Run.id.in_(base_ids)))).all()) + compares = {artifact: brief for artifact, brief in compares.items() + if bases.get(brief["base_run_id"]) in auto_compare.TERMINAL} return [{ "id": f.id, "fire_at": _iso(f.fire_at), diff --git a/backend/app/scheduler/auto_compare.py b/backend/app/scheduler/auto_compare.py index dc464b3f..90c15bc6 100644 --- a/backend/app/scheduler/auto_compare.py +++ b/backend/app/scheduler/auto_compare.py @@ -145,7 +145,8 @@ def _remember(artifact: str, brief: dict[str, Any]) -> None: def brief_of(artifact: str) -> dict[str, Any] | None: - """读一份对比工件的摘要。文件不在、内容和哈希对不上(另记一条警告)、形状不对的返回 None,不抛异常。""" + """读一份对比工件的摘要。文件不在、内容和哈希对不上、读不出来(后两种另记一条警告)、形状不对的返回 None, + 不抛异常:一份对比读不出来不能让整页触发历史打不开。""" with _BRIEFS_LOCK: known = _BRIEFS.get(artifact) if known is not None: @@ -155,6 +156,9 @@ def brief_of(artifact: str) -> dict[str, Any] | None: except ValueError: logger.warning("对比工件 %s… 内容与哈希不符,触发历史中不显示这次对比", artifact[:12]) return None + except OSError as e: # 权限、磁盘出错 + logger.warning("对比工件 %s… 读取失败(%s),触发历史中不显示这次对比", artifact[:12], type(e).__name__) + return None if brief is None: return None _remember(artifact, brief) diff --git a/backend/tests/test_scheduler_compare.py b/backend/tests/test_scheduler_compare.py index 480bfb53..d7f79238 100644 --- a/backend/tests/test_scheduler_compare.py +++ b/backend/tests/test_scheduler_compare.py @@ -25,6 +25,7 @@ import pytest from httpx import ASGITransport, AsyncClient from sqlalchemy import delete, select +from app.core import artifact_store from app.core.artifact_store import canonical_json, load, manifest_hash, put_json from app.core.crypto import encrypt from app.db.base import SessionLocal @@ -340,16 +341,56 @@ async def test_compare_is_hidden_once_the_run_is_running_again(client): assert item["compare"] == {"base_run_id": base_id, "summary": SUMMARY} -async def test_a_missing_artifact_reads_as_no_compare(client): - """工件文件不在了、被改过:接口给 null,不报错""" +async def test_compare_is_hidden_once_the_base_run_is_gone_or_running(client): + """基准那一次被删掉、或者被继续执行(不在终态):对比页比不了,链过去只会报错,接口同样给 null""" + sid, base_id, _, fire_id = await weekly_pair(url=None) + await webhook.notify_due() + assert (await fire_row(fire_id)).compare_artifact + + async def shown(): + items = (await client.get(f"/api/schedules/{sid}/fires")).json()["items"] + return {f["id"]: f for f in items}[fire_id]["compare"] + + assert (await shown())["base_run_id"] == base_id + await set_run(base_id, status="running") + assert await shown() is None + await set_run(base_id, status="succeeded") + assert (await shown())["base_run_id"] == base_id + # 封存过的正式运行要强制删除 + assert (await client.delete(f"/api/runs/{base_id}", params={"force": "true"})).status_code == 204 + assert await shown() is None + + +def unreadable(artifact_id): + raise PermissionError(13, "Permission denied") + + +@pytest.mark.parametrize("damage", ["missing", "tampered", "unreadable"]) +async def test_a_damaged_artifact_reads_as_no_compare(client, monkeypatch, caplog, damage): + """工件文件不在了、被改过、读不出来(权限、磁盘出错):接口给 null,触发历史照常打开,不回 500""" sid, _, _, fire_id = await weekly_pair(url=None) - async with SessionLocal() as session: - await session.execute(ScheduleFire.__table__.update().where(ScheduleFire.id == fire_id) - .values(compare_artifact="0" * 64)) - await session.commit() - resp = await client.get(f"/api/schedules/{sid}/fires") + await webhook.notify_due() + artifact = (await fire_row(fire_id)).compare_artifact + # 刚比完的摘要记在进程里,清掉才会去读盘 + auto_compare._BRIEFS.clear() + if damage == "missing": + async with SessionLocal() as session: + await session.execute(ScheduleFire.__table__.update().where(ScheduleFire.id == fire_id) + .values(compare_artifact="0" * 64)) + await session.commit() + elif damage == "tampered": + path = artifact_store._path_of(artifact) + text = path.read_text(encoding="utf-8") + assert '"data_only":2' in text + path.write_text(text.replace('"data_only":2', '"data_only":7'), encoding="utf-8") + else: + monkeypatch.setattr(artifact_store, "load", unreadable) + with caplog.at_level(logging.WARNING, logger="app.scheduler"): + resp = await client.get(f"/api/schedules/{sid}/fires") assert resp.status_code == 200 assert {f["id"]: f for f in resp.json()["items"]}[fire_id]["compare"] is None + if damage != "missing": + assert artifact[:12] in caplog.text # -------------------------------------------------------------------------- diff --git a/scripts/check-schedules.mjs b/scripts/check-schedules.mjs index d0ffbc2e..b720db49 100644 --- a/scripts/check-schedules.mjs +++ b/scripts/check-schedules.mjs @@ -671,6 +671,40 @@ const inViewX = (page, sel) => page.evaluate((s) => { const b = el.getBoundingClientRect() return b.left >= 0 && b.right <= innerWidth + 0.5 }, sel) +/** + * 元素的字和它实际落在上面的底色的对比度(WCAG 算法),{ratio, fg, bg},找不到元素为 null。底色顺着祖先往上找, + * 半透明的一层层叠上去,直到不透明的那层(都透明就按白底)。只比「字色和某个容器的底色不相等」不够:页头本身是 + * 透明的,那样比永远成立 + */ +const readable = (page, sel) => page.evaluate((s) => { + const el = document.querySelector(s) + if (!el) return null + const parse = (c) => { + const m = c.match(/color\(srgb ([\d.]+) ([\d.]+) ([\d.]+)(?: \/ ([\d.]+))?\)/) + if (m) return [m[1] * 255, m[2] * 255, m[3] * 255, m[4] == null ? 1 : Number(m[4])] + const n = c.match(/rgba?\(([\d.]+),\s*([\d.]+),\s*([\d.]+)(?:,\s*([\d.]+))?\)/) + return n ? [Number(n[1]), Number(n[2]), Number(n[3]), n[4] == null ? 1 : Number(n[4])] : null + } + const over = (top, under) => top.slice(0, 3).map((v, i) => v * top[3] + under[i] * (1 - top[3])) + const layers = [] + for (let n = el; n; n = n.parentElement) { + const c = parse(getComputedStyle(n).backgroundColor) + if (c && c[3] > 0) layers.push(c) + if (c && c[3] >= 1) break + } + let bg = [255, 255, 255] + for (const layer of layers.reverse()) bg = over(layer, bg) + const color = parse(getComputedStyle(el).color) + if (!color) return null + const fg = over(color, bg) + const lum = (rgb) => { + const [r, g, b] = rgb.map((v) => { const x = v / 255; return x <= 0.03928 ? x / 12.92 : ((x + 0.055) / 1.055) ** 2.4 }) + return 0.2126 * r + 0.7152 * g + 0.0722 * b + } + const [hi, lo] = [lum(fg), lum(bg)].sort((a, b) => b - a) + return { ratio: Math.round(((hi + 0.05) / (lo + 0.05)) * 100) / 100, fg: getComputedStyle(el).color, + bg: `rgb(${bg.map(Math.round).join(', ')})` } +}, sel) // ---------------------------------------------------------------- 列表 @@ -1432,6 +1466,8 @@ await section('运行详情:页头的定时任务链接', async () => { check('键盘:Tab 到得了页头的定时任务链接', presses > 0, `按了 ${presses} 次`) const ring = await page.evaluate(() => getComputedStyle(document.activeElement).boxShadow) check('……聚焦时有焦点环', !!ring && ring !== 'none', ring) + const darkTone = await readable(page, '[data-run-detail] header [data-run-schedule-link]') + check('深色:页头的定时任务链接对比度 ≥ 4.5', !!darkTone && darkTone.ratio >= 4.5, JSON.stringify(darkTone)) await shot(page, 'run-detail-schedule-link-dark') await page.keyboard.press('Enter') check('……回车去那个定时任务', await until(() => new URL(page.url()).pathname === '/schedules/s-weekly'), page.url()) @@ -1450,13 +1486,10 @@ await section('运行详情:页头的定时任务链接', async () => { await shot(manual.page, 'run-detail-manual-fire-dark') const light = await open('/runs/run-sched-1', { state: freshState({ runs: [...RUNS, RUN_MANUAL_FIRE] }), theme: 'light' }) await light.page.locator('[data-run-detail="run-sched-1"] header [data-run-schedule-link]').waitFor() - const lightTone = await light.page.evaluate(() => { - const link = document.querySelector('[data-run-detail] header [data-run-schedule-link]') - const head = document.querySelector('[data-run-detail] header') - return { theme: document.documentElement.dataset.theme, fg: getComputedStyle(link).color, - bg: getComputedStyle(head).backgroundColor } - }) - check('浅色:页头的定时任务链接字和底色分得开', lightTone.theme === 'light' && lightTone.fg !== lightTone.bg, JSON.stringify(lightTone)) + const lightTheme = await light.page.evaluate(() => document.documentElement.dataset.theme) + const lightTone = await readable(light.page, '[data-run-detail] header [data-run-schedule-link]') + check('浅色:页头的定时任务链接对比度 ≥ 4.5', lightTheme === 'light' && !!lightTone && lightTone.ratio >= 4.5, + `${lightTheme} ${JSON.stringify(lightTone)}`) await shot(light.page, 'run-detail-schedule-link-light') const plain = await open('/runs/run-manual-1', { state: freshState({ runs: [...RUNS, RUN_MANUAL_FIRE] }) }) await plain.page.locator('[data-run-detail="run-manual-1"]').waitFor({ timeout: 10000 }) @@ -1480,12 +1513,30 @@ await section('运行详情:页头的定时任务链接', async () => { }) check('长名字:链接截断,不超出页头、页头不横向溢出', !!fit && fit.linkRight <= fit.headRight && fit.scroll <= 1 && fit.clipped, JSON.stringify(fit)) + // 署名的立即运行(署名加链接,这一行最挤)配长名字:链接照样截断,署名不被挤掉 + const crowdedState = freshState({ runs: [...RUNS, RUN_MANUAL_FIRE] }) + crowdedState.schedules = longName.schedules + const crowded = await open('/runs/run-sched-2', { state: crowdedState, width: 900 }) + await crowded.page.locator('[data-run-detail="run-sched-2"]').waitFor({ timeout: 10000 }) + const cLink = crowded.page.locator('[data-run-detail] header [data-run-schedule-link]') + await until(async () => (await cLink.innerText().catch(() => '')).includes('复盘')) + const crowd = await crowded.page.evaluate(() => { + const head = document.querySelector('[data-run-detail] header') + const link = head?.querySelector('[data-run-schedule-link]') + if (!head || !link) return null + const l = link.getBoundingClientRect() + return { headRight: Math.round(head.getBoundingClientRect().right), linkRight: Math.round(l.right), + linkWidth: Math.round(l.width), scroll: head.scrollWidth - head.clientWidth, + starter: head.innerText.includes('检查员甲 发起') } + }) + check('……署名的立即运行配长名字:署名照写,链接截断、看得到,页头不横向溢出', !!crowd && crowd.starter + && crowd.linkRight <= crowd.headRight && crowd.linkWidth >= 24 && crowd.scroll <= 1, JSON.stringify(crowd)) // 回车打开的定时任务页会请求预览(POST,但只读),不算写 const writesSeen = [...writes(sent, /./), ...writes(manual.sent, /./), ...writes(light.sent, /./), ...writes(plain.sent, /./), - ...writes(narrow.sent, /./)] + ...writes(narrow.sent, /./), ...writes(crowded.sent, /./)] .filter((x) => x.key !== 'POST /schedules/preview') check('只读:没有写请求', writesSeen.length === 0, writesSeen.map((x) => x.key).join('、')) - const all = [...errors, ...manual.errors, ...light.errors, ...plain.errors, ...narrow.errors] + const all = [...errors, ...manual.errors, ...light.errors, ...plain.errors, ...narrow.errors, ...crowded.errors] check('没有页面报错', !all.length, all.join(' | ')) }) @@ -1571,12 +1622,8 @@ await section('窄屏 390px 与浅色', async () => { await hist.page.locator('[data-fire-id]').first().waitFor() check(`${theme}:触发历史不横向溢出`, await overflow(hist.page) <= 1, `${await overflow(hist.page)}px`) check(`${theme}:「对比上一次」在屏幕里`, await inViewX(hist.page, '[data-fire-id="f00"] [data-fire-compare]')) - const cmp = await hist.page.evaluate(() => { - const link = document.querySelector('[data-fire-id="f00"] [data-fire-compare]') - const list = document.querySelector('[data-fire-list] ul') - return link && list && { fg: getComputedStyle(link).color, bg: getComputedStyle(list).backgroundColor } - }) - check(`${theme}:「对比上一次」的字和底色分得开`, !!cmp && cmp.fg !== cmp.bg, JSON.stringify(cmp)) + const cmp = await readable(hist.page, '[data-fire-id="f00"] [data-fire-compare]') + check(`${theme}:「对比上一次」对比度 ≥ 4.5`, !!cmp && cmp.ratio >= 4.5, JSON.stringify(cmp)) check(`${theme}:页头的开关、立即运行、删除在屏幕里`, await inViewX(hist.page, 'header [data-schedule-run]') && await inViewX(hist.page, 'header [data-schedule-toggle]') && await inViewX(hist.page, 'header [data-schedule-delete]')) await shot(hist.page, `history-390-${theme}`) From 45fe01effe2a4d71d50b8539cbed673ff0f60397 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 08:48:56 +0800 Subject: [PATCH 5/6] =?UTF-8?q?=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E8=87=AA=E5=8A=A8=E5=AF=B9=E6=AF=94=E5=A4=8D=E6=A0=B8=E6=95=B4?= =?UTF-8?q?=E6=94=B9=EF=BC=88=E5=90=8E=E7=AB=AF=EF=BC=89=EF=BC=9A=E5=8F=AA?= =?UTF-8?q?=E6=AF=94=E6=88=90=E5=8A=9F=E7=9A=84=E8=BF=90=E8=A1=8C=E3=80=81?= =?UTF-8?q?=E6=8C=89=E5=8F=91=E8=B5=B7=E5=85=88=E5=90=8E=E6=8C=91=E4=B8=8A?= =?UTF-8?q?=E4=B8=80=E6=AC=A1=E6=88=90=E5=8A=9F=E7=9A=84=E5=9F=BA=E5=87=86?= =?UTF-8?q?=E3=80=81=E5=92=8C=E4=B8=A4=E8=BE=B9=E7=9A=84=E5=B0=81=E5=AD=98?= =?UTF-8?q?=E7=BB=91=E5=AE=9A=EF=BC=8C=E5=AF=B9=E6=AF=94=E6=8C=AA=E5=88=B0?= =?UTF-8?q?=E8=AE=A4=E9=A2=86=E4=B9=8B=E5=89=8D=E5=9C=A8=E9=94=81=E5=A4=96?= =?UTF-8?q?=E5=81=9A=E5=B9=B6=E9=99=90=E6=97=B6=E9=99=90=E5=B9=B6=E5=8F=91?= =?UTF-8?q?=EF=BC=9B=E5=A4=B1=E8=B4=A5=E9=80=9A=E7=9F=A5=E7=9A=84=E5=A4=84?= =?UTF-8?q?=E7=90=86=E8=A1=8C=E4=B8=8D=E5=86=8D=E9=87=8D=E5=A4=8D=E5=8E=9F?= =?UTF-8?q?=E5=9B=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 只比成功结束的运行,基准也只取成功的:失败、取消的运行指标不全,和上一次比只会得出「移除 N 个」,拿失败的那次当 基准每个指标都成了「新增」。基准改成自己查同一定时任务、同一工作流、比这次先发起的运行里成功并封存的最近一次, 不再用对比接口的 default_base(按结束时间挑:老的失败运行事后被接着跑完,会拿后一次触发当它的「上一次」)。 中间隔着失败、取消的运行时,通知写「与上一次成功的运行(#短编号,日期)相比」,Fire.compare 多一个 previous - 和封存绑定:运行的封存要盖住最后一条结束事件才算封存完(sealed_now)。runner 先写终态、再另起会话封存,失败后 接着跑完的运行在两步之间还带着失败时的清单哈希,以前这一拍会直接认领、按旧封存比、通知里写旧的清单哈希;现在 等新的封存,等不来就照常推送但不比、不写哈希。停在审批上的通知也不再写失败时留下的哈希。对比工件连同两边的清单 哈希一起存,触发历史发现任何一边不在终态或清单哈希变了就给 null;对比途中封存变了不记(条件写) - 时机和时限:锁里只挑出状态有变化的触发,认领挪到对比之后、锁外按条件 UPDATE 做。对比最多等 20 秒(含排队), 同一时刻最多 2 个;超时只记警告,通知不带对比照常推送。停机时扫描在对比途中被取消,这次触发还没认领,下次照常 比、照常推(以前先认领、后比,这条通知会永远丢掉)。同一份封存已经比过的直接用记下的结果 - 失败通知:从没开始执行的,原因刚写过,处理行不再让人「查看原因」,任务在排期上写「下一次触发会照常发起」,停用 的写怎么重新发起;开始执行过的改成「请在 AgentLab 的「记录」中查看失败原因」,和挂起那句指向同一个导航入口 - 新增 test_scheduler_compare_guards.py(封存绑定、时限、取消、并发、复用),test_scheduler_compare.py 补真实缺 指标的失败运行、上一次失败、补跑老运行几种用例;去掉实现逐项验证过会变红 --- backend/app/api/schedules.py | 45 ++- backend/app/scheduler/__init__.py | 3 +- backend/app/scheduler/auto_compare.py | 201 ++++++++--- backend/app/scheduler/webhook.py | 161 ++++++--- backend/tests/test_scheduler_compare.py | 196 ++++++++--- .../tests/test_scheduler_compare_guards.py | 312 ++++++++++++++++++ backend/tests/test_scheduler_notice_rules.py | 28 +- backend/tests/test_scheduler_webhook.py | 2 +- 8 files changed, 766 insertions(+), 182 deletions(-) create mode 100644 backend/tests/test_scheduler_compare_guards.py diff --git a/backend/app/api/schedules.py b/backend/app/api/schedules.py index 7931aaf3..e45c2e0a 100644 --- a/backend/app/api/schedules.py +++ b/backend/app/api/schedules.py @@ -1,4 +1,5 @@ -"""定时任务:按周期自动发起工作流运行,结束、失败、停在审批时推送 Webhook,运行结束后和同一任务的上一次自动对比。 +"""定时任务:按周期自动发起工作流运行,结束、失败、停在审批时推送 Webhook,运行成功结束后和同一任务上一次成功的运行 +自动对比。 形状按前后端约定(Schedule、Fire 两种对象,见 _schedule_out、_fire_outs)。Webhook 地址和签名密钥加密存放,接口 只给主机和末 4 位(url_hint)、有没有密钥(has_secret)。请求体自己解析、自己校验:错误一律回中文 detail,表单 @@ -279,13 +280,30 @@ def _webhook_out(row: Schedule) -> dict[str, Any] | None: async def _fire_outs(session: AsyncSession, fires: list[ScheduleFire]) -> list[dict[str, Any]]: """Fire 对象:run_status 是运行现在的状态,notified 是这次触发的推送记录 [{event, status, at}](按时间先后), - compare 是运行结束后和同一任务上一次的自动对比 {base_run_id, summary}(summary 同对比接口的计数),没有为 null。 + compare 是运行成功结束后和同一任务上一次成功的运行的自动对比 {base_run_id, summary, previous}(summary 同对比接口 + 的计数,previous 说基准是不是紧挨着的上一次),没有为 null。 + + 对比和比的时候两边的封存绑定(auto_compare):哪一边现在不在终态(被继续执行)、被删掉、清单哈希和比的时候不一样 + (重新封存过)都给 null——对比页按现在的封存现算,记下的计数和点进去看到的会对不上,或者干脆比不了。工件读不出来、 + 被改过的同样给 null。""" + compared = {f.id: f.compare_artifact for f in fires if f.compare_artifact and f.run_id} + compares = await auto_compare.briefs(list(compared.values())) + run_ids = {f.run_id for f in fires if f.run_id} | {brief["base_run_id"] for brief in compares.values()} + # 运行现在的状态和清单哈希:run_status 照写,对比按它核对两边的封存 + current = {run_id: (status, seal) for run_id, status, seal in (await session.execute( + select(Run.id, Run.status, Run.manifest_hash).where(Run.id.in_(run_ids)))).all()} if run_ids else {} + + def compare_of(f: ScheduleFire) -> dict[str, Any] | None: + brief = compares.get(compared.get(f.id) or "") + if brief is None or brief["target_run_id"] != f.run_id: + return None + for run_id, seal in ((brief["target_run_id"], brief["target_manifest"]), + (brief["base_run_id"], brief["base_manifest"])): + status, now_seal = current.get(run_id, (None, None)) + if status not in auto_compare.TERMINAL or now_seal != seal: + return None + return auto_compare.public(brief) - 运行又被继续执行、现在不在终态时 compare 也给 null:对比页只认结束并封存的运行,链过去只会被拒;再结束时 - 通知扫描会按新的封存重新比。基准那一次被删掉、被继续执行的,工件读不出来、被改过的,同样给 null。""" - run_ids = [f.run_id for f in fires if f.run_id] - statuses = dict((await session.execute(select(Run.id, Run.status).where(Run.id.in_(run_ids)))).all()) \ - if run_ids else {} notified: dict[str, list[dict[str, Any]]] = {} if fires: rows = (await session.execute( @@ -296,25 +314,16 @@ async def _fire_outs(session: AsyncSession, fires: list[ScheduleFire]) -> list[d notified.setdefault(d.fire_id, []).append({"event": d.event, "status": d.status, "at": _iso(d.created_at)}) # 正在发起的触发:库里是占位的「未能发起」(为了进程被强杀时照实说),进程还在、只是还没发起完,按已发起给 launching = {f.id for f in fires if service.launching(f.id)} - compared = {f.id: f.compare_artifact for f in fires - if f.compare_artifact and statuses.get(f.run_id) in auto_compare.TERMINAL} - compares = await auto_compare.briefs(list(compared.values())) - # 基准一侧同样要在终态:被删掉(强制删除封存的正式运行)、失败后被继续执行的,对比页比不了 - base_ids = list({brief["base_run_id"] for brief in compares.values()}) - if base_ids: - bases = dict((await session.execute(select(Run.id, Run.status).where(Run.id.in_(base_ids)))).all()) - compares = {artifact: brief for artifact, brief in compares.items() - if bases.get(brief["base_run_id"]) in auto_compare.TERMINAL} return [{ "id": f.id, "fire_at": _iso(f.fire_at), "kind": f.kind, "status": "started" if f.id in launching else f.status, "run_id": f.run_id, - "run_status": statuses.get(f.run_id) if f.run_id else None, + "run_status": current.get(f.run_id, (None, None))[0] if f.run_id else None, "reason": None if f.id in launching else f.reason, "notified": notified.get(f.id, []), - "compare": compares.get(compared[f.id]) if f.id in compared else None, + "compare": compare_of(f), "created_at": _iso(f.created_at), } for f in fires] diff --git a/backend/app/scheduler/__init__.py b/backend/app/scheduler/__init__.py index beac3ead..57b7c9d7 100644 --- a/backend/app/scheduler/__init__.py +++ b/backend/app/scheduler/__init__.py @@ -4,7 +4,8 @@ - service.py:输入模板、和工作流入口字段的对照、发起一次触发(调度循环和「立即运行」共用) - loop.py:调度循环,条件 UPDATE 认领到点的任务 - webhook.py:通知推送(通用 / 飞书 / 企业微信 / 钉钉) -- auto_compare.py:运行结束后和同一定时任务的上一次自动对比,结果记在触发记录上,通知里只带计数摘要和对比页链接 +- auto_compare.py:运行成功结束后和同一定时任务上一次成功的运行自动对比,结果记在触发记录上,通知里只带计数摘要和 + 对比页链接 服务启动时(main.py 的 lifespan)只在 scheduler_enabled() 时调 start,关闭时总是调 stop——stop 要能在没 start 过时安全返回。start 自己也看一次开关:拷来的数据目录(检查脚本的 --stacks、验收副本)不管从哪条路径起服务, diff --git a/backend/app/scheduler/auto_compare.py b/backend/app/scheduler/auto_compare.py index 90c15bc6..83a209b4 100644 --- a/backend/app/scheduler/auto_compare.py +++ b/backend/app/scheduler/auto_compare.py @@ -1,15 +1,24 @@ -"""定时运行结束后的自动对比:和同一定时任务的上一次运行比一次,结果存成工件,工件 id 记在触发记录上 +"""定时运行结束后的自动对比:和同一定时任务上一次成功的运行比一次,结果存成工件,工件 id 记在触发记录上 (ScheduleFire.compare_artifact)。 -对比本身调运行对比接口的现成函数(api/compare.py 的 default_base、compare_runs),这里只管什么时候比、跟谁比、结果 -放哪、怎么说给触发历史和通知听: +对比本身调运行对比接口的现成函数(api/compare.py 的 compare_runs,不改它),这里只管比哪些、跟谁比、什么时候比、 +结果放哪、怎么说给触发历史和通知听: -- 什么时候:通知扫描(webhook.notify_due)认领到运行进入终态(结束、失败、取消)的那一次之后、推送之前。和推送一样, - 同一个终态只处理一次;失败后被继续执行、又结束的,按新的封存重新比,覆盖旧的结果 -- 跟谁比:default_base 的「同一定时任务」那一档,即这个任务里更早结束、封存过的终态运行中最近的一次。它退回「同一 - 工作流、同一运行类别」时不比:那不是这个定时任务的上一次,别人手动改的输入、另一个任务的参数都会被算成变化 -- 结果:整份对比结果存成工件(kind=run_compare)。不挂在任何运行名下:它不是运行产出的,不该出现在运行的工件列表里。 - 没封存的不比(服务重启时还在排队、被改成失败的,永远不会封存);比不了、出错只记日志,通知照常推送 +- 比哪些:成功结束、封存完的运行。失败、取消的运行指标不全(口径卡常常没执行),和上一次比只会得出「移除 N 个」, + 收到通知的人会以为报告删了指标 +- 跟谁比:同一定时任务、同一工作流、比这次先发起的运行里成功结束并封存的最近一次(_previous)。失败的那次不当基准: + 拿它当基准,每个指标都成了「新增」。按发起先后挑,不按结束时间:老的失败运行事后被接着跑完,结束得比后一次晚, + 它的上一次仍是它前面那次。紧挨着的上一次不是成功的(失败、取消、还没结束)时 previous 为 False,通知和触发历史 + 写明比的是哪一次 +- 和封存绑定:运行的封存要盖住它最后一条结束事件才算封存完(sealed_now)。runner 先写终态、再另起会话封存,失败后 + 接着跑完的运行在两步之间还带着失败时的清单哈希,拿它去比只比得到旧封存范围里的事件。结果里记下两边的清单哈希: + 哪一边后来变了,触发历史就不再给这份对比(api/schedules._fire_outs);对比途中变了,结果不记 +- 什么时候:通知扫描(webhook.notify_due)挑出这次触发之后、认领之前,在锁外比,比完(或等过时限)再认领、推送。 + 比到一半服务停了,这次触发还没认领,下次启动照常比、照常推;同一份封存已经比过的,直接用记下的结果 +- 时限:一次对比最多等 WAIT_S 秒,同一时刻最多 SLOTS 个在算(读快照、重算都在线程里,不能一起占满接口请求也要用的 + 线程池)。超时、比不了(对比接口拒绝,如基准的封存核对不一致)、出错只记日志,通知照常推送、不带对比 +- 结果:整份对比结果连同两边的清单哈希存成工件(kind=run_compare)。不挂在任何运行名下:它不是运行产出的,不该出现 + 在运行的工件列表里 - 对外:触发历史(Fire.compare)和通知只给基准是哪一次、各类指标的个数,不给具体数字 """ from __future__ import annotations @@ -17,21 +26,29 @@ from __future__ import annotations import asyncio import logging import threading +import weakref from typing import Any from urllib.parse import urlencode from fastapi import HTTPException -from sqlalchemy import update +from sqlalchemy import func, select, update from app.core import artifact_store +from app.core.events import EventType from app.db.base import SessionLocal -from app.db.models import Run, ScheduleFire +from app.db.models import Run, RunEvent, ScheduleFire logger = logging.getLogger(__name__) KIND = "run_compare" -#: 终态,同对比接口(api/evidence._TERMINAL):只有这几种状态、封存过的运行能参与对比 +#: 终态,同对比接口(api/evidence._TERMINAL):触发历史里两边都要在终态,对比页才比得了 TERMINAL = ("succeeded", "failed", "cancelled") +#: 结束事件:封存要盖住这次运行的最后一条,才是这一次结束时封的 +_ENDS = (EventType.RUN_FINISHED.value, EventType.RUN_FAILED.value, EventType.RUN_CANCELLED.value) +#: 一次对比最多等多久(秒),排队等空位的时间也算在里面。等不到就不等了:通知不带对比照常推送 +WAIT_S = 20.0 +#: 同一时刻最多几个对比在算 +SLOTS = 2 #: 各类指标的叫法,和对比页一致(frontend/src/lib/terms.ts 的 RUN_COMPARE_TEXT.classLabel) CLASS_LABEL = { "data_only": "只数据变", @@ -56,6 +73,8 @@ NO_METRICS = "两次运行都没有指标" _BRIEFS: dict[str, dict[str, Any]] = {} _BRIEFS_MAX = 512 _BRIEFS_LOCK = threading.Lock() +#: 对比的空位,按事件循环分开(同 service.launch_lock):测试每个用例一个新循环,跨循环用同一个信号量会报错 +_SLOTS: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Semaphore]" = weakref.WeakKeyDictionary() def summary_text(summary: dict[str, Any]) -> str: @@ -73,68 +92,148 @@ def page_url(base_url: str, target: str, base: str) -> str: return f"{base_url}/runs/compare?{urlencode({'target': target, 'base': base})}" -async def compare_fire(fire_id: str, run_id: str) -> dict[str, Any] | None: - """这次触发发起的运行刚进入终态:和同一定时任务的上一次比一次,结果记进触发记录。返回给通知用的摘要 - {base_run_id, summary};没有可比的、比不了时返回 None,触发记录上之前的结果一并清掉(那是上一次封存的)。 +async def sealed_now(session: Any, run: Run) -> bool: + """这次运行的封存是不是这一次结束时封的:有清单哈希,而且封存范围盖住了最后一条结束事件。 - 不抛异常:比不了(对比接口拒绝,如基准的封存核对不一致)记一条警告,意外的异常记堆栈,都不影响通知。""" - artifact: str | None = None - brief: dict[str, Any] | None = None + runner 先在一个事务里写终态和结束事件,再另起会话封存。失败后接着跑完(或接着跑之后停在审批上被放弃)的运行,在 + 两步之间还带着失败时的清单哈希,只看「有没有哈希」会把旧的封存当成这一次的。没有 manifest_seq 的是加这一列之前 + 封存的老运行,只能看有没有哈希。""" + if not run.manifest_hash: + return False + if run.manifest_seq is None: + return True + last = await session.scalar(select(func.max(RunEvent.seq)).where( + RunEvent.run_id == run.id, RunEvent.type.in_(_ENDS))) + return last is None or run.manifest_seq >= last + + +async def compare_fire(fire_id: str, run_id: str) -> dict[str, Any] | None: + """这次触发发起的运行成功结束、封存完了:和同一定时任务上一次成功的运行比一次,结果记进触发记录。返回给通知用的 + 摘要 {base_run_id, summary, previous, base_started_at, …};不该比、没有可比的、比不了时返回 None。 + + 最多等 WAIT_S 秒(含排队)。超时、对比接口拒绝(基准的封存核对不一致等)记一条警告,意外的异常记堆栈,都不影响 + 通知。取消不吞:扫描被取消(停机)时这次触发还没认领,下次照常比。""" try: - artifact, brief = await _compare(run_id) + return await asyncio.wait_for(_compare(fire_id, run_id), timeout=WAIT_S) + except TimeoutError: + logger.warning("定时任务的运行 %s 自动对比超过 %g 秒仍未完成,通知不带对比照常推送", run_id, WAIT_S) except HTTPException as e: logger.warning("定时任务的运行 %s 自动对比未完成:%s", run_id, e.detail) except Exception: # noqa: BLE001 - 对比出错不能拖住通知,也不能让通知扫描停下 logger.exception("定时任务的运行 %s 自动对比出错", run_id) - await _store(fire_id, artifact) - return brief + return None -async def _compare(run_id: str) -> tuple[str | None, dict[str, Any] | None]: +async def _compare(fire_id: str, run_id: str) -> dict[str, Any] | None: # 现成的对比函数在用到时才导入:api/compare.py 连带证据、推演接口,调度模块被导入时不必一起加载 from app.api import compare as compare_api + # 挑基准、核对封存、看有没有比过都只是读库,不占对比的空位:前面排着慢的对比时,第一次运行这类没什么可比的 + # 不必跟着排队、等到超时 async with SessionLocal() as session: - run = await session.get(Run, run_id) - if run is None or run.status not in TERMINAL or not run.manifest_hash: - return None, None - found = await compare_api.default_base(run_id) - if found is None or found[1] != "same_schedule": - return None, None - base = found[0] - result = await compare_api.compare_runs(run_id, base.id) - brief = _brief(result) - if brief is None: - return None, None - artifact = await artifact_store.put_json(result, kind=KIND, meta={"target_run_id": run_id, "base_run_id": base.id}) + target = await session.get(Run, run_id) + stored = await session.scalar(select(ScheduleFire.compare_artifact).where(ScheduleFire.id == fire_id)) + if target is None or target.status != "succeeded" or not target.schedule_id or not target.workflow_id \ + or not await sealed_now(session, target): + return None + found = await _previous(session, target) + if found is None: + return None + base, previous = found + if not await sealed_now(session, base): + # 基准正好处在写完终态、还没封存的那一刻,或者封存没补上就停了机:拿它比只比得到旧封存里的事件 + logger.warning("作为基准的运行 %s 还没有封存到它的结束事件,运行 %s 这次不比", base.id, run_id) + return None + sides = ((target.id, target.manifest_hash), (base.id, base.manifest_hash)) + (target_id, target_seal), (base_id, base_seal) = sides + if stored: + # 同一份封存已经比过(比完还没认领就停了机):直接用 + known = await asyncio.to_thread(brief_of, stored) + if known is not None and _sides_of(known) == sides: + return _copy(known) + async with _slots(): + result = await compare_api.compare_runs(target_id, base_id) + payload = {"target_manifest": target_seal, "base_manifest": base_seal, "previous": previous, "result": result} + brief = _brief(payload) + if brief is None or _sides_of(brief) != sides: + return None + artifact = await artifact_store.put_json(payload, kind=KIND, + meta={"target_run_id": target_id, "base_run_id": base_id}) + if not await _store(fire_id, artifact, sides): + logger.warning("运行 %s 或作为基准的运行 %s 在对比途中重新封存(或触发记录已删除),这次的对比结果不记", + target_id, base_id) + return None _remember(artifact, brief) - return artifact, _copy(brief) + return _copy(brief) -async def _store(fire_id: str, artifact: str | None) -> None: - try: - async with SessionLocal() as session: - await session.execute(update(ScheduleFire).where(ScheduleFire.id == fire_id) - .values(compare_artifact=artifact)) - await session.commit() - except Exception: # noqa: BLE001 - 任务刚被删掉之类:记不下不影响通知 - logger.warning("触发记录 %s 的自动对比结果没能保存", fire_id) +async def _previous(session: Any, target: Run) -> tuple[Run, bool] | None: + """(基准, 是不是紧挨着的上一次):同一定时任务、同一工作流、比这次先发起的运行里成功结束并封存的最近一次。 - -def _brief(result: Any) -> dict[str, Any] | None: - """对比结果里给触发历史、通知的那一点:基准是哪一次、各类指标的个数(同对比接口的 summary)。""" - if not isinstance(result, dict): + 按发起先后挑(created_at),不按结束时间;同一工作流才比得了(任务改过工作流的,之前那个工作流的运行不算)。 + 中间隔着别的运行(失败、取消、还没结束的)时第二项为 False。""" + earlier = (Run.schedule_id == target.schedule_id, Run.workflow_id == target.workflow_id, Run.id != target.id, + Run.created_at < target.created_at) + base = (await session.execute( + select(Run).where(*earlier, Run.status == "succeeded", Run.manifest_hash.is_not(None)) + .order_by(Run.created_at.desc()).limit(1))).scalars().first() + if base is None: return None - base, summary = result.get("base"), result.get("summary") - if not isinstance(base, dict) or not isinstance(base.get("id"), str) or not isinstance(summary, dict): + between = await session.scalar(select(func.count()).select_from(Run).where( + *earlier, Run.created_at > base.created_at)) + return base, not between + + +def _slots() -> asyncio.Semaphore: + loop = asyncio.get_running_loop() + slots = _SLOTS.get(loop) + if slots is None: + slots = _SLOTS[loop] = asyncio.Semaphore(SLOTS) + return slots + + +async def _store(fire_id: str, artifact: str, sides: tuple[tuple[str, str | None], ...]) -> bool: + """把工件 id 记进触发记录。条件写:两边都还是成功结束、清单哈希还是比的时候那一份,才记。""" + same = [select(Run.id).where(Run.id == run_id, Run.status == "succeeded", Run.manifest_hash == seal).exists() + for run_id, seal in sides] + async with SessionLocal() as session: + result = await session.execute(update(ScheduleFire).where(ScheduleFire.id == fire_id, *same) + .values(compare_artifact=artifact)) + await session.commit() + return result.rowcount == 1 + + +def _sides_of(brief: dict[str, Any]) -> tuple[tuple[str, str | None], ...]: + return (brief["target_run_id"], brief["target_manifest"]), (brief["base_run_id"], brief["base_manifest"]) + + +def _brief(payload: Any) -> dict[str, Any] | None: + """对比工件里给触发历史、通知的那一点:两边是哪一次、比的时候两边的清单哈希、基准是不是紧挨着的上一次、基准 + 什么时候开始的、各类指标的个数(同对比接口的 summary)。形状不对的(这一版之前的工件、被改过的)返回 None。""" + if not isinstance(payload, dict) or not isinstance(payload.get("result"), dict): return None - return {"base_run_id": base["id"], + result = payload["result"] + target, base, summary = result.get("target"), result.get("base"), result.get("summary") + if not isinstance(target, dict) or not isinstance(base, dict) or not isinstance(summary, dict): + return None + keys = (target.get("id"), base.get("id"), payload.get("target_manifest"), payload.get("base_manifest")) + if not all(isinstance(key, str) and key for key in keys): + return None + started = base.get("started_at") + return {"target_run_id": keys[0], "base_run_id": keys[1], "target_manifest": keys[2], "base_manifest": keys[3], + "previous": payload.get("previous") is not False, + "base_started_at": started if isinstance(started, str) else None, "summary": {key: value for key, value in summary.items() if isinstance(value, int) and not isinstance(value, bool)}} +def public(brief: dict[str, Any]) -> dict[str, Any]: + """接口给前端的那一份(Fire.compare):基准是哪一次、各类指标的个数、基准是不是紧挨着的上一次。""" + return {"base_run_id": brief["base_run_id"], "summary": dict(brief["summary"]), "previous": brief["previous"]} + + def _copy(brief: dict[str, Any]) -> dict[str, Any]: - return {"base_run_id": brief["base_run_id"], "summary": dict(brief["summary"])} + return {**brief, "summary": dict(brief["summary"])} def _remember(artifact: str, brief: dict[str, Any]) -> None: diff --git a/backend/app/scheduler/webhook.py b/backend/app/scheduler/webhook.py index e74e12b7..8d8927d4 100644 --- a/backend/app/scheduler/webhook.py +++ b/backend/app/scheduler/webhook.py @@ -9,8 +9,8 @@ Webhook 地址本身就是凭证(企业微信的 key、钉钉的 access_token 一个过滤器把推送期间的那几行换成主机名。 通知里只有任务名、状态、出具档位、缺口条数、运行链接、封存哈希(界面上叫清单哈希)前 12 位和起止时间,失败时 -另写失败节点的名字和去哪里查看(不许诺能继续运行);运行结束后和同一任务上一次的对比(auto_compare)只写各类 -指标的个数和对比页链接。不放报告里的具体数字,也不放失败时的原始报错——报错里可能带着查询到的数据,Webhook +另写失败节点的名字和下一步(不许诺能继续运行);运行成功结束后和同一任务上一次成功的运行的对比(auto_compare)只写 +各类指标的个数和对比页链接。不放报告里的具体数字,也不放失败时的原始报错——报错里可能带着查询到的数据,Webhook 推去的群不一定有权看。 三个平台出错时都回 HTTP 200、错误码放在响应体里,所以送没送达要看响应体:按地址的主机认平台,不看选的格式—— @@ -67,14 +67,17 @@ _STATUS_LABEL = {"succeeded": "已完成", "failed": "失败", "cancelled": "已 #: 挂起的运行要人去处理:不处理,这个任务之后的触发都按「上一次还没结束」跳过 SUSPENDED_REASON = "运行停在断点上、没有待审批,通常是服务重启所致" SUSPENDED_ACTION = "请在 AgentLab 的「记录」中继续运行或放弃这次运行;它结束之前,这个定时任务之后的触发都会跳过" -#: 失败的运行怎么处理:不许诺「修正后可继续运行」。没开始执行的没有断点;开始执行过的也可能没有(执行第一个节点前就 -#: 失败)或已不兼容(系统升级),能不能继续以记录页的提示为准 -FAILED_ACTION = "请在 AgentLab 的运行记录中查看原因" +#: 失败的运行怎么处理:不许诺「修正后可继续运行」。开始执行过的也可能没有断点(执行第一个节点前就失败)或已不兼容 +#: (系统升级),能不能继续以记录页的提示为准。「记录」是导航里那一页的名字,和挂起那句一致 +FAILED_ACTION = "请在 AgentLab 的「记录」中查看失败原因" #: 从没开始执行就失败的(服务重启时还在排队,被改成失败、没有开始时间):没有可继续的进度。不照抄运行记录上的报错—— #: 通知一律不放原始报错,这一句是按「没有开始时间」认出来的,自己写 NOT_STARTED_REASON = "运行在开始执行前就已中止(通常是服务重启时仍在排队),没有可继续的进度" #: 失败的运行不挡之后的触发(挂起的才挡):任务还在排期上时补这一句,免得人以为这个定时任务就此停了 NEXT_AS_USUAL = "下一次触发会照常发起" +#: 从没开始执行、任务又不在排期上(停用了):原因刚写过,记录页上也没有更多可看的,下一步是重新发起。「立即运行」 +#: 是定时任务页上那个按钮的名字,停用的任务也能点 +RERUN_ACTION = "如需重新发起,请在 AgentLab 的「定时任务」中点击「立即运行」" # -------------------------------------------------------------------------- @@ -228,7 +231,8 @@ class Notice: suspended:运行停在断点上、没有待审批(通常是服务重启挂起的),按 failed 事件推。error_node:失败节点的名字。 next_fire:任务启用着、往后还会触发(失败通知据此补「下一次触发会照常发起」)。 - compare:和同一定时任务上一次的对比摘要 {base_run_id, summary}(只有个数),compare_url:对比页的链接。 + compare:和同一定时任务上一次成功的运行的对比摘要(auto_compare.compare_fire 的返回:基准是哪一次、只有个数的 + summary、基准是不是紧挨着的上一次、基准开始的时间),compare_url:对比页的链接。 fmt、signed 只给测试消息用:说明里只说这条消息证明得了的事。""" event: str @@ -293,7 +297,7 @@ def message(notice: Notice) -> tuple[str, list[tuple[str, str]], str | None]: lines.append(("出具档位", f"{ISSUANCE_LABEL.get(notice.tier, notice.tier)}{gaps}")) if notice.compare: # 只给各类指标的个数:具体的数在对比页上,跟着运行的权限走,不随消息推进群里 - lines.append(("与上一次相比", auto_compare.summary_text(notice.compare.get("summary") or {}))) + lines.append((_compare_head(notice), auto_compare.summary_text(notice.compare.get("summary") or {}))) if notice.pending_approvals: lines.append(("待审批", f"{notice.pending_approvals} 项,请到 AgentLab 处理;运行结束之前," "这个定时任务之后的触发都会跳过")) @@ -302,12 +306,14 @@ def message(notice: Notice) -> tuple[str, list[tuple[str, str]], str | None]: elif notice.suspended: lines += [("原因", SUSPENDED_REASON), ("处理", SUSPENDED_ACTION)] elif notice.status == "failed": - # 不写原始报错(可能带着查询到的数据),写失败节点的名字和去哪里处理 + # 不写原始报错(可能带着查询到的数据),写失败节点的名字和下一步 if notice.error_node: lines.append(("失败节点", f"「{notice.error_node}」")) if notice.never_started: - lines.append(("原因", NOT_STARTED_REASON)) - lines.append(("处理", f"{FAILED_ACTION};{NEXT_AS_USUAL}" if notice.next_fire else FAILED_ACTION)) + # 原因刚写过,也没有断点可续:不再让人去查看原因,说下一次照常,不在排期上的说怎么重新发起 + lines += [("原因", NOT_STARTED_REASON), ("处理", NEXT_AS_USUAL if notice.next_fire else RERUN_ACTION)] + else: + lines.append(("处理", f"{FAILED_ACTION};{NEXT_AS_USUAL}" if notice.next_fire else FAILED_ACTION)) if notice.fire_at is not None: when = f"{_local(notice.fire_at, notice.timezone, seconds=False)}({notice.timezone})" # 立即运行不写「计划触发」:群里的人会以为报告按时出了 @@ -329,6 +335,27 @@ def message(notice: Notice) -> tuple[str, list[tuple[str, str]], str | None]: return title, lines, notice.run_url +def _compare_head(notice: Notice) -> str: + """对比那一行的项名。基准是紧挨着的上一次时写「与上一次相比」;不是(上一次失败了、取消了)时写明比的是哪一次: + 短编号(同界面,#前 6 位)和开始的日期(按定时任务的时区)。""" + compare = notice.compare or {} + if compare.get("previous", True): + return "与上一次相比" + when = _date_of(compare.get("base_started_at"), notice.timezone) + return f"与上一次成功的运行(#{str(compare.get('base_run_id') or '')[:6]}{f',{when}' if when else ''})相比" + + +def _date_of(text: Any, tz: str) -> str | None: + """ISO 时间(对比结果里运行的开始时间)按定时任务的时区写成日期,读不出来的返回 None。""" + try: + moment = datetime.fromisoformat(str(text)) + except ValueError: + return None + if moment.tzinfo is None: + moment = moment.replace(tzinfo=timezone.utc) + return (_local(moment, tz, seconds=False) or "")[:10] or None + + def _status_label(notice: Notice) -> str: if notice.status == "interrupted": return "等待审批" if notice.pending_approvals else "已挂起,可继续运行" @@ -406,9 +433,10 @@ def _generic(notice: Notice, now: datetime) -> dict[str, Any]: else f"{SUSPENDED_REASON}。{SUSPENDED_ACTION}" if notice.suspended else NOT_STARTED_REASON if notice.never_started else None), "error_node": notice.error_node, - # 对比只给基准是哪一次、各类指标的个数和对比页链接 + # 对比只给基准是哪一次(是不是紧挨着的上一次)、各类指标的个数和对比页链接 "compare": ({"base_run_id": notice.compare.get("base_run_id"), - "summary": dict(notice.compare.get("summary") or {}), "url": notice.compare_url} + "summary": dict(notice.compare.get("summary") or {}), + "previous": notice.compare.get("previous", True) is not False, "url": notice.compare_url} if notice.compare else None), "sent_at": _iso(now), } @@ -592,12 +620,16 @@ class _Job: @dataclass class _Due: - """认领到的一次状态变化接下来要做的事:运行进入终态时和上一次比一次(compare_run),订阅了这个事件时推一条 - (job)。两样都在锁外做:对比要读快照、重算,推送要等接收端,都不能挡着发起。""" + """挑出来的一次状态变化接下来要做的事:运行成功结束时和上一次成功的运行比一次(compare_run),按条件 UPDATE + 认领(把 notified_status 从 old 改成 state),认领到了、订阅了这个事件时推一条(job)。都在锁外做:对比要读快照、 + 重算,推送要等接收端,都不能挡着发起。pushed:认领到了、发出了推送(含失败的)。""" fire_id: str + old: str | None + state: str compare_run: str | None job: _Job | None + pushed: bool = False async def notify_due(now: datetime | None = None) -> int: @@ -607,8 +639,11 @@ async def notify_due(now: datetime | None = None) -> int: 推一次。没订阅这个事件、任务没配 Webhook 的也记下已处理——之后再配上 Webhook,不会把旧的补推一遍。 停在断点上的运行每一拍都要重新看(见 _state_of):它的已处理状态带着第几段执行,和运行状态本身对不上。 - 运行进入终态的那一次,推送之前先和同一定时任务的上一次比一次(auto_compare),结束、失败的通知带上计数摘要和 - 对比页链接。没配 Webhook 的也比:触发历史里要用。比不了、出错不影响推送。 + 运行成功结束、封存完的那一次,认领之前先和同一定时任务上一次成功的运行比一次(auto_compare),结束的通知带上计数 + 摘要和对比页链接。没配 Webhook 的也比:触发历史里要用。对比在锁外、认领之前做,最多等 auto_compare.WAIT_S 秒, + 过了就不带对比推送:一次慢的对比最多让这一拍晚这么久,不会把之后所有的通知一直压着,也不拖着发起用的锁。比到一半 + 扫描被取消(停机只等通知扫描一个宽限),这次触发还没认领,下次照常比、照常推——以前先认领、后比,取消之后这条 + 通知再也不会推。比不了、出错不影响推送。 """ now = now or utcnow() dues: list[_Due] = [] @@ -617,7 +652,9 @@ async def notify_due(now: datetime | None = None) -> int: # 扫描窗口按触发记录的创建时间算,两种例外:运行记录窗口内有过变化(运行状态一变 updated_at 就跟着变),以及 # 停在审批上、有待审批的——以前只看触发记录,创建 30 天以后才停到审批上的运行收不到「等待审批」 waiting = select(Approval.id).where(Approval.run_id == Run.id, Approval.status == "pending").exists() - # 和发起共用一把锁:发起时先记一条「未能发起」再发起运行、成功了才改成 started,扫描不能撞在这中间 + # 和发起共用一把锁:发起时先记一条「未能发起」再发起运行、成功了才改成 started,扫描不能撞在这中间。锁里只挑出 + # 状态有变化的、拼好要推的消息;认领在锁外按条件 UPDATE 做,挑出来的是锁里看到的状态,和别的扫描撞上也只有一个 + # 认领得到 async with service.launch_lock(): async with SessionLocal() as session: rows = (await session.execute( @@ -636,39 +673,52 @@ async def notify_due(now: datetime | None = None) -> int: base_url = (await notify_settings(session))["public_base_url"] for fire, run in rows: - state, event, pending = await _state_of(session, fire, run, now) - if state is None or state == fire.notified_status \ - or not await _mark(fire.id, fire.notified_status, state): + seen = await _state_of(session, fire, run, now) + if seen.state is None or seen.state == fire.notified_status: continue - compare_run = run.id if run is not None and run.status in auto_compare.TERMINAL else None schedule = await session.get(Schedule, fire.schedule_id) url = decrypt(schedule.webhook_url) if schedule and schedule.webhook_url else None job = None - if event and url and schedule.webhook_format in FORMATS and event in (schedule.webhook_events or []): - notice = await _notice(session, schedule, fire, run, event, base_url, pending) + if seen.event and url and schedule.webhook_format in FORMATS \ + and seen.event in (schedule.webhook_events or []): + notice = await _notice(session, schedule, fire, run, seen, base_url) job = _Job(schedule.webhook_format, url, decrypt(schedule.webhook_secret), notice) - if compare_run or job: - dues.append(_Due(fire.id, compare_run, job)) + # 只比成功结束、封存完的(见 auto_compare):失败、取消的指标不全,等不来新封存的比了也是旧的 + compare_run = run.id if run is not None and run.status == "succeeded" and seen.sealed else None + dues.append(_Due(fire.id, fire.notified_status, seen.state, compare_run, job)) if dues: await asyncio.gather(*(_follow_up(due, base_url) for due in dues)) - return sum(1 for due in dues if due.job is not None) + return sum(1 for due in dues if due.pushed) async def _follow_up(due: _Due, base_url: str | None) -> None: - """先比(需要的话),再推:对比摘要要写进这条通知。对比自己兜住了异常,出什么事都不挡推送。""" - if due.compare_run: - brief = await auto_compare.compare_fire(due.fire_id, due.compare_run) - if brief is not None and due.job is not None: - due.job.notice.compare = brief - if base_url: - due.job.notice.compare_url = auto_compare.page_url(base_url, due.compare_run, brief["base_run_id"]) - if due.job is not None: - await _deliver(due.job) + """先比(需要的话),再认领,认领到了再推:对比摘要要写进这条通知;认领放在对比之后,比到一半被取消时这次触发 + 还没认领。对比自己兜住了超时和异常,出什么事都不挡推送。""" + brief = await auto_compare.compare_fire(due.fire_id, due.compare_run) if due.compare_run else None + if not await _mark(due.fire_id, due.old, due.state) or due.job is None: + return + if brief is not None: + due.job.notice.compare = brief + if base_url: + due.job.notice.compare_url = auto_compare.page_url(base_url, due.compare_run, brief["base_run_id"]) + due.pushed = True + await _deliver(due.job) -async def _state_of(session: Any, fire: ScheduleFire, run: Run | None, - now: datetime) -> tuple[str | None, str | None, int]: - """(要记下的已处理状态, 要推送的事件, 待审批数)。还不能下结论(没封存、审批刚批完还没接着执行)时状态为 None。 +@dataclass +class _Seen: + """_state_of 的结论。state:要记下的已处理状态,还不能下结论时为 None;event:要推送的事件;pending:待审批数; + sealed:终态的运行封存完了(封存盖住了这一次的结束事件,见 auto_compare.sealed_now),只有这时才比、通知里才写 + 清单哈希。""" + + state: str | None + event: str | None = None + pending: int = 0 + sealed: bool = False + + +async def _state_of(session: Any, fire: ScheduleFire, run: Run | None, now: datetime) -> _Seen: + """这次触发现在该记成什么状态、推什么。还不能下结论(没封存、审批刚批完还没接着执行)时状态为 None。 停在断点上(interrupted)的分两种,各推一次: - 有待审批:停在审批,推 awaiting_approval。批完接着执行、又停在下一个审批点时要再推,所以已处理状态里带着这是 @@ -678,18 +728,21 @@ async def _state_of(session: Any, fire: ScheduleFire, run: Run | None, 审批批完到运行改回 running 之间,审批是 answered、还没有 resolved:那不是挂起,等下一拍。 """ if fire.status == "failed_to_start": - return "failed_to_start", "failed", 0 + return _Seen("failed_to_start", "failed") if run is None: - return None, None, 0 + return _Seen(None) if run.status in ("succeeded", "failed", "cancelled"): - # runner 先写终态和结束时间、再封存,两步之间被这一拍撞上就等。没有结束时间的(服务重启时还在排队、被改成 - # failed 的)永远不会封存:按它最后一次更新的时间等,以前按「现在」等,每一拍都等、永远不推 + # runner 先写终态和结束时间、再另起会话封存,两步之间被这一拍撞上就等。失败后接着跑完的运行这时还带着失败时 + # 的清单哈希,不能只看有没有哈希,要看封存盖没盖住这一次的结束事件。没有结束时间的(服务重启时还在排队、被改 + # 成 failed 的)永远不会封存:按它最后一次更新的时间等,以前按「现在」等,每一拍都等、永远不推。等过时限仍没 + # 封存完的照常推,只是不比、不写清单哈希 + sealed = await auto_compare.sealed_now(session, run) since = run.finished_at or run.updated_at or fire.created_at - if not run.manifest_hash and (now - service.as_utc(since)).total_seconds() < SEAL_WAIT_S: - return None, None, 0 - return run.status, _EVENT_OF[run.status], 0 + if not sealed and (now - service.as_utc(since)).total_seconds() < SEAL_WAIT_S: + return _Seen(None) + return _Seen(run.status, _EVENT_OF[run.status], sealed=sealed) if run.status != "interrupted": - return None, None, 0 + return _Seen(None) counts = dict((await session.execute( select(Approval.status, func.count()).where( Approval.run_id == run.id, Approval.status.in_(("pending", "answered"))) @@ -697,10 +750,10 @@ async def _state_of(session: Any, fire: ScheduleFire, run: Run | None, segment = await session.scalar(select(func.count()).select_from(RunEvent).where( RunEvent.run_id == run.id, RunEvent.type == EventType.RUN_STARTED.value)) or 0 if counts.get("pending"): - return f"interrupted:approval:{segment}", "awaiting_approval", counts["pending"] + return _Seen(f"interrupted:approval:{segment}", "awaiting_approval", counts["pending"]) if counts.get("answered"): - return None, None, 0 - return f"interrupted:held:{segment}", "failed", 0 + return _Seen(None) + return _Seen(f"interrupted:held:{segment}", "failed") async def _mark(fire_id: str, old: str | None, state: str) -> bool: @@ -712,17 +765,19 @@ async def _mark(fire_id: str, old: str | None, state: str) -> bool: return result.rowcount == 1 -async def _notice(session: Any, schedule: Schedule, fire: ScheduleFire, run: Run | None, event: str, - base_url: str | None, pending: int) -> Notice: +async def _notice(session: Any, schedule: Schedule, fire: ScheduleFire, run: Run | None, seen: _Seen, + base_url: str | None) -> Notice: tier, gaps = await _issuance_of(session, run) if run is not None else (None, None) + pending = seen.pending return Notice( - event=event, schedule_id=schedule.id, schedule_name=schedule.name, timezone=schedule.timezone, + event=seen.event or "", schedule_id=schedule.id, schedule_name=schedule.name, timezone=schedule.timezone, fire_id=fire.id, fire_at=fire.fire_at, fire_kind=fire.kind, run_id=run.id if run else None, status=run.status if run else fire.status, tier=tier, gaps=gaps, run_url=f"{base_url}/runs/{run.id}" if base_url and run else None, - manifest_hash=run.manifest_hash if run else None, + # 只写这一次结束时封的:停在审批上、封存还没补上的运行,身上可能还留着上一次失败时的清单哈希 + manifest_hash=run.manifest_hash if run is not None and seen.sealed else None, started_at=run.started_at if run else None, finished_at=run.finished_at if run else None, reason=fire.reason if fire.status == "failed_to_start" else None, diff --git a/backend/tests/test_scheduler_compare.py b/backend/tests/test_scheduler_compare.py index d7f79238..3c9af745 100644 --- a/backend/tests/test_scheduler_compare.py +++ b/backend/tests/test_scheduler_compare.py @@ -1,17 +1,20 @@ -"""定时运行结束后的自动对比(app/scheduler/auto_compare.py,由通知扫描 webhook.notify_due 在推送之前触发): +"""定时运行结束后的自动对比(app/scheduler/auto_compare.py,由通知扫描 webhook.notify_due 在认领、推送之前触发): -- 运行进入终态且已封存、同一定时任务有更早的已封存运行时,用运行对比的现成函数(api/compare.py 的 default_base、 - compare_runs)和它比一次,整份结果存成工件(kind=run_compare,不挂在运行名下、不进运行的工件列表),工件 id 记进 - 触发记录的 compare_artifact。基准是同一定时任务的上一次:同一工作流更近的手动运行、别的定时任务的运行都不算 -- 这个任务的第一次(只有同一工作流别的运行可比)、还没封存时不比;对比出错(基准的封存核对不一致、意外的异常)只记 - 日志,通知照常推送、不带对比 -- 接口:Fire 多一个 compare: {base_run_id, summary} 或 null,summary 同对比接口的计数;运行又被继续执行、不在终态 - 时为 null(对比页只认封存的运行) -- 通知:结束、失败的通知多一行「与上一次相比:只数据变 2 个、口径与数据都变 0 个、SQL 结构变 0 个」,配了对外访问 - 地址时附对比页链接(/runs/compare?target=…&base=…);只给个数,报告、指标、查询里的具体数字一个不出现 +- 只比成功结束并封存的运行:失败、取消的指标不全,和上一次比只会得出「移除 N 个」,误导收到通知的人 +- 基准是同一定时任务、同一工作流、比这次先发起的运行里成功结束并封存的最近一次:同一工作流更近的手动运行、别的 + 定时任务的运行都不算;失败的那一次不当基准(拿它当基准每个指标都成了「新增」);按发起先后挑,不按结束时间—— + 老的失败运行事后被接着跑完,结束得比后一次晚,它的上一次仍是它前面那次。紧挨着的上一次不是成功的时,通知和触发 + 历史写明比的是哪一次 +- 整份结果连同两边的清单哈希存成工件(kind=run_compare,不挂在运行名下、不进运行的工件列表),工件 id 记进触发 + 记录的 compare_artifact。这个任务的第一次(只有同一工作流别的运行可比)、还没封存时不比;对比出错(基准的封存核对 + 不一致、意外的异常)只记日志,通知照常推送、不带对比 +- 接口:Fire 多一个 compare: {base_run_id, summary, previous} 或 null,summary 同对比接口的计数,previous 说基准是不是 + 紧挨着的上一次;基准被删、被继续执行(不在终态)时为 null(对比页只认封存的运行) +- 通知:结束的通知多一行「与上一次相比:只数据变 2 个、口径与数据都变 0 个、SQL 结构变 0 个」,配了对外访问地址时附 + 对比页链接(/runs/compare?target=…&base=…);只给个数,报告、指标、查询里的具体数字一个不出现 -两次周报运行用 tests/fixtures/compare_runs.py 的虚构运行(区域、金额、日期都是编的),按 runner 的算法封存,不真的 -执行;Webhook 用 httpx.MockTransport 接住,不连外网。 +封存绑定、时限、取消这几道保护另见 test_scheduler_compare_guards.py。两次周报运行用 tests/fixtures/compare_runs.py 的 +虚构运行(区域、金额、日期都是编的),按 runner 的算法封存,不真的执行;Webhook 用 httpx.MockTransport 接住,不连外网。 """ from __future__ import annotations @@ -32,7 +35,7 @@ from app.db.base import SessionLocal from app.db.models import Artifact, Run, RunEvent, Schedule, ScheduleFire, Setting, Workflow from app.main import app from app.scheduler import auto_compare, webhook -from tests.fixtures.compare_runs import NEXT_ROWS, Fake, week +from tests.fixtures.compare_runs import NEXT_ROWS, Fake, region_sql, week UTC = timezone.utc T0 = datetime(2026, 9, 7, 1, 0, tzinfo=UTC) @@ -111,8 +114,9 @@ async def new_schedule(workflow_id: str, *, fmt: str = "generic", url: str | Non async def persist(fake: Fake, workflow_id: str, *, finished: datetime, schedule_id: str | None = None, - status: str = "succeeded", seal: bool = True) -> str: - """把一次虚构运行落库:工件进工件库,事件进运行记录,按 runner 的算法封存(同 test_compare_api.py)。""" + status: str = "succeeded", seal: bool = True, seal_upto: int | None = None) -> str: + """把一次虚构运行落库:工件进工件库,事件进运行记录,按 runner 的算法封存(同 test_compare_api.py)。 + seal_upto 只封到第几条事件:失败时封存过、接着跑完又写下了终态,新的封存还没补上的那一刻。""" for content in fake.store.values(): await put_json(content, kind="compare_fixture") async with SessionLocal() as session: @@ -128,11 +132,39 @@ async def persist(fake: Fake, workflow_id: str, *, finished: datetime, schedule_ rows.append((seq, etype, node_id, data)) run.last_seq = len(rows) if seal: - run.manifest_hash, run.manifest_seq = manifest_hash(rows), rows[-1][0] + covered = [r for r in rows if seal_upto is None or r[0] <= seal_upto] + run.manifest_hash, run.manifest_seq = manifest_hash(covered), covered[-1][0] await session.commit() return run.id +async def reseal(run_id: str, fake: Fake, *, status: str, finished: datetime) -> None: + """一次运行被接着跑完(或者被重新封存):事件换成 fake 的那一份,按新的事件封存,状态、结束时间一并改。""" + for content in fake.store.values(): + await put_json(content, kind="compare_fixture") + async with SessionLocal() as session: + await session.execute(delete(RunEvent).where(RunEvent.run_id == run_id)) + rows = [] + for seq, etype, node_id, data in fake.events: + session.add(RunEvent(run_id=run_id, seq=seq, type=etype, node_id=node_id, ts=float(seq), data=data)) + rows.append((seq, etype, node_id, data)) + run = await session.get(Run, run_id) + run.last_seq = len(rows) + run.manifest_hash, run.manifest_seq = manifest_hash(rows), rows[-1][0] + run.status, run.finished_at = status, finished + await session.commit() + + +def failed_week(run_id: str, rows=NEXT_ROWS) -> Fake: + """在 Agent 节点失败的一次周报:取数做完了,Agent 没交回结果,口径卡没有执行,一个指标都没有。""" + run = Fake(run_id) + run.tool("fetch", region_sql(), rows=rows) + run.add("node.started", "bot", {"node_type": "agent"}) + run.add("node.failed", "bot", {"error": "模型接入超时"}) + run.add("run.failed", None, {"error": "节点「区域分析」失败"}) + return run + + async def seal(run_id: str) -> None: async with SessionLocal() as session: run = await session.get(Run, run_id) @@ -206,8 +238,14 @@ async def test_finished_run_is_compared_with_the_previous_run_of_its_schedule(re from app.api.compare import compare_runs stored = load(artifact) - assert stored == json.loads(canonical_json(await compare_runs(target_id, base_id))) - assert stored["base"]["id"] == base_id and stored["summary"] == SUMMARY + # 整份对比结果,连同比的时候两边的清单哈希(封存变了,这份结果就不再作数) + assert stored["result"] == json.loads(canonical_json(await compare_runs(target_id, base_id))) + assert stored["result"]["base"]["id"] == base_id and stored["result"]["summary"] == SUMMARY + async with SessionLocal() as session: + runs = await session.scalars(select(Run).where(Run.id.in_((target_id, base_id)))) + seals = {r.id: r.manifest_hash for r in runs} + assert (stored["target_manifest"], stored["base_manifest"], stored["previous"]) == ( + seals[target_id], seals[base_id], True) async with SessionLocal() as session: rows = list((await session.execute(select(Artifact).where(Artifact.id == artifact))).scalars()) # 对比结果不是运行产出的:不挂在运行名下,不进这次运行的工件列表 @@ -215,7 +253,7 @@ async def test_finished_run_is_compared_with_the_previous_run_of_its_schedule(re assert artifact not in [a["id"] for a in (await client.get(f"/api/runs/{target_id}/artifacts")).json()] items = {f["id"]: f for f in (await client.get(f"/api/schedules/{sid}/fires")).json()["items"]} - assert items[fire_id]["compare"] == {"base_run_id": base_id, "summary": SUMMARY} + assert items[fire_id]["compare"] == {"base_run_id": base_id, "summary": SUMMARY, "previous": True} assert items[base_fire]["compare"] is None [listed] = [s for s in (await client.get("/api/schedules")).json()["items"] if s["id"] == sid] assert listed["last_fire"]["id"] == fire_id and listed["last_fire"]["compare"]["base_run_id"] == base_id @@ -223,7 +261,7 @@ async def test_finished_run_is_compared_with_the_previous_run_of_its_schedule(re [body] = recorder.bodies() link = f"{BASE_URL}/runs/compare?target={target_id}&base={base_id}" assert body["event"] == "finished" and LINE in body["text"] and f"查看对比:{link}" in body["text"] - assert body["compare"] == {"base_run_id": base_id, "summary": SUMMARY, "url": link} + assert body["compare"] == {"base_run_id": base_id, "summary": SUMMARY, "previous": True, "url": link} query = parse_qs(urlsplit(body["compare"]["url"]).query) assert query == {"target": [target_id], "base": [base_id]} assert no_numbers(recorder.requests[0].content.decode()) == [] @@ -231,25 +269,79 @@ async def test_finished_run_is_compared_with_the_previous_run_of_its_schedule(re assert await webhook.notify_due() == 0 and (await fire_row(fire_id)).compare_artifact == artifact -@pytest.mark.parametrize("fmt, url", [("feishu", FEISHU), ("generic", GENERIC)]) -async def test_failed_run_notice_carries_the_counts_too(recorder, fmt, url): - """失败的运行也封存:和上一次比一次,失败通知同样带计数摘要""" +@pytest.mark.parametrize("status, fmt, url", [("failed", "feishu", FEISHU), ("failed", "generic", GENERIC), + ("cancelled", "generic", GENERIC)]) +async def test_failed_or_cancelled_run_is_not_compared(recorder, client, status, fmt, url): + """失败、取消的运行指标不全(这次在 Agent 节点失败,口径卡没有执行):和上一次比只会得出「移除 2 个」,收到通知 + 的人会以为报告删了两个指标。不比:触发记录上没有对比,失败通知不带计数那一行和对比页链接""" await public_address() - _, base_id, target_id, fire_id = await weekly_pair(target_status="failed", fmt=fmt, url=url) - assert await webhook.notify_due() == 1 - assert (await fire_row(fire_id)).compare_artifact - raw = recorder.requests[0].content.decode() - text = json.loads(raw)["content"]["text"] if fmt == "feishu" else json.loads(raw)["text"] - assert "运行失败" in text and LINE in text and f"base={base_id}" in text and f"target={target_id}" in text - assert no_numbers(raw) == [] - - -async def test_cancelled_run_is_compared_but_not_pushed(recorder, client): - """取消的运行不推送(有人手动停的),对比照样算:触发历史里一样能点「对比上一次」""" - sid, base_id, _, fire_id = await weekly_pair(target_status="cancelled") - assert await webhook.notify_due() == 0 and recorder.requests == [] + workflow = await new_workflow() + sid = await new_schedule(workflow, fmt=fmt, url=url) + base_id = await persist(week("r1"), workflow, finished=T0, schedule_id=sid) + await fire_for(sid, base_id, T0 - timedelta(minutes=2), notified="succeeded") + target_id = await persist(failed_week("r2"), workflow, finished=T0 + WEEK, schedule_id=sid, status=status) + fire_id = await fire_for(sid, target_id, T0 + WEEK - timedelta(minutes=2)) + # 取消的不推送(有人手动停的) + assert await webhook.notify_due() == (1 if status == "failed" else 0) + assert (await fire_row(fire_id)).compare_artifact is None items = {f["id"]: f for f in (await client.get(f"/api/schedules/{sid}/fires")).json()["items"]} - assert items[fire_id]["compare"]["base_run_id"] == base_id + assert items[fire_id]["compare"] is None + if status == "failed": + raw = recorder.requests[0].content.decode() + text = json.loads(raw)["content"]["text"] if fmt == "feishu" else json.loads(raw)["text"] + assert "运行失败" in text and f"{BASE_URL}/runs/{target_id}" in text + for absent in ("相比", "移除", "查看对比", "/runs/compare"): + assert absent not in text, text + if fmt == "generic": + assert json.loads(raw)["compare"] is None + + +async def test_after_a_failed_run_the_base_is_the_last_successful_one(recorder, client): + """上一次失败了:基准取再往前、成功的那一次——拿失败的那次当基准,每个指标都成了「新增」。紧挨着的上一次不是 + 它,通知和触发历史写明比的是哪一次(短编号和发起的日期,按定时任务的时区)""" + await public_address() + workflow = await new_workflow() + sid = await new_schedule(workflow) + first = await persist(week("r1"), workflow, finished=T0, schedule_id=sid) + await fire_for(sid, first, T0 - timedelta(minutes=2), notified="succeeded") + failed = await persist(failed_week("r2"), workflow, finished=T0 + WEEK, schedule_id=sid, status="failed") + await fire_for(sid, failed, T0 + WEEK - timedelta(minutes=2), notified="failed") + target_id = await persist(week("r3", NEXT_ROWS), workflow, finished=T0 + 2 * WEEK, schedule_id=sid) + fire_id = await fire_for(sid, target_id, T0 + 2 * WEEK - timedelta(minutes=2)) + + assert await webhook.notify_due() == 1 + stored = load((await fire_row(fire_id)).compare_artifact) + assert stored["result"]["base"]["id"] == first and stored["previous"] is False + [body] = recorder.bodies() + # 第一次发起于北京时间 2026-09-07 08:58 + line = f"与上一次成功的运行(#{first[:6]},2026-09-07)相比:只数据变 2 个、口径与数据都变 0 个、SQL 结构变 0 个" + assert line in body["text"] and "新增" not in body["text"], body["text"] + assert f"base={first}" in body["text"] and failed not in body["text"] + assert (body["compare"]["base_run_id"], body["compare"]["previous"]) == (first, False) + items = {f["id"]: f for f in (await client.get(f"/api/schedules/{sid}/fires")).json()["items"]} + assert items[fire_id]["compare"] == {"base_run_id": first, "summary": SUMMARY, "previous": False} + + +async def test_base_follows_launch_order_not_finish_time(recorder): + """第 2 周失败、第 3 周照常成功之后,有人把第 2 周接着跑完:它结束得比第 3 周晚,但它的上一次仍是第 1 周。以前 + 按结束时间挑,拿后一次触发的第 3 周当基准,「与上一次相比」的变化方向是反的。第 3 周已经比过的那一份不受影响""" + workflow = await new_workflow() + sid = await new_schedule(workflow, url=None) + first = await persist(week("r1"), workflow, finished=T0, schedule_id=sid) + await fire_for(sid, first, T0 - timedelta(minutes=2), notified="succeeded") + second = await persist(failed_week("r2"), workflow, finished=T0 + WEEK, schedule_id=sid, status="failed") + second_fire = await fire_for(sid, second, T0 + WEEK - timedelta(minutes=2)) + third = await persist(week("r3", NEXT_ROWS), workflow, finished=T0 + 2 * WEEK, schedule_id=sid) + third_fire = await fire_for(sid, third, T0 + 2 * WEEK - timedelta(minutes=2)) + await webhook.notify_due() + third_artifact = (await fire_row(third_fire)).compare_artifact + assert load(third_artifact)["result"]["base"]["id"] == first + + await reseal(second, week("r2", NEXT_ROWS), status="succeeded", finished=T0 + 2 * WEEK + timedelta(days=1)) + await webhook.notify_due() + stored = load((await fire_row(second_fire)).compare_artifact) + assert stored["result"]["base"]["id"] == first and stored["previous"] is True + assert (await fire_row(third_fire)).compare_artifact == third_artifact async def test_first_run_of_a_schedule_has_no_compare(recorder, client): @@ -276,7 +368,7 @@ async def test_waits_for_the_seal_before_comparing(recorder): assert (await fire_row(fire_id)).compare_artifact is None and recorder.requests == [] await seal(target_id) assert await webhook.notify_due() == 1 - assert load((await fire_row(fire_id)).compare_artifact)["base"]["id"] == base_id + assert load((await fire_row(fire_id)).compare_artifact)["result"]["base"]["id"] == base_id assert LINE in recorder.bodies()[0]["text"] @@ -327,18 +419,28 @@ async def test_compare_failure_only_logs_and_the_notice_still_goes_out(recorder, assert "封存核对不一致" in caplog.text -async def test_compare_is_hidden_once_the_run_is_running_again(client): - """失败后被继续执行(不在终态):对比页只认封存的运行,接口不给 compare;再次结束时重新比""" - sid, base_id, target_id, fire_id = await weekly_pair(target_status="failed", url=None) +async def test_a_failed_run_is_compared_once_it_is_continued_to_success(client): + """失败时不比;接着跑(不在终态)时也没有;跑完、成功并重新封存之后按新的封存比一次""" + workflow = await new_workflow() + sid = await new_schedule(workflow, url=None) + base_id = await persist(week("r1"), workflow, finished=T0, schedule_id=sid) + await fire_for(sid, base_id, T0 - timedelta(minutes=2), notified="succeeded") + target_id = await persist(failed_week("r2"), workflow, finished=T0 + WEEK, schedule_id=sid, status="failed") + fire_id = await fire_for(sid, target_id, T0 + WEEK - timedelta(minutes=2)) + + async def shown(): + items = (await client.get(f"/api/schedules/{sid}/fires")).json()["items"] + return {f["id"]: f for f in items}[fire_id] + await webhook.notify_due() - assert (await fire_row(fire_id)).compare_artifact + assert (await fire_row(fire_id)).compare_artifact is None and (await shown())["compare"] is None await set_run(target_id, status="running") - [item] = [f for f in (await client.get(f"/api/schedules/{sid}/fires")).json()["items"] if f["id"] == fire_id] - assert item["run_status"] == "running" and item["compare"] is None - await set_run(target_id, status="succeeded", finished_at=T0 + WEEK + timedelta(hours=1)) await webhook.notify_due() - [item] = [f for f in (await client.get(f"/api/schedules/{sid}/fires")).json()["items"] if f["id"] == fire_id] - assert item["compare"] == {"base_run_id": base_id, "summary": SUMMARY} + item = await shown() + assert item["run_status"] == "running" and item["compare"] is None + await reseal(target_id, week("r2", NEXT_ROWS), status="succeeded", finished=T0 + WEEK + timedelta(hours=1)) + await webhook.notify_due() + assert (await shown())["compare"] == {"base_run_id": base_id, "summary": SUMMARY, "previous": True} async def test_compare_is_hidden_once_the_base_run_is_gone_or_running(client): diff --git a/backend/tests/test_scheduler_compare_guards.py b/backend/tests/test_scheduler_compare_guards.py new file mode 100644 index 00000000..1e1754f7 --- /dev/null +++ b/backend/tests/test_scheduler_compare_guards.py @@ -0,0 +1,312 @@ +"""定时运行自动对比的几道保护(app/scheduler/auto_compare.py、webhook.notify_due): + +- 和封存绑定:运行的封存要盖住它最后一条结束事件才算封存完。runner 先写终态、再另起会话封存,失败后接着跑完的运行 + 在两步之间还带着失败时的清单哈希——这时去比,只比得到旧封存范围里的事件,通知里的清单哈希也是旧的。等不来新的 + 封存(两步之间进程没了)时照常推送,不带对比、不写旧的哈希 +- 对比结果记下两边的清单哈希:任何一边后来变了,触发历史就不再给这份对比(悬停的计数和点进去的对比页对不上); + 对比途中封存变了,结果不记 +- 时限:对比放在认领之前、锁外做,最多等 WAIT_S 秒,同一时刻最多 SLOTS 个。慢的对比不压住这条通知,也不压住别的 + 定时任务的通知;扫描在对比途中被取消(停机),这次触发还没认领,下次照常比、照常推。同一份封存已经比过的(比完 + 还没推送就停机了),下次直接用记下的结果 + +运行、任务名、金额都是虚构的(tests/fixtures/compare_runs.py),不真的执行;Webhook 用 httpx.MockTransport 接住。 +""" +from __future__ import annotations + +import asyncio +import logging +import time +from datetime import datetime, timedelta, timezone + +import httpx +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import select + +from app import scheduler +from app.api import compare as compare_api +from app.core.artifact_store import load +from app.core.config import Settings +from app.db.base import SessionLocal +from app.db.models import Run, RunEvent +from app.main import app +from app.scheduler import auto_compare, webhook +from tests.fixtures.compare_runs import BASE_ROWS, NEXT_ROWS, Fake, default_metrics, region_sql, week +from tests.test_scheduler_compare import ( # noqa: F401 - clean、recorder 是夹具,按名字引入 + BASE_URL, LINE, T0, WEEK, Recorder, clean, fire_for, fire_row, new_schedule, new_workflow, persist, + public_address, recorder, reseal, seal, set_run, weekly_pair, +) + +UTC = timezone.utc + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +@pytest.fixture +def slow_compare(monkeypatch): + """对比要 5 秒:真的对比之前先等着。返回原来的函数,用例里可以换回去""" + real = compare_api.compare_runs + + async def slow(*args, **kwargs): + await asyncio.sleep(5) + return await real(*args, **kwargs) + + monkeypatch.setattr(compare_api, "compare_runs", slow) + return real + + +def failed_then_continued(run_id: str, rows=NEXT_ROWS) -> tuple[Fake, int]: + """在 Agent 节点失败、之后从断点接着跑完的一次周报。返回 (运行, 失败时封存到第几条事件)。""" + run = Fake(run_id) + sql = region_sql() + fetch = run.tool("fetch", sql, rows=rows) + run.add("node.started", "bot", {"node_type": "agent"}) + run.add("node.failed", "bot", {"error": "模型接入超时"}) + run.add("run.failed", None, {"error": "节点「区域分析」失败"}) + failed_at = len(run.events) + run.add("run.resumed", None, {"from": ["bot"], "message": "从「区域分析」继续运行"}) + bot = run.agent("bot", sql, rows=rows) + run.card(default_metrics(fetch, bot, rows, rows)) + run.add("run.finished", None, {"output": {}}) + return run, failed_at + + +async def previous_week(*, url: str | None = None): + """一个定时任务和它上一次成功、已推送过的周报。返回 (工作流, 任务, 上一次)。""" + workflow = await new_workflow() + sid = await new_schedule(workflow, url=url) if url else await new_schedule(workflow) + base_id = await persist(week("r1"), workflow, finished=T0, schedule_id=sid) + await fire_for(sid, base_id, T0 - timedelta(minutes=2), notified="succeeded") + return workflow, sid, base_id + + +async def seal_of(run_id: str) -> str | None: + async with SessionLocal() as session: + return (await session.get(Run, run_id)).manifest_hash + + +# -------------------------------------------------------------------------- +# 和封存绑定 +# -------------------------------------------------------------------------- + + +async def test_waits_for_the_seal_of_this_ending_not_the_failed_one(recorder): + """失败后接着跑完:终态和结束时间已经写下,封存还是失败时那一份(只盖到 run.failed)。这一拍不比、不推,新的 + 封存补上之后再比,比的是完整的一次,通知里的清单哈希是新的""" + await public_address() + workflow, sid, base_id = await previous_week() + fake, failed_at = failed_then_continued("r2") + target_id = await persist(fake, workflow, finished=datetime.now(UTC), schedule_id=sid, seal_upto=failed_at) + fire_id = await fire_for(sid, target_id, T0 + WEEK - timedelta(minutes=2)) + stale = await seal_of(target_id) + + assert await webhook.notify_due() == 0 + assert recorder.requests == [] and (await fire_row(fire_id)).compare_artifact is None + await seal(target_id) + fresh = await seal_of(target_id) + assert fresh != stale + assert await webhook.notify_due() == 1 + stored = load((await fire_row(fire_id)).compare_artifact) + assert (stored["target_manifest"], stored["result"]["base"]["id"]) == (fresh, base_id) + [body] = recorder.bodies() + assert LINE in body["text"] and body["manifest_hash"] == fresh[:12] + + +async def test_a_seal_that_never_catches_up_is_not_used(recorder): + """新的封存一直没补上(写完终态、封存之前进程没了):等过时限照常推送,但不比,也不写失败时那份旧的清单哈希""" + await public_address() + workflow, sid, _ = await previous_week() + fake, failed_at = failed_then_continued("r2") + target_id = await persist(fake, workflow, finished=datetime.now(UTC), schedule_id=sid, seal_upto=failed_at) + fire_id = await fire_for(sid, target_id, T0 + WEEK - timedelta(minutes=2)) + later = datetime.now(UTC) + timedelta(seconds=webhook.SEAL_WAIT_S + 5) + assert await webhook.notify_due(later) == 1 + [body] = recorder.bodies() + assert (body["event"], body["manifest_hash"], body["compare"]) == ("finished", None, None) + assert "清单哈希" not in body["text"] and "相比" not in body["text"] + assert (await fire_row(fire_id)).compare_artifact is None + + +async def test_approval_notice_does_not_carry_the_seal_of_an_earlier_failure(recorder): + """失败后接着跑、停在审批上:运行上还留着失败时的清单哈希,那不是这一次的封存,「等待审批」里不写""" + workflow, sid, _ = await previous_week() + fake, failed_at = failed_then_continued("r2") + target_id = await persist(fake, workflow, finished=T0 + WEEK, schedule_id=sid, status="interrupted", + seal_upto=failed_at) + await set_run(target_id, finished_at=None) + async with SessionLocal() as session: + from app.db.models import Approval + + session.add(Approval(run_id=target_id, node_id="card", title="审核周报口径", status="pending")) + await session.commit() + await fire_for(sid, target_id, T0 + WEEK - timedelta(minutes=2)) + assert await webhook.notify_due() == 1 + [body] = recorder.bodies() + assert (body["event"], body["manifest_hash"]) == ("awaiting_approval", None) + assert "清单哈希" not in body["text"] + + +@pytest.mark.parametrize("side", ["target", "base"]) +async def test_fire_history_drops_a_compare_whose_seal_changed(client, side): + """比过之后哪一边被重新封存(内容变了):记下的计数已经不是对比页现场算出来的那样,触发历史不再给这份对比""" + sid, base_id, target_id, fire_id = await weekly_pair(url=None) + await webhook.notify_due() + + async def shown(): + items = (await client.get(f"/api/schedules/{sid}/fires")).json()["items"] + return {f["id"]: f for f in items}[fire_id]["compare"] + + assert (await shown())["base_run_id"] == base_id + # 换成和另一边一样的数:现场再比就是「未变」 + run_id, rows = (target_id, BASE_ROWS) if side == "target" else (base_id, NEXT_ROWS) + async with SessionLocal() as session: + finished = (await session.get(Run, run_id)).finished_at + await reseal(run_id, week("rx", rows), status="succeeded", finished=finished) + assert await shown() is None + # 对比页现场算出来的和记下的确实不一样 + assert (await compare_api.compare_runs(target_id, base_id))["summary"]["unchanged"] == 2 + + +async def test_result_is_dropped_when_a_seal_changes_mid_compare(recorder, monkeypatch): + """对比途中基准被重新封存:比出来的是旧封存的结果,不记进触发记录、不写进通知""" + await public_address() + _, base_id, _, fire_id = await weekly_pair() + real = compare_api.compare_runs + + async def resealing(run_id, other): + out = await real(run_id, other) + await set_run(base_id, manifest_hash="ab" * 32) + return out + + monkeypatch.setattr(compare_api, "compare_runs", resealing) + assert await webhook.notify_due() == 1 + assert (await fire_row(fire_id)).compare_artifact is None + [body] = recorder.bodies() + assert body["compare"] is None and "相比" not in body["text"] + + +# -------------------------------------------------------------------------- +# 时限、取消、并发 +# -------------------------------------------------------------------------- + + +async def test_a_slow_compare_is_cut_off_and_the_notice_goes_out(recorder, monkeypatch, caplog, slow_compare): + """对比超过时限:通知不等它,照常推送、不带对比,记一条警告""" + monkeypatch.setattr(auto_compare, "WAIT_S", 0.3, raising=False) + await public_address() + _, _, target_id, fire_id = await weekly_pair() + began = time.monotonic() + with caplog.at_level(logging.WARNING, logger="app.scheduler"): + assert await webhook.notify_due() == 1 + assert time.monotonic() - began < 2 + [body] = recorder.bodies() + assert (body["event"], body["compare"]) == ("finished", None) and "相比" not in body["text"] + assert f"{BASE_URL}/runs/{target_id}" in body["text"] + assert (await fire_row(fire_id)).compare_artifact is None + assert "超过" in caplog.text and target_id in caplog.text + + +async def test_scan_cancelled_mid_compare_pushes_next_time(recorder, monkeypatch, slow_compare): + """停机时通知扫描只等 1.5 秒就被取消:撞在对比途中的那次触发还没认领,下次启动照常比、照常推。以前先认领、后比, + 取消之后这条「运行结束」再也不会推""" + await public_address() + _, _, _, fire_id = await weekly_pair() + with pytest.raises(TimeoutError): + await asyncio.wait_for(webhook.notify_due(), timeout=0.5) + assert (await fire_row(fire_id)).notified_status is None and recorder.requests == [] + monkeypatch.setattr(compare_api, "compare_runs", slow_compare) + assert await webhook.notify_due() == 1 + [body] = recorder.bodies() + assert LINE in body["text"] + assert (await fire_row(fire_id)).notified_status == "succeeded" + + +async def test_a_slow_compare_does_not_hold_other_schedules(monkeypatch, slow_compare): + """任务 A 的对比慢:调度循环里通知扫描同一时刻只跑一份,以前 A 比多久,之后才结束的任务 B(第一次,不用比) + 的通知就要等多久。现在 A 的对比等过时限就不等了,B 的通知在下一拍推出""" + rec = Recorder() + monkeypatch.setattr(webhook, "_transport", httpx.MockTransport(rec)) + monkeypatch.setattr(webhook, "RETRY_DELAYS", (0, 0, 0)) + monkeypatch.setattr(auto_compare, "WAIT_S", 0.5, raising=False) + await weekly_pair() + on = Settings(_env_file=None, scheduler=True) + assert await scheduler.start(interval=0.05, config=on) is True + try: + await asyncio.sleep(0.2) + workflow = await new_workflow("仓库日报") + sid = await new_schedule(workflow, name="仓库日报 · 每天") + run_id = await persist(week("b1"), workflow, finished=T0, schedule_id=sid) + await fire_for(sid, run_id, T0 - timedelta(minutes=2)) + began = time.monotonic() + while not any(b.get("run_id") == run_id for b in rec.bodies()) and time.monotonic() - began < 8: + await asyncio.sleep(0.02) + waited = time.monotonic() - began + finally: + await scheduler.stop() + assert any(b.get("run_id") == run_id for b in rec.bodies()) + assert waited < 2, f"任务 B 的通知等了 {waited:.2f} 秒" + + +async def test_compares_take_turns(monkeypatch): + """同一拍里好几个任务都要比:同一时刻最多 SLOTS 个,不一起占满线程池(接口请求也要用)""" + monkeypatch.setattr(auto_compare, "SLOTS", 2, raising=False) + real = compare_api.compare_runs + active = peak = 0 + + async def counted(*args, **kwargs): + nonlocal active, peak + active += 1 + peak = max(peak, active) + try: + await asyncio.sleep(0.2) + return await real(*args, **kwargs) + finally: + active -= 1 + + monkeypatch.setattr(compare_api, "compare_runs", counted) + fires = [(await weekly_pair(url=None))[3] for _ in range(4)] + await webhook.notify_due() + assert peak == 2 + assert all([(await fire_row(f)).compare_artifact for f in fires]) + + +async def test_a_compare_already_made_for_this_seal_is_reused(recorder, monkeypatch): + """比完、还没认领就停机了,结果已经记在触发记录上:下次直接用,不再算一遍""" + await public_address() + _, _, target_id, fire_id = await weekly_pair() + assert await auto_compare.compare_fire(fire_id, target_id) is not None + artifact = (await fire_row(fire_id)).compare_artifact + assert artifact + calls = 0 + + async def counted(*args, **kwargs): + nonlocal calls + calls += 1 + raise RuntimeError("不该再算一遍") + + monkeypatch.setattr(compare_api, "compare_runs", counted) + auto_compare._BRIEFS.clear() + assert await webhook.notify_due() == 1 + assert calls == 0 and LINE in recorder.bodies()[0]["text"] + assert (await fire_row(fire_id)).compare_artifact == artifact + + +async def test_seal_covers_the_last_ending(): + """封存盖没盖住最后一条结束事件:老运行(没有 manifest_seq)只看有没有哈希""" + workflow = await new_workflow() + fake, failed_at = failed_then_continued("r9") + partial = await persist(fake, workflow, finished=T0, seal_upto=failed_at) + full = await persist(failed_then_continued("r8")[0], workflow, finished=T0) + legacy = await persist(week("r7"), workflow, finished=T0) + await set_run(legacy, manifest_seq=None) + unsealed = await persist(week("r6"), workflow, finished=T0, seal=False) + async with SessionLocal() as session: + got = {rid: await auto_compare.sealed_now(session, await session.get(Run, rid)) + for rid in (partial, full, legacy, unsealed)} + assert await session.scalar(select(RunEvent.seq).where( + RunEvent.run_id == partial, RunEvent.type == "run.finished")) > failed_at + assert got == {partial: False, full: True, legacy: True, unsealed: False} diff --git a/backend/tests/test_scheduler_notice_rules.py b/backend/tests/test_scheduler_notice_rules.py index dfc11af7..6d098db4 100644 --- a/backend/tests/test_scheduler_notice_rules.py +++ b/backend/tests/test_scheduler_notice_rules.py @@ -1,9 +1,10 @@ """定时任务通知(app/scheduler/webhook.py)的两条规则: - 失败通知的「处理」行不许诺能继续运行:服务重启时还在排队、从没开始执行的运行(被改成失败、没有开始时间)没有 - 断点,「修正后可继续运行」是错的。按有没有开始执行分开写——没开始的另写一行原因(没有可继续的进度),处理一律是 - 中性的「请在运行记录中查看原因」;任务还在排期上时补一句「下一次触发会照常发起」:失败的运行不挡之后的触发, - 和挂起的那种(之后的触发都会跳过)说清区别 + 断点,「修正后可继续运行」是错的。按有没有开始执行分开写——没开始的另写一行原因(没有可继续的进度),处理行不再 + 让人去「查看原因」(原因刚写过),任务还在排期上时只写「下一次触发会照常发起」,不在排期上时写怎么重新发起;开始 + 执行过的写「请在 AgentLab 的「记录」中查看失败原因」(和挂起那句一样指到导航里的「记录」),任务还在排期上时补 + 「下一次触发会照常发起」:失败的运行不挡之后的触发,和挂起的那种(之后的触发都会跳过)说清区别 - 扫描窗口(SCAN_DAYS)之外的触发:停在审批、有待审批的运行不受窗口限制——创建 30 天以后才停到审批上的照样推送 「等待审批」;运行记录在窗口内有变化的(等了一个多月的审批批完、跑完)照样推送结束。早就结束、记录多日没有变化 的旧触发仍不扫 @@ -27,6 +28,8 @@ from app.scheduler import webhook UTC = timezone.utc GENERIC = "https://hooks.example.com/agentlab/inbox-3e9d" NOT_STARTED = "运行在开始执行前就已中止(通常是服务重启时仍在排队),没有可继续的进度" +LOOK = "请在 AgentLab 的「记录」中查看失败原因" +RERUN = "如需重新发起,请在 AgentLab 的「定时任务」中点击「立即运行」" class Recorder: @@ -106,12 +109,15 @@ async def notified(fire_id: str) -> str | None: def test_a_run_that_never_started_is_not_promised_a_continue(): - """服务重启时还在排队的运行没有断点:不写「修正后可继续运行」,另写一行原因说没有可继续的进度""" + """服务重启时还在排队的运行没有断点:不写「修正后可继续运行」,另写一行原因说没有可继续的进度。处理行不再让人 + 去「查看原因」(原因刚写过):还在排期上的只说下一次照常,不在排期上的说怎么重新发起""" _, lines, _ = webhook.message(notice(started_at=None, next_fire=True)) shown = dict(lines) assert shown["原因"] == NOT_STARTED - assert shown["处理"] == "请在 AgentLab 的运行记录中查看原因;下一次触发会照常发起" - assert not any("继续运行" in value for _, value in lines), lines + assert shown["处理"] == "下一次触发会照常发起" + assert not any("继续运行" in value or "查看" in value for _, value in lines), lines + _, lines, _ = webhook.message(notice(started_at=None, next_fire=False)) + assert dict(lines)["处理"] == RERUN def test_a_failed_run_gets_a_neutral_action_without_promising_a_continue(): @@ -120,14 +126,14 @@ def test_a_failed_run_gets_a_neutral_action_without_promising_a_continue(): _, lines, _ = webhook.message(notice(error_node="汇总库存", next_fire=True)) shown = dict(lines) assert shown["失败节点"] == "「汇总库存」" and "原因" not in shown - assert shown["处理"] == "请在 AgentLab 的运行记录中查看原因;下一次触发会照常发起" + assert shown["处理"] == f"{LOOK};下一次触发会照常发起" assert "修正后可继续运行" not in webhook._text(notice(error_node="汇总库存")) def test_next_fire_is_only_promised_while_the_schedule_is_on(): """任务已暂停(或往后不再触发)时不说「下一次触发会照常发起」""" _, lines, _ = webhook.message(notice(next_fire=False)) - assert dict(lines)["处理"] == "请在 AgentLab 的运行记录中查看原因" + assert dict(lines)["处理"] == LOOK async def test_restart_failure_notice_end_to_end(recorder): @@ -140,12 +146,12 @@ async def test_restart_failure_notice_end_to_end(recorder): by_fire = {b["fire_id"]: b for b in recorder.bodies()} on, off = by_fire[on_fire], by_fire[off_fire] assert (on["event"], on["status"], on["reason"]) == ("failed", "failed", NOT_STARTED) - assert "处理:请在 AgentLab 的运行记录中查看原因;下一次触发会照常发起" in on["text"] - assert "处理:请在 AgentLab 的运行记录中查看原因\n" in off["text"] + "\n" and "照常发起" not in off["text"] + assert "处理:下一次触发会照常发起" in on["text"] + assert f"处理:{RERUN}" in off["text"] and "照常发起" not in off["text"] for body in (on, off): assert "继续运行" not in body["text"], body["text"] # 运行记录上的原始报错不进通知 - assert "请重新发起运行" not in body["text"] + assert "没有可恢复的进度" not in body["text"] # -------------------------------------------------------------------------- diff --git a/backend/tests/test_scheduler_webhook.py b/backend/tests/test_scheduler_webhook.py index ca32dca3..635a59ab 100644 --- a/backend/tests/test_scheduler_webhook.py +++ b/backend/tests/test_scheduler_webhook.py @@ -179,7 +179,7 @@ def test_failed_run_text_points_to_the_node_without_repeating_itself(): shown = dict(lines) assert title.endswith("运行失败") and shown["状态"] == "失败" # 处理只说去哪里看,不许诺能继续运行(没有断点的运行续不了,见 test_scheduler_notice_rules.py) - assert shown["失败节点"] == "「查询门店销售」" and shown["处理"] == "请在 AgentLab 的运行记录中查看原因" + assert shown["失败节点"] == "「查询门店销售」" and shown["处理"] == "请在 AgentLab 的「记录」中查看失败原因" # 不再写「原因:运行失败,详情请查看运行记录」这种同义反复 assert "原因" not in shown From 5e8d2c9c362ec20d0b9f48cbf99af486f119035a Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 08:49:10 +0800 Subject: [PATCH 6/6] =?UTF-8?q?=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E8=87=AA=E5=8A=A8=E5=AF=B9=E6=AF=94=E5=A4=8D=E6=A0=B8=E6=95=B4?= =?UTF-8?q?=E6=94=B9=EF=BC=88=E5=89=8D=E7=AB=AF=EF=BC=89=EF=BC=9A=E9=A1=B5?= =?UTF-8?q?=E5=A4=B4=E7=9A=84=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1=E9=93=BE?= =?UTF-8?q?=E6=8E=A5=E5=9C=A8=E7=AA=84=E7=AA=97=E5=8F=A3=E9=87=8C=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E8=A2=AB=E6=8C=A4=E6=B2=A1=E3=80=81=E5=B8=A6=E6=97=A5?= =?UTF-8?q?=E5=8E=86=E5=9B=BE=E6=A0=87=E3=80=81=E4=BB=BB=E5=8A=A1=E5=88=A0?= =?UTF-8?q?=E4=BA=86=E5=86=99=E7=BA=AF=E6=96=87=E6=9C=AC=EF=BC=9B=E8=A7=A6?= =?UTF-8?q?=E5=8F=91=E5=8E=86=E5=8F=B2=E7=9A=84=E5=AF=B9=E6=AF=94=E5=85=A5?= =?UTF-8?q?=E5=8F=A3=E6=94=B9=E5=8F=AB=E3=80=8C=E4=B8=8E=E4=B8=8A=E4=B8=80?= =?UTF-8?q?=E6=AC=A1=E5=AF=B9=E6=AF=94=E3=80=8D=E5=B9=B6=E5=86=99=E5=87=BA?= =?UTF-8?q?=E6=9C=89=E5=87=A0=E4=B8=AA=E6=8C=87=E6=A0=87=E6=9C=89=E5=8F=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 运行详情页头:元信息这一行改成可换行,定时任务链接至少留 7 个字宽。以前这一行不换行、只有链接能缩,820 宽的探索 运行只剩「由…」,768 宽为 0,Tab 停在看不见的链接上,「2 条事件」被压成竖排。链接前加和「定时」标记一样的日历 图标,平时看得出能点。取回的定时任务列表里没有这个任务(已删除)时写成纯文本「由定时任务「X」发起(已删除)」, 不再链到「不存在」页;列表取不到时不算删除 - 触发历史:「对比上一次」改成和运行详情「更多」菜单同一个叫法「与上一次对比」(复用 RUN_COMPARE_TEXT.menu),旁边 写「N 个指标有变化」,触屏和键盘也看得到变化的规模,悬停照旧写各类个数。基准不是紧挨着的上一次(服务端给 previous: false)时悬停写明是哪一次,也不再带「默认基准:同一定时任务的上一次运行」的来历 - 「通知中的运行链接」改成「通知中的链接」,说明里写清对比页链接也靠这个地址 - check-schedules:对比入口的叫法、计数、previous 为 false 时的悬停和来历;页头链接在 768、820 宽(含「提取为草稿」 的探索运行、署名的立即运行)写得下「由定时任务」且「条事件」不竖排、Tab 到时看得见;图标;定时任务删除后的纯文本 和列表取不到时照旧是链接;390 宽深浅色下计数在屏幕里、对比度够 --- frontend/src/lib/terms.ts | 46 ++++- frontend/src/pages/runs/RunDetailView.tsx | 3 +- frontend/src/pages/schedules/FireHistory.tsx | 26 ++- .../src/pages/schedules/WebhookFields.tsx | 4 +- frontend/src/pages/schedules/parts.tsx | 59 ++++-- frontend/src/types.ts | 12 +- scripts/check-schedules.mjs | 187 ++++++++++++++---- 7 files changed, 268 insertions(+), 69 deletions(-) diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index c5645c78..7d850bd5 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -2968,14 +2968,14 @@ export const SCHEDULE_TEXT = { testResult: (head: string, status: number | null, detail: string) => `${head}${status != null ? `(HTTP ${status})` : ''}${detail ? `:${detail}` : ''}`, - // ---- 通知里的运行链接(设置 notify 组,所有定时任务共用) - linkBase: '通知中的运行链接', + // ---- 通知里的链接:运行、对比页都按这个地址拼(设置 notify 组,所有定时任务共用) + linkBase: '通知中的链接', linkBaseValue: (url: string) => `按 ${url} 拼接`, - linkBaseUnset: '未设置服务访问地址,通知中不附带运行链接', + linkBaseUnset: '未设置服务访问地址,通知中不附带运行和对比页链接', linkBaseEdit: '修改', linkBaseSet: '设置', linkBaseLabel: '服务访问地址', - linkBaseHint: '收件人打开运行链接用的地址,例如 https://agentlab.example.com。所有定时任务共用', + linkBaseHint: '收件人打开运行、对比页链接用的地址,例如 https://agentlab.example.com。所有定时任务共用', linkBasePlaceholder: 'https://agentlab.example.com', linkBaseInvalid: '请输入以 http:// 或 https:// 开头的地址', linkBaseSaved: '已保存服务访问地址', @@ -3050,21 +3050,40 @@ export const SCHEDULE_TEXT = { allLoaded: '已全部加载', shown: (n: number) => `已显示 ${n} 条`, - // ---- 触发历史里的对比:运行结束后服务端自动和同一定时任务的上一次比一次 - compareLink: '对比上一次', + // ---- 触发历史里的对比:运行成功结束后,服务端自动和同一定时任务上一次成功的运行比一次 + /** + * 和运行详情「更多」菜单里的入口是同一件事,用同一个叫法。RUN_COMPARE_TEXT 定义在本对象之后,模块初始化时还取不到, + * 写成 getter 用到时再取 + */ + get compareLink() { return RUN_COMPARE_TEXT.menu }, /** * 悬停写各类指标的个数,和 Webhook 通知里那一行同一个写法(服务端 scheduler/auto_compare.py 的 summary_text): * 变化三分法的三类总写上,0 个也写;其余各类有才写,次序同对比页;两边都没有指标时不写一串 0 个。只有个数, - * 不写数值。各类的叫法用对比页那一套(RUN_COMPARE_TEXT.classLabel) + * 不写数值。各类的叫法用对比页那一套(RUN_COMPARE_TEXT.classLabel)。base 是基准的短编号,基准不是紧挨着的 + * 上一次(上一次失败了、取消了)时给,写明比的是哪一次,同通知 */ - compareTitle: (summary: Partial>) => { + compareTitle: (summary: Partial>, base?: string | null) => { const n = (key: string) => (typeof summary[key] === 'number' ? summary[key] as number : 0) const core = ['data_only', 'caliber_and_data', 'sql_changed'] const rest = ['params_only', 'added', 'removed', 'incomparable', 'unchanged'] - if (![...core, ...rest].some((key) => n(key) > 0)) return '与上一次相比:两次运行都没有指标' + const head = base ? `与上一次成功的运行(${base})相比` : '与上一次相比' + if (![...core, ...rest].some((key) => n(key) > 0)) return `${head}:两次运行都没有指标` const parts = [...core, ...rest.filter((key) => n(key) > 0)] .map((key) => `${RUN_COMPARE_TEXT.classLabel[key] ?? key} ${n(key)} 个`) - return `与上一次相比:${parts.join('、')}` + return `${head}:${parts.join('、')}` + }, + /** + * 链接旁看得见的简短计数:触屏没有悬停,键盘聚焦时也不显示 title,只靠悬停看不出变化的规模。有变化的几类(只数据变、 + * 口径与数据都变、SQL 结构变、只参数变、新增、移除)合起来写一个数;都没变化时说有几个无法比较,或者都未变;两边都 + * 没有指标时不写(悬停里有) + */ + compareCount: (summary: Partial>) => { + const n = (key: string) => (typeof summary[key] === 'number' ? summary[key] as number : 0) + const changed = ['data_only', 'caliber_and_data', 'sql_changed', 'params_only', 'added', 'removed'] + .reduce((sum, key) => sum + n(key), 0) + if (changed) return `${changed} 个指标有变化` + if (n('incomparable')) return `${n('incomparable')} 个指标无法比较` + return n('unchanged') ? '指标都未变' : null }, // ---- 运行详情页头的来源 @@ -3080,6 +3099,13 @@ export const SCHEDULE_TEXT = { /** 悬停写全名(页头窄时链接会截断)和点开去哪 */ runSourceTitle: (name?: string | null) => (name ? `定时任务「${name}」:查看设置和触发历史` : '查看这个定时任务的设置和触发历史'), + /** + * 定时任务已经删除(取回的定时任务列表里没有它):不再链过去——只会落到「不存在」页,页上的返回又回到定时任务列表。 + * 和同一行的「工作流已删除」一样写成纯文本 + */ + runSourceGone: (name?: string | null) => (name ? `由定时任务「${name}」发起(已删除)` : '由已删除的定时任务发起'), + runSourceManualGone: (name?: string | null) => (name ? `来自定时任务「${name}」(已删除)` : '来自已删除的定时任务'), + runSourceGoneTitle: '这个定时任务已删除,它发起的运行记录仍然保留', // ---- 记录页的定时标记 runTag: '定时', diff --git a/frontend/src/pages/runs/RunDetailView.tsx b/frontend/src/pages/runs/RunDetailView.tsx index 3a32e12d..271f0ef8 100644 --- a/frontend/src/pages/runs/RunDetailView.tsx +++ b/frontend/src/pages/runs/RunDetailView.tsx @@ -555,7 +555,8 @@ export function RunDetailView({ runId, onChange, onDeleted }: { -
+ {/* 窄时换行:不换行时这一行只有定时任务链接能缩,会被挤成 0 宽,其余各项被压成竖排 */} +
{shortId(run.id)} ·