Files
AgentLab/backend/app/scheduler/auto_compare.py
T
ThearchyHelios 45fe01effe 定时任务自动对比复核整改(后端):只比成功的运行、按发起先后挑上一次成功的基准、和两边的封存绑定,对比挪到认领之前在锁外做并限时限并发;失败通知的处理行不再重复原因
- 只比成功结束的运行,基准也只取成功的:失败、取消的运行指标不全,和上一次比只会得出「移除 N 个」,拿失败的那次当
  基准每个指标都成了「新增」。基准改成自己查同一定时任务、同一工作流、比这次先发起的运行里成功并封存的最近一次,
  不再用对比接口的 default_base(按结束时间挑:老的失败运行事后被接着跑完,会拿后一次触发当它的「上一次」)。
  中间隔着失败、取消的运行时,通知写「与上一次成功的运行(#短编号,日期)相比」,Fire.compare 多一个 previous
- 和封存绑定:运行的封存要盖住最后一条结束事件才算封存完(sealed_now)。runner 先写终态、再另起会话封存,失败后
  接着跑完的运行在两步之间还带着失败时的清单哈希,以前这一拍会直接认领、按旧封存比、通知里写旧的清单哈希;现在
  等新的封存,等不来就照常推送但不比、不写哈希。停在审批上的通知也不再写失败时留下的哈希。对比工件连同两边的清单
  哈希一起存,触发历史发现任何一边不在终态或清单哈希变了就给 null;对比途中封存变了不记(条件写)
- 时机和时限:锁里只挑出状态有变化的触发,认领挪到对比之后、锁外按条件 UPDATE 做。对比最多等 20 秒(含排队),
  同一时刻最多 2 个;超时只记警告,通知不带对比照常推送。停机时扫描在对比途中被取消,这次触发还没认领,下次照常
  比、照常推(以前先认领、后比,这条通知会永远丢掉)。同一份封存已经比过的直接用记下的结果
- 失败通知:从没开始执行的,原因刚写过,处理行不再让人「查看原因」,任务在排期上写「下一次触发会照常发起」,停用
  的写怎么重新发起;开始执行过的改成「请在 AgentLab 的「记录」中查看失败原因」,和挂起那句指向同一个导航入口
- 新增 test_scheduler_compare_guards.py(封存绑定、时限、取消、并发、复用),test_scheduler_compare.py 补真实缺
  指标的失败运行、上一次失败、补跑老运行几种用例;去掉实现逐项验证过会变红
2026-10-06 08:48:56 +08:00

282 lines
16 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""定时运行结束后的自动对比:和同一定时任务上一次成功的运行比一次,结果存成工件,工件 id 记在触发记录上
(ScheduleFire.compare_artifact)。
对比本身调运行对比接口的现成函数(api/compare.py 的 compare_runs,不改它),这里只管比哪些、跟谁比、什么时候比、
结果放哪、怎么说给触发历史和通知听:
- 比哪些:成功结束、封存完的运行。失败、取消的运行指标不全(口径卡常常没执行),和上一次比只会得出「移除 N 个」,
收到通知的人会以为报告删了指标
- 跟谁比:同一定时任务、同一工作流、比这次先发起的运行里成功结束并封存的最近一次(_previous)。失败的那次不当基准:
拿它当基准,每个指标都成了「新增」。按发起先后挑,不按结束时间:老的失败运行事后被接着跑完,结束得比后一次晚,
它的上一次仍是它前面那次。紧挨着的上一次不是成功的(失败、取消、还没结束)时 previous 为 False,通知和触发历史
写明比的是哪一次
- 和封存绑定:运行的封存要盖住它最后一条结束事件才算封存完(sealed_now)。runner 先写终态、再另起会话封存,失败后
接着跑完的运行在两步之间还带着失败时的清单哈希,拿它去比只比得到旧封存范围里的事件。结果里记下两边的清单哈希:
哪一边后来变了,触发历史就不再给这份对比(api/schedules._fire_outs);对比途中变了,结果不记
- 什么时候:通知扫描(webhook.notify_due)挑出这次触发之后、认领之前,在锁外比,比完(或等过时限)再认领、推送。
比到一半服务停了,这次触发还没认领,下次启动照常比、照常推;同一份封存已经比过的,直接用记下的结果
- 时限:一次对比最多等 WAIT_S 秒,同一时刻最多 SLOTS 个在算(读快照、重算都在线程里,不能一起占满接口请求也要用的
线程池)。超时、比不了(对比接口拒绝,如基准的封存核对不一致)、出错只记日志,通知照常推送、不带对比
- 结果:整份对比结果连同两边的清单哈希存成工件(kind=run_compare)。不挂在任何运行名下:它不是运行产出的,不该出现
在运行的工件列表里
- 对外:触发历史(Fire.compare)和通知只给基准是哪一次、各类指标的个数,不给具体数字
"""
from __future__ import annotations
import asyncio
import logging
import threading
import weakref
from typing import Any
from urllib.parse import urlencode
from fastapi import HTTPException
from sqlalchemy import func, select, update
from app.core import artifact_store
from app.core.events import EventType
from app.db.base import SessionLocal
from app.db.models import Run, RunEvent, ScheduleFire
logger = logging.getLogger(__name__)
KIND = "run_compare"
#: 终态,同对比接口(api/evidence._TERMINAL):触发历史里两边都要在终态,对比页才比得了
TERMINAL = ("succeeded", "failed", "cancelled")
#: 结束事件:封存要盖住这次运行的最后一条,才是这一次结束时封的
_ENDS = (EventType.RUN_FINISHED.value, EventType.RUN_FAILED.value, EventType.RUN_CANCELLED.value)
#: 一次对比最多等多久(秒),排队等空位的时间也算在里面。等不到就不等了:通知不带对比照常推送
WAIT_S = 20.0
#: 同一时刻最多几个对比在算
SLOTS = 2
#: 各类指标的叫法,和对比页一致(frontend/src/lib/terms.ts 的 RUN_COMPARE_TEXT.classLabel)
CLASS_LABEL = {
"data_only": "只数据变",
"caliber_and_data": "口径与数据都变",
"sql_changed": "SQL 结构变",
"params_only": "只参数变",
"added": "新增",
"removed": "移除",
"incomparable": "无法比较",
"unchanged": "未变",
}
#: 变化三分法的三类,0 个也写:「SQL 结构变 0 个」本身就是要告诉人的事
CORE = ("data_only", "caliber_and_data", "sql_changed")
#: 其余各类有才写,次序同对比页的摘要
OTHERS = ("params_only", "added", "removed", "incomparable", "unchanged")
#: 两边都没有指标时的说法:不写一串「0 个」,免得读成「什么都没变」
NO_METRICS = "两次运行都没有指标"
#: 读过的对比工件:工件 id → 摘要。工件按内容寻址、内容不会变,记下来不会过期;触发历史、任务列表每次打开都要读,
#: 不必每次都去盘上读一份几十 KB 的 JSON。只记读成功的,满了丢最早的。读盘在线程里、记下新算的在事件循环里,
#: 两边都动这个字典,加一把线程锁
_BRIEFS: dict[str, dict[str, Any]] = {}
_BRIEFS_MAX = 512
_BRIEFS_LOCK = threading.Lock()
#: 对比的空位,按事件循环分开(同 service.launch_lock):测试每个用例一个新循环,跨循环用同一个信号量会报错
_SLOTS: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Semaphore]" = weakref.WeakKeyDictionary()
def summary_text(summary: dict[str, Any]) -> str:
"""计数摘要:「只数据变 3 个、口径与数据都变 1 个、SQL 结构变 0 个」,其余各类有才接在后面。只有个数。"""
counts = {key: value for key, value in summary.items() if isinstance(value, int) and not isinstance(value, bool)}
if not any(counts.values()):
return NO_METRICS
parts = [f"{CLASS_LABEL[key]} {counts.get(key, 0)} 个" for key in CORE]
parts += [f"{CLASS_LABEL[key]} {counts[key]} 个" for key in OTHERS if counts.get(key)]
return "、".join(parts)
def page_url(base_url: str, target: str, base: str) -> str:
"""对比页的地址(对外访问地址 + /runs/compare?target=…&base=…),和界面上运行详情跳过去的是同一个页面。"""
return f"{base_url}/runs/compare?{urlencode({'target': target, 'base': base})}"
async def sealed_now(session: Any, run: Run) -> bool:
"""这次运行的封存是不是这一次结束时封的:有清单哈希,而且封存范围盖住了最后一条结束事件。
runner 先在一个事务里写终态和结束事件,再另起会话封存。失败后接着跑完(或接着跑之后停在审批上被放弃)的运行,在
两步之间还带着失败时的清单哈希,只看「有没有哈希」会把旧的封存当成这一次的。没有 manifest_seq 的是加这一列之前
封存的老运行,只能看有没有哈希。"""
if not run.manifest_hash:
return False
if run.manifest_seq is None:
return True
last = await session.scalar(select(func.max(RunEvent.seq)).where(
RunEvent.run_id == run.id, RunEvent.type.in_(_ENDS)))
return last is None or run.manifest_seq >= last
async def compare_fire(fire_id: str, run_id: str) -> dict[str, Any] | None:
"""这次触发发起的运行成功结束、封存完了:和同一定时任务上一次成功的运行比一次,结果记进触发记录。返回给通知用的
摘要 {base_run_id, summary, previous, base_started_at, …};不该比、没有可比的、比不了时返回 None。
最多等 WAIT_S 秒(含排队)。超时、对比接口拒绝(基准的封存核对不一致等)记一条警告,意外的异常记堆栈,都不影响
通知。取消不吞:扫描被取消(停机)时这次触发还没认领,下次照常比。"""
try:
return await asyncio.wait_for(_compare(fire_id, run_id), timeout=WAIT_S)
except TimeoutError:
logger.warning("定时任务的运行 %s 自动对比超过 %g 秒仍未完成,通知不带对比照常推送", run_id, WAIT_S)
except HTTPException as e:
logger.warning("定时任务的运行 %s 自动对比未完成:%s", run_id, e.detail)
except Exception: # noqa: BLE001 - 对比出错不能拖住通知,也不能让通知扫描停下
logger.exception("定时任务的运行 %s 自动对比出错", run_id)
return None
async def _compare(fire_id: str, run_id: str) -> dict[str, Any] | None:
# 现成的对比函数在用到时才导入:api/compare.py 连带证据、推演接口,调度模块被导入时不必一起加载
from app.api import compare as compare_api
# 挑基准、核对封存、看有没有比过都只是读库,不占对比的空位:前面排着慢的对比时,第一次运行这类没什么可比的
# 不必跟着排队、等到超时
async with SessionLocal() as session:
target = await session.get(Run, run_id)
stored = await session.scalar(select(ScheduleFire.compare_artifact).where(ScheduleFire.id == fire_id))
if target is None or target.status != "succeeded" or not target.schedule_id or not target.workflow_id \
or not await sealed_now(session, target):
return None
found = await _previous(session, target)
if found is None:
return None
base, previous = found
if not await sealed_now(session, base):
# 基准正好处在写完终态、还没封存的那一刻,或者封存没补上就停了机:拿它比只比得到旧封存里的事件
logger.warning("作为基准的运行 %s 还没有封存到它的结束事件,运行 %s 这次不比", base.id, run_id)
return None
sides = ((target.id, target.manifest_hash), (base.id, base.manifest_hash))
(target_id, target_seal), (base_id, base_seal) = sides
if stored:
# 同一份封存已经比过(比完还没认领就停了机):直接用
known = await asyncio.to_thread(brief_of, stored)
if known is not None and _sides_of(known) == sides:
return _copy(known)
async with _slots():
result = await compare_api.compare_runs(target_id, base_id)
payload = {"target_manifest": target_seal, "base_manifest": base_seal, "previous": previous, "result": result}
brief = _brief(payload)
if brief is None or _sides_of(brief) != sides:
return None
artifact = await artifact_store.put_json(payload, kind=KIND,
meta={"target_run_id": target_id, "base_run_id": base_id})
if not await _store(fire_id, artifact, sides):
logger.warning("运行 %s 或作为基准的运行 %s 在对比途中重新封存(或触发记录已删除),这次的对比结果不记",
target_id, base_id)
return None
_remember(artifact, brief)
return _copy(brief)
async def _previous(session: Any, target: Run) -> tuple[Run, bool] | None:
"""(基准, 是不是紧挨着的上一次):同一定时任务、同一工作流、比这次先发起的运行里成功结束并封存的最近一次。
按发起先后挑(created_at),不按结束时间;同一工作流才比得了(任务改过工作流的,之前那个工作流的运行不算)。
中间隔着别的运行(失败、取消、还没结束的)时第二项为 False。"""
earlier = (Run.schedule_id == target.schedule_id, Run.workflow_id == target.workflow_id, Run.id != target.id,
Run.created_at < target.created_at)
base = (await session.execute(
select(Run).where(*earlier, Run.status == "succeeded", Run.manifest_hash.is_not(None))
.order_by(Run.created_at.desc()).limit(1))).scalars().first()
if base is None:
return None
between = await session.scalar(select(func.count()).select_from(Run).where(
*earlier, Run.created_at > base.created_at))
return base, not between
def _slots() -> asyncio.Semaphore:
loop = asyncio.get_running_loop()
slots = _SLOTS.get(loop)
if slots is None:
slots = _SLOTS[loop] = asyncio.Semaphore(SLOTS)
return slots
async def _store(fire_id: str, artifact: str, sides: tuple[tuple[str, str | None], ...]) -> bool:
"""把工件 id 记进触发记录。条件写:两边都还是成功结束、清单哈希还是比的时候那一份,才记。"""
same = [select(Run.id).where(Run.id == run_id, Run.status == "succeeded", Run.manifest_hash == seal).exists()
for run_id, seal in sides]
async with SessionLocal() as session:
result = await session.execute(update(ScheduleFire).where(ScheduleFire.id == fire_id, *same)
.values(compare_artifact=artifact))
await session.commit()
return result.rowcount == 1
def _sides_of(brief: dict[str, Any]) -> tuple[tuple[str, str | None], ...]:
return (brief["target_run_id"], brief["target_manifest"]), (brief["base_run_id"], brief["base_manifest"])
def _brief(payload: Any) -> dict[str, Any] | None:
"""对比工件里给触发历史、通知的那一点:两边是哪一次、比的时候两边的清单哈希、基准是不是紧挨着的上一次、基准
什么时候开始的、各类指标的个数(同对比接口的 summary)。形状不对的(这一版之前的工件、被改过的)返回 None。"""
if not isinstance(payload, dict) or not isinstance(payload.get("result"), dict):
return None
result = payload["result"]
target, base, summary = result.get("target"), result.get("base"), result.get("summary")
if not isinstance(target, dict) or not isinstance(base, dict) or not isinstance(summary, dict):
return None
keys = (target.get("id"), base.get("id"), payload.get("target_manifest"), payload.get("base_manifest"))
if not all(isinstance(key, str) and key for key in keys):
return None
started = base.get("started_at")
return {"target_run_id": keys[0], "base_run_id": keys[1], "target_manifest": keys[2], "base_manifest": keys[3],
"previous": payload.get("previous") is not False,
"base_started_at": started if isinstance(started, str) else None,
"summary": {key: value for key, value in summary.items()
if isinstance(value, int) and not isinstance(value, bool)}}
def public(brief: dict[str, Any]) -> dict[str, Any]:
"""接口给前端的那一份(Fire.compare):基准是哪一次、各类指标的个数、基准是不是紧挨着的上一次。"""
return {"base_run_id": brief["base_run_id"], "summary": dict(brief["summary"]), "previous": brief["previous"]}
def _copy(brief: dict[str, Any]) -> dict[str, Any]:
return {**brief, "summary": dict(brief["summary"])}
def _remember(artifact: str, brief: dict[str, Any]) -> None:
with _BRIEFS_LOCK:
if artifact not in _BRIEFS and len(_BRIEFS) >= _BRIEFS_MAX:
_BRIEFS.pop(next(iter(_BRIEFS)))
_BRIEFS[artifact] = brief
def brief_of(artifact: str) -> dict[str, Any] | None:
"""读一份对比工件的摘要。文件不在、内容和哈希对不上、读不出来(后两种另记一条警告)、形状不对的返回 None,
不抛异常:一份对比读不出来不能让整页触发历史打不开。"""
with _BRIEFS_LOCK:
known = _BRIEFS.get(artifact)
if known is not None:
return _copy(known)
try:
brief = _brief(artifact_store.load(artifact))
except ValueError:
logger.warning("对比工件 %s… 内容与哈希不符,触发历史中不显示这次对比", artifact[:12])
return None
except OSError as e: # 权限、磁盘出错
logger.warning("对比工件 %s… 读取失败(%s),触发历史中不显示这次对比", artifact[:12], type(e).__name__)
return None
if brief is None:
return None
_remember(artifact, brief)
return _copy(brief)
async def briefs(artifacts: list[str]) -> dict[str, dict[str, Any]]:
"""几份对比工件的摘要:工件 id → 摘要,读不出来的不在里面。读盘在线程里做。"""
wanted = list(dict.fromkeys(a for a in artifacts if a))
if not wanted:
return {}
def read() -> dict[str, dict[str, Any]]:
out = {}
for artifact in wanted:
brief = brief_of(artifact)
if brief is not None:
out[artifact] = brief
return out
return await asyncio.to_thread(read)