mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-18 08:11:58 +00:00
合入期 4 定时任务 × 运行对比:定时运行结束后自动和同一定时任务上一次成功的运行对比,结果存成工件记在触发记录上;触发历史和通知只给计数摘要与对比页链接,附封存哈希前 12 位;运行详情页头的定时任务来源改成键盘可达的链接
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
"""定时任务:按周期自动发起工作流运行,结束、失败、停在审批时推送 Webhook。
|
||||
"""定时任务:按周期自动发起工作流运行,结束、失败、停在审批时推送 Webhook,运行成功结束后和同一任务上一次成功的运行
|
||||
自动对比。
|
||||
|
||||
形状按前后端约定(Schedule、Fire 两种对象,见 _schedule_out、_fire_outs)。Webhook 地址和签名密钥加密存放,接口
|
||||
只给主机和末 4 位(url_hint)、有没有密钥(has_secret)。请求体自己解析、自己校验:错误一律回中文 detail,表单
|
||||
@@ -18,7 +19,7 @@ from app.api.runs import actor_of
|
||||
from app.core.crypto import decrypt, encrypt
|
||||
from app.db.base import get_session, utcnow
|
||||
from app.db.models import Run, Schedule, ScheduleFire, WebhookDelivery, Workflow, WorkflowVersion
|
||||
from app.scheduler import cron, service, webhook
|
||||
from app.scheduler import auto_compare, cron, service, webhook
|
||||
|
||||
router = APIRouter(prefix="/api/schedules", tags=["schedules"])
|
||||
|
||||
@@ -278,10 +279,31 @@ def _webhook_out(row: Schedule) -> dict[str, Any] | None:
|
||||
|
||||
|
||||
async def _fire_outs(session: AsyncSession, fires: list[ScheduleFire]) -> list[dict[str, Any]]:
|
||||
"""Fire 对象:run_status 是运行现在的状态,notified 是这次触发的推送记录 [{event, status, at}](按时间先后)。"""
|
||||
run_ids = [f.run_id for f in fires if f.run_id]
|
||||
statuses = dict((await session.execute(select(Run.id, Run.status).where(Run.id.in_(run_ids)))).all()) \
|
||||
if run_ids else {}
|
||||
"""Fire 对象:run_status 是运行现在的状态,notified 是这次触发的推送记录 [{event, status, at}](按时间先后),
|
||||
compare 是运行成功结束后和同一任务上一次成功的运行的自动对比 {base_run_id, summary, previous}(summary 同对比接口
|
||||
的计数,previous 说基准是不是紧挨着的上一次),没有为 null。
|
||||
|
||||
对比和比的时候两边的封存绑定(auto_compare):哪一边现在不在终态(被继续执行)、被删掉、清单哈希和比的时候不一样
|
||||
(重新封存过)都给 null——对比页按现在的封存现算,记下的计数和点进去看到的会对不上,或者干脆比不了。工件读不出来、
|
||||
被改过的同样给 null。"""
|
||||
compared = {f.id: f.compare_artifact for f in fires if f.compare_artifact and f.run_id}
|
||||
compares = await auto_compare.briefs(list(compared.values()))
|
||||
run_ids = {f.run_id for f in fires if f.run_id} | {brief["base_run_id"] for brief in compares.values()}
|
||||
# 运行现在的状态和清单哈希:run_status 照写,对比按它核对两边的封存
|
||||
current = {run_id: (status, seal) for run_id, status, seal in (await session.execute(
|
||||
select(Run.id, Run.status, Run.manifest_hash).where(Run.id.in_(run_ids)))).all()} if run_ids else {}
|
||||
|
||||
def compare_of(f: ScheduleFire) -> dict[str, Any] | None:
|
||||
brief = compares.get(compared.get(f.id) or "")
|
||||
if brief is None or brief["target_run_id"] != f.run_id:
|
||||
return None
|
||||
for run_id, seal in ((brief["target_run_id"], brief["target_manifest"]),
|
||||
(brief["base_run_id"], brief["base_manifest"])):
|
||||
status, now_seal = current.get(run_id, (None, None))
|
||||
if status not in auto_compare.TERMINAL or now_seal != seal:
|
||||
return None
|
||||
return auto_compare.public(brief)
|
||||
|
||||
notified: dict[str, list[dict[str, Any]]] = {}
|
||||
if fires:
|
||||
rows = (await session.execute(
|
||||
@@ -298,9 +320,10 @@ async def _fire_outs(session: AsyncSession, fires: list[ScheduleFire]) -> list[d
|
||||
"kind": f.kind,
|
||||
"status": "started" if f.id in launching else f.status,
|
||||
"run_id": f.run_id,
|
||||
"run_status": statuses.get(f.run_id) if f.run_id else None,
|
||||
"run_status": current.get(f.run_id, (None, None))[0] if f.run_id else None,
|
||||
"reason": None if f.id in launching else f.reason,
|
||||
"notified": notified.get(f.id, []),
|
||||
"compare": compare_of(f),
|
||||
"created_at": _iso(f.created_at),
|
||||
} for f in fires]
|
||||
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
- service.py:输入模板、和工作流入口字段的对照、发起一次触发(调度循环和「立即运行」共用)
|
||||
- loop.py:调度循环,条件 UPDATE 认领到点的任务
|
||||
- webhook.py:通知推送(通用 / 飞书 / 企业微信 / 钉钉)
|
||||
- auto_compare.py:运行成功结束后和同一定时任务上一次成功的运行自动对比,结果记在触发记录上,通知里只带计数摘要和
|
||||
对比页链接
|
||||
|
||||
服务启动时(main.py 的 lifespan)只在 scheduler_enabled() 时调 start,关闭时总是调 stop——stop 要能在没 start
|
||||
过时安全返回。start 自己也看一次开关:拷来的数据目录(检查脚本的 --stacks、验收副本)不管从哪条路径起服务,
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
"""定时运行结束后的自动对比:和同一定时任务上一次成功的运行比一次,结果存成工件,工件 id 记在触发记录上
|
||||
(ScheduleFire.compare_artifact)。
|
||||
|
||||
对比本身调运行对比接口的现成函数(api/compare.py 的 compare_runs,不改它),这里只管比哪些、跟谁比、什么时候比、
|
||||
结果放哪、怎么说给触发历史和通知听:
|
||||
|
||||
- 比哪些:成功结束、封存完的运行。失败、取消的运行指标不全(口径卡常常没执行),和上一次比只会得出「移除 N 个」,
|
||||
收到通知的人会以为报告删了指标
|
||||
- 跟谁比:同一定时任务、同一工作流、比这次先发起的运行里成功结束并封存的最近一次(_previous)。失败的那次不当基准:
|
||||
拿它当基准,每个指标都成了「新增」。按发起先后挑,不按结束时间:老的失败运行事后被接着跑完,结束得比后一次晚,
|
||||
它的上一次仍是它前面那次。紧挨着的上一次不是成功的(失败、取消、还没结束)时 previous 为 False,通知和触发历史
|
||||
写明比的是哪一次
|
||||
- 和封存绑定:运行的封存要盖住它最后一条结束事件才算封存完(sealed_now)。runner 先写终态、再另起会话封存,失败后
|
||||
接着跑完的运行在两步之间还带着失败时的清单哈希,拿它去比只比得到旧封存范围里的事件。结果里记下两边的清单哈希:
|
||||
哪一边后来变了,触发历史就不再给这份对比(api/schedules._fire_outs);对比途中变了,结果不记
|
||||
- 什么时候:通知扫描(webhook.notify_due)挑出这次触发之后、认领之前,在锁外比,比完(或等过时限)再认领、推送。
|
||||
比到一半服务停了,这次触发还没认领,下次启动照常比、照常推;同一份封存已经比过的,直接用记下的结果
|
||||
- 时限:一次对比最多等 WAIT_S 秒,同一时刻最多 SLOTS 个在算(读快照、重算都在线程里,不能一起占满接口请求也要用的
|
||||
线程池)。超时、比不了(对比接口拒绝,如基准的封存核对不一致)、出错只记日志,通知照常推送、不带对比
|
||||
- 结果:整份对比结果连同两边的清单哈希存成工件(kind=run_compare)。不挂在任何运行名下:它不是运行产出的,不该出现
|
||||
在运行的工件列表里
|
||||
- 对外:触发历史(Fire.compare)和通知只给基准是哪一次、各类指标的个数,不给具体数字
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import threading
|
||||
import weakref
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import func, select, update
|
||||
|
||||
from app.core import artifact_store
|
||||
from app.core.events import EventType
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Run, RunEvent, ScheduleFire
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
KIND = "run_compare"
|
||||
#: 终态,同对比接口(api/evidence._TERMINAL):触发历史里两边都要在终态,对比页才比得了
|
||||
TERMINAL = ("succeeded", "failed", "cancelled")
|
||||
#: 结束事件:封存要盖住这次运行的最后一条,才是这一次结束时封的
|
||||
_ENDS = (EventType.RUN_FINISHED.value, EventType.RUN_FAILED.value, EventType.RUN_CANCELLED.value)
|
||||
#: 一次对比最多等多久(秒),排队等空位的时间也算在里面。等不到就不等了:通知不带对比照常推送
|
||||
WAIT_S = 20.0
|
||||
#: 同一时刻最多几个对比在算
|
||||
SLOTS = 2
|
||||
#: 各类指标的叫法,和对比页一致(frontend/src/lib/terms.ts 的 RUN_COMPARE_TEXT.classLabel)
|
||||
CLASS_LABEL = {
|
||||
"data_only": "只数据变",
|
||||
"caliber_and_data": "口径与数据都变",
|
||||
"sql_changed": "SQL 结构变",
|
||||
"params_only": "只参数变",
|
||||
"added": "新增",
|
||||
"removed": "移除",
|
||||
"incomparable": "无法比较",
|
||||
"unchanged": "未变",
|
||||
}
|
||||
#: 变化三分法的三类,0 个也写:「SQL 结构变 0 个」本身就是要告诉人的事
|
||||
CORE = ("data_only", "caliber_and_data", "sql_changed")
|
||||
#: 其余各类有才写,次序同对比页的摘要
|
||||
OTHERS = ("params_only", "added", "removed", "incomparable", "unchanged")
|
||||
#: 两边都没有指标时的说法:不写一串「0 个」,免得读成「什么都没变」
|
||||
NO_METRICS = "两次运行都没有指标"
|
||||
|
||||
#: 读过的对比工件:工件 id → 摘要。工件按内容寻址、内容不会变,记下来不会过期;触发历史、任务列表每次打开都要读,
|
||||
#: 不必每次都去盘上读一份几十 KB 的 JSON。只记读成功的,满了丢最早的。读盘在线程里、记下新算的在事件循环里,
|
||||
#: 两边都动这个字典,加一把线程锁
|
||||
_BRIEFS: dict[str, dict[str, Any]] = {}
|
||||
_BRIEFS_MAX = 512
|
||||
_BRIEFS_LOCK = threading.Lock()
|
||||
#: 对比的空位,按事件循环分开(同 service.launch_lock):测试每个用例一个新循环,跨循环用同一个信号量会报错
|
||||
_SLOTS: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Semaphore]" = weakref.WeakKeyDictionary()
|
||||
|
||||
|
||||
def summary_text(summary: dict[str, Any]) -> str:
|
||||
"""计数摘要:「只数据变 3 个、口径与数据都变 1 个、SQL 结构变 0 个」,其余各类有才接在后面。只有个数。"""
|
||||
counts = {key: value for key, value in summary.items() if isinstance(value, int) and not isinstance(value, bool)}
|
||||
if not any(counts.values()):
|
||||
return NO_METRICS
|
||||
parts = [f"{CLASS_LABEL[key]} {counts.get(key, 0)} 个" for key in CORE]
|
||||
parts += [f"{CLASS_LABEL[key]} {counts[key]} 个" for key in OTHERS if counts.get(key)]
|
||||
return "、".join(parts)
|
||||
|
||||
|
||||
def page_url(base_url: str, target: str, base: str) -> str:
|
||||
"""对比页的地址(对外访问地址 + /runs/compare?target=…&base=…),和界面上运行详情跳过去的是同一个页面。"""
|
||||
return f"{base_url}/runs/compare?{urlencode({'target': target, 'base': base})}"
|
||||
|
||||
|
||||
async def sealed_now(session: Any, run: Run) -> bool:
|
||||
"""这次运行的封存是不是这一次结束时封的:有清单哈希,而且封存范围盖住了最后一条结束事件。
|
||||
|
||||
runner 先在一个事务里写终态和结束事件,再另起会话封存。失败后接着跑完(或接着跑之后停在审批上被放弃)的运行,在
|
||||
两步之间还带着失败时的清单哈希,只看「有没有哈希」会把旧的封存当成这一次的。没有 manifest_seq 的是加这一列之前
|
||||
封存的老运行,只能看有没有哈希。"""
|
||||
if not run.manifest_hash:
|
||||
return False
|
||||
if run.manifest_seq is None:
|
||||
return True
|
||||
last = await session.scalar(select(func.max(RunEvent.seq)).where(
|
||||
RunEvent.run_id == run.id, RunEvent.type.in_(_ENDS)))
|
||||
return last is None or run.manifest_seq >= last
|
||||
|
||||
|
||||
async def compare_fire(fire_id: str, run_id: str) -> dict[str, Any] | None:
|
||||
"""这次触发发起的运行成功结束、封存完了:和同一定时任务上一次成功的运行比一次,结果记进触发记录。返回给通知用的
|
||||
摘要 {base_run_id, summary, previous, base_started_at, …};不该比、没有可比的、比不了时返回 None。
|
||||
|
||||
最多等 WAIT_S 秒(含排队)。超时、对比接口拒绝(基准的封存核对不一致等)记一条警告,意外的异常记堆栈,都不影响
|
||||
通知。取消不吞:扫描被取消(停机)时这次触发还没认领,下次照常比。"""
|
||||
try:
|
||||
return await asyncio.wait_for(_compare(fire_id, run_id), timeout=WAIT_S)
|
||||
except TimeoutError:
|
||||
logger.warning("定时任务的运行 %s 自动对比超过 %g 秒仍未完成,通知不带对比照常推送", run_id, WAIT_S)
|
||||
except HTTPException as e:
|
||||
logger.warning("定时任务的运行 %s 自动对比未完成:%s", run_id, e.detail)
|
||||
except Exception: # noqa: BLE001 - 对比出错不能拖住通知,也不能让通知扫描停下
|
||||
logger.exception("定时任务的运行 %s 自动对比出错", run_id)
|
||||
return None
|
||||
|
||||
|
||||
async def _compare(fire_id: str, run_id: str) -> dict[str, Any] | None:
|
||||
# 现成的对比函数在用到时才导入:api/compare.py 连带证据、推演接口,调度模块被导入时不必一起加载
|
||||
from app.api import compare as compare_api
|
||||
|
||||
# 挑基准、核对封存、看有没有比过都只是读库,不占对比的空位:前面排着慢的对比时,第一次运行这类没什么可比的
|
||||
# 不必跟着排队、等到超时
|
||||
async with SessionLocal() as session:
|
||||
target = await session.get(Run, run_id)
|
||||
stored = await session.scalar(select(ScheduleFire.compare_artifact).where(ScheduleFire.id == fire_id))
|
||||
if target is None or target.status != "succeeded" or not target.schedule_id or not target.workflow_id \
|
||||
or not await sealed_now(session, target):
|
||||
return None
|
||||
found = await _previous(session, target)
|
||||
if found is None:
|
||||
return None
|
||||
base, previous = found
|
||||
if not await sealed_now(session, base):
|
||||
# 基准正好处在写完终态、还没封存的那一刻,或者封存没补上就停了机:拿它比只比得到旧封存里的事件
|
||||
logger.warning("作为基准的运行 %s 还没有封存到它的结束事件,运行 %s 这次不比", base.id, run_id)
|
||||
return None
|
||||
sides = ((target.id, target.manifest_hash), (base.id, base.manifest_hash))
|
||||
(target_id, target_seal), (base_id, base_seal) = sides
|
||||
if stored:
|
||||
# 同一份封存已经比过(比完还没认领就停了机):直接用
|
||||
known = await asyncio.to_thread(brief_of, stored)
|
||||
if known is not None and _sides_of(known) == sides:
|
||||
return _copy(known)
|
||||
async with _slots():
|
||||
result = await compare_api.compare_runs(target_id, base_id)
|
||||
payload = {"target_manifest": target_seal, "base_manifest": base_seal, "previous": previous, "result": result}
|
||||
brief = _brief(payload)
|
||||
if brief is None or _sides_of(brief) != sides:
|
||||
return None
|
||||
artifact = await artifact_store.put_json(payload, kind=KIND,
|
||||
meta={"target_run_id": target_id, "base_run_id": base_id})
|
||||
if not await _store(fire_id, artifact, sides):
|
||||
logger.warning("运行 %s 或作为基准的运行 %s 在对比途中重新封存(或触发记录已删除),这次的对比结果不记",
|
||||
target_id, base_id)
|
||||
return None
|
||||
_remember(artifact, brief)
|
||||
return _copy(brief)
|
||||
|
||||
|
||||
async def _previous(session: Any, target: Run) -> tuple[Run, bool] | None:
|
||||
"""(基准, 是不是紧挨着的上一次):同一定时任务、同一工作流、比这次先发起的运行里成功结束并封存的最近一次。
|
||||
|
||||
按发起先后挑(created_at),不按结束时间;同一工作流才比得了(任务改过工作流的,之前那个工作流的运行不算)。
|
||||
中间隔着别的运行(失败、取消、还没结束的)时第二项为 False。"""
|
||||
earlier = (Run.schedule_id == target.schedule_id, Run.workflow_id == target.workflow_id, Run.id != target.id,
|
||||
Run.created_at < target.created_at)
|
||||
base = (await session.execute(
|
||||
select(Run).where(*earlier, Run.status == "succeeded", Run.manifest_hash.is_not(None))
|
||||
.order_by(Run.created_at.desc()).limit(1))).scalars().first()
|
||||
if base is None:
|
||||
return None
|
||||
between = await session.scalar(select(func.count()).select_from(Run).where(
|
||||
*earlier, Run.created_at > base.created_at))
|
||||
return base, not between
|
||||
|
||||
|
||||
def _slots() -> asyncio.Semaphore:
|
||||
loop = asyncio.get_running_loop()
|
||||
slots = _SLOTS.get(loop)
|
||||
if slots is None:
|
||||
slots = _SLOTS[loop] = asyncio.Semaphore(SLOTS)
|
||||
return slots
|
||||
|
||||
|
||||
async def _store(fire_id: str, artifact: str, sides: tuple[tuple[str, str | None], ...]) -> bool:
|
||||
"""把工件 id 记进触发记录。条件写:两边都还是成功结束、清单哈希还是比的时候那一份,才记。"""
|
||||
same = [select(Run.id).where(Run.id == run_id, Run.status == "succeeded", Run.manifest_hash == seal).exists()
|
||||
for run_id, seal in sides]
|
||||
async with SessionLocal() as session:
|
||||
result = await session.execute(update(ScheduleFire).where(ScheduleFire.id == fire_id, *same)
|
||||
.values(compare_artifact=artifact))
|
||||
await session.commit()
|
||||
return result.rowcount == 1
|
||||
|
||||
|
||||
def _sides_of(brief: dict[str, Any]) -> tuple[tuple[str, str | None], ...]:
|
||||
return (brief["target_run_id"], brief["target_manifest"]), (brief["base_run_id"], brief["base_manifest"])
|
||||
|
||||
|
||||
def _brief(payload: Any) -> dict[str, Any] | None:
|
||||
"""对比工件里给触发历史、通知的那一点:两边是哪一次、比的时候两边的清单哈希、基准是不是紧挨着的上一次、基准
|
||||
什么时候开始的、各类指标的个数(同对比接口的 summary)。形状不对的(这一版之前的工件、被改过的)返回 None。"""
|
||||
if not isinstance(payload, dict) or not isinstance(payload.get("result"), dict):
|
||||
return None
|
||||
result = payload["result"]
|
||||
target, base, summary = result.get("target"), result.get("base"), result.get("summary")
|
||||
if not isinstance(target, dict) or not isinstance(base, dict) or not isinstance(summary, dict):
|
||||
return None
|
||||
keys = (target.get("id"), base.get("id"), payload.get("target_manifest"), payload.get("base_manifest"))
|
||||
if not all(isinstance(key, str) and key for key in keys):
|
||||
return None
|
||||
started = base.get("started_at")
|
||||
return {"target_run_id": keys[0], "base_run_id": keys[1], "target_manifest": keys[2], "base_manifest": keys[3],
|
||||
"previous": payload.get("previous") is not False,
|
||||
"base_started_at": started if isinstance(started, str) else None,
|
||||
"summary": {key: value for key, value in summary.items()
|
||||
if isinstance(value, int) and not isinstance(value, bool)}}
|
||||
|
||||
|
||||
def public(brief: dict[str, Any]) -> dict[str, Any]:
|
||||
"""接口给前端的那一份(Fire.compare):基准是哪一次、各类指标的个数、基准是不是紧挨着的上一次。"""
|
||||
return {"base_run_id": brief["base_run_id"], "summary": dict(brief["summary"]), "previous": brief["previous"]}
|
||||
|
||||
|
||||
def _copy(brief: dict[str, Any]) -> dict[str, Any]:
|
||||
return {**brief, "summary": dict(brief["summary"])}
|
||||
|
||||
|
||||
def _remember(artifact: str, brief: dict[str, Any]) -> None:
|
||||
with _BRIEFS_LOCK:
|
||||
if artifact not in _BRIEFS and len(_BRIEFS) >= _BRIEFS_MAX:
|
||||
_BRIEFS.pop(next(iter(_BRIEFS)))
|
||||
_BRIEFS[artifact] = brief
|
||||
|
||||
|
||||
def brief_of(artifact: str) -> dict[str, Any] | None:
|
||||
"""读一份对比工件的摘要。文件不在、内容和哈希对不上、读不出来(后两种另记一条警告)、形状不对的返回 None,
|
||||
不抛异常:一份对比读不出来不能让整页触发历史打不开。"""
|
||||
with _BRIEFS_LOCK:
|
||||
known = _BRIEFS.get(artifact)
|
||||
if known is not None:
|
||||
return _copy(known)
|
||||
try:
|
||||
brief = _brief(artifact_store.load(artifact))
|
||||
except ValueError:
|
||||
logger.warning("对比工件 %s… 内容与哈希不符,触发历史中不显示这次对比", artifact[:12])
|
||||
return None
|
||||
except OSError as e: # 权限、磁盘出错
|
||||
logger.warning("对比工件 %s… 读取失败(%s),触发历史中不显示这次对比", artifact[:12], type(e).__name__)
|
||||
return None
|
||||
if brief is None:
|
||||
return None
|
||||
_remember(artifact, brief)
|
||||
return _copy(brief)
|
||||
|
||||
|
||||
async def briefs(artifacts: list[str]) -> dict[str, dict[str, Any]]:
|
||||
"""几份对比工件的摘要:工件 id → 摘要,读不出来的不在里面。读盘在线程里做。"""
|
||||
wanted = list(dict.fromkeys(a for a in artifacts if a))
|
||||
if not wanted:
|
||||
return {}
|
||||
|
||||
def read() -> dict[str, dict[str, Any]]:
|
||||
out = {}
|
||||
for artifact in wanted:
|
||||
brief = brief_of(artifact)
|
||||
if brief is not None:
|
||||
out[artifact] = brief
|
||||
return out
|
||||
|
||||
return await asyncio.to_thread(read)
|
||||
@@ -9,8 +9,9 @@ Webhook 地址本身就是凭证(企业微信的 key、钉钉的 access_token
|
||||
一个过滤器把推送期间的那几行换成主机名。
|
||||
|
||||
通知里只有任务名、状态、出具档位、缺口条数、运行链接、封存哈希(界面上叫清单哈希)前 12 位和起止时间,失败时
|
||||
另写失败节点的名字。不放报告里的具体数字,也不放失败时的原始报错——报错里可能带着查询到的数据,Webhook 推去的群
|
||||
不一定有权看。
|
||||
另写失败节点的名字和下一步(不许诺能继续运行);运行成功结束后和同一任务上一次成功的运行的对比(auto_compare)只写
|
||||
各类指标的个数和对比页链接。不放报告里的具体数字,也不放失败时的原始报错——报错里可能带着查询到的数据,Webhook
|
||||
推去的群不一定有权看。
|
||||
|
||||
三个平台出错时都回 HTTP 200、错误码放在响应体里,所以送没送达要看响应体:按地址的主机认平台,不看选的格式——
|
||||
格式选错了(飞书的地址选了通用格式)正是最需要认出来的情形。群机器人的地址只认自己平台的格式,保存时就拦下
|
||||
@@ -39,7 +40,7 @@ from app.core.events import EventType
|
||||
from app.db.base import SessionLocal, utcnow
|
||||
from app.db.models import Approval, Run, RunEvent, Schedule, ScheduleFire, WebhookDelivery
|
||||
from app.engine.labels import ISSUANCE_LABEL
|
||||
from app.scheduler import service
|
||||
from app.scheduler import auto_compare, service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -52,7 +53,8 @@ EVENT_LABEL = {"finished": "运行结束", "failed": "运行失败", "awaiting_a
|
||||
TIMEOUT_S = 10.0
|
||||
#: 失败后再试 3 次,每次之前等这么久(秒)。4xx(429 除外)不重试:地址或参数错了,重试也一样
|
||||
RETRY_DELAYS: tuple[float, ...] = (1.0, 2.0, 4.0)
|
||||
#: 只看这么多天内的触发:更早的运行状态不会再变,不用每一拍都扫
|
||||
#: 只看这么多天内的触发:更早结束的运行状态不会再变,不用每一拍都扫。例外见 notify_due:停在审批上、有待审批的不受
|
||||
#: 这个限制,运行记录这些天内有过变化的也照扫——审批可以等上一个多月,批完接着执行、停到下一个审批点或跑完都要推
|
||||
SCAN_DAYS = 30
|
||||
#: 运行结束后等封存(清单哈希)最多等多久。runner 先写终态再封存,两步之间被这一拍撞上就等下一拍
|
||||
SEAL_WAIT_S = 60
|
||||
@@ -65,6 +67,17 @@ _STATUS_LABEL = {"succeeded": "已完成", "failed": "失败", "cancelled": "已
|
||||
#: 挂起的运行要人去处理:不处理,这个任务之后的触发都按「上一次还没结束」跳过
|
||||
SUSPENDED_REASON = "运行停在断点上、没有待审批,通常是服务重启所致"
|
||||
SUSPENDED_ACTION = "请在 AgentLab 的「记录」中继续运行或放弃这次运行;它结束之前,这个定时任务之后的触发都会跳过"
|
||||
#: 失败的运行怎么处理:不许诺「修正后可继续运行」。开始执行过的也可能没有断点(执行第一个节点前就失败)或已不兼容
|
||||
#: (系统升级),能不能继续以记录页的提示为准。「记录」是导航里那一页的名字,和挂起那句一致
|
||||
FAILED_ACTION = "请在 AgentLab 的「记录」中查看失败原因"
|
||||
#: 从没开始执行就失败的(服务重启时还在排队,被改成失败、没有开始时间):没有可继续的进度。不照抄运行记录上的报错——
|
||||
#: 通知一律不放原始报错,这一句是按「没有开始时间」认出来的,自己写
|
||||
NOT_STARTED_REASON = "运行在开始执行前就已中止(通常是服务重启时仍在排队),没有可继续的进度"
|
||||
#: 失败的运行不挡之后的触发(挂起的才挡):任务还在排期上时补这一句,免得人以为这个定时任务就此停了
|
||||
NEXT_AS_USUAL = "下一次触发会照常发起"
|
||||
#: 从没开始执行、任务又不在排期上(停用了):原因刚写过,记录页上也没有更多可看的,下一步是重新发起。「立即运行」
|
||||
#: 是定时任务页上那个按钮的名字,停用的任务也能点
|
||||
RERUN_ACTION = "如需重新发起,请在 AgentLab 的「定时任务」中点击「立即运行」"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -217,6 +230,9 @@ class Notice:
|
||||
failed_to_start。没有的项为 None。
|
||||
|
||||
suspended:运行停在断点上、没有待审批(通常是服务重启挂起的),按 failed 事件推。error_node:失败节点的名字。
|
||||
next_fire:任务启用着、往后还会触发(失败通知据此补「下一次触发会照常发起」)。
|
||||
compare:和同一定时任务上一次成功的运行的对比摘要(auto_compare.compare_fire 的返回:基准是哪一次、只有个数的
|
||||
summary、基准是不是紧挨着的上一次、基准开始的时间),compare_url:对比页的链接。
|
||||
fmt、signed 只给测试消息用:说明里只说这条消息证明得了的事。"""
|
||||
|
||||
event: str
|
||||
@@ -238,9 +254,17 @@ class Notice:
|
||||
pending_approvals: int | None = None
|
||||
suspended: bool = False
|
||||
error_node: str | None = None
|
||||
next_fire: bool = False
|
||||
compare: dict[str, Any] | None = None
|
||||
compare_url: str | None = None
|
||||
fmt: str | None = None
|
||||
signed: bool = False
|
||||
|
||||
@property
|
||||
def never_started(self) -> bool:
|
||||
"""失败的运行从没开始执行(没有开始时间):服务重启时还在排队、被改成失败的就是这种,没有断点可续。"""
|
||||
return self.status == "failed" and self.run_id is not None and self.started_at is None
|
||||
|
||||
|
||||
def _local(moment: datetime | None, tz: str, *, seconds: bool = True) -> str | None:
|
||||
if moment is None:
|
||||
@@ -271,6 +295,9 @@ def message(notice: Notice) -> tuple[str, list[tuple[str, str]], str | None]:
|
||||
if notice.tier:
|
||||
gaps = f"(缺口 {notice.gaps} 条)" if notice.gaps is not None else ""
|
||||
lines.append(("出具档位", f"{ISSUANCE_LABEL.get(notice.tier, notice.tier)}{gaps}"))
|
||||
if notice.compare:
|
||||
# 只给各类指标的个数:具体的数在对比页上,跟着运行的权限走,不随消息推进群里
|
||||
lines.append((_compare_head(notice), auto_compare.summary_text(notice.compare.get("summary") or {})))
|
||||
if notice.pending_approvals:
|
||||
lines.append(("待审批", f"{notice.pending_approvals} 项,请到 AgentLab 处理;运行结束之前,"
|
||||
"这个定时任务之后的触发都会跳过"))
|
||||
@@ -279,10 +306,14 @@ def message(notice: Notice) -> tuple[str, list[tuple[str, str]], str | None]:
|
||||
elif notice.suspended:
|
||||
lines += [("原因", SUSPENDED_REASON), ("处理", SUSPENDED_ACTION)]
|
||||
elif notice.status == "failed":
|
||||
# 不写原始报错(可能带着查询到的数据),写失败节点的名字和去哪里处理
|
||||
# 不写原始报错(可能带着查询到的数据),写失败节点的名字和下一步
|
||||
if notice.error_node:
|
||||
lines.append(("失败节点", f"「{notice.error_node}」"))
|
||||
lines.append(("处理", "请在 AgentLab 的运行记录中查看报错,修正后可继续运行"))
|
||||
if notice.never_started:
|
||||
# 原因刚写过,也没有断点可续:不再让人去查看原因,说下一次照常,不在排期上的说怎么重新发起
|
||||
lines += [("原因", NOT_STARTED_REASON), ("处理", NEXT_AS_USUAL if notice.next_fire else RERUN_ACTION)]
|
||||
else:
|
||||
lines.append(("处理", f"{FAILED_ACTION};{NEXT_AS_USUAL}" if notice.next_fire else FAILED_ACTION))
|
||||
if notice.fire_at is not None:
|
||||
when = f"{_local(notice.fire_at, notice.timezone, seconds=False)}({notice.timezone})"
|
||||
# 立即运行不写「计划触发」:群里的人会以为报告按时出了
|
||||
@@ -304,6 +335,27 @@ def message(notice: Notice) -> tuple[str, list[tuple[str, str]], str | None]:
|
||||
return title, lines, notice.run_url
|
||||
|
||||
|
||||
def _compare_head(notice: Notice) -> str:
|
||||
"""对比那一行的项名。基准是紧挨着的上一次时写「与上一次相比」;不是(上一次失败了、取消了)时写明比的是哪一次:
|
||||
短编号(同界面,#前 6 位)和开始的日期(按定时任务的时区)。"""
|
||||
compare = notice.compare or {}
|
||||
if compare.get("previous", True):
|
||||
return "与上一次相比"
|
||||
when = _date_of(compare.get("base_started_at"), notice.timezone)
|
||||
return f"与上一次成功的运行(#{str(compare.get('base_run_id') or '')[:6]}{f',{when}' if when else ''})相比"
|
||||
|
||||
|
||||
def _date_of(text: Any, tz: str) -> str | None:
|
||||
"""ISO 时间(对比结果里运行的开始时间)按定时任务的时区写成日期,读不出来的返回 None。"""
|
||||
try:
|
||||
moment = datetime.fromisoformat(str(text))
|
||||
except ValueError:
|
||||
return None
|
||||
if moment.tzinfo is None:
|
||||
moment = moment.replace(tzinfo=timezone.utc)
|
||||
return (_local(moment, tz, seconds=False) or "")[:10] or None
|
||||
|
||||
|
||||
def _status_label(notice: Notice) -> str:
|
||||
if notice.status == "interrupted":
|
||||
return "等待审批" if notice.pending_approvals else "已挂起,可继续运行"
|
||||
@@ -320,31 +372,34 @@ def _test_text(notice: Notice) -> str:
|
||||
return "这是一条测试消息。收到即表示 Webhook 地址配置正确"
|
||||
|
||||
|
||||
def _links(notice: Notice) -> list[tuple[str, str]]:
|
||||
"""消息末尾的链接:运行,有对比时再加对比页。没设对外访问地址时都没有。"""
|
||||
return [(label, url) for label, url in (("查看运行", notice.run_url), ("查看对比", notice.compare_url)) if url]
|
||||
|
||||
|
||||
def _text(notice: Notice) -> str:
|
||||
title, lines, link = message(notice)
|
||||
title, lines, _ = message(notice)
|
||||
out = [f"【AgentLab】{title}", *(f"{k}:{v}" for k, v in lines)]
|
||||
if link:
|
||||
out.append(f"查看运行:{link}")
|
||||
out += [f"{label}:{url}" for label, url in _links(notice)]
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def _wecom_markdown(notice: Notice) -> str:
|
||||
title, lines, link = message(notice)
|
||||
title, lines, _ = message(notice)
|
||||
color = "info" if notice.event in ("finished", "test") else "warning"
|
||||
out = [f"**【AgentLab】{title}**"]
|
||||
for key, value in lines:
|
||||
shown = f'<font color="{color}">{value}</font>' if key == "状态" else value
|
||||
out.append(f"> {key}:{shown}")
|
||||
if link:
|
||||
out.append(f"[查看运行]({link})")
|
||||
out += [f"[{label}]({url})" for label, url in _links(notice)]
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def _dingtalk_markdown(notice: Notice) -> tuple[str, str]:
|
||||
title, lines, link = message(notice)
|
||||
title, lines, _ = message(notice)
|
||||
out = [f"#### 【AgentLab】{title}", "", *(f"- {k}:{v}" for k, v in lines)]
|
||||
if link:
|
||||
out += ["", f"[查看运行]({link})"]
|
||||
for label, url in _links(notice):
|
||||
out += ["", f"[{label}]({url})"]
|
||||
return title, "\n".join(out)
|
||||
|
||||
|
||||
@@ -375,8 +430,14 @@ def _generic(notice: Notice, now: datetime) -> dict[str, Any]:
|
||||
"started_at": _iso(notice.started_at),
|
||||
"finished_at": _iso(notice.finished_at),
|
||||
"reason": (notice.reason if notice.status == "failed_to_start"
|
||||
else f"{SUSPENDED_REASON}。{SUSPENDED_ACTION}" if notice.suspended else None),
|
||||
else f"{SUSPENDED_REASON}。{SUSPENDED_ACTION}" if notice.suspended
|
||||
else NOT_STARTED_REASON if notice.never_started else None),
|
||||
"error_node": notice.error_node,
|
||||
# 对比只给基准是哪一次(是不是紧挨着的上一次)、各类指标的个数和对比页链接
|
||||
"compare": ({"base_run_id": notice.compare.get("base_run_id"),
|
||||
"summary": dict(notice.compare.get("summary") or {}),
|
||||
"previous": notice.compare.get("previous", True) is not False, "url": notice.compare_url}
|
||||
if notice.compare else None),
|
||||
"sent_at": _iso(now),
|
||||
}
|
||||
|
||||
@@ -557,53 +618,107 @@ class _Job:
|
||||
notice: Notice
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Due:
|
||||
"""挑出来的一次状态变化接下来要做的事:运行成功结束时和上一次成功的运行比一次(compare_run),按条件 UPDATE
|
||||
认领(把 notified_status 从 old 改成 state),认领到了、订阅了这个事件时推一条(job)。都在锁外做:对比要读快照、
|
||||
重算,推送要等接收端,都不能挡着发起。pushed:认领到了、发出了推送(含失败的)。"""
|
||||
|
||||
fire_id: str
|
||||
old: str | None
|
||||
state: str
|
||||
compare_run: str | None
|
||||
job: _Job | None
|
||||
pushed: bool = False
|
||||
|
||||
|
||||
async def notify_due(now: datetime | None = None) -> int:
|
||||
"""查已发起的触发,运行状态有变化的按订阅推送一次。返回这一拍推送了几条(含失败的)。
|
||||
|
||||
处理过的状态记在 notified_status 上,用条件 UPDATE 认领(WHERE notified_status = 旧值):两拍撞在一起也只
|
||||
推一次。没订阅这个事件、任务没配 Webhook 的也记下已处理——之后再配上 Webhook,不会把旧的补推一遍。
|
||||
停在断点上的运行每一拍都要重新看(见 _state_of):它的已处理状态带着第几段执行,和运行状态本身对不上。
|
||||
|
||||
运行成功结束、封存完的那一次,认领之前先和同一定时任务上一次成功的运行比一次(auto_compare),结束的通知带上计数
|
||||
摘要和对比页链接。没配 Webhook 的也比:触发历史里要用。对比在锁外、认领之前做,最多等 auto_compare.WAIT_S 秒,
|
||||
过了就不带对比推送:一次慢的对比最多让这一拍晚这么久,不会把之后所有的通知一直压着,也不拖着发起用的锁。比到一半
|
||||
扫描被取消(停机只等通知扫描一个宽限),这次触发还没认领,下次照常比、照常推——以前先认领、后比,取消之后这条
|
||||
通知再也不会推。比不了、出错不影响推送。
|
||||
"""
|
||||
now = now or utcnow()
|
||||
jobs: list[_Job] = []
|
||||
# 和发起共用一把锁:发起时先记一条「未能发起」再发起运行、成功了才改成 started,扫描不能撞在这中间
|
||||
dues: list[_Due] = []
|
||||
base_url: str | None = None
|
||||
cutoff = now - timedelta(days=SCAN_DAYS)
|
||||
# 扫描窗口按触发记录的创建时间算,两种例外:运行记录窗口内有过变化(运行状态一变 updated_at 就跟着变),以及
|
||||
# 停在审批上、有待审批的——以前只看触发记录,创建 30 天以后才停到审批上的运行收不到「等待审批」
|
||||
waiting = select(Approval.id).where(Approval.run_id == Run.id, Approval.status == "pending").exists()
|
||||
# 和发起共用一把锁:发起时先记一条「未能发起」再发起运行、成功了才改成 started,扫描不能撞在这中间。锁里只挑出
|
||||
# 状态有变化的、拼好要推的消息;认领在锁外按条件 UPDATE 做,挑出来的是锁里看到的状态,和别的扫描撞上也只有一个
|
||||
# 认领得到
|
||||
async with service.launch_lock():
|
||||
async with SessionLocal() as session:
|
||||
rows = (await session.execute(
|
||||
select(ScheduleFire, Run)
|
||||
.outerjoin(Run, Run.id == ScheduleFire.run_id)
|
||||
.where(ScheduleFire.created_at >= now - timedelta(days=SCAN_DAYS), or_(
|
||||
.where(or_(ScheduleFire.created_at >= cutoff, Run.updated_at >= cutoff,
|
||||
and_(Run.status == "interrupted", waiting)), or_(
|
||||
and_(ScheduleFire.status == "failed_to_start", ScheduleFire.notified_status.is_(None)),
|
||||
and_(ScheduleFire.status == "started", Run.status.in_(_WATCHED),
|
||||
or_(ScheduleFire.notified_status.is_(None), ScheduleFire.notified_status != Run.status)),
|
||||
))
|
||||
.order_by(ScheduleFire.created_at)
|
||||
)).all()
|
||||
base_url: str | None = None
|
||||
if rows:
|
||||
from app.api.settings import notify_settings
|
||||
|
||||
base_url = (await notify_settings(session))["public_base_url"]
|
||||
for fire, run in rows:
|
||||
state, event, pending = await _state_of(session, fire, run, now)
|
||||
if state is None or state == fire.notified_status \
|
||||
or not await _mark(fire.id, fire.notified_status, state):
|
||||
seen = await _state_of(session, fire, run, now)
|
||||
if seen.state is None or seen.state == fire.notified_status:
|
||||
continue
|
||||
schedule = await session.get(Schedule, fire.schedule_id)
|
||||
url = decrypt(schedule.webhook_url) if schedule and schedule.webhook_url else None
|
||||
if not (event and url and schedule.webhook_format in FORMATS
|
||||
and event in (schedule.webhook_events or [])):
|
||||
continue
|
||||
notice = await _notice(session, schedule, fire, run, event, base_url, pending)
|
||||
jobs.append(_Job(schedule.webhook_format, url, decrypt(schedule.webhook_secret), notice))
|
||||
if jobs:
|
||||
await asyncio.gather(*(_deliver(job) for job in jobs))
|
||||
return len(jobs)
|
||||
job = None
|
||||
if seen.event and url and schedule.webhook_format in FORMATS \
|
||||
and seen.event in (schedule.webhook_events or []):
|
||||
notice = await _notice(session, schedule, fire, run, seen, base_url)
|
||||
job = _Job(schedule.webhook_format, url, decrypt(schedule.webhook_secret), notice)
|
||||
# 只比成功结束、封存完的(见 auto_compare):失败、取消的指标不全,等不来新封存的比了也是旧的
|
||||
compare_run = run.id if run is not None and run.status == "succeeded" and seen.sealed else None
|
||||
dues.append(_Due(fire.id, fire.notified_status, seen.state, compare_run, job))
|
||||
if dues:
|
||||
await asyncio.gather(*(_follow_up(due, base_url) for due in dues))
|
||||
return sum(1 for due in dues if due.pushed)
|
||||
|
||||
|
||||
async def _state_of(session: Any, fire: ScheduleFire, run: Run | None,
|
||||
now: datetime) -> tuple[str | None, str | None, int]:
|
||||
"""(要记下的已处理状态, 要推送的事件, 待审批数)。还不能下结论(没封存、审批刚批完还没接着执行)时状态为 None。
|
||||
async def _follow_up(due: _Due, base_url: str | None) -> None:
|
||||
"""先比(需要的话),再认领,认领到了再推:对比摘要要写进这条通知;认领放在对比之后,比到一半被取消时这次触发
|
||||
还没认领。对比自己兜住了超时和异常,出什么事都不挡推送。"""
|
||||
brief = await auto_compare.compare_fire(due.fire_id, due.compare_run) if due.compare_run else None
|
||||
if not await _mark(due.fire_id, due.old, due.state) or due.job is None:
|
||||
return
|
||||
if brief is not None:
|
||||
due.job.notice.compare = brief
|
||||
if base_url:
|
||||
due.job.notice.compare_url = auto_compare.page_url(base_url, due.compare_run, brief["base_run_id"])
|
||||
due.pushed = True
|
||||
await _deliver(due.job)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Seen:
|
||||
"""_state_of 的结论。state:要记下的已处理状态,还不能下结论时为 None;event:要推送的事件;pending:待审批数;
|
||||
sealed:终态的运行封存完了(封存盖住了这一次的结束事件,见 auto_compare.sealed_now),只有这时才比、通知里才写
|
||||
清单哈希。"""
|
||||
|
||||
state: str | None
|
||||
event: str | None = None
|
||||
pending: int = 0
|
||||
sealed: bool = False
|
||||
|
||||
|
||||
async def _state_of(session: Any, fire: ScheduleFire, run: Run | None, now: datetime) -> _Seen:
|
||||
"""这次触发现在该记成什么状态、推什么。还不能下结论(没封存、审批刚批完还没接着执行)时状态为 None。
|
||||
|
||||
停在断点上(interrupted)的分两种,各推一次:
|
||||
- 有待审批:停在审批,推 awaiting_approval。批完接着执行、又停在下一个审批点时要再推,所以已处理状态里带着这是
|
||||
@@ -613,18 +728,21 @@ async def _state_of(session: Any, fire: ScheduleFire, run: Run | None,
|
||||
审批批完到运行改回 running 之间,审批是 answered、还没有 resolved:那不是挂起,等下一拍。
|
||||
"""
|
||||
if fire.status == "failed_to_start":
|
||||
return "failed_to_start", "failed", 0
|
||||
return _Seen("failed_to_start", "failed")
|
||||
if run is None:
|
||||
return None, None, 0
|
||||
return _Seen(None)
|
||||
if run.status in ("succeeded", "failed", "cancelled"):
|
||||
# runner 先写终态和结束时间、再封存,两步之间被这一拍撞上就等。没有结束时间的(服务重启时还在排队、被改成
|
||||
# failed 的)永远不会封存:按它最后一次更新的时间等,以前按「现在」等,每一拍都等、永远不推
|
||||
# runner 先写终态和结束时间、再另起会话封存,两步之间被这一拍撞上就等。失败后接着跑完的运行这时还带着失败时
|
||||
# 的清单哈希,不能只看有没有哈希,要看封存盖没盖住这一次的结束事件。没有结束时间的(服务重启时还在排队、被改
|
||||
# 成 failed 的)永远不会封存:按它最后一次更新的时间等,以前按「现在」等,每一拍都等、永远不推。等过时限仍没
|
||||
# 封存完的照常推,只是不比、不写清单哈希
|
||||
sealed = await auto_compare.sealed_now(session, run)
|
||||
since = run.finished_at or run.updated_at or fire.created_at
|
||||
if not run.manifest_hash and (now - service.as_utc(since)).total_seconds() < SEAL_WAIT_S:
|
||||
return None, None, 0
|
||||
return run.status, _EVENT_OF[run.status], 0
|
||||
if not sealed and (now - service.as_utc(since)).total_seconds() < SEAL_WAIT_S:
|
||||
return _Seen(None)
|
||||
return _Seen(run.status, _EVENT_OF[run.status], sealed=sealed)
|
||||
if run.status != "interrupted":
|
||||
return None, None, 0
|
||||
return _Seen(None)
|
||||
counts = dict((await session.execute(
|
||||
select(Approval.status, func.count()).where(
|
||||
Approval.run_id == run.id, Approval.status.in_(("pending", "answered")))
|
||||
@@ -632,10 +750,10 @@ async def _state_of(session: Any, fire: ScheduleFire, run: Run | None,
|
||||
segment = await session.scalar(select(func.count()).select_from(RunEvent).where(
|
||||
RunEvent.run_id == run.id, RunEvent.type == EventType.RUN_STARTED.value)) or 0
|
||||
if counts.get("pending"):
|
||||
return f"interrupted:approval:{segment}", "awaiting_approval", counts["pending"]
|
||||
return _Seen(f"interrupted:approval:{segment}", "awaiting_approval", counts["pending"])
|
||||
if counts.get("answered"):
|
||||
return None, None, 0
|
||||
return f"interrupted:held:{segment}", "failed", 0
|
||||
return _Seen(None)
|
||||
return _Seen(f"interrupted:held:{segment}", "failed")
|
||||
|
||||
|
||||
async def _mark(fire_id: str, old: str | None, state: str) -> bool:
|
||||
@@ -647,23 +765,26 @@ async def _mark(fire_id: str, old: str | None, state: str) -> bool:
|
||||
return result.rowcount == 1
|
||||
|
||||
|
||||
async def _notice(session: Any, schedule: Schedule, fire: ScheduleFire, run: Run | None, event: str,
|
||||
base_url: str | None, pending: int) -> Notice:
|
||||
async def _notice(session: Any, schedule: Schedule, fire: ScheduleFire, run: Run | None, seen: _Seen,
|
||||
base_url: str | None) -> Notice:
|
||||
tier, gaps = await _issuance_of(session, run) if run is not None else (None, None)
|
||||
pending = seen.pending
|
||||
return Notice(
|
||||
event=event, schedule_id=schedule.id, schedule_name=schedule.name, timezone=schedule.timezone,
|
||||
event=seen.event or "", schedule_id=schedule.id, schedule_name=schedule.name, timezone=schedule.timezone,
|
||||
fire_id=fire.id, fire_at=fire.fire_at, fire_kind=fire.kind,
|
||||
run_id=run.id if run else None,
|
||||
status=run.status if run else fire.status,
|
||||
tier=tier, gaps=gaps,
|
||||
run_url=f"{base_url}/runs/{run.id}" if base_url and run else None,
|
||||
manifest_hash=run.manifest_hash if run else None,
|
||||
# 只写这一次结束时封的:停在审批上、封存还没补上的运行,身上可能还留着上一次失败时的清单哈希
|
||||
manifest_hash=run.manifest_hash if run is not None and seen.sealed else None,
|
||||
started_at=run.started_at if run else None,
|
||||
finished_at=run.finished_at if run else None,
|
||||
reason=fire.reason if fire.status == "failed_to_start" else None,
|
||||
pending_approvals=pending or None,
|
||||
suspended=run is not None and run.status == "interrupted" and not pending,
|
||||
error_node=_failed_node(run) if run is not None and run.status == "failed" else None,
|
||||
next_fire=bool(schedule.enabled and schedule.next_run_at is not None),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,538 @@
|
||||
"""定时运行结束后的自动对比(app/scheduler/auto_compare.py,由通知扫描 webhook.notify_due 在认领、推送之前触发):
|
||||
|
||||
- 只比成功结束并封存的运行:失败、取消的指标不全,和上一次比只会得出「移除 N 个」,误导收到通知的人
|
||||
- 基准是同一定时任务、同一工作流、比这次先发起的运行里成功结束并封存的最近一次:同一工作流更近的手动运行、别的
|
||||
定时任务的运行都不算;失败的那一次不当基准(拿它当基准每个指标都成了「新增」);按发起先后挑,不按结束时间——
|
||||
老的失败运行事后被接着跑完,结束得比后一次晚,它的上一次仍是它前面那次。紧挨着的上一次不是成功的时,通知和触发
|
||||
历史写明比的是哪一次
|
||||
- 整份结果连同两边的清单哈希存成工件(kind=run_compare,不挂在运行名下、不进运行的工件列表),工件 id 记进触发
|
||||
记录的 compare_artifact。这个任务的第一次(只有同一工作流别的运行可比)、还没封存时不比;对比出错(基准的封存核对
|
||||
不一致、意外的异常)只记日志,通知照常推送、不带对比
|
||||
- 接口:Fire 多一个 compare: {base_run_id, summary, previous} 或 null,summary 同对比接口的计数,previous 说基准是不是
|
||||
紧挨着的上一次;基准被删、被继续执行(不在终态)时为 null(对比页只认封存的运行)
|
||||
- 通知:结束的通知多一行「与上一次相比:只数据变 2 个、口径与数据都变 0 个、SQL 结构变 0 个」,配了对外访问地址时附
|
||||
对比页链接(/runs/compare?target=…&base=…);只给个数,报告、指标、查询里的具体数字一个不出现
|
||||
|
||||
封存绑定、时限、取消这几道保护另见 test_scheduler_compare_guards.py。两次周报运行用 tests/fixtures/compare_runs.py 的
|
||||
虚构运行(区域、金额、日期都是编的),按 runner 的算法封存,不真的执行;Webhook 用 httpx.MockTransport 接住,不连外网。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from app.core import artifact_store
|
||||
from app.core.artifact_store import canonical_json, load, manifest_hash, put_json
|
||||
from app.core.crypto import encrypt
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Artifact, Run, RunEvent, Schedule, ScheduleFire, Setting, Workflow
|
||||
from app.main import app
|
||||
from app.scheduler import auto_compare, webhook
|
||||
from tests.fixtures.compare_runs import NEXT_ROWS, Fake, region_sql, week
|
||||
|
||||
UTC = timezone.utc
|
||||
T0 = datetime(2026, 9, 7, 1, 0, tzinfo=UTC)
|
||||
WEEK = timedelta(days=7)
|
||||
GENERIC = "https://hooks.example.com/agentlab/inbox-5a7c"
|
||||
FEISHU = "https://open.feishu.cn/open-apis/bot/v2/hook/1a2b3c4d-aaaa-bbbb-cccc-0d9e8f7a6b5c"
|
||||
BASE_URL = "https://agentlab.example.com"
|
||||
#: 两次虚构周报(BASE_ROWS → NEXT_ROWS)之间两个指标都只是数据变了
|
||||
SUMMARY = {"unchanged": 0, "data_only": 2, "caliber_and_data": 0, "sql_changed": 0, "params_only": 0, "added": 0,
|
||||
"removed": 0, "incomparable": 0}
|
||||
LINE = "与上一次相比:只数据变 2 个、口径与数据都变 0 个、SQL 结构变 0 个"
|
||||
#: 两次运行查到、算出的数(金额、退款、合计、变化量),通知里一个都不能有
|
||||
NUMBERS = ("1200.5", "1,200.5", "1500.5", "1,500.5", "2600.5", "2,600.5", "2800.5", "2,800.5", "300.0", "800.0",
|
||||
"700.0", "600.0")
|
||||
|
||||
|
||||
class Recorder:
|
||||
def __init__(self) -> None:
|
||||
self.requests: list[httpx.Request] = []
|
||||
|
||||
def __call__(self, request: httpx.Request) -> httpx.Response:
|
||||
self.requests.append(request)
|
||||
return httpx.Response(200, json={"code": 0, "errcode": 0})
|
||||
|
||||
def bodies(self) -> list[dict]:
|
||||
return [json.loads(r.content) for r in self.requests]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def recorder(monkeypatch):
|
||||
rec = Recorder()
|
||||
monkeypatch.setattr(webhook, "_transport", httpx.MockTransport(rec))
|
||||
monkeypatch.setattr(webhook, "RETRY_DELAYS", (0, 0, 0))
|
||||
return rec
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def clean():
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(delete(Schedule))
|
||||
await session.execute(delete(Setting).where(Setting.key == "notify"))
|
||||
await session.commit()
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
async def public_address() -> None:
|
||||
async with SessionLocal() as session:
|
||||
session.add(Setting(key="notify", value={"public_base_url": BASE_URL}))
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def new_workflow(name: str = "区域周报") -> str:
|
||||
async with SessionLocal() as session:
|
||||
workflow = Workflow(name=name)
|
||||
session.add(workflow)
|
||||
await session.commit()
|
||||
return workflow.id
|
||||
|
||||
|
||||
async def new_schedule(workflow_id: str, *, fmt: str = "generic", url: str | None = GENERIC,
|
||||
name: str = "区域周报 · 每周") -> str:
|
||||
async with SessionLocal() as session:
|
||||
row = Schedule(name=name, workflow_id=workflow_id, cron="0 9 * * 1", timezone="Asia/Shanghai",
|
||||
enabled=True, next_run_at=datetime.now(UTC) + timedelta(days=3),
|
||||
webhook_format=fmt if url else None, webhook_url=encrypt(url) if url else None,
|
||||
webhook_events=["finished", "failed", "awaiting_approval"])
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
return row.id
|
||||
|
||||
|
||||
async def persist(fake: Fake, workflow_id: str, *, finished: datetime, schedule_id: str | None = None,
|
||||
status: str = "succeeded", seal: bool = True, seal_upto: int | None = None) -> str:
|
||||
"""把一次虚构运行落库:工件进工件库,事件进运行记录,按 runner 的算法封存(同 test_compare_api.py)。
|
||||
seal_upto 只封到第几条事件:失败时封存过、接着跑完又写下了终态,新的封存还没补上的那一刻。"""
|
||||
for content in fake.store.values():
|
||||
await put_json(content, kind="compare_fixture")
|
||||
async with SessionLocal() as session:
|
||||
run = Run(workflow_id=workflow_id, workflow_name="区域周报", status=status, graph=fake.graph,
|
||||
input=dict(fake.inputs), run_class="formal", version=fake.version, schedule_id=schedule_id,
|
||||
started_at=finished - timedelta(minutes=2), finished_at=finished, agent_limits={},
|
||||
started_by="定时任务:区域周报 · 每周" if schedule_id else "周报编辑")
|
||||
session.add(run)
|
||||
await session.flush()
|
||||
rows = []
|
||||
for seq, etype, node_id, data in fake.events:
|
||||
session.add(RunEvent(run_id=run.id, seq=seq, type=etype, node_id=node_id, ts=float(seq), data=data))
|
||||
rows.append((seq, etype, node_id, data))
|
||||
run.last_seq = len(rows)
|
||||
if seal:
|
||||
covered = [r for r in rows if seal_upto is None or r[0] <= seal_upto]
|
||||
run.manifest_hash, run.manifest_seq = manifest_hash(covered), covered[-1][0]
|
||||
await session.commit()
|
||||
return run.id
|
||||
|
||||
|
||||
async def reseal(run_id: str, fake: Fake, *, status: str, finished: datetime) -> None:
|
||||
"""一次运行被接着跑完(或者被重新封存):事件换成 fake 的那一份,按新的事件封存,状态、结束时间一并改。"""
|
||||
for content in fake.store.values():
|
||||
await put_json(content, kind="compare_fixture")
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(delete(RunEvent).where(RunEvent.run_id == run_id))
|
||||
rows = []
|
||||
for seq, etype, node_id, data in fake.events:
|
||||
session.add(RunEvent(run_id=run_id, seq=seq, type=etype, node_id=node_id, ts=float(seq), data=data))
|
||||
rows.append((seq, etype, node_id, data))
|
||||
run = await session.get(Run, run_id)
|
||||
run.last_seq = len(rows)
|
||||
run.manifest_hash, run.manifest_seq = manifest_hash(rows), rows[-1][0]
|
||||
run.status, run.finished_at = status, finished
|
||||
await session.commit()
|
||||
|
||||
|
||||
def failed_week(run_id: str, rows=NEXT_ROWS) -> Fake:
|
||||
"""在 Agent 节点失败的一次周报:取数做完了,Agent 没交回结果,口径卡没有执行,一个指标都没有。"""
|
||||
run = Fake(run_id)
|
||||
run.tool("fetch", region_sql(), rows=rows)
|
||||
run.add("node.started", "bot", {"node_type": "agent"})
|
||||
run.add("node.failed", "bot", {"error": "模型接入超时"})
|
||||
run.add("run.failed", None, {"error": "节点「区域分析」失败"})
|
||||
return run
|
||||
|
||||
|
||||
async def seal(run_id: str) -> None:
|
||||
async with SessionLocal() as session:
|
||||
run = await session.get(Run, run_id)
|
||||
rows = [tuple(r) for r in await session.execute(
|
||||
select(RunEvent.seq, RunEvent.type, RunEvent.node_id, RunEvent.data)
|
||||
.where(RunEvent.run_id == run_id).order_by(RunEvent.seq))]
|
||||
run.manifest_hash, run.manifest_seq = manifest_hash(rows), rows[-1][0]
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def fire_for(schedule_id: str, run_id: str, at: datetime, *, notified: str | None = None) -> str:
|
||||
async with SessionLocal() as session:
|
||||
fire = ScheduleFire(schedule_id=schedule_id, fire_at=at, kind="scheduled", status="started", run_id=run_id,
|
||||
notified_status=notified)
|
||||
session.add(fire)
|
||||
await session.commit()
|
||||
return fire.id
|
||||
|
||||
|
||||
async def fire_row(fire_id: str) -> ScheduleFire:
|
||||
async with SessionLocal() as session:
|
||||
return await session.get(ScheduleFire, fire_id)
|
||||
|
||||
|
||||
async def set_run(run_id: str, **values) -> None:
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(Run.__table__.update().where(Run.id == run_id).values(**values))
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def weekly_pair(*, target_status: str = "succeeded", target_seal: bool = True,
|
||||
target_finished: datetime | None = None, fmt: str = "generic", url: str | None = GENERIC):
|
||||
"""同一个定时任务的两次周报:上一次(已推送过)和这一次(还没处理),这次的数变了。返回 (任务, 上一次, 这一次,
|
||||
这一次的触发)。"""
|
||||
workflow = await new_workflow()
|
||||
sid = await new_schedule(workflow, fmt=fmt, url=url)
|
||||
base_id = await persist(week("r1"), workflow, finished=T0, schedule_id=sid)
|
||||
await fire_for(sid, base_id, T0 - timedelta(minutes=2), notified="succeeded")
|
||||
target_id = await persist(week("r2", NEXT_ROWS), workflow, finished=target_finished or T0 + WEEK,
|
||||
schedule_id=sid, status=target_status, seal=target_seal)
|
||||
fire_id = await fire_for(sid, target_id, T0 + WEEK - timedelta(minutes=2))
|
||||
return sid, base_id, target_id, fire_id
|
||||
|
||||
|
||||
def no_numbers(raw: str) -> list[str]:
|
||||
return [n for n in NUMBERS if n in raw]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 什么时候比、跟谁比、存在哪
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_finished_run_is_compared_with_the_previous_run_of_its_schedule(recorder, client):
|
||||
await public_address()
|
||||
workflow = await new_workflow()
|
||||
sid = await new_schedule(workflow)
|
||||
other = await new_schedule(workflow, url=None, name="区域周报 · 补跑")
|
||||
base_id = await persist(week("r1"), workflow, finished=T0, schedule_id=sid)
|
||||
base_fire = await fire_for(sid, base_id, T0 - timedelta(minutes=2), notified="succeeded")
|
||||
# 同一工作流、比上一次更近的两次:手动发起的、别的定时任务发起的。都不是「同一定时任务的上一次」
|
||||
await persist(week("m1"), workflow, finished=T0 + timedelta(days=3))
|
||||
await persist(week("o1"), workflow, finished=T0 + timedelta(days=5), schedule_id=other)
|
||||
target_id = await persist(week("r2", NEXT_ROWS), workflow, finished=T0 + WEEK, schedule_id=sid)
|
||||
fire_id = await fire_for(sid, target_id, T0 + WEEK - timedelta(minutes=2))
|
||||
|
||||
assert await webhook.notify_due() == 1
|
||||
|
||||
artifact = (await fire_row(fire_id)).compare_artifact
|
||||
assert artifact
|
||||
from app.api.compare import compare_runs
|
||||
|
||||
stored = load(artifact)
|
||||
# 整份对比结果,连同比的时候两边的清单哈希(封存变了,这份结果就不再作数)
|
||||
assert stored["result"] == json.loads(canonical_json(await compare_runs(target_id, base_id)))
|
||||
assert stored["result"]["base"]["id"] == base_id and stored["result"]["summary"] == SUMMARY
|
||||
async with SessionLocal() as session:
|
||||
runs = await session.scalars(select(Run).where(Run.id.in_((target_id, base_id))))
|
||||
seals = {r.id: r.manifest_hash for r in runs}
|
||||
assert (stored["target_manifest"], stored["base_manifest"], stored["previous"]) == (
|
||||
seals[target_id], seals[base_id], True)
|
||||
async with SessionLocal() as session:
|
||||
rows = list((await session.execute(select(Artifact).where(Artifact.id == artifact))).scalars())
|
||||
# 对比结果不是运行产出的:不挂在运行名下,不进这次运行的工件列表
|
||||
assert [(a.kind, a.run_id) for a in rows] == [("run_compare", "")]
|
||||
assert artifact not in [a["id"] for a in (await client.get(f"/api/runs/{target_id}/artifacts")).json()]
|
||||
|
||||
items = {f["id"]: f for f in (await client.get(f"/api/schedules/{sid}/fires")).json()["items"]}
|
||||
assert items[fire_id]["compare"] == {"base_run_id": base_id, "summary": SUMMARY, "previous": True}
|
||||
assert items[base_fire]["compare"] is None
|
||||
[listed] = [s for s in (await client.get("/api/schedules")).json()["items"] if s["id"] == sid]
|
||||
assert listed["last_fire"]["id"] == fire_id and listed["last_fire"]["compare"]["base_run_id"] == base_id
|
||||
|
||||
[body] = recorder.bodies()
|
||||
link = f"{BASE_URL}/runs/compare?target={target_id}&base={base_id}"
|
||||
assert body["event"] == "finished" and LINE in body["text"] and f"查看对比:{link}" in body["text"]
|
||||
assert body["compare"] == {"base_run_id": base_id, "summary": SUMMARY, "previous": True, "url": link}
|
||||
query = parse_qs(urlsplit(body["compare"]["url"]).query)
|
||||
assert query == {"target": [target_id], "base": [base_id]}
|
||||
assert no_numbers(recorder.requests[0].content.decode()) == []
|
||||
# 下一拍:同一状态不再比、不再推
|
||||
assert await webhook.notify_due() == 0 and (await fire_row(fire_id)).compare_artifact == artifact
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status, fmt, url", [("failed", "feishu", FEISHU), ("failed", "generic", GENERIC),
|
||||
("cancelled", "generic", GENERIC)])
|
||||
async def test_failed_or_cancelled_run_is_not_compared(recorder, client, status, fmt, url):
|
||||
"""失败、取消的运行指标不全(这次在 Agent 节点失败,口径卡没有执行):和上一次比只会得出「移除 2 个」,收到通知
|
||||
的人会以为报告删了两个指标。不比:触发记录上没有对比,失败通知不带计数那一行和对比页链接"""
|
||||
await public_address()
|
||||
workflow = await new_workflow()
|
||||
sid = await new_schedule(workflow, fmt=fmt, url=url)
|
||||
base_id = await persist(week("r1"), workflow, finished=T0, schedule_id=sid)
|
||||
await fire_for(sid, base_id, T0 - timedelta(minutes=2), notified="succeeded")
|
||||
target_id = await persist(failed_week("r2"), workflow, finished=T0 + WEEK, schedule_id=sid, status=status)
|
||||
fire_id = await fire_for(sid, target_id, T0 + WEEK - timedelta(minutes=2))
|
||||
# 取消的不推送(有人手动停的)
|
||||
assert await webhook.notify_due() == (1 if status == "failed" else 0)
|
||||
assert (await fire_row(fire_id)).compare_artifact is None
|
||||
items = {f["id"]: f for f in (await client.get(f"/api/schedules/{sid}/fires")).json()["items"]}
|
||||
assert items[fire_id]["compare"] is None
|
||||
if status == "failed":
|
||||
raw = recorder.requests[0].content.decode()
|
||||
text = json.loads(raw)["content"]["text"] if fmt == "feishu" else json.loads(raw)["text"]
|
||||
assert "运行失败" in text and f"{BASE_URL}/runs/{target_id}" in text
|
||||
for absent in ("相比", "移除", "查看对比", "/runs/compare"):
|
||||
assert absent not in text, text
|
||||
if fmt == "generic":
|
||||
assert json.loads(raw)["compare"] is None
|
||||
|
||||
|
||||
async def test_after_a_failed_run_the_base_is_the_last_successful_one(recorder, client):
|
||||
"""上一次失败了:基准取再往前、成功的那一次——拿失败的那次当基准,每个指标都成了「新增」。紧挨着的上一次不是
|
||||
它,通知和触发历史写明比的是哪一次(短编号和发起的日期,按定时任务的时区)"""
|
||||
await public_address()
|
||||
workflow = await new_workflow()
|
||||
sid = await new_schedule(workflow)
|
||||
first = await persist(week("r1"), workflow, finished=T0, schedule_id=sid)
|
||||
await fire_for(sid, first, T0 - timedelta(minutes=2), notified="succeeded")
|
||||
failed = await persist(failed_week("r2"), workflow, finished=T0 + WEEK, schedule_id=sid, status="failed")
|
||||
await fire_for(sid, failed, T0 + WEEK - timedelta(minutes=2), notified="failed")
|
||||
target_id = await persist(week("r3", NEXT_ROWS), workflow, finished=T0 + 2 * WEEK, schedule_id=sid)
|
||||
fire_id = await fire_for(sid, target_id, T0 + 2 * WEEK - timedelta(minutes=2))
|
||||
|
||||
assert await webhook.notify_due() == 1
|
||||
stored = load((await fire_row(fire_id)).compare_artifact)
|
||||
assert stored["result"]["base"]["id"] == first and stored["previous"] is False
|
||||
[body] = recorder.bodies()
|
||||
# 第一次发起于北京时间 2026-09-07 08:58
|
||||
line = f"与上一次成功的运行(#{first[:6]},2026-09-07)相比:只数据变 2 个、口径与数据都变 0 个、SQL 结构变 0 个"
|
||||
assert line in body["text"] and "新增" not in body["text"], body["text"]
|
||||
assert f"base={first}" in body["text"] and failed not in body["text"]
|
||||
assert (body["compare"]["base_run_id"], body["compare"]["previous"]) == (first, False)
|
||||
items = {f["id"]: f for f in (await client.get(f"/api/schedules/{sid}/fires")).json()["items"]}
|
||||
assert items[fire_id]["compare"] == {"base_run_id": first, "summary": SUMMARY, "previous": False}
|
||||
|
||||
|
||||
async def test_base_follows_launch_order_not_finish_time(recorder):
|
||||
"""第 2 周失败、第 3 周照常成功之后,有人把第 2 周接着跑完:它结束得比第 3 周晚,但它的上一次仍是第 1 周。以前
|
||||
按结束时间挑,拿后一次触发的第 3 周当基准,「与上一次相比」的变化方向是反的。第 3 周已经比过的那一份不受影响"""
|
||||
workflow = await new_workflow()
|
||||
sid = await new_schedule(workflow, url=None)
|
||||
first = await persist(week("r1"), workflow, finished=T0, schedule_id=sid)
|
||||
await fire_for(sid, first, T0 - timedelta(minutes=2), notified="succeeded")
|
||||
second = await persist(failed_week("r2"), workflow, finished=T0 + WEEK, schedule_id=sid, status="failed")
|
||||
second_fire = await fire_for(sid, second, T0 + WEEK - timedelta(minutes=2))
|
||||
third = await persist(week("r3", NEXT_ROWS), workflow, finished=T0 + 2 * WEEK, schedule_id=sid)
|
||||
third_fire = await fire_for(sid, third, T0 + 2 * WEEK - timedelta(minutes=2))
|
||||
await webhook.notify_due()
|
||||
third_artifact = (await fire_row(third_fire)).compare_artifact
|
||||
assert load(third_artifact)["result"]["base"]["id"] == first
|
||||
|
||||
await reseal(second, week("r2", NEXT_ROWS), status="succeeded", finished=T0 + 2 * WEEK + timedelta(days=1))
|
||||
await webhook.notify_due()
|
||||
stored = load((await fire_row(second_fire)).compare_artifact)
|
||||
assert stored["result"]["base"]["id"] == first and stored["previous"] is True
|
||||
assert (await fire_row(third_fire)).compare_artifact == third_artifact
|
||||
|
||||
|
||||
async def test_first_run_of_a_schedule_has_no_compare(recorder, client):
|
||||
"""这个任务的第一次:同一工作流有更早的手动运行,对比接口会退回「同一工作流」那一档,那不是「上一次」,不比"""
|
||||
await public_address()
|
||||
workflow = await new_workflow()
|
||||
sid = await new_schedule(workflow)
|
||||
await persist(week("m1"), workflow, finished=T0)
|
||||
target_id = await persist(week("r2", NEXT_ROWS), workflow, finished=T0 + WEEK, schedule_id=sid)
|
||||
fire_id = await fire_for(sid, target_id, T0 + WEEK - timedelta(minutes=2))
|
||||
assert await webhook.notify_due() == 1
|
||||
assert (await fire_row(fire_id)).compare_artifact is None
|
||||
[item] = (await client.get(f"/api/schedules/{sid}/fires")).json()["items"]
|
||||
assert item["compare"] is None
|
||||
[body] = recorder.bodies()
|
||||
assert body["compare"] is None and "与上一次相比" not in body["text"] and "查看对比" not in body["text"]
|
||||
assert f"查看运行:{BASE_URL}/runs/{target_id}" in body["text"]
|
||||
|
||||
|
||||
async def test_waits_for_the_seal_before_comparing(recorder):
|
||||
"""runner 先写终态、再封存:两步之间被这一拍撞上就不比、不推,封存好了下一拍再比"""
|
||||
_, base_id, target_id, fire_id = await weekly_pair(target_seal=False, target_finished=datetime.now(UTC))
|
||||
assert await webhook.notify_due() == 0
|
||||
assert (await fire_row(fire_id)).compare_artifact is None and recorder.requests == []
|
||||
await seal(target_id)
|
||||
assert await webhook.notify_due() == 1
|
||||
assert load((await fire_row(fire_id)).compare_artifact)["result"]["base"]["id"] == base_id
|
||||
assert LINE in recorder.bodies()[0]["text"]
|
||||
|
||||
|
||||
async def test_no_public_address_means_counts_without_a_link(recorder):
|
||||
_, _, _, fire_id = await weekly_pair()
|
||||
assert await webhook.notify_due() == 1
|
||||
[body] = recorder.bodies()
|
||||
assert LINE in body["text"] and "查看对比" not in body["text"] and body["compare"]["url"] is None
|
||||
assert (await fire_row(fire_id)).compare_artifact
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 比不了、出错
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def break_seal(run_id: str) -> None:
|
||||
async with SessionLocal() as session:
|
||||
event = (await session.execute(select(RunEvent).where(
|
||||
RunEvent.run_id == run_id, RunEvent.type == "tool.end"))).scalars().first()
|
||||
event.data = {**event.data, "duration_ms": 999}
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def explode(*args, **kwargs):
|
||||
raise RuntimeError("快照读取失败:磁盘错误")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["broken_seal", "unexpected"])
|
||||
async def test_compare_failure_only_logs_and_the_notice_still_goes_out(recorder, caplog, monkeypatch, failure):
|
||||
"""基准的封存核对不一致(对比接口回 409)、对比中途抛了意外的异常:只记日志,通知照常推送、不带对比"""
|
||||
await public_address()
|
||||
_, base_id, target_id, fire_id = await weekly_pair()
|
||||
if failure == "broken_seal":
|
||||
await break_seal(base_id)
|
||||
else:
|
||||
from app.api import compare as compare_api
|
||||
|
||||
monkeypatch.setattr(compare_api, "compare_runs", explode)
|
||||
with caplog.at_level(logging.INFO, logger="app.scheduler"):
|
||||
assert await webhook.notify_due() == 1
|
||||
assert (await fire_row(fire_id)).compare_artifact is None
|
||||
[body] = recorder.bodies()
|
||||
assert (body["event"], body["run_id"], body["compare"]) == ("finished", target_id, None)
|
||||
assert "与上一次相比" not in body["text"] and f"{BASE_URL}/runs/{target_id}" in body["text"]
|
||||
assert "自动对比" in caplog.text and target_id in caplog.text
|
||||
if failure == "broken_seal":
|
||||
assert "封存核对不一致" in caplog.text
|
||||
|
||||
|
||||
async def test_a_failed_run_is_compared_once_it_is_continued_to_success(client):
|
||||
"""失败时不比;接着跑(不在终态)时也没有;跑完、成功并重新封存之后按新的封存比一次"""
|
||||
workflow = await new_workflow()
|
||||
sid = await new_schedule(workflow, url=None)
|
||||
base_id = await persist(week("r1"), workflow, finished=T0, schedule_id=sid)
|
||||
await fire_for(sid, base_id, T0 - timedelta(minutes=2), notified="succeeded")
|
||||
target_id = await persist(failed_week("r2"), workflow, finished=T0 + WEEK, schedule_id=sid, status="failed")
|
||||
fire_id = await fire_for(sid, target_id, T0 + WEEK - timedelta(minutes=2))
|
||||
|
||||
async def shown():
|
||||
items = (await client.get(f"/api/schedules/{sid}/fires")).json()["items"]
|
||||
return {f["id"]: f for f in items}[fire_id]
|
||||
|
||||
await webhook.notify_due()
|
||||
assert (await fire_row(fire_id)).compare_artifact is None and (await shown())["compare"] is None
|
||||
await set_run(target_id, status="running")
|
||||
await webhook.notify_due()
|
||||
item = await shown()
|
||||
assert item["run_status"] == "running" and item["compare"] is None
|
||||
await reseal(target_id, week("r2", NEXT_ROWS), status="succeeded", finished=T0 + WEEK + timedelta(hours=1))
|
||||
await webhook.notify_due()
|
||||
assert (await shown())["compare"] == {"base_run_id": base_id, "summary": SUMMARY, "previous": True}
|
||||
|
||||
|
||||
async def test_compare_is_hidden_once_the_base_run_is_gone_or_running(client):
|
||||
"""基准那一次被删掉、或者被继续执行(不在终态):对比页比不了,链过去只会报错,接口同样给 null"""
|
||||
sid, base_id, _, fire_id = await weekly_pair(url=None)
|
||||
await webhook.notify_due()
|
||||
assert (await fire_row(fire_id)).compare_artifact
|
||||
|
||||
async def shown():
|
||||
items = (await client.get(f"/api/schedules/{sid}/fires")).json()["items"]
|
||||
return {f["id"]: f for f in items}[fire_id]["compare"]
|
||||
|
||||
assert (await shown())["base_run_id"] == base_id
|
||||
await set_run(base_id, status="running")
|
||||
assert await shown() is None
|
||||
await set_run(base_id, status="succeeded")
|
||||
assert (await shown())["base_run_id"] == base_id
|
||||
# 封存过的正式运行要强制删除
|
||||
assert (await client.delete(f"/api/runs/{base_id}", params={"force": "true"})).status_code == 204
|
||||
assert await shown() is None
|
||||
|
||||
|
||||
def unreadable(artifact_id):
|
||||
raise PermissionError(13, "Permission denied")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("damage", ["missing", "tampered", "unreadable"])
|
||||
async def test_a_damaged_artifact_reads_as_no_compare(client, monkeypatch, caplog, damage):
|
||||
"""工件文件不在了、被改过、读不出来(权限、磁盘出错):接口给 null,触发历史照常打开,不回 500"""
|
||||
sid, _, _, fire_id = await weekly_pair(url=None)
|
||||
await webhook.notify_due()
|
||||
artifact = (await fire_row(fire_id)).compare_artifact
|
||||
# 刚比完的摘要记在进程里,清掉才会去读盘
|
||||
auto_compare._BRIEFS.clear()
|
||||
if damage == "missing":
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(ScheduleFire.__table__.update().where(ScheduleFire.id == fire_id)
|
||||
.values(compare_artifact="0" * 64))
|
||||
await session.commit()
|
||||
elif damage == "tampered":
|
||||
path = artifact_store._path_of(artifact)
|
||||
text = path.read_text(encoding="utf-8")
|
||||
assert '"data_only":2' in text
|
||||
path.write_text(text.replace('"data_only":2', '"data_only":7'), encoding="utf-8")
|
||||
else:
|
||||
monkeypatch.setattr(artifact_store, "load", unreadable)
|
||||
with caplog.at_level(logging.WARNING, logger="app.scheduler"):
|
||||
resp = await client.get(f"/api/schedules/{sid}/fires")
|
||||
assert resp.status_code == 200
|
||||
assert {f["id"]: f for f in resp.json()["items"]}[fire_id]["compare"] is None
|
||||
if damage != "missing":
|
||||
assert artifact[:12] in caplog.text
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 计数摘要的写法
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def counts(**over) -> dict[str, int]:
|
||||
return {**{k: 0 for k in SUMMARY}, **over}
|
||||
|
||||
|
||||
def test_summary_text_always_names_the_three_kinds_of_change():
|
||||
assert auto_compare.summary_text(counts(data_only=3, caliber_and_data=1)) == \
|
||||
"只数据变 3 个、口径与数据都变 1 个、SQL 结构变 0 个"
|
||||
# 其余各类有才写,次序同对比页
|
||||
assert auto_compare.summary_text(counts(sql_changed=1, params_only=4, added=1, removed=2, incomparable=1,
|
||||
unchanged=5)) == \
|
||||
"只数据变 0 个、口径与数据都变 0 个、SQL 结构变 1 个、只参数变 4 个、新增 1 个、移除 2 个、无法比较 1 个、未变 5 个"
|
||||
# 两边都没有指标:不写一串 0 个,免得读成「什么都没变」
|
||||
assert auto_compare.summary_text(counts()) == "两次运行都没有指标"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fmt, url, link_text", [
|
||||
("generic", GENERIC, "查看对比:{link}"),
|
||||
("feishu", FEISHU, "查看对比:{link}"),
|
||||
("wecom", "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=0a1b2c3d-1111-2222-3333-444455556666",
|
||||
"[查看对比]({link})"),
|
||||
("dingtalk", "https://oapi.dingtalk.com/robot/send?access_token=0123456789abcdef0123456789abcdef",
|
||||
"[查看对比]({link})"),
|
||||
])
|
||||
def test_every_format_carries_the_counts_and_the_compare_link(fmt, url, link_text):
|
||||
link = f"{BASE_URL}/runs/compare?target=r2&base=r1"
|
||||
notice = webhook.Notice(event="finished", schedule_id="s1", schedule_name="区域周报", timezone="Asia/Shanghai",
|
||||
run_id="r2", status="succeeded", run_url=f"{BASE_URL}/runs/r2",
|
||||
compare={"base_run_id": "r1", "summary": counts(data_only=3, caliber_and_data=1)},
|
||||
compare_url=link)
|
||||
_, _, body = webhook.build_request(fmt, url, None, notice)
|
||||
# 请求体是 ensure_ascii=False 的 JSON:中文、链接原样出现在里面
|
||||
text = body.decode()
|
||||
assert "与上一次相比:只数据变 3 个、口径与数据都变 1 个、SQL 结构变 0 个" in text
|
||||
assert link_text.format(link=link) in text
|
||||
# 运行链接照旧在前
|
||||
assert text.index("/runs/r2") < text.index("/runs/compare")
|
||||
@@ -0,0 +1,312 @@
|
||||
"""定时运行自动对比的几道保护(app/scheduler/auto_compare.py、webhook.notify_due):
|
||||
|
||||
- 和封存绑定:运行的封存要盖住它最后一条结束事件才算封存完。runner 先写终态、再另起会话封存,失败后接着跑完的运行
|
||||
在两步之间还带着失败时的清单哈希——这时去比,只比得到旧封存范围里的事件,通知里的清单哈希也是旧的。等不来新的
|
||||
封存(两步之间进程没了)时照常推送,不带对比、不写旧的哈希
|
||||
- 对比结果记下两边的清单哈希:任何一边后来变了,触发历史就不再给这份对比(悬停的计数和点进去的对比页对不上);
|
||||
对比途中封存变了,结果不记
|
||||
- 时限:对比放在认领之前、锁外做,最多等 WAIT_S 秒,同一时刻最多 SLOTS 个。慢的对比不压住这条通知,也不压住别的
|
||||
定时任务的通知;扫描在对比途中被取消(停机),这次触发还没认领,下次照常比、照常推。同一份封存已经比过的(比完
|
||||
还没推送就停机了),下次直接用记下的结果
|
||||
|
||||
运行、任务名、金额都是虚构的(tests/fixtures/compare_runs.py),不真的执行;Webhook 用 httpx.MockTransport 接住。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy import select
|
||||
|
||||
from app import scheduler
|
||||
from app.api import compare as compare_api
|
||||
from app.core.artifact_store import load
|
||||
from app.core.config import Settings
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Run, RunEvent
|
||||
from app.main import app
|
||||
from app.scheduler import auto_compare, webhook
|
||||
from tests.fixtures.compare_runs import BASE_ROWS, NEXT_ROWS, Fake, default_metrics, region_sql, week
|
||||
from tests.test_scheduler_compare import ( # noqa: F401 - clean、recorder 是夹具,按名字引入
|
||||
BASE_URL, LINE, T0, WEEK, Recorder, clean, fire_for, fire_row, new_schedule, new_workflow, persist,
|
||||
public_address, recorder, reseal, seal, set_run, weekly_pair,
|
||||
)
|
||||
|
||||
UTC = timezone.utc
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def slow_compare(monkeypatch):
|
||||
"""对比要 5 秒:真的对比之前先等着。返回原来的函数,用例里可以换回去"""
|
||||
real = compare_api.compare_runs
|
||||
|
||||
async def slow(*args, **kwargs):
|
||||
await asyncio.sleep(5)
|
||||
return await real(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(compare_api, "compare_runs", slow)
|
||||
return real
|
||||
|
||||
|
||||
def failed_then_continued(run_id: str, rows=NEXT_ROWS) -> tuple[Fake, int]:
|
||||
"""在 Agent 节点失败、之后从断点接着跑完的一次周报。返回 (运行, 失败时封存到第几条事件)。"""
|
||||
run = Fake(run_id)
|
||||
sql = region_sql()
|
||||
fetch = run.tool("fetch", sql, rows=rows)
|
||||
run.add("node.started", "bot", {"node_type": "agent"})
|
||||
run.add("node.failed", "bot", {"error": "模型接入超时"})
|
||||
run.add("run.failed", None, {"error": "节点「区域分析」失败"})
|
||||
failed_at = len(run.events)
|
||||
run.add("run.resumed", None, {"from": ["bot"], "message": "从「区域分析」继续运行"})
|
||||
bot = run.agent("bot", sql, rows=rows)
|
||||
run.card(default_metrics(fetch, bot, rows, rows))
|
||||
run.add("run.finished", None, {"output": {}})
|
||||
return run, failed_at
|
||||
|
||||
|
||||
async def previous_week(*, url: str | None = None):
|
||||
"""一个定时任务和它上一次成功、已推送过的周报。返回 (工作流, 任务, 上一次)。"""
|
||||
workflow = await new_workflow()
|
||||
sid = await new_schedule(workflow, url=url) if url else await new_schedule(workflow)
|
||||
base_id = await persist(week("r1"), workflow, finished=T0, schedule_id=sid)
|
||||
await fire_for(sid, base_id, T0 - timedelta(minutes=2), notified="succeeded")
|
||||
return workflow, sid, base_id
|
||||
|
||||
|
||||
async def seal_of(run_id: str) -> str | None:
|
||||
async with SessionLocal() as session:
|
||||
return (await session.get(Run, run_id)).manifest_hash
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 和封存绑定
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_waits_for_the_seal_of_this_ending_not_the_failed_one(recorder):
|
||||
"""失败后接着跑完:终态和结束时间已经写下,封存还是失败时那一份(只盖到 run.failed)。这一拍不比、不推,新的
|
||||
封存补上之后再比,比的是完整的一次,通知里的清单哈希是新的"""
|
||||
await public_address()
|
||||
workflow, sid, base_id = await previous_week()
|
||||
fake, failed_at = failed_then_continued("r2")
|
||||
target_id = await persist(fake, workflow, finished=datetime.now(UTC), schedule_id=sid, seal_upto=failed_at)
|
||||
fire_id = await fire_for(sid, target_id, T0 + WEEK - timedelta(minutes=2))
|
||||
stale = await seal_of(target_id)
|
||||
|
||||
assert await webhook.notify_due() == 0
|
||||
assert recorder.requests == [] and (await fire_row(fire_id)).compare_artifact is None
|
||||
await seal(target_id)
|
||||
fresh = await seal_of(target_id)
|
||||
assert fresh != stale
|
||||
assert await webhook.notify_due() == 1
|
||||
stored = load((await fire_row(fire_id)).compare_artifact)
|
||||
assert (stored["target_manifest"], stored["result"]["base"]["id"]) == (fresh, base_id)
|
||||
[body] = recorder.bodies()
|
||||
assert LINE in body["text"] and body["manifest_hash"] == fresh[:12]
|
||||
|
||||
|
||||
async def test_a_seal_that_never_catches_up_is_not_used(recorder):
|
||||
"""新的封存一直没补上(写完终态、封存之前进程没了):等过时限照常推送,但不比,也不写失败时那份旧的清单哈希"""
|
||||
await public_address()
|
||||
workflow, sid, _ = await previous_week()
|
||||
fake, failed_at = failed_then_continued("r2")
|
||||
target_id = await persist(fake, workflow, finished=datetime.now(UTC), schedule_id=sid, seal_upto=failed_at)
|
||||
fire_id = await fire_for(sid, target_id, T0 + WEEK - timedelta(minutes=2))
|
||||
later = datetime.now(UTC) + timedelta(seconds=webhook.SEAL_WAIT_S + 5)
|
||||
assert await webhook.notify_due(later) == 1
|
||||
[body] = recorder.bodies()
|
||||
assert (body["event"], body["manifest_hash"], body["compare"]) == ("finished", None, None)
|
||||
assert "清单哈希" not in body["text"] and "相比" not in body["text"]
|
||||
assert (await fire_row(fire_id)).compare_artifact is None
|
||||
|
||||
|
||||
async def test_approval_notice_does_not_carry_the_seal_of_an_earlier_failure(recorder):
|
||||
"""失败后接着跑、停在审批上:运行上还留着失败时的清单哈希,那不是这一次的封存,「等待审批」里不写"""
|
||||
workflow, sid, _ = await previous_week()
|
||||
fake, failed_at = failed_then_continued("r2")
|
||||
target_id = await persist(fake, workflow, finished=T0 + WEEK, schedule_id=sid, status="interrupted",
|
||||
seal_upto=failed_at)
|
||||
await set_run(target_id, finished_at=None)
|
||||
async with SessionLocal() as session:
|
||||
from app.db.models import Approval
|
||||
|
||||
session.add(Approval(run_id=target_id, node_id="card", title="审核周报口径", status="pending"))
|
||||
await session.commit()
|
||||
await fire_for(sid, target_id, T0 + WEEK - timedelta(minutes=2))
|
||||
assert await webhook.notify_due() == 1
|
||||
[body] = recorder.bodies()
|
||||
assert (body["event"], body["manifest_hash"]) == ("awaiting_approval", None)
|
||||
assert "清单哈希" not in body["text"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("side", ["target", "base"])
|
||||
async def test_fire_history_drops_a_compare_whose_seal_changed(client, side):
|
||||
"""比过之后哪一边被重新封存(内容变了):记下的计数已经不是对比页现场算出来的那样,触发历史不再给这份对比"""
|
||||
sid, base_id, target_id, fire_id = await weekly_pair(url=None)
|
||||
await webhook.notify_due()
|
||||
|
||||
async def shown():
|
||||
items = (await client.get(f"/api/schedules/{sid}/fires")).json()["items"]
|
||||
return {f["id"]: f for f in items}[fire_id]["compare"]
|
||||
|
||||
assert (await shown())["base_run_id"] == base_id
|
||||
# 换成和另一边一样的数:现场再比就是「未变」
|
||||
run_id, rows = (target_id, BASE_ROWS) if side == "target" else (base_id, NEXT_ROWS)
|
||||
async with SessionLocal() as session:
|
||||
finished = (await session.get(Run, run_id)).finished_at
|
||||
await reseal(run_id, week("rx", rows), status="succeeded", finished=finished)
|
||||
assert await shown() is None
|
||||
# 对比页现场算出来的和记下的确实不一样
|
||||
assert (await compare_api.compare_runs(target_id, base_id))["summary"]["unchanged"] == 2
|
||||
|
||||
|
||||
async def test_result_is_dropped_when_a_seal_changes_mid_compare(recorder, monkeypatch):
|
||||
"""对比途中基准被重新封存:比出来的是旧封存的结果,不记进触发记录、不写进通知"""
|
||||
await public_address()
|
||||
_, base_id, _, fire_id = await weekly_pair()
|
||||
real = compare_api.compare_runs
|
||||
|
||||
async def resealing(run_id, other):
|
||||
out = await real(run_id, other)
|
||||
await set_run(base_id, manifest_hash="ab" * 32)
|
||||
return out
|
||||
|
||||
monkeypatch.setattr(compare_api, "compare_runs", resealing)
|
||||
assert await webhook.notify_due() == 1
|
||||
assert (await fire_row(fire_id)).compare_artifact is None
|
||||
[body] = recorder.bodies()
|
||||
assert body["compare"] is None and "相比" not in body["text"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 时限、取消、并发
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_slow_compare_is_cut_off_and_the_notice_goes_out(recorder, monkeypatch, caplog, slow_compare):
|
||||
"""对比超过时限:通知不等它,照常推送、不带对比,记一条警告"""
|
||||
monkeypatch.setattr(auto_compare, "WAIT_S", 0.3, raising=False)
|
||||
await public_address()
|
||||
_, _, target_id, fire_id = await weekly_pair()
|
||||
began = time.monotonic()
|
||||
with caplog.at_level(logging.WARNING, logger="app.scheduler"):
|
||||
assert await webhook.notify_due() == 1
|
||||
assert time.monotonic() - began < 2
|
||||
[body] = recorder.bodies()
|
||||
assert (body["event"], body["compare"]) == ("finished", None) and "相比" not in body["text"]
|
||||
assert f"{BASE_URL}/runs/{target_id}" in body["text"]
|
||||
assert (await fire_row(fire_id)).compare_artifact is None
|
||||
assert "超过" in caplog.text and target_id in caplog.text
|
||||
|
||||
|
||||
async def test_scan_cancelled_mid_compare_pushes_next_time(recorder, monkeypatch, slow_compare):
|
||||
"""停机时通知扫描只等 1.5 秒就被取消:撞在对比途中的那次触发还没认领,下次启动照常比、照常推。以前先认领、后比,
|
||||
取消之后这条「运行结束」再也不会推"""
|
||||
await public_address()
|
||||
_, _, _, fire_id = await weekly_pair()
|
||||
with pytest.raises(TimeoutError):
|
||||
await asyncio.wait_for(webhook.notify_due(), timeout=0.5)
|
||||
assert (await fire_row(fire_id)).notified_status is None and recorder.requests == []
|
||||
monkeypatch.setattr(compare_api, "compare_runs", slow_compare)
|
||||
assert await webhook.notify_due() == 1
|
||||
[body] = recorder.bodies()
|
||||
assert LINE in body["text"]
|
||||
assert (await fire_row(fire_id)).notified_status == "succeeded"
|
||||
|
||||
|
||||
async def test_a_slow_compare_does_not_hold_other_schedules(monkeypatch, slow_compare):
|
||||
"""任务 A 的对比慢:调度循环里通知扫描同一时刻只跑一份,以前 A 比多久,之后才结束的任务 B(第一次,不用比)
|
||||
的通知就要等多久。现在 A 的对比等过时限就不等了,B 的通知在下一拍推出"""
|
||||
rec = Recorder()
|
||||
monkeypatch.setattr(webhook, "_transport", httpx.MockTransport(rec))
|
||||
monkeypatch.setattr(webhook, "RETRY_DELAYS", (0, 0, 0))
|
||||
monkeypatch.setattr(auto_compare, "WAIT_S", 0.5, raising=False)
|
||||
await weekly_pair()
|
||||
on = Settings(_env_file=None, scheduler=True)
|
||||
assert await scheduler.start(interval=0.05, config=on) is True
|
||||
try:
|
||||
await asyncio.sleep(0.2)
|
||||
workflow = await new_workflow("仓库日报")
|
||||
sid = await new_schedule(workflow, name="仓库日报 · 每天")
|
||||
run_id = await persist(week("b1"), workflow, finished=T0, schedule_id=sid)
|
||||
await fire_for(sid, run_id, T0 - timedelta(minutes=2))
|
||||
began = time.monotonic()
|
||||
while not any(b.get("run_id") == run_id for b in rec.bodies()) and time.monotonic() - began < 8:
|
||||
await asyncio.sleep(0.02)
|
||||
waited = time.monotonic() - began
|
||||
finally:
|
||||
await scheduler.stop()
|
||||
assert any(b.get("run_id") == run_id for b in rec.bodies())
|
||||
assert waited < 2, f"任务 B 的通知等了 {waited:.2f} 秒"
|
||||
|
||||
|
||||
async def test_compares_take_turns(monkeypatch):
|
||||
"""同一拍里好几个任务都要比:同一时刻最多 SLOTS 个,不一起占满线程池(接口请求也要用)"""
|
||||
monkeypatch.setattr(auto_compare, "SLOTS", 2, raising=False)
|
||||
real = compare_api.compare_runs
|
||||
active = peak = 0
|
||||
|
||||
async def counted(*args, **kwargs):
|
||||
nonlocal active, peak
|
||||
active += 1
|
||||
peak = max(peak, active)
|
||||
try:
|
||||
await asyncio.sleep(0.2)
|
||||
return await real(*args, **kwargs)
|
||||
finally:
|
||||
active -= 1
|
||||
|
||||
monkeypatch.setattr(compare_api, "compare_runs", counted)
|
||||
fires = [(await weekly_pair(url=None))[3] for _ in range(4)]
|
||||
await webhook.notify_due()
|
||||
assert peak == 2
|
||||
assert all([(await fire_row(f)).compare_artifact for f in fires])
|
||||
|
||||
|
||||
async def test_a_compare_already_made_for_this_seal_is_reused(recorder, monkeypatch):
|
||||
"""比完、还没认领就停机了,结果已经记在触发记录上:下次直接用,不再算一遍"""
|
||||
await public_address()
|
||||
_, _, target_id, fire_id = await weekly_pair()
|
||||
assert await auto_compare.compare_fire(fire_id, target_id) is not None
|
||||
artifact = (await fire_row(fire_id)).compare_artifact
|
||||
assert artifact
|
||||
calls = 0
|
||||
|
||||
async def counted(*args, **kwargs):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
raise RuntimeError("不该再算一遍")
|
||||
|
||||
monkeypatch.setattr(compare_api, "compare_runs", counted)
|
||||
auto_compare._BRIEFS.clear()
|
||||
assert await webhook.notify_due() == 1
|
||||
assert calls == 0 and LINE in recorder.bodies()[0]["text"]
|
||||
assert (await fire_row(fire_id)).compare_artifact == artifact
|
||||
|
||||
|
||||
async def test_seal_covers_the_last_ending():
|
||||
"""封存盖没盖住最后一条结束事件:老运行(没有 manifest_seq)只看有没有哈希"""
|
||||
workflow = await new_workflow()
|
||||
fake, failed_at = failed_then_continued("r9")
|
||||
partial = await persist(fake, workflow, finished=T0, seal_upto=failed_at)
|
||||
full = await persist(failed_then_continued("r8")[0], workflow, finished=T0)
|
||||
legacy = await persist(week("r7"), workflow, finished=T0)
|
||||
await set_run(legacy, manifest_seq=None)
|
||||
unsealed = await persist(week("r6"), workflow, finished=T0, seal=False)
|
||||
async with SessionLocal() as session:
|
||||
got = {rid: await auto_compare.sealed_now(session, await session.get(Run, rid))
|
||||
for rid in (partial, full, legacy, unsealed)}
|
||||
assert await session.scalar(select(RunEvent.seq).where(
|
||||
RunEvent.run_id == partial, RunEvent.type == "run.finished")) > failed_at
|
||||
assert got == {partial: False, full: True, legacy: True, unsealed: False}
|
||||
@@ -369,9 +369,20 @@ async def test_start_does_nothing_when_switched_off():
|
||||
await scheduler.stop()
|
||||
|
||||
|
||||
async def test_loop_ticks_until_stopped():
|
||||
# 下一次放在整分上(和 cron 对齐):放在「现在 - 1 秒」的话,这一秒里恰好跨过整分时会多记一条错过的跳过
|
||||
async def fire_count(schedule_id: str) -> int:
|
||||
async with SessionLocal() as session:
|
||||
return await session.scalar(select(func.count()).select_from(ScheduleFire)
|
||||
.where(ScheduleFire.schedule_id == schedule_id))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("crossed", [False, True], ids=["on_the_minute", "minute_crossed"])
|
||||
async def test_loop_ticks_until_stopped(crossed):
|
||||
"""循环按拍执行、stop 之后不再有拍。只看「发起了」和停下前后的条数差,不看总条数:建任务和第一拍之间恰好跨过
|
||||
整分时,第一拍会先记一条错过的跳过、再发起最近这一次——以前按总条数断言,跨整分时偶发失败。crossed 把下一次
|
||||
放在上一个整分,必然走到这一种"""
|
||||
due = cron.last_due("* * * * *", "UTC", datetime.now(UTC))
|
||||
if crossed:
|
||||
due -= timedelta(minutes=1)
|
||||
sid = await schedule(await workflow(), next_run_at=due, cron="* * * * *")
|
||||
on = Settings(_env_file=None, scheduler=True)
|
||||
assert await scheduler.start(interval=0.05, config=on) is True
|
||||
@@ -382,18 +393,20 @@ async def test_loop_ticks_until_stopped():
|
||||
await asyncio.sleep(0.02)
|
||||
await scheduler.stop()
|
||||
assert scheduler.running() is False
|
||||
# 只认发起的那一条:建任务和第一拍之间跨过整分时,前面还会有一条错过的跳过,那不是这里要测的
|
||||
[fire] = [f for f in await fires(sid) if f.status == "started"]
|
||||
await settle(fire.run_id)
|
||||
# 停下之后不再有拍:把下一次拨回过去也不会再发起
|
||||
started = [f for f in await fires(sid) if f.status == "started"]
|
||||
assert started
|
||||
if crossed:
|
||||
assert [f.status for f in await fires(sid)][0] == "skipped_misfire"
|
||||
for fire in started:
|
||||
await settle(fire.run_id)
|
||||
# 停下之后不再有拍:把下一次拨回过去,过一会儿条数一条不多
|
||||
before = await fire_count(sid)
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(Schedule.__table__.update().where(Schedule.id == sid)
|
||||
.values(next_run_at=datetime.now(UTC) - timedelta(seconds=1)))
|
||||
await session.commit()
|
||||
await asyncio.sleep(0.2)
|
||||
async with SessionLocal() as session:
|
||||
assert await session.scalar(select(func.count()).select_from(ScheduleFire)
|
||||
.where(ScheduleFire.schedule_id == sid)) == 1
|
||||
assert await fire_count(sid) == before
|
||||
|
||||
|
||||
async def test_start_twice_keeps_one_loop():
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
"""定时任务通知(app/scheduler/webhook.py)的两条规则:
|
||||
|
||||
- 失败通知的「处理」行不许诺能继续运行:服务重启时还在排队、从没开始执行的运行(被改成失败、没有开始时间)没有
|
||||
断点,「修正后可继续运行」是错的。按有没有开始执行分开写——没开始的另写一行原因(没有可继续的进度),处理行不再
|
||||
让人去「查看原因」(原因刚写过),任务还在排期上时只写「下一次触发会照常发起」,不在排期上时写怎么重新发起;开始
|
||||
执行过的写「请在 AgentLab 的「记录」中查看失败原因」(和挂起那句一样指到导航里的「记录」),任务还在排期上时补
|
||||
「下一次触发会照常发起」:失败的运行不挡之后的触发,和挂起的那种(之后的触发都会跳过)说清区别
|
||||
- 扫描窗口(SCAN_DAYS)之外的触发:停在审批、有待审批的运行不受窗口限制——创建 30 天以后才停到审批上的照样推送
|
||||
「等待审批」;运行记录在窗口内有变化的(等了一个多月的审批批完、跑完)照样推送结束。早就结束、记录多日没有变化
|
||||
的旧触发仍不扫
|
||||
|
||||
用 httpx.MockTransport 接住请求,不连外网。任务名、地址、数字都是虚构的。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from sqlalchemy import delete
|
||||
|
||||
from app.core.crypto import encrypt
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Approval, Run, RunEvent, Schedule, ScheduleFire, Setting
|
||||
from app.scheduler import webhook
|
||||
|
||||
UTC = timezone.utc
|
||||
GENERIC = "https://hooks.example.com/agentlab/inbox-3e9d"
|
||||
NOT_STARTED = "运行在开始执行前就已中止(通常是服务重启时仍在排队),没有可继续的进度"
|
||||
LOOK = "请在 AgentLab 的「记录」中查看失败原因"
|
||||
RERUN = "如需重新发起,请在 AgentLab 的「定时任务」中点击「立即运行」"
|
||||
|
||||
|
||||
class Recorder:
|
||||
def __init__(self) -> None:
|
||||
self.requests: list[httpx.Request] = []
|
||||
|
||||
def __call__(self, request: httpx.Request) -> httpx.Response:
|
||||
self.requests.append(request)
|
||||
return httpx.Response(200, json={})
|
||||
|
||||
def bodies(self) -> list[dict]:
|
||||
return [json.loads(r.content) for r in self.requests]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def recorder(monkeypatch):
|
||||
rec = Recorder()
|
||||
monkeypatch.setattr(webhook, "_transport", httpx.MockTransport(rec))
|
||||
monkeypatch.setattr(webhook, "RETRY_DELAYS", (0, 0, 0))
|
||||
return rec
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def clean():
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(delete(Schedule))
|
||||
await session.execute(delete(Setting).where(Setting.key == "notify"))
|
||||
await session.commit()
|
||||
yield
|
||||
|
||||
|
||||
def notice(**over) -> webhook.Notice:
|
||||
base = dict(event="failed", schedule_id="s1", schedule_name="仓库日报", timezone="Asia/Shanghai",
|
||||
fire_id="f1", fire_at=datetime(2026, 10, 5, 1, 0, tzinfo=UTC), fire_kind="scheduled",
|
||||
run_id="r9876543210fedcba", status="failed", run_url=None, manifest_hash=None,
|
||||
started_at=datetime(2026, 10, 5, 1, 0, 2, tzinfo=UTC), finished_at=None)
|
||||
base.update(over)
|
||||
return webhook.Notice(**base)
|
||||
|
||||
|
||||
async def make(*, run_status: str, started: bool = True, fire_age: timedelta = timedelta(0),
|
||||
run_age: timedelta = timedelta(0), enabled: bool = True, sealed: bool = True) -> tuple[str, str, str]:
|
||||
"""一个配了通用 Webhook 的任务、一次触发和它发起的运行(不真的执行,状态直接写库)。fire_age、run_age 是触发
|
||||
记录、运行记录最后一次更新离现在多久;started=False 是从没开始执行的(服务重启时还在排队)。"""
|
||||
now = datetime.now(UTC)
|
||||
async with SessionLocal() as session:
|
||||
sched = Schedule(name="仓库日报", workflow_id="wf-demo", cron="0 9 * * *", timezone="Asia/Shanghai",
|
||||
enabled=enabled, next_run_at=now + timedelta(hours=3) if enabled else None,
|
||||
webhook_format="generic", webhook_url=encrypt(GENERIC),
|
||||
webhook_events=["finished", "failed", "awaiting_approval"])
|
||||
session.add(sched)
|
||||
await session.flush()
|
||||
ended = run_status in ("succeeded", "failed", "cancelled") and started
|
||||
touched = now - run_age
|
||||
run = Run(workflow_id=None, workflow_name="仓库日报", status=run_status, schedule_id=sched.id,
|
||||
run_class="formal", started_at=touched - timedelta(minutes=1) if started else None,
|
||||
finished_at=touched if ended else None, manifest_hash=("d00d" * 16) if sealed and ended else None,
|
||||
created_at=now - fire_age, updated_at=touched,
|
||||
error="服务重启时这次运行仍在排队,没有可恢复的进度,请重新发起运行" if not started else None)
|
||||
session.add(run)
|
||||
await session.flush()
|
||||
fire = ScheduleFire(schedule_id=sched.id, fire_at=(now - fire_age).replace(microsecond=0), kind="scheduled",
|
||||
status="started", run_id=run.id, created_at=now - fire_age)
|
||||
session.add(fire)
|
||||
await session.commit()
|
||||
return sched.id, fire.id, run.id
|
||||
|
||||
|
||||
async def notified(fire_id: str) -> str | None:
|
||||
async with SessionLocal() as session:
|
||||
return (await session.get(ScheduleFire, fire_id)).notified_status
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 失败通知的「处理」行
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_run_that_never_started_is_not_promised_a_continue():
|
||||
"""服务重启时还在排队的运行没有断点:不写「修正后可继续运行」,另写一行原因说没有可继续的进度。处理行不再让人
|
||||
去「查看原因」(原因刚写过):还在排期上的只说下一次照常,不在排期上的说怎么重新发起"""
|
||||
_, lines, _ = webhook.message(notice(started_at=None, next_fire=True))
|
||||
shown = dict(lines)
|
||||
assert shown["原因"] == NOT_STARTED
|
||||
assert shown["处理"] == "下一次触发会照常发起"
|
||||
assert not any("继续运行" in value or "查看" in value for _, value in lines), lines
|
||||
_, lines, _ = webhook.message(notice(started_at=None, next_fire=False))
|
||||
assert dict(lines)["处理"] == RERUN
|
||||
|
||||
|
||||
def test_a_failed_run_gets_a_neutral_action_without_promising_a_continue():
|
||||
"""开始执行过的运行也不许诺能继续:断点可能没有(执行第一个节点前就失败)、可能已不兼容(系统升级),能不能继续
|
||||
以记录页为准。失败节点照写,不另写原因(原始报错可能带着查询到的数据)"""
|
||||
_, lines, _ = webhook.message(notice(error_node="汇总库存", next_fire=True))
|
||||
shown = dict(lines)
|
||||
assert shown["失败节点"] == "「汇总库存」" and "原因" not in shown
|
||||
assert shown["处理"] == f"{LOOK};下一次触发会照常发起"
|
||||
assert "修正后可继续运行" not in webhook._text(notice(error_node="汇总库存"))
|
||||
|
||||
|
||||
def test_next_fire_is_only_promised_while_the_schedule_is_on():
|
||||
"""任务已暂停(或往后不再触发)时不说「下一次触发会照常发起」"""
|
||||
_, lines, _ = webhook.message(notice(next_fire=False))
|
||||
assert dict(lines)["处理"] == LOOK
|
||||
|
||||
|
||||
async def test_restart_failure_notice_end_to_end(recorder):
|
||||
"""服务重启时还在排队、被改成失败的运行:不会封存,等过封存时限后推一次失败,原因、处理按上面写;通用格式的
|
||||
reason 字段也写这句原因。任务停用时不补「下一次触发会照常发起」"""
|
||||
_, on_fire, _ = await make(run_status="failed", started=False, sealed=False)
|
||||
_, off_fire, _ = await make(run_status="failed", started=False, sealed=False, enabled=False)
|
||||
later = datetime.now(UTC) + timedelta(seconds=webhook.SEAL_WAIT_S + 5)
|
||||
assert await webhook.notify_due(later) == 2
|
||||
by_fire = {b["fire_id"]: b for b in recorder.bodies()}
|
||||
on, off = by_fire[on_fire], by_fire[off_fire]
|
||||
assert (on["event"], on["status"], on["reason"]) == ("failed", "failed", NOT_STARTED)
|
||||
assert "处理:下一次触发会照常发起" in on["text"]
|
||||
assert f"处理:{RERUN}" in off["text"] and "照常发起" not in off["text"]
|
||||
for body in (on, off):
|
||||
assert "继续运行" not in body["text"], body["text"]
|
||||
# 运行记录上的原始报错不进通知
|
||||
assert "没有可恢复的进度" not in body["text"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 扫描窗口之外
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_old_fire_stopped_at_an_approval_is_still_pushed(recorder):
|
||||
"""触发是 40 天前的,运行刚停到审批上(或者停下时服务没在推送):有待审批的运行不受 30 天的窗口限制。
|
||||
运行记录本身也很久没有变化,照样推「等待审批」"""
|
||||
_, fire_id, run_id = await make(run_status="interrupted", fire_age=timedelta(days=40), run_age=timedelta(days=35))
|
||||
async with SessionLocal() as session:
|
||||
session.add(RunEvent(run_id=run_id, seq=1, type="run.started", ts=1.0, data={}))
|
||||
session.add(Approval(run_id=run_id, node_id="gate", title="审核日报", status="pending"))
|
||||
await session.commit()
|
||||
assert await webhook.notify_due() == 1
|
||||
[body] = recorder.bodies()
|
||||
assert (body["event"], body["pending_approvals"]) == ("awaiting_approval", 1)
|
||||
assert await notified(fire_id) == "interrupted:approval:1"
|
||||
assert await webhook.notify_due() == 0
|
||||
|
||||
|
||||
async def test_old_fire_whose_run_just_finished_is_pushed(recorder):
|
||||
"""等了一个多月的审批批完、刚跑完:运行记录在窗口内有变化,推「运行结束」"""
|
||||
_, fire_id, _ = await make(run_status="succeeded", fire_age=timedelta(days=45))
|
||||
assert await webhook.notify_due() == 1
|
||||
assert recorder.bodies()[0]["event"] == "finished" and await notified(fire_id) == "succeeded"
|
||||
|
||||
|
||||
async def test_old_fire_whose_run_ended_long_ago_is_not_scanned(recorder):
|
||||
"""早就结束、多日没有变化的旧触发仍不扫:窗口本来就是为了不每一拍把全部历史翻一遍"""
|
||||
_, fire_id, _ = await make(run_status="succeeded", fire_age=timedelta(days=45), run_age=timedelta(days=40))
|
||||
assert await webhook.notify_due() == 0
|
||||
assert recorder.requests == [] and await notified(fire_id) is None
|
||||
|
||||
|
||||
async def test_old_suspended_run_without_approval_is_not_rescanned_forever(recorder):
|
||||
"""挂起(没有待审批)、多日没有变化的旧触发不在例外里:例外只给停在审批上的"""
|
||||
_, fire_id, _ = await make(run_status="interrupted", fire_age=timedelta(days=40), run_age=timedelta(days=40))
|
||||
assert await webhook.notify_due() == 0 and await notified(fire_id) is None
|
||||
@@ -178,7 +178,8 @@ def test_failed_run_text_points_to_the_node_without_repeating_itself():
|
||||
error_node="查询门店销售"))
|
||||
shown = dict(lines)
|
||||
assert title.endswith("运行失败") and shown["状态"] == "失败"
|
||||
assert shown["失败节点"] == "「查询门店销售」" and "继续运行" in shown["处理"]
|
||||
# 处理只说去哪里看,不许诺能继续运行(没有断点的运行续不了,见 test_scheduler_notice_rules.py)
|
||||
assert shown["失败节点"] == "「查询门店销售」" and shown["处理"] == "请在 AgentLab 的「记录」中查看失败原因"
|
||||
# 不再写「原因:运行失败,详情请查看运行记录」这种同义反复
|
||||
assert "原因" not in shown
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ SECRET = "SECdemo-api-not-real-7"
|
||||
SCHEDULE_KEYS = {"id", "name", "workflow_id", "workflow_name", "cron", "timezone", "run_class", "input_template",
|
||||
"enabled", "overlap", "misfire_grace_s", "webhook", "next_run_at", "last_fire", "description_text",
|
||||
"created_by", "created_at", "updated_at", "version"}
|
||||
FIRE_KEYS = {"id", "fire_at", "kind", "status", "run_id", "run_status", "reason", "notified", "created_at"}
|
||||
FIRE_KEYS = {"id", "fire_at", "kind", "status", "run_id", "run_status", "reason", "notified", "compare", "created_at"}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
||||
@@ -2968,14 +2968,14 @@ export const SCHEDULE_TEXT = {
|
||||
testResult: (head: string, status: number | null, detail: string) =>
|
||||
`${head}${status != null ? `(HTTP ${status})` : ''}${detail ? `:${detail}` : ''}`,
|
||||
|
||||
// ---- 通知里的运行链接(设置 notify 组,所有定时任务共用)
|
||||
linkBase: '通知中的运行链接',
|
||||
// ---- 通知里的链接:运行、对比页都按这个地址拼(设置 notify 组,所有定时任务共用)
|
||||
linkBase: '通知中的链接',
|
||||
linkBaseValue: (url: string) => `按 ${url} 拼接`,
|
||||
linkBaseUnset: '未设置服务访问地址,通知中不附带运行链接',
|
||||
linkBaseUnset: '未设置服务访问地址,通知中不附带运行和对比页链接',
|
||||
linkBaseEdit: '修改',
|
||||
linkBaseSet: '设置',
|
||||
linkBaseLabel: '服务访问地址',
|
||||
linkBaseHint: '收件人打开运行链接用的地址,例如 https://agentlab.example.com。所有定时任务共用',
|
||||
linkBaseHint: '收件人打开运行、对比页链接用的地址,例如 https://agentlab.example.com。所有定时任务共用',
|
||||
linkBasePlaceholder: 'https://agentlab.example.com',
|
||||
linkBaseInvalid: '请输入以 http:// 或 https:// 开头的地址',
|
||||
linkBaseSaved: '已保存服务访问地址',
|
||||
@@ -3050,6 +3050,63 @@ export const SCHEDULE_TEXT = {
|
||||
allLoaded: '已全部加载',
|
||||
shown: (n: number) => `已显示 ${n} 条`,
|
||||
|
||||
// ---- 触发历史里的对比:运行成功结束后,服务端自动和同一定时任务上一次成功的运行比一次
|
||||
/**
|
||||
* 和运行详情「更多」菜单里的入口是同一件事,用同一个叫法。RUN_COMPARE_TEXT 定义在本对象之后,模块初始化时还取不到,
|
||||
* 写成 getter 用到时再取
|
||||
*/
|
||||
get compareLink() { return RUN_COMPARE_TEXT.menu },
|
||||
/**
|
||||
* 悬停写各类指标的个数,和 Webhook 通知里那一行同一个写法(服务端 scheduler/auto_compare.py 的 summary_text):
|
||||
* 变化三分法的三类总写上,0 个也写;其余各类有才写,次序同对比页;两边都没有指标时不写一串 0 个。只有个数,
|
||||
* 不写数值。各类的叫法用对比页那一套(RUN_COMPARE_TEXT.classLabel)。base 是基准的短编号,基准不是紧挨着的
|
||||
* 上一次(上一次失败了、取消了)时给,写明比的是哪一次,同通知
|
||||
*/
|
||||
compareTitle: (summary: Partial<Record<string, number>>, base?: string | null) => {
|
||||
const n = (key: string) => (typeof summary[key] === 'number' ? summary[key] as number : 0)
|
||||
const core = ['data_only', 'caliber_and_data', 'sql_changed']
|
||||
const rest = ['params_only', 'added', 'removed', 'incomparable', 'unchanged']
|
||||
const head = base ? `与上一次成功的运行(${base})相比` : '与上一次相比'
|
||||
if (![...core, ...rest].some((key) => n(key) > 0)) return `${head}:两次运行都没有指标`
|
||||
const parts = [...core, ...rest.filter((key) => n(key) > 0)]
|
||||
.map((key) => `${RUN_COMPARE_TEXT.classLabel[key] ?? key} ${n(key)} 个`)
|
||||
return `${head}:${parts.join('、')}`
|
||||
},
|
||||
/**
|
||||
* 链接旁看得见的简短计数:触屏没有悬停,键盘聚焦时也不显示 title,只靠悬停看不出变化的规模。有变化的几类(只数据变、
|
||||
* 口径与数据都变、SQL 结构变、只参数变、新增、移除)合起来写一个数;都没变化时说有几个无法比较,或者都未变;两边都
|
||||
* 没有指标时不写(悬停里有)
|
||||
*/
|
||||
compareCount: (summary: Partial<Record<string, number>>) => {
|
||||
const n = (key: string) => (typeof summary[key] === 'number' ? summary[key] as number : 0)
|
||||
const changed = ['data_only', 'caliber_and_data', 'sql_changed', 'params_only', 'added', 'removed']
|
||||
.reduce((sum, key) => sum + n(key), 0)
|
||||
if (changed) return `${changed} 个指标有变化`
|
||||
if (n('incomparable')) return `${n('incomparable')} 个指标无法比较`
|
||||
return n('unchanged') ? '指标都未变' : null
|
||||
},
|
||||
|
||||
// ---- 运行详情页头的来源
|
||||
/**
|
||||
* 到点发起(以及没有署名的立即运行)的运行记在「定时任务:名字」名下,前缀和服务端 scheduler/service.py 的
|
||||
* actor_label 一致。页头据此不再写「定时任务:X 发起」,换成下面那条链接
|
||||
*/
|
||||
actorPrefix: '定时任务:',
|
||||
runSource: (name?: string | null) => (name ? `由定时任务「${name}」发起` : '由定时任务发起'),
|
||||
/** 有人署名点「立即运行」发起的:署名照写(runStarter),后面接这一句 */
|
||||
runSourceManual: (name?: string | null) => (name ? `来自定时任务「${name}」` : '来自定时任务'),
|
||||
runStarter: (who: string) => `${who} 发起`,
|
||||
/** 悬停写全名(页头窄时链接会截断)和点开去哪 */
|
||||
runSourceTitle: (name?: string | null) =>
|
||||
(name ? `定时任务「${name}」:查看设置和触发历史` : '查看这个定时任务的设置和触发历史'),
|
||||
/**
|
||||
* 定时任务已经删除(取回的定时任务列表里没有它):不再链过去——只会落到「不存在」页,页上的返回又回到定时任务列表。
|
||||
* 和同一行的「工作流已删除」一样写成纯文本
|
||||
*/
|
||||
runSourceGone: (name?: string | null) => (name ? `由定时任务「${name}」发起(已删除)` : '由已删除的定时任务发起'),
|
||||
runSourceManualGone: (name?: string | null) => (name ? `来自定时任务「${name}」(已删除)` : '来自已删除的定时任务'),
|
||||
runSourceGoneTitle: '这个定时任务已删除,它发起的运行记录仍然保留',
|
||||
|
||||
// ---- 记录页的定时标记
|
||||
runTag: '定时',
|
||||
runTagTitle: (name?: string | null) => (name ? `由定时任务「${name}」发起,点击查看` : '由定时任务发起,点击查看'),
|
||||
|
||||
@@ -31,6 +31,7 @@ import { ProvenanceBar, Telemetry, runClocks } from './Telemetry'
|
||||
import { TracePane } from './TracePane'
|
||||
import { useRunDetail } from './useRunDetail'
|
||||
import { useCompareEntry } from './compare/entry'
|
||||
import { RunScheduleSource } from '../schedules/parts'
|
||||
|
||||
/**
|
||||
* 一次运行的详情:头部读数、凭证、排错路径,下面分三个视图——
|
||||
@@ -554,16 +555,20 @@ export function RunDetailView({ runId, onChange, onDeleted }: {
|
||||
<ClassChip runClass={run.run_class} version={run.version} always />
|
||||
<TierChip tier={(run.output as any)?._issuance?.tier} />
|
||||
</div>
|
||||
<div className="mt-0.5 flex min-w-0 items-center gap-1 text-2xs text-faint">
|
||||
{/* 窄时换行:不换行时这一行只有定时任务链接能缩,会被挤成 0 宽,其余各项被压成竖排 */}
|
||||
<div className="mt-0.5 flex min-w-0 flex-wrap items-center gap-x-1 gap-y-0.5 text-2xs text-faint">
|
||||
<span className="mono">{shortId(run.id)}</span>
|
||||
<span>·</span>
|
||||
<time className="tnum" title={`发起于 ${formatDateTime(run.created_at ?? null)}`}>
|
||||
{formatTime(run.created_at ?? null, new Date(now))}
|
||||
</time>
|
||||
<span>·</span>
|
||||
<span title={run.started_by ? undefined : '发起时未设置署名'}>
|
||||
{run.started_by || '未署名'} 发起
|
||||
</span>
|
||||
{/* 定时任务发起的:写成到那个定时任务的链接(键盘可达;记录页的「定时」标记只能鼠标点) */}
|
||||
{run.schedule_id ? <RunScheduleSource scheduleId={run.schedule_id} startedBy={run.started_by} /> : (
|
||||
<span title={run.started_by ? undefined : '发起时未设置署名'}>
|
||||
{run.started_by || '未署名'} 发起
|
||||
</span>
|
||||
)}
|
||||
<span>·</span>
|
||||
<span className="tnum">{formatNumber(d.events.length)} 条事件</span>
|
||||
{workflowGone && <span>· 工作流已删除</span>}
|
||||
|
||||
@@ -1,18 +1,20 @@
|
||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { Link } from 'react-router-dom'
|
||||
import { ChevronRight, History, RotateCw } from 'lucide-react'
|
||||
import { ChevronRight, GitCompareArrows, History, RotateCw } from 'lucide-react'
|
||||
import { api } from '../../api/client'
|
||||
import { EmptyState, ErrorState, Skeleton, Spinner, StatusPill, toast, useTicker } from '../../components/ui'
|
||||
import { formatDateTime, parseServerTime } from '../../lib/format'
|
||||
import { formatDateTime, parseServerTime, shortId } from '../../lib/format'
|
||||
import { SCHEDULE_TEXT as T } from '../../lib/terms'
|
||||
import type { Schedule, ScheduleFire } from '../../types'
|
||||
import type { Schedule, ScheduleFire, ScheduleFireCompare } from '../../types'
|
||||
import { comparePath, type CompareVia } from '../runs/compare/entry'
|
||||
import { FireKindChip, FireStatusChip, ZonedTime } from './parts'
|
||||
|
||||
const PAGE = 20
|
||||
|
||||
/**
|
||||
* 触发历史:每次触发一行,新的在前,20 条一页、「加载更多」往后接。计划时间按定时任务的时区写;跳过、发起失败
|
||||
* 写原因;发起了的链到那次运行、写运行现在的状态;发出去的通知逐条写结果。refreshKey 变了(立即运行之后)从头重取
|
||||
* 写原因;发起了的链到那次运行、写运行现在的状态;运行成功结束后服务端和同一任务上一次成功的运行比过的,另给
|
||||
* 「与上一次对比」;发出去的通知逐条写结果。refreshKey 变了(立即运行之后)从头重取
|
||||
*/
|
||||
export function FireHistory({ schedule, refreshKey }: { schedule: Schedule; refreshKey: number }) {
|
||||
const [items, setItems] = useState<ScheduleFire[] | null>(null)
|
||||
@@ -110,6 +112,7 @@ function FireRow({ fire: f, tz, now }: { fire: ScheduleFire; tz: string; now: nu
|
||||
<FireKindChip fire={f} />
|
||||
<FireStatusChip fire={f} />
|
||||
<span className="flex-1" />
|
||||
{f.run_id && f.compare && <CompareLink runId={f.run_id} compare={f.compare} />}
|
||||
{f.run_id ? (
|
||||
<Link to={`/runs/${encodeURIComponent(f.run_id)}`} className="inline-flex shrink-0 items-center gap-1.5 text-2xs text-dim hover:text-fg hover:underline">
|
||||
{f.run_status && <StatusPill status={f.run_status} plain />}
|
||||
@@ -137,3 +140,33 @@ function FireRow({ fire: f, tz, now }: { fire: ScheduleFire; tz: string; now: nu
|
||||
</li>
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* 「与上一次对比」:去对比页,本次是这次触发的运行、基准是服务端自动对比时取的同一任务上一次成功的运行。旁边写有几个
|
||||
* 指标有变化(触屏没有悬停,键盘聚焦也不显示悬停),悬停写各类指标的个数(和通知里那一行同一个写法),读屏把它当
|
||||
* 说明念。基准就是紧挨着的上一次时带上来历,对比页据此写「默认基准:同一定时任务的上一次运行」;不是(上一次失败了、
|
||||
* 取消了)时不带,免得说错,悬停写明比的是哪一次
|
||||
*/
|
||||
function CompareLink({ runId, compare }: { runId: string; compare: ScheduleFireCompare }) {
|
||||
const previous = compare.previous !== false
|
||||
const via: CompareVia | undefined = previous ? { base: compare.base_run_id, reason: 'same_schedule' } : undefined
|
||||
const count = T.compareCount(compare.summary)
|
||||
return (
|
||||
<Link
|
||||
to={comparePath(runId, compare.base_run_id)}
|
||||
state={via}
|
||||
title={T.compareTitle(compare.summary, previous ? null : shortId(compare.base_run_id))}
|
||||
data-fire-compare={compare.base_run_id}
|
||||
className="inline-flex shrink-0 items-center gap-1 rounded-sm text-2xs text-dim outline-none hover:text-fg hover:underline focus-visible:ring-2 focus-visible:ring-[var(--accent)]"
|
||||
>
|
||||
<GitCompareArrows size={11} aria-hidden />
|
||||
<span data-fire-compare-label="">{T.compareLink}</span>
|
||||
{count && (
|
||||
<>
|
||||
<span aria-hidden>·</span>
|
||||
<span className="tnum" data-fire-compare-count="">{count}</span>
|
||||
</>
|
||||
)}
|
||||
</Link>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ import type { ScheduleEvent, ScheduleWebhook, ScheduleWebhookTest } from '../../
|
||||
import { SCHEDULE_EVENTS, WEBHOOK_FORMATS, isHttpUrl, webhookTestText, type FormField, type WebhookForm } from './model'
|
||||
|
||||
/**
|
||||
* Webhook 通知:格式、地址、签名密钥、订阅事件、发送测试消息,外加通知里运行链接用的服务访问地址。
|
||||
* Webhook 通知:格式、地址、签名密钥、订阅事件、发送测试消息,外加通知里运行、对比页链接用的服务访问地址。
|
||||
*
|
||||
* 地址和密钥保存之后服务端不再返回明文:地址只显示主机和末 4 位(url_hint),可以替换或删除;密钥只说「已设置」,
|
||||
* 可以替换或清除。没点替换时请求里不带它们,改别的字段不会把地址清掉。测试消息用的是已保存的设置,
|
||||
@@ -195,7 +195,7 @@ function SecretField({ value: w, saved, onChange, error }: {
|
||||
}
|
||||
|
||||
/**
|
||||
* 通知里运行链接用的服务访问地址:设置里 notify 一组的 public_base_url,所有定时任务共用。这里给出当前值和修改入口,
|
||||
* 通知里运行、对比页链接用的服务访问地址:设置里 notify 一组的 public_base_url,所有定时任务共用。这里给出当前值和修改入口,
|
||||
* 写的时候先重读一遍 notify 组、只换这一项,不把同组别的设置冲掉
|
||||
*/
|
||||
function LinkBase() {
|
||||
|
||||
@@ -223,6 +223,11 @@ export const isUnsupported = (e: unknown) => e instanceof ApiError && e.status =
|
||||
* 不认识的 id 再取一次列表(一分钟内最多一次)。取不到(老服务端、暂时连不上)不要紧,标记照样显示,只是不写名字
|
||||
*/
|
||||
const names = new Map<string, string>()
|
||||
/**
|
||||
* 已删除的定时任务:取回的列表里没有的(列表是全部定时任务,服务端不分页),或者刚在本页删掉的。运行详情页头据此不再
|
||||
* 链过去。列表没取到时谁都不算删除:只是不知道
|
||||
*/
|
||||
const missing = new Set<string>()
|
||||
let fetchedAt = 0
|
||||
let inflight: Promise<void> | null = null
|
||||
let namesVersion = 0
|
||||
@@ -236,16 +241,30 @@ const emitNames = () => {
|
||||
|
||||
/** 记下读到的定时任务名字(gone 里的是刚删掉的) */
|
||||
export function rememberScheduleNames(items: Pick<Schedule, 'id' | 'name'>[], gone: string[] = []) {
|
||||
for (const s of items) names.set(s.id, s.name)
|
||||
for (const id of gone) names.delete(id)
|
||||
for (const s of items) {
|
||||
names.set(s.id, s.name)
|
||||
missing.delete(s.id)
|
||||
}
|
||||
for (const id of gone) {
|
||||
names.delete(id)
|
||||
missing.add(id)
|
||||
}
|
||||
emitNames()
|
||||
}
|
||||
|
||||
function ensureNames(ids: string[]) {
|
||||
if (inflight || !ids.some((id) => !names.has(id)) || Date.now() - fetchedAt < NAMES_RETRY_MS) return
|
||||
if (inflight || !ids.some((id) => !names.has(id) && !missing.has(id)) || Date.now() - fetchedAt < NAMES_RETRY_MS) return
|
||||
inflight = api.schedules.list({ timeoutMs: 10_000 })
|
||||
.then((r) => { for (const s of r.items ?? []) names.set(s.id, s.name) })
|
||||
.catch(() => { /* 取不到就不写名字 */ })
|
||||
.then((r) => {
|
||||
const items = r.items ?? []
|
||||
for (const s of items) {
|
||||
names.set(s.id, s.name)
|
||||
missing.delete(s.id)
|
||||
}
|
||||
const listed = new Set(items.map((s) => s.id))
|
||||
for (const id of ids) if (!listed.has(id)) missing.add(id)
|
||||
})
|
||||
.catch(() => { /* 取不到就不写名字,也不算删除 */ })
|
||||
.finally(() => {
|
||||
inflight = null
|
||||
fetchedAt = Date.now()
|
||||
@@ -253,6 +272,9 @@ function ensureNames(ids: string[]) {
|
||||
})
|
||||
}
|
||||
|
||||
/** 这个定时任务已删除(取回的列表里没有它、或者刚在本页删掉)。和 useScheduleNames 一起用:取回来后会重渲染 */
|
||||
export const scheduleGone = (id: string) => missing.has(id)
|
||||
|
||||
/** 这些定时任务的名字(id → 名字)。没记过的去取一次,取回来后重渲染 */
|
||||
export function useScheduleNames(ids: (string | null | undefined)[]): ReadonlyMap<string, string> {
|
||||
useSyncExternalStore(
|
||||
@@ -269,8 +291,8 @@ export function useScheduleNames(ids: (string | null | undefined)[]): ReadonlyMa
|
||||
* 这里拦住冒泡,只去定时任务、不打开运行。
|
||||
*
|
||||
* 按钮里放不了可聚焦的链接(读屏会把按钮里的内容压平,Tab 也到不了),所以键盘到不了这个标记:读屏改念
|
||||
* 「由定时任务「X」发起」,至少在整行的名字里听到来源。键盘可达的入口要放到运行详情的页头,或者把整行改成
|
||||
* 「打开运行」按钮和这个链接并排,都不在本页的文件里,见交付说明
|
||||
* 「由定时任务「X」发起」,至少在整行的名字里听到来源。键盘可达的入口在运行详情的页头(RunScheduleSource):
|
||||
* 打开这次运行之后,Tab 就能到那条链接
|
||||
*/
|
||||
export function ScheduleTag({ scheduleId, name }: { scheduleId: string; name?: string | null }) {
|
||||
const navigate = useNavigate()
|
||||
@@ -291,3 +313,55 @@ export function ScheduleTag({ scheduleId, name }: { scheduleId: string; name?: s
|
||||
</span>
|
||||
)
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// 运行详情页头的来源
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 运行详情页头里「谁发起的」那一段,定时任务发起的运行用它:「由定时任务「X」发起」是真的链接,Tab 到得了、回车能
|
||||
* 打开,去那个定时任务的设置和触发历史。记录页的「定时」标记在整行按钮里,只能用鼠标点,键盘的入口在这里。
|
||||
*
|
||||
* 到点发起的运行(以及没有署名的立即运行)记在「定时任务:X」名下:原来那句「定时任务:X 发起」整句换成这条链接,
|
||||
* 不写两遍。有人署名点「立即运行」的,署名照写,后面接「来自定时任务「X」」。名字取定时任务现在的名字(改过名的写
|
||||
* 新名字),取不到时用署名里记的那个,再没有就不写名字。
|
||||
*
|
||||
* 前面带和「定时」标记一样的日历图标:页头这一行都是灰字,不带记号看不出这一段能点。页头窄时这一行换行,链接至少留
|
||||
* 几个字宽(以前这一行不换行,只有链接能缩,窄窗口里被挤成 0 宽,Tab 停在一个看不见的链接上),再长截断,全名在悬停里。
|
||||
* 定时任务已经删除的写成纯文本(链过去只有「不存在」页),同一行的「工作流已删除」也是这样
|
||||
*/
|
||||
export function RunScheduleSource({ scheduleId, startedBy }: { scheduleId: string; startedBy?: string | null }) {
|
||||
const names = useScheduleNames([scheduleId])
|
||||
const recorded = startedBy?.startsWith(T.actorPrefix) ? startedBy.slice(T.actorPrefix.length).trim() : null
|
||||
const auto = !startedBy || recorded != null
|
||||
const name = names.get(scheduleId) || recorded || null
|
||||
const body = (text: string) => (
|
||||
<>
|
||||
<CalendarClock size={10} aria-hidden className="shrink-0" />
|
||||
<span className="min-w-0 truncate">{text}</span>
|
||||
</>
|
||||
)
|
||||
const source = scheduleGone(scheduleId) ? (
|
||||
<span title={T.runSourceGoneTitle} data-run-schedule-gone={scheduleId}
|
||||
className="inline-flex min-w-[7em] max-w-full items-center gap-1">
|
||||
{body(auto ? T.runSourceGone(name) : T.runSourceManualGone(name))}
|
||||
</span>
|
||||
) : (
|
||||
<Link
|
||||
to={`/schedules/${encodeURIComponent(scheduleId)}`}
|
||||
title={T.runSourceTitle(name)}
|
||||
data-run-schedule-link={scheduleId}
|
||||
className="inline-flex min-w-[7em] max-w-full items-center gap-1 rounded-sm text-dim outline-none hover:text-fg hover:underline focus-visible:ring-2 focus-visible:ring-[var(--accent)]"
|
||||
>
|
||||
{body(auto ? T.runSource(name) : T.runSourceManual(name))}
|
||||
</Link>
|
||||
)
|
||||
if (auto || !startedBy) return source
|
||||
return (
|
||||
<>
|
||||
<span className="shrink-0">{T.runStarter(startedBy)}</span>
|
||||
<span>·</span>
|
||||
{source}
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -3207,6 +3207,20 @@ export interface ScheduleFireNotice {
|
||||
at: string
|
||||
}
|
||||
|
||||
/**
|
||||
* 运行成功结束后和同一定时任务上一次成功的运行的自动对比:基准是哪一次、各类指标的个数(同对比接口的 summary)。
|
||||
* 只有个数,具体的数在对比页上
|
||||
*/
|
||||
export interface ScheduleFireCompare {
|
||||
base_run_id: string
|
||||
summary: RunCompareSummary
|
||||
/**
|
||||
* 基准是不是紧挨着的上一次。上一次失败了、取消了时,基准是再往前成功的那一次,为 false:界面写明比的是哪一次。
|
||||
* 老服务端不给,按 true 看
|
||||
*/
|
||||
previous?: boolean
|
||||
}
|
||||
|
||||
/** 一次触发(触发历史的一行) */
|
||||
export interface ScheduleFire {
|
||||
id: string
|
||||
@@ -3220,6 +3234,11 @@ export interface ScheduleFire {
|
||||
/** 跳过、发起失败的原因(中文) */
|
||||
reason: string | null
|
||||
notified: ScheduleFireNotice[]
|
||||
/**
|
||||
* 和同一定时任务上一次成功的运行的对比。运行没有成功结束、没有可比的、还没算、比不了、任何一边又在执行(不在终态)
|
||||
* 或重新封存过时为 null;老服务端不给
|
||||
*/
|
||||
compare?: ScheduleFireCompare | null
|
||||
created_at: string
|
||||
}
|
||||
|
||||
|
||||
+332
-5
@@ -3,7 +3,11 @@
|
||||
// 守的是这些「看着没坏、其实不能用」的地方:预设生成的 cron 不对(选了每周一 09:00,交上去的是每天)、时区按本机写
|
||||
// 而不是按定时任务的时区、预览不防抖、非法 cron 只弹一句不知道指哪一项、启用开关不带版本号(别人改过也照样覆盖)、
|
||||
// 立即运行之后没有下文、触发历史只有一页、保存过的 Webhook 地址又被原样显示出来、改了别的字段却把地址清掉、
|
||||
// 记录页看不出哪些运行来自定时任务、390px 窄屏被表单撑出横向滚动。
|
||||
// 记录页看不出哪些运行来自定时任务、390px 窄屏被表单撑出横向滚动。运行成功结束后的自动对比:触发历史里「与上一次
|
||||
// 对比」(和运行详情「更多」菜单同一个叫法)链到对比页,旁边写有几个指标有变化,悬停只写各类指标的个数,基准不是
|
||||
// 紧挨着的上一次时写明是哪一次、不带「默认基准」的来历;运行详情页头的「由定时任务「X」发起」是键盘到得了的真链接
|
||||
// (记录页的「定时」标记在整行按钮里,只能鼠标点),带日历图标,窄窗口里(768、820、900 宽,探索运行多一个「提取为
|
||||
// 草稿」)不被挤没,定时任务删掉之后写成纯文本。
|
||||
// 还有几处时序和口径:晚到的刷新把刚切换的开关盖回去、先开触发历史再开设置时带着旧版本撞假的 409、保存途中又改了
|
||||
// 别的字段时替换过的地址明文留在框里、服务端会拒的组合(未发布 + 正式运行、必填没填、已发布版本里没有的字段)
|
||||
// 照样放行、测试消息的结果把服务端的整句又套一层、错过已跳过的那条悬停另说一个原因(和本条记的原因对不上)。
|
||||
@@ -263,6 +267,24 @@ await section('纯函数:服务端会拒的组合、保存回来怎么接、
|
||||
check(`测试结果:服务端给整句、给补充说明都只说一遍(${tests.length} 种)`, !wrong.length, wrong.map(([r]) => t(r)).join(';'))
|
||||
})
|
||||
|
||||
await section('纯函数:触发历史对比的文案', async () => {
|
||||
const { SCHEDULE_TEXT: T, RUN_COMPARE_TEXT: C } = await import(await load('/src/lib/terms.ts'))
|
||||
check('「与上一次对比」和运行详情「更多」菜单同一个叫法', T.compareLink === C.menu && T.compareLink === '与上一次对比', T.compareLink)
|
||||
const zero = { unchanged: 0, data_only: 0, caliber_and_data: 0, sql_changed: 0, params_only: 0, added: 0, removed: 0, incomparable: 0 }
|
||||
const cases = [
|
||||
['有变化的几类合起来', { ...zero, data_only: 3, caliber_and_data: 1, params_only: 2, unchanged: 4 }, '6 个指标有变化'],
|
||||
['新增、移除也算变化', { ...zero, added: 1, removed: 2 }, '3 个指标有变化'],
|
||||
['没有变化、有无法比较的', { ...zero, incomparable: 2, unchanged: 3 }, '2 个指标无法比较'],
|
||||
['都未变', { ...zero, unchanged: 5 }, '指标都未变'],
|
||||
['两边都没有指标:不写', zero, null],
|
||||
]
|
||||
const bad = cases.filter(([, sum, want]) => T.compareCount(sum) !== want)
|
||||
check(`看得见的计数:${cases.length} 种情况`, !bad.length, bad.map(([n, sum]) => `${n}:${T.compareCount(sum)}`).join(';'))
|
||||
check('悬停:基准不是紧挨着的上一次时写明是哪一次', T.compareTitle({ ...zero, data_only: 1 }, '#run-f1')
|
||||
=== '与上一次成功的运行(#run-f1)相比:只数据变 1 个、口径与数据都变 0 个、SQL 结构变 0 个'
|
||||
&& T.compareTitle(zero, '#run-f1') === '与上一次成功的运行(#run-f1)相比:两次运行都没有指标')
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------- 夹具:虚构的门店周报、库存日报
|
||||
|
||||
const NOW = Date.now()
|
||||
@@ -375,6 +397,16 @@ const S2 = {
|
||||
* 原因却有两种,各放一条、原因照服务端的原话写:f03 是停机一周多、在宽限内恢复时积压的那一次(最近一次就是补发的
|
||||
* f02),f05 是超出补发宽限的。f06、f07 没有记原因(约定里 reason 可以是 null),看兜底的说明
|
||||
*/
|
||||
/** 服务端自动对比给的计数(只有个数):f00 和同一任务的上一次(run-f02)比过 */
|
||||
const COMPARE_F00 = { base_run_id: 'run-f02', previous: true, summary: { unchanged: 4, data_only: 3, caliber_and_data: 1,
|
||||
sql_changed: 0, params_only: 2, added: 0, removed: 0, incomparable: 0 } }
|
||||
const COMPARE_F00_TITLE = '与上一次相比:只数据变 3 个、口径与数据都变 1 个、SQL 结构变 0 个、只参数变 2 个、未变 4 个'
|
||||
const NO_METRICS = { base_run_id: 'run-f09', previous: true, summary: { unchanged: 0, data_only: 0, caliber_and_data: 0,
|
||||
sql_changed: 0, params_only: 0, added: 0, removed: 0, incomparable: 0 } }
|
||||
/** f10 的上一次(f11)失败了:基准是再往前成功的 f12,不是紧挨着的上一次 */
|
||||
const COMPARE_F10 = { base_run_id: 'run-f12', previous: false, summary: { unchanged: 1, data_only: 1, caliber_and_data: 0,
|
||||
sql_changed: 0, params_only: 0, added: 0, removed: 0, incomparable: 0 } }
|
||||
|
||||
function makeFires() {
|
||||
const out = []
|
||||
const local = (at) => {
|
||||
@@ -385,14 +417,17 @@ function makeFires() {
|
||||
for (let i = 0; i < 25; i++) {
|
||||
const at = iso(nextMonday9(-(i + 1)))
|
||||
let f = fire(`f${String(i).padStart(2, '0')}`, { fire_at: at, created_at: at })
|
||||
if (i === 0) f = { ...f, kind: 'manual', notified: [{ event: 'finished', status: 'ok', at }, { event: 'failed', status: 'failed', at }] }
|
||||
if (i === 0) f = { ...f, kind: 'manual', compare: COMPARE_F00, notified: [{ event: 'finished', status: 'ok', at }, { event: 'failed', status: 'failed', at }] }
|
||||
if (i === 1) f = { ...f, status: 'skipped_overlap', run_id: null, run_status: null, reason: '上一次运行(#9f8e7d)尚未结束' }
|
||||
if (i === 2) f = { ...f, kind: 'misfire', run_status: 'failed' }
|
||||
if (i === 2) f = { ...f, kind: 'misfire', run_status: 'failed', compare: null }
|
||||
if (i === 3) f = { ...f, ...missed, reason: `服务未运行,错过了 ${local(at)} 的触发;积压的触发不补,只发起最近一次` }
|
||||
if (i === 4) f = { ...f, status: 'failed_to_start', run_id: null, run_status: null, reason: '工作流「门店周报」还没有发布:正式运行需要已发布的版本' }
|
||||
if (i === 5) f = { ...f, ...missed, reason: `服务未运行,错过了 ${local(at)} 的触发,超出补发宽限(1 小时),没有发起` }
|
||||
if (i === 6) f = { ...f, ...missed, reason: null }
|
||||
if (i === 7) f = { ...f, status: 'failed_to_start', run_id: null, run_status: null, reason: null }
|
||||
if (i === 8) f = { ...f, compare: NO_METRICS }
|
||||
if (i === 10) f = { ...f, compare: COMPARE_F10 }
|
||||
if (i === 11) f = { ...f, run_status: 'failed', compare: null }
|
||||
out.push(f)
|
||||
}
|
||||
return out
|
||||
@@ -401,12 +436,30 @@ function makeFires() {
|
||||
const RUNS = [
|
||||
{ id: 'run-sched-1', workflow_id: 'wf-weekly', workflow_name: '门店周报', status: 'succeeded', input: { date: '2026-10-04' },
|
||||
output: { 周报: '上周销售额 1,234 万元' }, error: null, usage: { duration_ms: 4200 }, run_class: 'formal', version: 3,
|
||||
started_by: '定时任务:门店周报(旧名)',
|
||||
schedule_id: 's-weekly', created_at: iso(NOW - 2 * H), started_at: iso(NOW - 2 * H), finished_at: iso(NOW - 2 * H + 4200) },
|
||||
{ id: 'run-manual-1', workflow_id: 'wf-weekly', workflow_name: '门店周报', status: 'succeeded', input: { date: '2026-10-03' },
|
||||
output: { 周报: '手动补一份' }, error: null, usage: { duration_ms: 3900 }, run_class: 'exploratory', version: null,
|
||||
schedule_id: null, created_at: iso(NOW - 3 * H), started_at: iso(NOW - 3 * H), finished_at: iso(NOW - 3 * H + 3900) },
|
||||
]
|
||||
|
||||
/** 有人署名点了「立即运行」:署名照写,页头另接到定时任务的链接 */
|
||||
const RUN_MANUAL_FIRE = { ...RUNS[0], id: 'run-sched-2', started_by: '检查员甲', created_at: iso(NOW - H), started_at: iso(NOW - H),
|
||||
finished_at: iso(NOW - H + 4200) }
|
||||
/** 探索运行成功了,页头多一个「提取为草稿」,这一行更挤:到点发起的、署名点了立即运行的各一次 */
|
||||
const RUN_EXPLORE = { ...RUNS[0], id: 'run-sched-3', run_class: 'exploratory', version: null, started_by: '定时任务:门店周报 · 每周' }
|
||||
const RUN_EXPLORE_MANUAL = { ...RUN_EXPLORE, id: 'run-sched-4', started_by: '检查员甲' }
|
||||
/** 运行详情要的图和事件:一个节点、开始到结束两条,够页头和时间线画出来 */
|
||||
const RUN_GRAPH = { nodes: [{ id: 'out', type: 'output', position: { x: 0, y: 0 }, data: { label: '成果', config: {} } }], edges: [] }
|
||||
function runEvents(run) {
|
||||
const t0 = Date.parse(run.started_at) / 1000
|
||||
return [
|
||||
{ seq: 1, type: 'run.started', node_id: null, ts: t0, data: { nodes: 1 } },
|
||||
{ seq: 2, type: 'run.finished', node_id: null, ts: t0 + 4.2,
|
||||
data: { output: run.output, usage: {}, timing: { wall_ms: 4200, active_ms: 4200, wait_ms: 0 } } },
|
||||
]
|
||||
}
|
||||
|
||||
function freshState(patch = {}) {
|
||||
return {
|
||||
schedules: [structuredClone(S1), structuredClone(S2)],
|
||||
@@ -538,7 +591,13 @@ async function reply(state, method, path, url, body) {
|
||||
if (state.runs && method === 'GET' && path.startsWith('/runs/')) {
|
||||
const run = state.runs.find((r) => path === `/runs/${r.id}`)
|
||||
if (run) return { json: run }
|
||||
const m = path.match(/^\/runs\/([^/]+)\/(events|graph|artifacts)$/)
|
||||
const of = m && state.runs.find((r) => r.id === m[1])
|
||||
if (of && m[2] === 'events') return { json: runEvents(of) }
|
||||
if (of && m[2] === 'graph') return { json: { graph: RUN_GRAPH, workflow_id: of.workflow_id, version: of.version } }
|
||||
if (of) return { json: [] }
|
||||
}
|
||||
if (state.runs && method === 'GET' && path === '/approvals' && url.searchParams.get('run_id')) return { json: [] }
|
||||
return null
|
||||
}
|
||||
|
||||
@@ -621,6 +680,18 @@ const saveDone = (page) => until(async () => {
|
||||
/** 等预览算完、说的是这个频率 */
|
||||
const previewSays = (page, words) => until(async () => (await page.locator('[data-schedule-preview][data-status="ok"]').count()) === 1
|
||||
&& (await page.locator('[data-schedule-preview]').innerText()).includes(words))
|
||||
/**
|
||||
* 只按 Tab 键把焦点移到 sel 上:从页面开头往后按,最多按 max 次。按到了返回按了几次,按不到返回 0。键盘可达就是
|
||||
* 这个意思——鼠标能点、程序能 focus() 的不算
|
||||
*/
|
||||
const tabTo = async (page, sel, max = 160) => {
|
||||
await page.evaluate(() => document.activeElement?.blur?.())
|
||||
for (let i = 1; i <= max; i++) {
|
||||
await page.keyboard.press('Tab')
|
||||
if (await page.evaluate((s) => !!document.activeElement?.matches?.(s), sel)) return i
|
||||
}
|
||||
return 0
|
||||
}
|
||||
/** 元素整个在视口的横向范围里 */
|
||||
const inViewX = (page, sel) => page.evaluate((s) => {
|
||||
const el = document.querySelector(s)
|
||||
@@ -628,6 +699,40 @@ const inViewX = (page, sel) => page.evaluate((s) => {
|
||||
const b = el.getBoundingClientRect()
|
||||
return b.left >= 0 && b.right <= innerWidth + 0.5
|
||||
}, sel)
|
||||
/**
|
||||
* 元素的字和它实际落在上面的底色的对比度(WCAG 算法),{ratio, fg, bg},找不到元素为 null。底色顺着祖先往上找,
|
||||
* 半透明的一层层叠上去,直到不透明的那层(都透明就按白底)。只比「字色和某个容器的底色不相等」不够:页头本身是
|
||||
* 透明的,那样比永远成立
|
||||
*/
|
||||
const readable = (page, sel) => page.evaluate((s) => {
|
||||
const el = document.querySelector(s)
|
||||
if (!el) return null
|
||||
const parse = (c) => {
|
||||
const m = c.match(/color\(srgb ([\d.]+) ([\d.]+) ([\d.]+)(?: \/ ([\d.]+))?\)/)
|
||||
if (m) return [m[1] * 255, m[2] * 255, m[3] * 255, m[4] == null ? 1 : Number(m[4])]
|
||||
const n = c.match(/rgba?\(([\d.]+),\s*([\d.]+),\s*([\d.]+)(?:,\s*([\d.]+))?\)/)
|
||||
return n ? [Number(n[1]), Number(n[2]), Number(n[3]), n[4] == null ? 1 : Number(n[4])] : null
|
||||
}
|
||||
const over = (top, under) => top.slice(0, 3).map((v, i) => v * top[3] + under[i] * (1 - top[3]))
|
||||
const layers = []
|
||||
for (let n = el; n; n = n.parentElement) {
|
||||
const c = parse(getComputedStyle(n).backgroundColor)
|
||||
if (c && c[3] > 0) layers.push(c)
|
||||
if (c && c[3] >= 1) break
|
||||
}
|
||||
let bg = [255, 255, 255]
|
||||
for (const layer of layers.reverse()) bg = over(layer, bg)
|
||||
const color = parse(getComputedStyle(el).color)
|
||||
if (!color) return null
|
||||
const fg = over(color, bg)
|
||||
const lum = (rgb) => {
|
||||
const [r, g, b] = rgb.map((v) => { const x = v / 255; return x <= 0.03928 ? x / 12.92 : ((x + 0.055) / 1.055) ** 2.4 })
|
||||
return 0.2126 * r + 0.7152 * g + 0.0722 * b
|
||||
}
|
||||
const [hi, lo] = [lum(fg), lum(bg)].sort((a, b) => b - a)
|
||||
return { ratio: Math.round(((hi + 0.05) / (lo + 0.05)) * 100) / 100, fg: getComputedStyle(el).color,
|
||||
bg: `rgb(${bg.map(Math.round).join(', ')})` }
|
||||
}, sel)
|
||||
|
||||
// ---------------------------------------------------------------- 列表
|
||||
|
||||
@@ -1089,8 +1194,10 @@ await section('详情:Webhook 与运行链接地址', async () => {
|
||||
&& !('secret' in (p2?.webhook ?? {})), JSON.stringify(p2))
|
||||
check('……保存后又只显示 url_hint', await until(async () => (await text(page, '[data-webhook-url-hint]').catch(() => '')).includes('…cd34'))
|
||||
&& await page.locator('[data-webhook-url]').count() === 0)
|
||||
// 运行链接地址:读写设置的 notify 组
|
||||
check('写出运行链接地址还没设置', (await text(page, '[data-link-base]')).includes('未设置'))
|
||||
// 运行链接地址:读写设置的 notify 组。对比页链接也按它拼,说明里一并说到
|
||||
const unset = await text(page, '[data-link-base]')
|
||||
check('写出运行链接地址还没设置,没有它时运行和对比页链接都不附', unset.includes('未设置') && unset.includes('运行和对比页链接'),
|
||||
oneLine([unset]))
|
||||
await page.locator('[data-link-base-edit]').click()
|
||||
await page.locator('[data-link-base-input]').fill('ftp://bad')
|
||||
check('地址不对时不让存', await page.locator('[data-link-base-save]').isDisabled())
|
||||
@@ -1335,6 +1442,220 @@ await section('详情:触发历史分页', async () => {
|
||||
&& await gone.page.locator('[data-schedule-missing] a[href="/schedules"]').count() === 1)
|
||||
})
|
||||
|
||||
await section('详情:触发历史里的「与上一次对比」', async () => {
|
||||
const { page, sent, errors } = await open('/schedules/s-weekly?tab=history')
|
||||
await page.locator('[data-fire-id]').first().waitFor()
|
||||
const row = (id) => page.locator(`[data-fire-id="${id}"]`)
|
||||
const link = row('f00').locator('[data-fire-compare]')
|
||||
const label = await link.locator('[data-fire-compare-label]').innerText().catch(() => '没有链接')
|
||||
check('有对比的那次:给「与上一次对比」(和运行详情「更多」菜单同一个叫法)', (await link.count()) === 1
|
||||
&& label.trim() === '与上一次对比', label)
|
||||
check('……链到对比页,本次是这次的运行、基准是服务端比过的那一次',
|
||||
await link.getAttribute('href') === '/runs/compare?target=run-f00&base=run-f02', (await link.getAttribute('href')) ?? '')
|
||||
const count = (await link.locator('[data-fire-compare-count]').innerText().catch(() => '')).trim()
|
||||
check('……旁边看得见有几个指标有变化(触屏、键盘看不到悬停)', count === '6 个指标有变化', count)
|
||||
const title = (await link.getAttribute('title')) ?? ''
|
||||
check('……悬停写各类指标的个数:三类总写,其余有才写,没有数值', title === COMPARE_F00_TITLE, title)
|
||||
const f08 = row('f08').locator('[data-fire-compare]')
|
||||
check('两边都没有指标:悬停不写一串 0 个,旁边也不写计数', ((await f08.getAttribute('title')) ?? '')
|
||||
=== '与上一次相比:两次运行都没有指标' && await f08.locator('[data-fire-compare-count]').count() === 0)
|
||||
const f10 = row('f10').locator('[data-fire-compare]')
|
||||
const t10 = (await f10.getAttribute('title').catch(() => '')) ?? ''
|
||||
check('上一次失败了、基准是再往前成功的那次:悬停写明是哪一次', t10.startsWith('与上一次成功的运行(#run-f1)相比:只数据变 1 个'), t10)
|
||||
const linked = await page.locator('[data-fire-compare]').evaluateAll((els) => els.map((e) => e.closest('[data-fire-id]')?.getAttribute('data-fire-id')))
|
||||
check('没有对比的(compare 为 null、老服务端不给、没有运行)不给链接', JSON.stringify(linked) === '["f00","f08","f10"]', JSON.stringify(linked))
|
||||
check('运行链接照旧在', await row('f00').locator('a[href="/runs/run-f00"]').count() === 1)
|
||||
const presses = await tabTo(page, '[data-fire-id="f00"] [data-fire-compare]')
|
||||
check('键盘:Tab 到得了「与上一次对比」', presses > 0, `按了 ${presses} 次`)
|
||||
await shot(page, 'history-compare-focus-dark')
|
||||
await page.keyboard.press('Enter')
|
||||
const url = await until(() => {
|
||||
const u = new URL(page.url())
|
||||
return u.pathname === '/runs/compare' ? u : null
|
||||
})
|
||||
check('……回车打开对比页,带着本次和基准', !!url && url.searchParams.get('target') === 'run-f00'
|
||||
&& url.searchParams.get('base') === 'run-f02', page.url())
|
||||
// 来历跟着这一次跳转走(路由的 state),对比页据此写「默认基准:同一定时任务的上一次运行」
|
||||
const via = await page.evaluate(() => history.state?.usr ?? null)
|
||||
check('……紧挨着的上一次:带上「同一定时任务」的来历', via?.reason === 'same_schedule' && via?.base === 'run-f02', JSON.stringify(via))
|
||||
|
||||
const second = await open('/schedules/s-weekly?tab=history')
|
||||
await second.page.locator('[data-fire-id="f10"] [data-fire-compare]').click()
|
||||
await until(() => new URL(second.page.url()).pathname === '/runs/compare')
|
||||
const via10 = await second.page.evaluate(() => history.state?.usr ?? null)
|
||||
check('……基准不是紧挨着的上一次:不带来历,对比页不说「上一次」', new URL(second.page.url()).searchParams.get('base') === 'run-f12'
|
||||
&& via10 == null, JSON.stringify(via10))
|
||||
const real = [...writes(sent, /./), ...writes(second.sent, /./)].filter((x) => x.key !== 'POST /schedules/preview')
|
||||
check('只读:没有写请求', real.length === 0, real.map((x) => x.key).join('、'))
|
||||
check('没有页面报错', ![...errors, ...second.errors].length, [...errors, ...second.errors].join(' | '))
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------- 运行详情
|
||||
|
||||
/**
|
||||
* 运行详情页头元信息那一行:定时任务的链接(或删除后的纯文本)可见的宽度够不够写下图标加「由定时任务」五个字(fits),
|
||||
* 「N 条事件」是不是还在一行里(eventsFlat,以前被挤成一字一行),页头有没有横向溢出,链接在不在视口里,署名在不在
|
||||
*/
|
||||
const headRow = (page) => page.evaluate(() => {
|
||||
const head = document.querySelector('[data-run-detail] header')
|
||||
const link = head?.querySelector('[data-run-schedule-link], [data-run-schedule-gone]')
|
||||
if (!head || !link) return null
|
||||
const font = parseFloat(getComputedStyle(link).fontSize)
|
||||
const icon = link.querySelector('svg')?.getBoundingClientRect().width ?? 0
|
||||
const l = link.getBoundingClientRect()
|
||||
const events = [...head.querySelectorAll('span')].find((el) => /^[\d,]+ 条事件$/.test(el.textContent.trim()))
|
||||
const eh = events?.getBoundingClientRect().height ?? 0
|
||||
return { width: Math.round(l.width), need: Math.round(icon + 4 + font * 5), fits: l.width >= icon + 4 + font * 5 - 0.5,
|
||||
eventsFlat: !!events && eh > 0 && eh < font * 2.2, scroll: head.scrollWidth - head.clientWidth,
|
||||
inView: l.left >= 0 && l.right <= innerWidth + 0.5 && l.width > 0, starter: head.innerText.includes('检查员甲 发起') }
|
||||
})
|
||||
|
||||
await section('运行详情:页头的定时任务链接', async () => {
|
||||
const writesSeenExtra = []
|
||||
const errorsExtra = []
|
||||
const state = freshState({ runs: [...RUNS, RUN_MANUAL_FIRE] })
|
||||
const { page, sent, errors } = await open('/runs/run-sched-1', { state })
|
||||
await page.locator('[data-run-detail="run-sched-1"]').waitFor({ timeout: 10000 })
|
||||
const head = page.locator('[data-run-detail] header').first()
|
||||
const link = head.locator('[data-run-schedule-link]')
|
||||
// 署名里记的是旧名字:取到定时任务列表之后写现在的名字
|
||||
const said = await until(async () => {
|
||||
const t = (await link.innerText().catch(() => '')).trim()
|
||||
return t === '由定时任务「门店周报 · 每周」发起' ? t : null
|
||||
})
|
||||
check('页头写「由定时任务「X」发起」,X 是定时任务现在的名字', !!said, (await link.innerText().catch(() => '没有链接')).trim())
|
||||
check('……是到那个定时任务的真链接', (await link.evaluate((el) => el.tagName)) === 'A'
|
||||
&& await link.getAttribute('href') === '/schedules/s-weekly', (await link.getAttribute('href')) ?? '')
|
||||
const headText = await head.innerText()
|
||||
check('……不再另写「定时任务:X 发起」', !headText.includes('定时任务:') && !headText.includes('旧名'), oneLine([headText]).slice(0, 160))
|
||||
check('……悬停写全名和点开去哪', ((await link.getAttribute('title')) ?? '').includes('门店周报 · 每周'))
|
||||
check('……前面带和「定时」标记一样的日历图标,平时看得出能点', await link.locator('svg').count() === 1)
|
||||
const presses = await tabTo(page, '[data-run-detail] header [data-run-schedule-link]')
|
||||
check('键盘:Tab 到得了页头的定时任务链接', presses > 0, `按了 ${presses} 次`)
|
||||
const ring = await page.evaluate(() => getComputedStyle(document.activeElement).boxShadow)
|
||||
check('……聚焦时有焦点环', !!ring && ring !== 'none', ring)
|
||||
const darkTone = await readable(page, '[data-run-detail] header [data-run-schedule-link]')
|
||||
check('深色:页头的定时任务链接对比度 ≥ 4.5', !!darkTone && darkTone.ratio >= 4.5, JSON.stringify(darkTone))
|
||||
await shot(page, 'run-detail-schedule-link-dark')
|
||||
await page.keyboard.press('Enter')
|
||||
check('……回车去那个定时任务', await until(() => new URL(page.url()).pathname === '/schedules/s-weekly'), page.url())
|
||||
await page.locator('[data-schedule-form], [data-schedules-page] header').first().waitFor()
|
||||
|
||||
const manual = await open('/runs/run-sched-2', { state: freshState({ runs: [...RUNS, RUN_MANUAL_FIRE] }) })
|
||||
await manual.page.locator('[data-run-detail="run-sched-2"]').waitFor({ timeout: 10000 })
|
||||
const mHead = manual.page.locator('[data-run-detail] header').first()
|
||||
const mLink = mHead.locator('[data-run-schedule-link]')
|
||||
await until(async () => (await mLink.innerText().catch(() => '')).includes('门店周报 · 每周'))
|
||||
const mText = oneLine([await mHead.innerText()])
|
||||
check('有人署名点了立即运行:署名照写,后面接「来自定时任务「X」」的链接', mText.includes('检查员甲 发起')
|
||||
&& (await mLink.innerText()).trim() === '来自定时任务「门店周报 · 每周」' && await mLink.getAttribute('href') === '/schedules/s-weekly',
|
||||
mText.slice(0, 160))
|
||||
|
||||
await shot(manual.page, 'run-detail-manual-fire-dark')
|
||||
const light = await open('/runs/run-sched-1', { state: freshState({ runs: [...RUNS, RUN_MANUAL_FIRE] }), theme: 'light' })
|
||||
await light.page.locator('[data-run-detail="run-sched-1"] header [data-run-schedule-link]').waitFor()
|
||||
const lightTheme = await light.page.evaluate(() => document.documentElement.dataset.theme)
|
||||
const lightTone = await readable(light.page, '[data-run-detail] header [data-run-schedule-link]')
|
||||
check('浅色:页头的定时任务链接对比度 ≥ 4.5', lightTheme === 'light' && !!lightTone && lightTone.ratio >= 4.5,
|
||||
`${lightTheme} ${JSON.stringify(lightTone)}`)
|
||||
await shot(light.page, 'run-detail-schedule-link-light')
|
||||
const plain = await open('/runs/run-manual-1', { state: freshState({ runs: [...RUNS, RUN_MANUAL_FIRE] }) })
|
||||
await plain.page.locator('[data-run-detail="run-manual-1"]').waitFor({ timeout: 10000 })
|
||||
check('手动发起的运行:页头没有定时任务链接', await plain.page.locator('[data-run-detail] header [data-run-schedule-link]').count() === 0)
|
||||
|
||||
// 名字很长、详情区不宽时链接截断,不把页头撑宽(全名在悬停里)
|
||||
const longName = freshState({ runs: [...RUNS, RUN_MANUAL_FIRE] })
|
||||
longName.schedules = longName.schedules.map((x) => (x.id === 's-weekly' ? { ...x, name: '华东区全部门店按品类与渠道拆分的周度销售与退款复盘' } : x))
|
||||
const narrow = await open('/runs/run-sched-1', { state: longName, width: 900 })
|
||||
await narrow.page.locator('[data-run-detail="run-sched-1"]').waitFor({ timeout: 10000 })
|
||||
const nLink = narrow.page.locator('[data-run-detail] header [data-run-schedule-link]')
|
||||
await until(async () => (await nLink.innerText().catch(() => '')).includes('复盘'))
|
||||
const fit = await narrow.page.evaluate(() => {
|
||||
const head = document.querySelector('[data-run-detail] header')
|
||||
const link = head?.querySelector('[data-run-schedule-link]')
|
||||
const label = link?.querySelector('span')
|
||||
if (!head || !link || !label) return null
|
||||
const h = head.getBoundingClientRect()
|
||||
const l = link.getBoundingClientRect()
|
||||
return { headRight: Math.round(h.right), linkRight: Math.round(l.right), scroll: head.scrollWidth - head.clientWidth,
|
||||
clipped: label.scrollWidth > label.clientWidth }
|
||||
})
|
||||
check('长名字:链接截断,不超出页头、页头不横向溢出', !!fit && fit.linkRight <= fit.headRight && fit.scroll <= 1 && fit.clipped,
|
||||
JSON.stringify(fit))
|
||||
// 署名的立即运行(署名加链接,这一行最挤)配长名字:链接照样截断,署名不被挤掉
|
||||
const crowdedState = freshState({ runs: [...RUNS, RUN_MANUAL_FIRE] })
|
||||
crowdedState.schedules = longName.schedules
|
||||
const crowded = await open('/runs/run-sched-2', { state: crowdedState, width: 900 })
|
||||
await crowded.page.locator('[data-run-detail="run-sched-2"]').waitFor({ timeout: 10000 })
|
||||
const cLink = crowded.page.locator('[data-run-detail] header [data-run-schedule-link]')
|
||||
await until(async () => (await cLink.innerText().catch(() => '')).includes('复盘'))
|
||||
const crowd = await headRow(crowded.page)
|
||||
check('……署名的立即运行配长名字:署名照写,链接截断、看得到,页头不横向溢出', !!crowd && crowd.starter
|
||||
&& crowd.fits && crowd.scroll <= 1 && crowd.inView, JSON.stringify(crowd))
|
||||
|
||||
// 分屏常见的窄窗口:探索运行的页头多一个「提取为草稿」,这一行最挤。以前这一行不换行、只有链接能缩,820 宽时链接
|
||||
// 只剩「由…」,768 宽时 0 宽,Tab 停在一个看不见的链接上,「2 条事件」被压成竖排
|
||||
const tight = []
|
||||
for (const [width, run, extract] of [[820, RUN_EXPLORE, true], [820, RUN_EXPLORE_MANUAL, true],
|
||||
[768, RUN_EXPLORE, true], [768, RUN_EXPLORE_MANUAL, true], [768, RUN_MANUAL_FIRE, false]]) {
|
||||
const one = await open(`/runs/${run.id}`, { state: freshState({ runs: [...RUNS, RUN_MANUAL_FIRE, RUN_EXPLORE, RUN_EXPLORE_MANUAL] }), width })
|
||||
await one.page.locator(`[data-run-detail="${run.id}"] header [data-run-schedule-link]`).waitFor({ timeout: 10000 })
|
||||
await until(async () => (await one.page.locator('[data-run-detail] header [data-run-schedule-link]').innerText()).includes('门店周报 · 每周'))
|
||||
const got = await headRow(one.page)
|
||||
const hasExtract = await one.page.locator('[data-run-detail] header [data-action="extract"]').count() === 1
|
||||
const manual = run.started_by === '检查员甲'
|
||||
const ok = !!got && got.fits && got.eventsFlat && got.scroll <= 1 && got.inView && hasExtract === extract
|
||||
&& (!manual || got.starter)
|
||||
tight.push(ok)
|
||||
check(`${width} 宽、${run.run_class === 'exploratory' ? '探索' : '正式'}运行(${manual ? '署名点了立即运行' : '到点发起'}):链接至少写得下「由定时任务」,「条事件」不被压成竖排`,
|
||||
ok, JSON.stringify({ ...got, hasExtract }))
|
||||
const presses = await tabTo(one.page, '[data-run-detail] header [data-run-schedule-link]')
|
||||
if (width === 768 && manual && extract) {
|
||||
check('……最挤的那种 Tab 到链接时看得见(有宽度、在视口里)', presses > 0 && !!got && got.width > 0 && got.inView, `按了 ${presses} 次`)
|
||||
await shot(one.page, 'run-detail-schedule-link-768')
|
||||
}
|
||||
writesSeenExtra.push(...writes(one.sent, /./))
|
||||
errorsExtra.push(...one.errors)
|
||||
}
|
||||
|
||||
// 定时任务已经删了:取回的列表里没有它,不再链过去(只会落到「不存在」页),写成纯文本,和「工作流已删除」一样
|
||||
const goneState = freshState({ runs: [...RUNS, RUN_MANUAL_FIRE] })
|
||||
goneState.schedules = goneState.schedules.filter((x) => x.id !== 's-weekly')
|
||||
const gone = await open('/runs/run-sched-1', { state: goneState })
|
||||
await gone.page.locator('[data-run-detail="run-sched-1"]').waitFor({ timeout: 10000 })
|
||||
const goneEl = gone.page.locator('[data-run-detail] header [data-run-schedule-gone]')
|
||||
await goneEl.waitFor({ timeout: 6000 }).catch(() => {})
|
||||
check('定时任务已删除:写「由定时任务「X」发起(已删除)」,不是链接', (await goneEl.count()) === 1
|
||||
&& (await goneEl.innerText()).trim() === '由定时任务「门店周报(旧名)」发起(已删除)'
|
||||
&& await gone.page.locator('[data-run-detail] header [data-run-schedule-link]').count() === 0
|
||||
&& (await goneEl.evaluate((el) => el.tagName)) !== 'A', (await goneEl.innerText().catch(() => '没有')).trim())
|
||||
check('……悬停说已删除、运行记录仍然保留', ((await goneEl.getAttribute('title')) ?? '').includes('已删除'))
|
||||
const goneManual = await open('/runs/run-sched-2', { state: goneState })
|
||||
await goneManual.page.locator('[data-run-detail="run-sched-2"]').waitFor({ timeout: 10000 })
|
||||
const gmEl = goneManual.page.locator('[data-run-detail] header [data-run-schedule-gone]')
|
||||
await gmEl.waitFor({ timeout: 6000 }).catch(() => {})
|
||||
check('……署名点了立即运行的:署名照写,后面是「来自已删除的定时任务」', oneLine([await goneManual.page.locator('[data-run-detail] header').first().innerText()]).includes('检查员甲 发起')
|
||||
&& (await gmEl.innerText().catch(() => '')).trim() === '来自已删除的定时任务', (await gmEl.innerText().catch(() => '没有')).trim())
|
||||
// 列表取不到(老服务端、暂时连不上)不算删除:照样是链接,用署名里记的名字
|
||||
const downState = freshState({ runs: [...RUNS, RUN_MANUAL_FIRE] })
|
||||
downState.override = (method, path) => (method === 'GET' && path === '/schedules' ? { status: 503, json: { detail: '暂时不可用' } } : null)
|
||||
const down = await open('/runs/run-sched-1', { state: downState })
|
||||
const downLink = down.page.locator('[data-run-detail="run-sched-1"] header [data-run-schedule-link]')
|
||||
await downLink.waitFor({ timeout: 10000 })
|
||||
await sleep(300)
|
||||
check('……定时任务列表取不到时不算删除,照样是链接', await down.page.locator('[data-run-detail] header [data-run-schedule-gone]').count() === 0
|
||||
&& (await downLink.innerText()).trim() === '由定时任务「门店周报(旧名)」发起')
|
||||
writesSeenExtra.push(...writes(gone.sent, /./), ...writes(goneManual.sent, /./), ...writes(down.sent, /./))
|
||||
errorsExtra.push(...gone.errors, ...goneManual.errors, ...down.errors)
|
||||
// 回车打开的定时任务页会请求预览(POST,但只读),不算写
|
||||
const writesSeen = [...writes(sent, /./), ...writes(manual.sent, /./), ...writes(light.sent, /./), ...writes(plain.sent, /./),
|
||||
...writes(narrow.sent, /./), ...writes(crowded.sent, /./), ...writesSeenExtra]
|
||||
.filter((x) => x.key !== 'POST /schedules/preview')
|
||||
check('只读:没有写请求', writesSeen.length === 0, writesSeen.map((x) => x.key).join('、'))
|
||||
const all = [...errors, ...manual.errors, ...light.errors, ...plain.errors, ...narrow.errors, ...crowded.errors, ...errorsExtra]
|
||||
check('没有页面报错', !all.length, all.join(' | '))
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------- 记录页
|
||||
|
||||
await section('记录页:定时标记', async () => {
|
||||
@@ -1416,6 +1737,12 @@ await section('窄屏 390px 与浅色', async () => {
|
||||
const hist = await open('/schedules/s-weekly?tab=history', { ...narrow, theme })
|
||||
await hist.page.locator('[data-fire-id]').first().waitFor()
|
||||
check(`${theme}:触发历史不横向溢出`, await overflow(hist.page) <= 1, `${await overflow(hist.page)}px`)
|
||||
check(`${theme}:「与上一次对比」和旁边的计数在屏幕里`, await inViewX(hist.page, '[data-fire-id="f00"] [data-fire-compare]')
|
||||
&& await inViewX(hist.page, '[data-fire-id="f00"] [data-fire-compare-count]'))
|
||||
const cmp = await readable(hist.page, '[data-fire-id="f00"] [data-fire-compare]')
|
||||
check(`${theme}:「与上一次对比」对比度 ≥ 4.5`, !!cmp && cmp.ratio >= 4.5, JSON.stringify(cmp))
|
||||
const cnt = await readable(hist.page, '[data-fire-id="f00"] [data-fire-compare-count]')
|
||||
check(`${theme}:旁边的计数对比度 ≥ 4.5`, !!cnt && cnt.ratio >= 4.5, JSON.stringify(cnt))
|
||||
check(`${theme}:页头的开关、立即运行、删除在屏幕里`, await inViewX(hist.page, 'header [data-schedule-run]')
|
||||
&& await inViewX(hist.page, 'header [data-schedule-toggle]') && await inViewX(hist.page, 'header [data-schedule-delete]'))
|
||||
await shot(hist.page, `history-390-${theme}`)
|
||||
|
||||
Reference in New Issue
Block a user