From 6a53a76694f52162b50259e7479daf0ef6f7f976 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 12:52:54 +0800 Subject: [PATCH] =?UTF-8?q?=E5=90=84=E6=9D=A1=E7=94=A8=E5=88=B0=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E7=9A=84=E8=B7=AF=E5=BE=84=E7=85=A7=E5=AE=9E=E6=8A=A5?= =?UTF-8?q?=E5=87=BA=E6=8E=A5=E5=85=A5=E4=B8=8D=E5=AD=98=E5=9C=A8=E6=88=96?= =?UTF-8?q?=E5=B7=B2=E5=81=9C=E7=94=A8=EF=BC=8C=E6=9F=A5=E5=8A=A9=E6=89=8B?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E7=9A=84=E6=8E=A5=E5=8F=A3=E4=B8=8D=E5=86=8D?= =?UTF-8?q?=20500?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 逐条核对 get_chat_model / resolve_provider 的调用方:模型调用、Agent、报告撰写节点按 NodeError 失败, 双盲复算记无法复算、反方审稿记未能反查、审批质询回答失败、结论句裁判记未裁判,原因都是「核查模型 不可用(…)」「裁判模型不可用(…)」这类整句;助手生成报 400,门控模型转人工审批,定时任务发起的 正式运行失败、通知照常推送。这些调用方原本就接了 ProviderNotConfigured,测试逐条钉住:产出、事件、 llm.end 里都不出现 mock-fast。 GET /api/copilot/model 原来直接调 resolve_provider 不接异常:存的接入不存在(现在会抛)、或只存了 模型名而它的接入已停用(以前就会抛)时整个接口 500。现在照常返回,effective_* 为空,error 写明原因, 能用时 error 为 None。 --- backend/app/api/copilot.py | 13 +- backend/tests/test_provider_ref_runtime.py | 463 +++++++++++++++++++++ 2 files changed, 474 insertions(+), 2 deletions(-) create mode 100644 backend/tests/test_provider_ref_runtime.py diff --git a/backend/app/api/copilot.py b/backend/app/api/copilot.py index 33ea5d50..b6c9a1a2 100644 --- a/backend/app/api/copilot.py +++ b/backend/app/api/copilot.py @@ -518,20 +518,29 @@ async def copilot_model_spec( @router.get("/model") async def get_copilot_model(session: AsyncSession = Depends(get_session)) -> dict[str, Any]: - """当前 Copilot 会用哪个模型,以及是显式配的还是兜底来的。""" + """当前 Copilot 会用哪个模型,以及是显式配的还是兜底来的。 + + 存的接入不存在、已停用(或者只存了模型名、提供它的接入已停用)时实际什么都用不了:effective_* 为空, + error 写明原因(和生成时报的同一句),能用时 error 为 None。""" from app.db.models import Setting from app.providers.factory import resolve_provider row = await session.get(Setting, COPILOT_SETTING_KEY) saved = row.value if row else {} configured = bool(saved.get("provider") or saved.get("model")) - resolved = await resolve_provider(session, saved.get("provider"), saved.get("model")) + try: + resolved = await resolve_provider(session, saved.get("provider"), saved.get("model")) + except ProviderNotConfigured as e: + return {"configured": configured, "provider": saved.get("provider") or None, + "model": saved.get("model") or None, "effective_provider": None, "effective_model": None, + "error": _unconfigured(e)} return { "configured": configured, "provider": saved.get("provider") or None, "model": saved.get("model") or None, "effective_provider": resolved.name if resolved else None, "effective_model": saved.get("model") or (resolved.default_model if resolved else None), + "error": None, } diff --git a/backend/tests/test_provider_ref_runtime.py b/backend/tests/test_provider_ref_runtime.py new file mode 100644 index 00000000..8ee6ed9f --- /dev/null +++ b/backend/tests/test_provider_ref_runtime.py @@ -0,0 +1,463 @@ +"""节点上、设置里点名的模型接入不存在或已停用时,每条用到模型的路径都照实报给人,不落成演示模型。 + +以前点名的接入不存在时 get_chat_model 悄悄换成演示模型(mock-fast):报告由它写成假文本,双盲复算、反方审稿、审批 +质询由它来「核查」,正式运行照样出具。现在(providers/factory.named_provider): + +- 模型调用、Agent、报告撰写节点:节点失败,报错带原因和去哪里改,一次模型调用都不发; +- 双盲复算:该指标无法复算,原因「核查模型不可用(…)」,记缺口;反方审稿:这几句未能反查,原因同上; +- 审批质询:回答 ok 为假,error 说明原因;结论句裁判:按「裁判模型不可用」照实记未裁判和缺口; +- 助手:报错给用户,查当前用哪个模型的接口照常返回、写明原因;门控模型:转人工审批、写明原因; +- 定时任务发起的正式运行:运行失败,通知照常推送。 + +每条路径都看一遍:产出、事件、llm.end 里不出现 mock-fast。没点名接入的节点(写作者)用一个显式建的演示接入,模型名是 +「写作演示模型」,所以事件里一出现 mock-fast 就是回退到了内置的演示模型。接入名、数据源、数字都是虚构的。 +""" +from __future__ import annotations + +import asyncio +import json +import sqlite3 +import uuid +from datetime import datetime, timedelta, timezone + +import httpx +import pytest +from httpx import ASGITransport, AsyncClient +from langchain_core.messages import AIMessage +from sqlalchemy import delete, select + +from app.core import artifact_store +from app.core.crypto import encrypt +from app.data import introspect +from app.db.base import SessionLocal +from app.db.models import ( + Approval, DataSource, Provider, Run, RunEvent, Schedule, ScheduleFire, Setting, Workflow, WorkflowVersion, +) +from app.engine import spend +from app.engine.evidence import iter_units +from app.engine.judge import SPEND_KEY as JUDGE_SPEND_KEY +from app.engine.runner import run_manager +from app.main import app +from app.providers.mock_model import MockChatModel +from tests.fixtures.dual_check_shop import DEFINITION, RIGHT, add_source, build +from tests.fixtures.sources import drop_source + +#: 没点名接入的节点用的演示接入:正常的调用在事件里记「写作演示模型」 +WRITER_PROVIDER, WRITER_MODEL = "测试演示接入", "写作演示模型" +#: 写错一个字的接入名;一个停用了的演示接入(停用前照样能调用:以前点名它会照常返回、照常调用) +MISSING, DISABLED = "测试演示接人", "停用演示接入" +REASONS = {MISSING: f"模型接入「{MISSING}」不存在", DISABLED: f"模型接入「{DISABLED}」已停用"} +HINTS = {MISSING: "请在「设置 → 模型接入」中核对名称,或换一个接入", + DISABLED: "请在「设置 → 模型接入」中启用它,或换一个接入"} +SETTING_KEYS = ["verify", "judge", "copilot", "run", spend.SPEND_KEY, JUDGE_SPEND_KEY] +UTC = timezone.utc + + +async def _clean() -> None: + async with SessionLocal() as session: + await session.execute(delete(Setting).where(Setting.key.in_(SETTING_KEYS))) + await session.execute(delete(Provider).where(Provider.name.in_([WRITER_PROVIDER, DISABLED]))) + await session.commit() + + +@pytest.fixture(autouse=True) +async def engine_up(): + await _clean() + async with SessionLocal() as session: + session.add_all([ + Provider(name=WRITER_PROVIDER, kind="mock", models=[{"id": WRITER_MODEL}], default_model=WRITER_MODEL, + enabled=True, extra={}), + Provider(name=DISABLED, kind="mock", models=[{"id": "停用演示模型"}], default_model="停用演示模型", + enabled=False, extra={}), + ]) + await session.commit() + await run_manager.setup() + yield + await run_manager.shutdown() + # 建过的接入、写过的设置不留给同一进程里后面的测试:留着的启用接入会改变别处「没点名」时的回退 + await _clean() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +def script(monkeypatch, writer: str = "本周情况平稳。") -> list[str]: + """演示模型的剧本:写作者回 writer;按结构交结果的调用交空对象、绑了工具的直接收尾(核查、裁判、质询的模型照理 + 一次都不该被调用到)。返回每次调用的模型名。""" + called: list[str] = [] + + def decide(model, messages): + called.append(model.model_name) + if model.response_format: + return AIMessage(content="{}") + if model.tools: + return AIMessage(content="查完了。") + return AIMessage(content=writer) + + monkeypatch.setattr(MockChatModel, "_decide", decide) + return called + + +async def put_setting(key: str, value: dict) -> None: + """直接写库:PUT /api/settings 现在会拦下不存在、已停用的接入,这里模拟的是写好之后接入才被删掉、停用。""" + async with SessionLocal() as session: + row = await session.get(Setting, key) + if row: + row.value = value + else: + session.add(Setting(key=key, value=value)) + await session.commit() + + +def node(nid, ntype, label=None, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}} + + +def chain(*nodes): + return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +async def wait(run_id: str, statuses=("succeeded", "failed")) -> Run: + for _ in range(600): + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row.status in statuses and (row.status == "interrupted" or row.manifest_seq is not None): + await run_manager.wait_idle(run_id) + return row + await asyncio.sleep(0.03) + raise AssertionError(f"运行 {run_id} 停在 {row.status}:{row.error}") + + +async def start(spec: dict, *, run_class: str = "exploratory", statuses=("succeeded", "failed")) -> Run: + run = await run_manager.start(graph=spec, input_payload={}, run_class=run_class) + return await wait(run.id, statuses) + + +async def events(run_id: str) -> list[RunEvent]: + async with SessionLocal() as session: + return list((await session.execute( + select(RunEvent).where(RunEvent.run_id == run_id).order_by(RunEvent.seq))).scalars()) + + +def output_of(evs: list[RunEvent], node_id: str) -> dict: + finished = [e for e in evs if e.type == "node.finished" and e.node_id == node_id] + assert finished, f"节点 {node_id} 没有执行完" + return artifact_store.load(finished[-1].data["artifact"]) + + +def logs(evs: list[RunEvent], code: str) -> list[dict]: + return [e.data for e in evs if e.type == "log" and e.data.get("code") == code] + + +def no_mock(evs: list[RunEvent], *more) -> None: + """事件和给出的产出里都不出现内置演示模型的名字。""" + blob = json.dumps([e.data for e in evs] + list(more), ensure_ascii=False, default=str) + assert "mock-fast" not in blob + + +# -------------------------------------------------------------------------- +# 模型调用、Agent、报告撰写节点:节点失败,说清原因和去哪里改 +# -------------------------------------------------------------------------- + +NODES = { + "llm": lambda ref: node("ask", "llm", "提问", provider=ref, prompt="总结本周情况"), + "agent": lambda ref: node("ask", "agent", "查数作答", provider=ref, prompt="回答问题"), + "report": lambda ref: node("ask", "report", "写周报", provider=ref, instructions="写周报"), +} + + +@pytest.mark.parametrize("ref", [MISSING, DISABLED]) +@pytest.mark.parametrize("kind", list(NODES)) +async def test_a_node_naming_a_broken_provider_fails_and_says_why(monkeypatch, kind, ref): + called = script(monkeypatch) + spec = chain(node("start", "input"), NODES[kind](ref), + node("out", "output", fields=[{"name": "结果", "value": "{{ last_message }}"}])) + run = await start(spec) + assert run.status == "failed" + assert REASONS[ref] in run.error and HINTS[ref] in run.error, run.error + evs = await events(run.id) + [failed] = [e for e in evs if e.type == "node.failed"] + assert failed.node_id == "ask" and REASONS[ref] in failed.data["error"] and HINTS[ref] in failed.data["error"] + assert not [e for e in evs if e.type in ("llm.start", "llm.end")] and called == [] + no_mock(evs, run.output, run.error) + + +# -------------------------------------------------------------------------- +# 双盲复算:无法复算,记缺口 +# -------------------------------------------------------------------------- + +GMV = {"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "cell(nodes.fetch, 0, 'gmv')"} + + +@pytest.fixture +async def shop(tmp_path): + name, source_id = await add_source(build(tmp_path / "shop.db")) + yield name + await drop_source(source_id) + + +def dual_graph(source: str, **dual) -> dict: + return chain( + node("start", "input"), + node("fetch", "tool", tool=f"db_query__{source}", args={"sql": RIGHT}), + node("card", "metrics", caliber="销售口径", caliber_version="v1", metrics=[GMV]), + node("dual", "dual_check", "复核销售额", definitions={"gmv": DEFINITION}, **dual), + node("out", "output", fields=[{"name": "结论", "value": "本周销售额见口径卡"}], + contract={"metrics_from": "card", "narrative": "本周销售额见口径卡"}), + ) + + +@pytest.mark.parametrize("where", ["settings", "node"]) +@pytest.mark.parametrize("ref", [MISSING, DISABLED]) +async def test_dual_check_with_a_broken_verify_provider_records_the_metric_as_unchecked(monkeypatch, shop, ref, where): + called = script(monkeypatch) + if where == "settings": + await put_setting("verify", {"provider": ref}) + run = await start(dual_graph(shop, **({"provider": ref} if where == "node" else {})), run_class="formal") + assert run.status == "succeeded", run.error + evs = await events(run.id) + out = output_of(evs, "dual") + assert out["status"] == "unchecked" + [metric] = out["metrics"] + assert metric["status"] == "unchecked" + assert metric["reason"].startswith("核查模型不可用") and REASONS[ref] in metric["reason"], metric["reason"] + assert out["issuance_gaps"] == [f"指标「销售额」无法复算:{metric['reason']}"] + assert any(REASONS[ref] in gap for gap in run.output["_issuance"]["gaps"]) + [said] = logs(evs, "dual_check_unchecked") + assert REASONS[ref] in said["metrics"][0]["reason"] + assert called == [] and not [e for e in evs if e.type == "llm.end"] + no_mock(evs, run.output, out) + + +# -------------------------------------------------------------------------- +# 反方审稿:未能反查;审批质询:回答失败、说明原因 +# -------------------------------------------------------------------------- + +SALES = ("CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, amount INTEGER, refunded INTEGER);" + "INSERT INTO sales VALUES (1, '华东', 1200, 0), (2, '华东', 300, 1), (3, '华南', 900, 0), (4, '华北', 400, 0);") +QUERY = "SELECT region AS 区域, SUM(amount) AS 销售额 FROM sales GROUP BY region ORDER BY 销售额 DESC" +#: 一句有依据的结论、一句预埋的错误结论(华南其实低于华东) +TEXT = "华东销售额 [[v:Q1.r0.销售额]],居各区域之首。[[see:Q1]]华南销售额超过华东。[[see:Q1]]" + + +@pytest.fixture +async def sales(tmp_path): + path = tmp_path / f"{uuid.uuid4().hex[:8]}.db" + db = sqlite3.connect(path) + db.executescript(SALES) + db.commit() + db.close() + async with SessionLocal() as session: + row = DataSource(name=f"sales_{uuid.uuid4().hex[:8]}", kind="sqlite", database=str(path), readonly=True, + enabled=True, options={}) + session.add(row) + await session.flush() + row.schema_cache = await introspect.introspect(row) + await session.commit() + name, source_id = row.name, row.id + yield name + await drop_source(source_id) + + +def report_graph(source: str, *between) -> dict: + return chain( + node("start", "input"), + node("fetch", "tool", tool=f"db_query__{source}", args={"sql": QUERY}), + node("write", "report", "写周报", instructions="写周报"), + *between, + node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}], + contract={"report_from": "write"}), + ) + + +async def test_challenge_with_a_broken_verify_provider_reviews_no_claim_and_says_why(monkeypatch, sales): + called = script(monkeypatch, writer=TEXT) + await put_setting("verify", {"provider": MISSING}) + run = await start(report_graph(sales, node("chal", "challenge", "审稿", report="write")), run_class="formal") + assert run.status == "succeeded", run.error + evs = await events(run.id) + out = output_of(evs, "chal") + assert out["counts"]["inconclusive"] == out["unreviewed"] == 2 + assert out["counts"]["refuted"] == out["counts"]["stands"] == 0 + [said] = logs(evs, "challenge_unreviewed") + assert said["reason"].startswith("核查模型不可用") and REASONS[MISSING] in said["reason"], said["reason"] + record = artifact_store.load(out["challenge_artifact"]) + assert all(REASONS[MISSING] in c["rationale"] for c in record["claims"]) + # 只有写作者调用了模型 + assert called == [WRITER_MODEL] + no_mock(evs, run.output, out, record) + + +async def test_an_inquiry_with_a_broken_verify_provider_answers_with_the_reason(monkeypatch, sales, client): + called = script(monkeypatch, writer=TEXT) + await put_setting("verify", {"provider": DISABLED}) + paused = await start(report_graph(sales, node("gate", "human", mode="approve", title="周报可以出具吗")), + statuses=("interrupted",)) + async with SessionLocal() as session: + [approval] = list((await session.execute(select(Approval).where( + Approval.run_id == paused.id, Approval.status == "pending"))).scalars()) + r = await client.post(f"/api/runs/{paused.id}/inquiries", + json={"report": "write", "unit": None, "question": "华南真的超过华东吗?", + "approval_id": approval.id}) + assert r.status_code == 200, r.text + inquiry = r.json()["inquiry"] + answer = inquiry["answer"] + assert answer["ok"] is False and answer["text"] == "" + assert answer["error"].startswith("核查模型不可用") and REASONS[DISABLED] in answer["error"], answer["error"] + assert inquiry["model"] is None and inquiry["usage"]["calls"] == 0 + evs = await events(paused.id) + [event] = [e for e in evs if e.type == "approval.inquiry"] + assert event.data["ok"] is False and event.data["error"] == answer["error"] + assert called == [WRITER_MODEL] + no_mock(evs, r.json()) + + +# -------------------------------------------------------------------------- +# 结论句裁判:照「裁判模型不可用」记未裁判和缺口 +# -------------------------------------------------------------------------- + +METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}] +JUDGED = "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。" + + +def weekly(**report) -> dict: + return chain( + node("start", "input"), + node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"), + node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=METRICS), + node("write", "report", instructions="写周报", **report), + node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]), + ) + + +@pytest.mark.parametrize("where,ref", [("node", MISSING), ("settings", DISABLED)]) +async def test_a_broken_judge_provider_leaves_the_claims_unjudged_and_says_why(monkeypatch, where, ref): + called = script(monkeypatch, writer=JUDGED) + if where == "settings": + await put_setting("judge", {"provider": ref}) + report = {"claims": "judge", **({"judge": {"provider": ref}} if where == "node" else {})} + run = await start(weekly(**report), run_class="formal") + assert run.status == "succeeded", run.error + evs = await events(run.id) + [checked] = [e for e in evs if e.type == "report.checked"] + summary = checked.data["judge"] + assert summary["complete"] is False and summary["counts"]["unjudged"] == 2 + assert summary["gaps"] and all("裁判模型不可用" in g and REASONS[ref] in g for g in summary["gaps"]), summary["gaps"] + doc = artifact_store.load(checked.data["doc_artifact"]) + verdicts = [u["verdict"] for _, u in iter_units(doc) if u.get("verdict")] + assert len(verdicts) == 2 and all(v["status"] == "unjudged" and v["reason"] == "error" for v in verdicts) + assert not [e for e in evs if e.type == "llm.end" and e.data.get("purpose") == "judge"] + assert called == [WRITER_MODEL] + no_mock(evs, run.output, doc) + + +# -------------------------------------------------------------------------- +# 助手、门控模型 +# -------------------------------------------------------------------------- + + +async def test_the_assistant_with_a_broken_provider_tells_the_user(monkeypatch, client): + called = script(monkeypatch) + await put_setting("copilot", {"provider": MISSING, "model": ""}) + r = await client.post("/api/copilot/generate", json={"instruction": "做一张门店周报"}) + assert r.status_code == 400 + assert r.json()["detail"] == f"助手使用的模型尚未配置完成:{REASONS[MISSING]}。请到「设置 → 模型接入」检查" + assert called == [] + # 查当前用哪个模型:照常返回,不 500,写明为什么用不了 + got = await client.get("/api/copilot/model") + assert got.status_code == 200, got.text + body = got.json() + assert (body["configured"], body["provider"]) == (True, MISSING) + assert body["effective_provider"] is None and body["effective_model"] is None + assert REASONS[MISSING] in body["error"] + + +async def test_the_assistant_model_lookup_survives_a_model_whose_provider_is_disabled(client): + """只存了模型名、提供它的接入已停用:以前这里直接 500。""" + await put_setting("copilot", {"provider": "", "model": "停用演示模型"}) + got = await client.get("/api/copilot/model") + assert got.status_code == 200, got.text + body = got.json() + assert body["effective_provider"] is None and body["effective_model"] is None + assert f"所属的模型接入「{DISABLED}」已停用" in body["error"] + + +async def test_the_assistant_model_lookup_reports_no_error_when_it_works(client): + await put_setting("copilot", {"provider": WRITER_PROVIDER, "model": ""}) + body = (await client.get("/api/copilot/model")).json() + assert body["effective_provider"] == WRITER_PROVIDER and body["effective_model"] == WRITER_MODEL + assert body["error"] is None + + +async def test_a_broken_gate_provider_hands_the_call_to_a_person_and_says_why(monkeypatch): + from langchain_core.tools import StructuredTool + + from app.tools.trust import ask_gate + + called = script(monkeypatch) + await put_setting("run", {"tool_gate_provider": MISSING}) + + def lookup(name: str) -> str: + return name + + tool = StructuredTool.from_function(lookup, name="crm_lookup", description="按客户名称查询客户档案") + verdict = await ask_gate(tool=tool, args={"name": "甲公司"}, node_title="查客户", task="查甲公司的档案") + assert verdict.allowed is False + assert verdict.reason.startswith("门控模型未给出判断") and REASONS[MISSING] in verdict.reason, verdict.reason + assert verdict.model == "" and called == [] + + +# -------------------------------------------------------------------------- +# 定时任务发起的正式运行:运行失败,通知照常 +# -------------------------------------------------------------------------- + +#: 上海 2026-10-05(周一)09:00 +NINE = datetime(2026, 10, 5, 1, 0, tzinfo=UTC) +HOOK = "https://hooks.example.com/agentlab/inbox-k1" + + +async def test_a_scheduled_formal_run_naming_a_broken_provider_fails_and_is_still_notified(monkeypatch): + from app.scheduler import loop, webhook + + called = script(monkeypatch) + sent: list[dict] = [] + + def receive(request: httpx.Request) -> httpx.Response: + sent.append(json.loads(request.content)) + return httpx.Response(200, json={}) + + monkeypatch.setattr(webhook, "_transport", httpx.MockTransport(receive)) + monkeypatch.setattr(webhook, "RETRY_DELAYS", (0, 0, 0)) + spec = chain(node("start", "input"), node("ask", "llm", "写日报", provider=MISSING, prompt="写今天的门店日报"), + node("out", "output", fields=[{"name": "日报", "value": "{{ last_message }}"}])) + async with SessionLocal() as session: + wf = Workflow(name=f"门店日报-{uuid.uuid4().hex[:6]}", graph=spec, status="published", published_version=1) + session.add(wf) + await session.flush() + session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=spec)) + sched = Schedule(name="门店日报", workflow_id=wf.id, cron="0 9 * * *", timezone="Asia/Shanghai", + run_class="formal", input_template={}, enabled=True, misfire_grace_s=3600, next_run_at=NINE, + webhook_format="generic", webhook_url=encrypt(HOOK), webhook_events=["finished", "failed"]) + session.add(sched) + await session.commit() + sid = sched.id + try: + await loop.fire_due(NINE + timedelta(seconds=5)) + async with SessionLocal() as session: + [fire] = list((await session.execute(select(ScheduleFire).where(ScheduleFire.schedule_id == sid))).scalars()) + assert fire.status == "started" and fire.run_id + run = await wait(fire.run_id, ("succeeded", "failed", "cancelled")) + assert run.status == "failed" and run.run_class == "formal" + assert REASONS[MISSING] in run.error and HINTS[MISSING] in run.error, run.error + assert await webhook.notify_due(datetime.now(UTC) + timedelta(seconds=webhook.SEAL_WAIT_S + 5)) == 1 + [body] = sent + assert (body["event"], body["status"], body["run_id"]) == ("failed", "failed", run.id) + assert called == [] + no_mock(await events(run.id), run.output, body) + finally: + async with SessionLocal() as session: + await session.execute(delete(Schedule).where(Schedule.id == sid)) + await session.commit()