合入 W3-b 口径账本闭环:caliber_from.nodes 改名映射;问数据的助手 use_caliber 选用已发布口径(服务端确定性内联实例化);按口径正式出具(自动只给问数据的答案)与口径对账;认证队列、固化成口径草稿、发布后补比;账本统计补上对账与认证转化;使用者的原始运行记录只给问数据的运行

This commit is contained in:
ThearchyHelios
2026-10-16 18:45:45 +08:00
34 changed files with 3985 additions and 100 deletions
+247
View File
@@ -0,0 +1,247 @@
"""口径卡钉住别处口径卡时的节点改名映射(caliber_from.nodes,next-round 02 做法 4 ③)。
那一版的口径卡读 cell(nodes.fetch, 0, 'gmv');拷进这张图时取数节点改名成 c1_fetch(组合式问数、对账、召回同一张图里放
两份口径会撞 id),caliber_from 带 nodes={fetch: c1_fetch}。要一起认这个映射的几处:
- 运行时(metrics._pinned_caliber):式子按语法树把 nodes.fetch 的根换成 c1_fetch 再算,指标集的 source 照记映射;
- 画布校验(schema._caliber_from_issues):映射的形状、改名后的节点在不在图里、在不在口径卡上游;
- 升版检查(governance.unresolved_caliber_upgrades):照常挡、照常记事件,事件带上映射;
- 证据(指标步骤的 source、输入链到改名后的那条查询)、推演(按改名后的式子原样重算);
- 运行对比(compare):同一份口径换了节点名不算口径变了。
没写 nodes 的老图行为一字不差:式子原文、指标集、升版事件都和以前一样(单独测)。数据是虚构店铺库。
"""
from __future__ import annotations
import pytest
from httpx import ASGITransport, AsyncClient
from sqlalchemy import select
from app.core import artifact_store
from app.db.base import SessionLocal
from app.db.models import RunEvent
from app.engine.nodes.metrics import rename_expression, rename_problem, source_expression
from app.engine.runner import run_manager
from app.engine.schema import GraphSpec, validate_graph
from app.main import app
from tests import ledger_samples as S
RENDERED = ("SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' "
"AND paid_at >= '2026-09-01' AND paid_at < '2026-09-08'")
@pytest.fixture(autouse=True)
async def engine_up(tmp_path):
from app.ledger import match as ledger_match
await run_manager.setup()
await S.fresh_source(tmp_path / "shop.db")
yield
await ledger_match.drain()
await run_manager.shutdown()
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
def composed(wf: str, *, rename: dict | None = None, fetch_id: str = "c1_fetch", card_id: str = "c1",
version: int = 1, **card) -> dict:
ref = {"workflow_id": wf, "workflow_version": version, "node_id": "card"}
if rename is not None:
ref["nodes"] = rename
return S.chain(S.node("start", "input"), S.node(fetch_id, "tool", tool=S.tool(), args={"sql": RENDERED}),
S.node(card_id, "metrics", caliber_from=ref, **card),
S.node("out", "output", fields=[{"name": "r", "value": f"{{{{ nodes.{card_id}.text }}}}"}]))
async def finished(run_id: str, node_id: str) -> RunEvent:
async with SessionLocal() as session:
return (await session.execute(select(RunEvent).where(
RunEvent.run_id == run_id, RunEvent.type == "node.finished", RunEvent.node_id == node_id))).scalars().one()
async def run(graph: dict):
return await S.wait((await run_manager.start(graph=graph, input_payload={})).id)
# --------------------------------------------------------------------------
# 改写式子
# --------------------------------------------------------------------------
def test_renaming_rewrites_only_the_roots_of_node_chains():
rename = {"fetch": "c1_fetch"}
assert rename_expression("cell(nodes.fetch, 0, 'gmv')", rename) == "cell(nodes.c1_fetch, 0, 'gmv')"
assert rename_expression("nodes.fetch.rows[0]['gmv'] / 2", rename) == "nodes.c1_fetch.rows[0]['gmv'] / 2"
assert rename_expression("nodes['fetch'].row_count", rename) == "nodes['c1_fetch'].row_count"
# 字符串里的字、别的根、别的节点不动
assert rename_expression("cell(nodes.other, 0, 'nodes.fetch') + vars.fetch", rename) == \
"cell(nodes.other, 0, 'nodes.fetch') + vars.fetch"
# 一处都没换到的原样交回(不按 unparse 改写空白)
assert rename_expression("cell(nodes.other,0,'gmv')", rename) == "cell(nodes.other,0,'gmv')"
assert rename_expression("cell(nodes.fetch,", rename) == "cell(nodes.fetch,"
# 反着换回那一版里的写法:对比口径按它比
assert source_expression("cell(nodes.c1_fetch, 0, 'gmv')", {"nodes": rename}) == "cell(nodes.fetch, 0, 'gmv')"
assert source_expression("cell(nodes.c1_fetch, 0, 'gmv')", {"workflow_id": "w"}) == "cell(nodes.c1_fetch, 0, 'gmv')"
def test_mapping_shape_problems_are_named():
assert rename_problem({"fetch": "c1_fetch"}) is None
assert rename_problem([]) and rename_problem({})
assert "节点 id" in rename_problem({"fetch": "c1 fetch"})
assert "两个节点" in rename_problem({"a": "x", "b": "x"})
# --------------------------------------------------------------------------
# 运行时、证据、推演
# --------------------------------------------------------------------------
async def test_a_renamed_fetch_feeds_the_pinned_card(client):
wf = await S.publish()
row = await run(composed(wf, rename={"fetch": "c1_fetch"}))
assert row.status == "succeeded", row.error
event = await finished(row.id, "c1")
card = artifact_store.load(event.data["artifact"])
assert (card["caliber"], card["caliber_version"]) == ("周销售", "v1")
[metric] = card["metrics"]
assert metric["value"] == 170.5 and metric["rendered"] == "170.50元"
# 卡上记的是这张图里实际算的式子;来源照记映射
assert metric["expression"] == "cell(nodes.c1_fetch, 0, 'gmv')"
assert card["source"] == {"workflow_id": wf, "workflow_version": 1, "node_id": "card",
"nodes": {"fetch": "c1_fetch"}}
[entry] = event.data["evidence"]
assert entry["source"]["nodes"] == {"fetch": "c1_fetch"}
[item] = metric["inputs"]
assert item["via"] == "tool_cell" and item["node_id"] == "c1_fetch" and item["artifact"]
# 证据:指标步骤的来源带映射,输入链到改名后的那条查询的那一格
resp = await client.get(f"/api/runs/{row.id}/evidence/metrics/c1/gmv")
assert resp.status_code == 200, resp.text
body = resp.json()
[step] = [s for s in body["chain"] if s["step"] == "metric"]
assert step["source"]["nodes"] == {"fetch": "c1_fetch"} and step["source"]["workflow_name"] == "周销售"
[inp] = [s for s in body["chain"] if s["step"] == "input"]
assert inp["node_id"] == "c1_fetch" and inp["artifact"] == item["artifact"]
# 推演:按改名后的式子原样重算,改一个输入照常算
resp = await client.post(f"/api/runs/{row.id}/evidence/whatif",
json={"card": "c1", "overrides": {"inputs": {"cell(nodes.c1_fetch, 0, 'gmv')": 200}}})
assert resp.status_code == 200, resp.text
[m] = resp.json()["metrics"]
assert m["old"] == 170.5 and m["new"] == 200 and m["rendered_new"] == "200.00元"
async def test_without_the_mapping_the_renamed_fetch_is_not_found():
"""没写映射:式子照那一版读 nodes.fetch,这张图里没有它——缺输入,照实失败,不会算错。"""
wf = await S.publish()
row = await run(composed(wf))
assert row.status == "failed" and "口径卡计算失败" in (row.error or "")
async def test_old_graphs_without_the_mapping_are_unchanged():
"""没写 nodes:式子原文一个字不改(不经 unparse),来源只有那三项。"""
graph = S.caliber_graph(expression="cell(nodes.fetch,0,'gmv')")
wf = await S.publish(graph=graph)
pinned = {"workflow_id": wf, "workflow_version": 1, "node_id": "card"}
row = await run(S.chain(S.node("start", "input"), S.node("fetch", "tool", tool=S.tool(), args={"sql": RENDERED}),
S.node("mine", "metrics", caliber_from=pinned),
S.node("out", "output", fields=[{"name": "r", "value": "{{ nodes.mine.text }}"}])))
assert row.status == "succeeded", row.error
[entry] = (await finished(row.id, "mine")).data["evidence"]
card = artifact_store.load(entry["artifact"])
assert card["source"] == pinned and entry["source"] == pinned
assert [m["expression"] for m in card["metrics"]] == ["cell(nodes.fetch,0,'gmv')"]
assert set(card) == {"kind", "caliber", "caliber_version", "metrics", "source"}
async def test_a_malformed_mapping_is_refused_before_running():
wf = await S.publish()
with pytest.raises(ValueError, match="节点改名映射有误"):
await run_manager.start(graph=composed(wf, rename={"fetch": "c1 fetch"}), input_payload={})
# --------------------------------------------------------------------------
# 画布校验、升版检查
# --------------------------------------------------------------------------
def _codes(graph: dict) -> list[tuple[str | None, str]]:
return [(i.field, i.message) for i in validate_graph(GraphSpec.model_validate(graph)).issues if i.level == "error"]
def test_validation_checks_the_mapping():
good = composed("wf1", rename={"fetch": "c1_fetch"})
assert not [f for f, _ in _codes(good) if (f or "").startswith("caliber_from")]
shape = _codes(composed("wf1", rename={"fetch": ""}))
assert any(f == "caliber_from.nodes" and "映射有误" in m for f, m in shape)
missing = _codes(composed("wf1", rename={"fetch": "nowhere"}))
assert any(f == "caliber_from.nodes" and "没有节点「nowhere」" in m for f, m in missing)
# 改名后的节点在口径卡下游:算卡的时候它还没执行
late = composed("wf1", rename={"fetch": "c1_fetch"})
late["edges"] = [{"source": "start", "target": "c1"}, {"source": "c1", "target": "c1_fetch"},
{"source": "c1_fetch", "target": "out"}]
assert any(f == "caliber_from.nodes" and "不在口径卡的上游" in m for f, m in _codes(late))
async def test_upgrade_checks_treat_renamed_cards_like_any_pinned_card():
from app.engine.governance import unresolved_caliber_upgrades
wf = await S.publish()
await S.publish(version=2, workflow_id=wf)
async with SessionLocal() as session:
errors, _ = await unresolved_caliber_upgrades(
session, GraphSpec.model_validate(composed(wf, rename={"fetch": "c1_fetch"})))
assert errors and "上游已有 v2" in errors[0]
_, [event] = await unresolved_caliber_upgrades(session, GraphSpec.model_validate(
composed(wf, rename={"fetch": "c1_fetch"}, upgrade_policy="dual")))
assert event["caliber_nodes"] == {"fetch": "c1_fetch"} and event["pinned"] == 1 and event["latest"] == 2
# 没改名的卡:事件和以前一样,没有这个键
plain = composed(wf, fetch_id="fetch", card_id="mine", upgrade_policy="dual")
_, [old] = await unresolved_caliber_upgrades(session, GraphSpec.model_validate(plain))
assert "caliber_nodes" not in old
# --------------------------------------------------------------------------
# 运行对比
# --------------------------------------------------------------------------
def test_compare_sees_the_same_caliber_under_different_node_names():
from app.engine.compare import _caliber_reasons
def side(prefix: str) -> tuple[dict, dict]:
metric = {"id": "gmv", "expression": f"cell(nodes.{prefix}fetch, 0, 'gmv')", "decimals": 2,
"format": "thousands", "unit": "元"}
card = {"caliber": "周销售", "caliber_version": "v1", "metrics": [metric],
"source": {"workflow_id": "w", "workflow_version": 1, "node_id": "card",
**({"nodes": {"fetch": f"{prefix}fetch"}} if prefix else {})}}
return metric, card
(bm, bc), (tm, tc) = side(""), side("c1_")
assert _caliber_reasons(bm, tm, bc, tc) == []
# 映射反着换回来之后式子还是不同的,照旧说表达式改了
tm2 = {**tm, "expression": "cell(nodes.c1_fetch, 0, 'gmv') * 2"}
assert any("表达式由" in r for r in _caliber_reasons(bm, tm2, bc, tc))
async def test_runs_of_the_same_composed_graph_compare_unchanged():
from app.db.models import Workflow
wf = await S.publish()
graph = composed(wf, rename={"fetch": "c1_fetch"})
async with SessionLocal() as session:
saved = Workflow(name="组合的答案", graph=graph)
session.add(saved)
await session.commit()
saved_id = saved.id
a, b = [await S.wait((await run_manager.start(graph=graph, workflow_id=saved_id, input_payload={})).id)
for _ in range(2)]
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
resp = await c.get(f"/api/runs/{b.id}/compare", params={"base": a.id})
assert resp.status_code == 200, resp.text
[metric] = resp.json()["metrics"]
assert metric["classification"] == "unchanged", metric
+2 -1
View File
@@ -61,7 +61,8 @@ async def test_a_matched_run_gives_labels_and_the_whole_judgement(client, monkey
body = await ledger_of(client, row.id)
assert body["state"] == "matched" and body["note"] == "封存后比对,不在封存范围内" and body["post_seal"] is True
assert body["trigger"] == "sealed" and body["retro"] is False and "retro_note" not in body
assert body["actions"] == [] and body["primary"] == "write"
# 第 3 波起整题等价时有「按口径出具」这一项(详细的在 tests/test_ledger_issue.py)
assert [a["kind"] for a in body["actions"]] == ["issue"] and body["primary"] == "write"
[caliber] = body["calibers"]
assert caliber["workflow_name"] == "周销售" and caliber["level_text"] == "受管" and caliber["caliber"] == f"{wf}@1"
[label] = body["reports"]["write"]["labels"].values()
+224
View File
@@ -0,0 +1,224 @@
"""沉淀:可沉淀的问题(认证队列)和「固化成口径草稿」(next-round 02 做法 8,app/ledger/certify.py)。
真的跑三次问数据(两段对话、同一个 SQL 形状、日期不同,虚构店铺库),等后台比对做完:
- 队列:确定性重复进队(3 次运行、2 段对话),变了的字面量是参数候选、不变的是常量,形状里不带取值;同一个问题问了 3 次
进问题聚类;导出过两次的、做成过应用的答案进「被反复用的答案」;点「不是口径」写审计后不再进队;
- 固化:报告引用的查询变成固定 SQL 的取数节点,跨轮变化的日期成为入口参数(缺省值取代表运行的),常量 'paid' 照写;单行
结果的数值列成口径卡指标;出口契约 report_from + metrics_from;过 validate_graph 和受管级别的发布门禁;origin.sources 记
三个来源问题;
- 按受管级别发布之后:来源运行在后台按它补比(trigger=certified),查询变成等价已证明、出了队;再问一次同样的形状,标
「等价已证明」;
- 统计 E3:认证队列的转化(固化、发布、不是口径)有数了。
数据是虚构店铺库。
"""
from __future__ import annotations
import pytest
from httpx import ASGITransport, AsyncClient
from app.db.base import SessionLocal
from app.db.models import Conversation, ConversationTurn, Run, Workflow
from app.engine.runner import run_manager
from app.engine.schema import GraphSpec, validate_graph
from app.ledger import certify
from app.ledger import match as ledger_match
from app.ledger import store
from app.main import app
from tests import ledger_samples as S
QUESTION = "九月第一周卖了多少"
WEEKS = [("2026-09-01", "2026-09-08"), ("2026-09-02", "2026-09-09"), ("2026-09-03", "2026-09-10")]
def weekly(start: str, end: str) -> str:
return (f"SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' AND paid_at >= '{start}' "
f"AND paid_at < '{end}'")
@pytest.fixture(autouse=True)
async def engine_up(tmp_path):
await run_manager.setup()
await S.fresh_source(tmp_path / "shop.db")
yield
await ledger_match.drain()
await run_manager.shutdown()
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
async def conversation() -> str:
async with SessionLocal() as session:
conv = Conversation(title="问销售", kind="chat")
session.add(conv)
await session.commit()
return conv.id
async def ask(monkeypatch, conversation_id: str, sql: str, question: str = QUESTION) -> Run:
"""问数据的一轮:建轮次、跑一次问数据运行(来源记着对话和轮次)、挂回轮次、等比对做完。"""
async with SessionLocal() as session:
turn = ConversationTurn(conversation_id=conversation_id, question=question)
session.add(turn)
await session.commit()
turn_id = turn.id
S.script(monkeypatch, "这一周销售额 [[v:Q1.r0.gmv]]。")
run = await run_manager.start(graph=S.explore_graph(sql), input_payload={"question": question}, origin_kind="chat",
origin={"conversation_id": conversation_id, "turn_id": turn_id})
row = await S.wait(run.id)
assert row.status == "succeeded", row.error
async with SessionLocal() as session:
(await session.get(ConversationTurn, turn_id)).run_id = row.id
await session.commit()
await ledger_match.drain()
return row
async def three_asks(monkeypatch) -> list[Run]:
a, b = await conversation(), await conversation()
return [await ask(monkeypatch, conv, weekly(*week)) for conv, week in zip((a, a, b), WEEKS)]
def shape_items(body: dict) -> list[dict]:
return [i for i in body["items"] if i["kind"] == "shape" and i["source"] == S.CURRENT["source"]]
# --------------------------------------------------------------------------
# 队列
# --------------------------------------------------------------------------
def test_varying_literals_are_parameters_and_the_template_puts_them_back():
shape = certify.Shape("shop", "SELECT SUM(amount) AS gmv FROM orders WHERE status = ? AND paid_at >= ? AND x = ?",
("'paid'", "'2026-09-01'", "3"))
assert certify.varying([shape.literals, ("'paid'", "'2026-09-02'", "4")]) == [1, 2]
template, fields = certify.template_of(shape, [1, 2], "sqlite", start=3)
assert template == ("SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' AND paid_at >= '{{ input.p3 }}' "
"AND x = {{ input.p4 }}")
assert fields == [{"name": "p3", "type": "string", "required": True, "default": "2026-09-01"},
{"name": "p4", "type": "number", "required": True, "default": 3}]
async def test_repeated_shapes_questions_and_reused_answers_queue_up(client, monkeypatch):
runs = await three_asks(monkeypatch)
body = (await client.get("/api/ledger/certify-queue")).json()
[item] = shape_items(body)
assert item["runs"] == 3 and item["conversations"] == 2 and item["question"] == QUESTION
assert item["params"] == [{"index": 1, "distinct": 3}, {"index": 2, "distinct": 3}] and item["constants"] == 1
assert "2026-09" not in item["shape"] and "'paid'" not in item["shape"] and "?" in item["shape"]
assert {t["run_id"] for t in item["turns"]} == {r.id for r in runs}
assert all(t["turn_id"] and t["conversation_id"] for t in item["turns"])
assert item["representative"]["run_id"] in {r.id for r in runs} and len(item["peers"]) == 2
assert item["related"] == [] and item["frozen_into"] == []
clusters = [i for i in body["items"] if i["kind"] == "cluster" and i["question"] == QUESTION]
assert clusters and clusters[0]["count"] >= 3
# 被反复用的答案:导出过两次的、做成过应用的
from app.db.models_blueprints import Blueprint
async with SessionLocal() as session:
from app.auth.audit import record
for _ in range(2):
await record(session, "report.exported", "run", runs[0].id, data={"format": "html"})
session.add(Blueprint(name="周销售应用", source={"kind": "run", "id": runs[1].id}, spec={}))
await session.commit()
body = (await client.get("/api/ledger/certify-queue")).json()
reused = {i["representative"]["run_id"]: i for i in body["items"] if i["kind"] == "reused"}
assert reused[runs[0].id]["uses"]["exports"] == 2 and reused[runs[1].id]["uses"]["app"] is True
assert runs[2].id not in reused
# 「不是口径」:记审计,之后不再进队
res = await client.post("/api/ledger/certify-queue/dismiss", json={"item_id": item["id"], "kind": "shape"})
assert res.status_code == 200
assert shape_items((await client.get("/api/ledger/certify-queue")).json()) == []
stats = (await client.get("/api/ledger/stats")).json()["e3"]["certify"]
assert stats["available"] is True and stats["dismissed"] >= 1
async def test_two_runs_or_one_conversation_are_not_enough(client, monkeypatch):
conv = await conversation()
for week in WEEKS:
await ask(monkeypatch, conv, weekly(*week))
assert shape_items((await client.get("/api/ledger/certify-queue")).json()) == [], "只来自一段对话"
# --------------------------------------------------------------------------
# 固化、发布、再问
# --------------------------------------------------------------------------
async def test_freezing_makes_a_fixed_sql_draft_that_passes_the_governed_gate(client, monkeypatch):
from app.engine.governance import publish_issues
runs = await three_asks(monkeypatch)
[item] = shape_items((await client.get("/api/ledger/certify-queue")).json())
rep = item["representative"]["run_id"]
res = await client.post("/api/copilot/from-run", json={"run_id": rep, "freeze": True, "peers": item["peers"]})
assert res.status_code == 200, res.text
made = res.json()
assert made["frozen"] is True and made["params"] == ["p1", "p2"] and made["metrics"] == ["m1"]
async with SessionLocal() as session:
wf = await session.get(Workflow, made["workflow_id"])
assert wf.status == "draft" and "certified" in wf.tags
nodes = {n["id"]: n for n in wf.graph["nodes"]}
assert set(nodes) == {"start", "fetch", "card", "write", "out"}
rep_run = next(r for r in runs if r.id == rep)
start, end = WEEKS[[r.id for r in runs].index(rep)]
assert nodes["start"]["data"]["config"]["fields"] == [
{"name": "p1", "type": "string", "required": True, "default": start},
{"name": "p2", "type": "string", "required": True, "default": end}]
sql = nodes["fetch"]["data"]["config"]["args"]["sql"]
assert "'{{ input.p1 }}'" in sql and "'{{ input.p2 }}'" in sql and "'paid'" in sql
assert nodes["fetch"]["data"]["config"]["tool"] == S.tool()
assert nodes["card"]["data"]["config"]["metrics"] == [
{"id": "m1", "name": "gmv", "expression": "cell(nodes.fetch, 0, 'gmv')"}]
assert nodes["out"]["data"]["config"]["contract"] == {"report_from": "write", "metrics_from": "card",
"required": ["m1"]}
assert {k: nodes["write"]["data"]["config"].get(k) for k in ("numbers", "on_violation", "claims")} == {
"numbers": "strict", "on_violation": "fail", "claims": "require_citation"}
assert nodes["write"]["data"]["config"]["instructions"] == f"回答:{QUESTION}"
spec = GraphSpec.model_validate(wf.graph)
assert not [i for i in validate_graph(spec).issues if i.level == "error"]
assert not [i for i in publish_issues(spec, level="governed") if i.level == "error"], \
[i.message for i in publish_issues(spec, level="governed") if i.level == "error"]
assert [s["run_id"] for s in wf.origin["sources"]][0] == rep and len(wf.origin["sources"]) == 3
assert all(s["question"] == QUESTION for s in wf.origin["sources"])
assert wf.origin["run_id"] == rep_run.id and wf.origin["freeze"] is True
# 队列上标着已经固化过
[again] = shape_items((await client.get("/api/ledger/certify-queue")).json())
assert {w["workflow_id"] for w in again["frozen_into"]} == {wf.id}
# 按受管级别发布:来源运行在后台补比,变成等价已证明、出了队
res = await client.post(f"/api/workflows/{wf.id}/publish", json={"level": "governed"})
assert res.status_code == 200, res.text
await ledger_match.drain()
for run in runs:
recorded = await store.latest(run.id)
assert recorded.trigger == "certified" and recorded.retro is True
[query] = recorded.content["queries"]
assert query["status"] == "proven" and query["candidates"][0]["caliber"] == f"{wf.id}@1"
body = (await client.get(f"/api/runs/{runs[0].id}/ledger")).json()
assert body["retro_note"] == "按之后发布的口径回看"
assert shape_items((await client.get("/api/ledger/certify-queue")).json()) == []
# 再问一次同样的形状:标等价已证明
later = await ask(monkeypatch, await conversation(), weekly("2026-09-05", "2026-09-12"))
[label] = (await store.latest(later.id)).content["reports"]["write"]["labels"].values()
assert label["kind"] == "caliber" and label["how"] == "independent" and label["caliber"] == f"{wf.id}@1"
assert label["binding"] == {"p1": "2026-09-05", "p2": "2026-09-12"}
stats = (await client.get("/api/ledger/stats")).json()["e3"]["certify"]
assert stats["frozen"] >= 1 and stats["published"] >= 1
async def test_freezing_needs_a_matched_successful_run(client, monkeypatch):
res = await client.post("/api/copilot/from-run", json={"run_id": "nope", "freeze": True})
assert res.status_code == 404
row = await S.explore(monkeypatch, S.ELSEWHERE, text="一共几家店,没有引用。")
res = await client.post("/api/copilot/from-run", json={"run_id": row.id, "freeze": True})
assert res.status_code == 409 and res.json()["detail"] == certify.NO_QUERIES
+294
View File
@@ -0,0 +1,294 @@
"""组合式问数:use_caliber 选用已发布口径,服务端按那一版确定性内联实例化(next-round 02 做法 4,ledger/compose.py)。
- 实例化是确定的:同一版、同一个操作得到一模一样的节点;取数节点照抄、id 加前缀、模板按参数渲染成字面 SQL,口径卡 id
就是操作 id、caliber_from 带改名映射;取数首尾相接再接口径卡,入口连到第一个取数节点;
- 参数按口径入口的字段校验(未知字段、缺必填、类型、引号里的值带引号、裸写的值不是数),不合规拒收、原因交回模型;
只能选这一轮查看到的那一版、能组合的卡、卡上的指标,id 不能撞;
- 问数据的查看对象多了口径:给入口字段和「能不能组合」,不给 SQL;「什么时候该看」换成问数据那一份;使用者不给;
- 走一遍问数据的生成(剧本模型)再真跑出来:比对器把这些数标成 composed「助手选用」,整题 how=composed。
数据是虚构店铺库。
"""
from __future__ import annotations
import json
import pytest
from httpx import ASGITransport, AsyncClient
from app.api import copilot_context
from app.api.copilot_gen import GraphApplier, GraphChecker
from app.db.base import SessionLocal
from app.engine.runner import run_manager
from app.engine.schema import GraphSpec, validate_graph
from app.ledger import compose
from app.ledger import match as ledger_match
from app.ledger import store
from app.look.calibers import CaliberKind
from app.main import app
from tests import ledger_samples as S
@pytest.fixture(autouse=True)
async def engine_up(tmp_path):
await run_manager.setup()
await S.fresh_source(tmp_path / "shop.db")
yield
await ledger_match.drain()
await run_manager.shutdown()
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
def use(wf: str, **extra) -> dict:
return {"op": "use_caliber", "id": "c1", "workflow_id": wf, "version": 1, "metrics": ["gmv"],
"inputs": {"start": "2026-09-01", "end": "2026-09-08"}, **extra}
async def composer() -> compose.Composer:
async with SessionLocal() as session:
kind = await CaliberKind.load(session, compose=True)
return compose.Composer(list(kind.options))
# --------------------------------------------------------------------------
# 实例化
# --------------------------------------------------------------------------
async def test_instantiation_is_deterministic_and_copies_the_pinned_fetch():
wf = await S.publish()
made = (await composer()).compose(use(wf), taken={"start"})
again = (await composer()).compose(use(wf), taken={"start"})
assert made.nodes == again.nodes and made.edges == again.edges
fetch, card = made.nodes
assert fetch["id"] == "c1_fetch" and fetch["type"] == "tool"
assert fetch["data"]["config"]["tool"] == S.tool()
assert fetch["data"]["config"]["args"]["sql"] == S.TEMPLATE.replace("{{ input.start }}", "2026-09-01").replace(
"{{ input.end }}", "2026-09-08")
assert card["id"] == "c1" and card["type"] == "metrics" and card["data"]["label"] == "card"
assert card["data"]["config"]["caliber_from"] == {"workflow_id": wf, "workflow_version": 1, "node_id": "card",
"nodes": {"fetch": "c1_fetch"}}
assert made.edges == [{"source": "c1_fetch", "target": "c1"}]
# 缺省值补上了(store 有缺省 all),不进结果
assert made.inputs == {"start": "2026-09-01", "end": "2026-09-08", "store": "all"}
result = made.result()
assert result["status"] == "ok" and result["card_id"] == "c1" and result["fetches"] == ["c1_fetch"]
assert result["level_text"] == "受管" and result["metrics"] == [{"id": "gmv", "name": "销售额"}]
assert "sql" not in json.dumps(result).lower()
@pytest.mark.parametrize(("change", "words"), [
({"workflow_id": "nope"}, "只能选用这一轮查看到的已发布口径"),
({"version": 2}, "能选用的是 v1"),
({"card": "other"}, "没有口径卡「other」"),
({"id": "C 1"}, "简短的英文小写"),
({"metrics": []}, "metrics 要写"),
({"metrics": ["profit"]}, "没有指标「profit」"),
({"inputs": {"start": "2026-09-01", "end": "2026-09-08", "city": "x"}}, "入口没有字段「city」"),
({"inputs": {"start": "2026-09-01"}}, "请填写「end」"),
({"inputs": {"start": "2026-09-01' OR '1'='1", "end": "2026-09-08"}}, "引号里,值里不能有引号"),
({"inputs": "start=2026-09-01"}, "inputs 要写成"),
])
async def test_bad_choices_are_refused_with_a_reason(change, words):
wf = await S.publish()
op = use(wf, **change) if change.get("workflow_id") != "nope" else {**use(wf), **change}
with pytest.raises(compose.ComposeRefused) as e:
(await composer()).compose(op, taken={"start"})
assert words in e.value.message, e.value.message
async def test_bare_slots_take_numbers_only_and_ids_must_not_clash():
graph = S.caliber_graph(sql="SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' AND id > {{ input.after }}")
# 字段没写类型(按文字收):表单校验放行,裸写进 SQL 的只能是数
graph["nodes"][0]["data"]["config"]["fields"] = [{"name": "after", "default": "0"}]
wf = await S.publish(graph=graph)
pool = await composer()
with pytest.raises(compose.ComposeRefused, match="只能是数"):
pool.compose({**use(wf), "inputs": {"after": "1 OR 1=1"}}, taken=set())
made = pool.compose({**use(wf), "inputs": {"after": "3"}}, taken=set())
assert made.nodes[0]["data"]["config"]["args"]["sql"].endswith("id > 3")
with pytest.raises(compose.ComposeRefused, match="撞了"):
pool.compose({**use(wf), "inputs": {"after": "3"}}, taken={"c1"})
async def test_a_card_that_reads_model_output_is_listed_but_not_composable():
graph = S.chain(S.node("start", "input"), S.node("ask", "agent", tools=[S.tool()], assign_to="kpi"),
S.node("card", "metrics", caliber="周销售", metrics=[{"id": "gmv", "expression": "vars.kpi.gmv"}]),
S.node("out", "output", fields=[{"name": "r", "value": "{{ nodes.card.text }}"}]))
wf = await S.publish(graph=graph)
async with SessionLocal() as session:
kind = await CaliberKind.load(session, compose=True)
[option] = [o for o in kind.options if o.workflow_id == wf]
assert not option.composable and "不能复用" in option.reason
text, _ = await kind.detail([(i, "named") for i in kind.items if i.workflow_id == wf], 4000)
assert "不能组合" in text
with pytest.raises(compose.ComposeRefused, match="不能组合"):
compose.Composer(list(kind.options)).compose(use(wf), taken=set())
# --------------------------------------------------------------------------
# 查看:入口字段、能不能组合,不给 SQL
# --------------------------------------------------------------------------
async def test_the_answer_look_shows_fields_and_how_to_compose_but_no_sql():
wf = await S.publish()
async with SessionLocal() as session:
kind = await CaliberKind.load(session, compose=True)
plain = await CaliberKind.load(session)
objects = [(i, "named") for i in kind.items if i.workflow_id == wf]
text, [record] = await kind.detail(objects, 4000)
assert "入口字段:start(string,必填);end(string,必填);store(string,缺省 all)" in text
assert '"op": "use_caliber"' in text and f'"workflow_id": "{wf}"' in text and '"card": "card"' in text
assert "SELECT" not in text.upper() and "orders" not in text
assert record["kind"] == "caliber" and record["named"]
# 蓝图助手那一支(不组合)照旧给复用写法,没有组合写法
plain_text, _ = await plain.detail([(i, "named") for i in plain.items if i.workflow_id == wf], 4000)
assert "复用写法" in plain_text and "use_caliber" not in plain_text
async def test_the_answer_prompt_switches_only_when_calibers_can_be_composed():
from app.look.service import LookService
assert copilot_context.LOOK_WHEN_ANSWER_PROMPT != copilot_context.LOOK_WHEN_PROMPT
assert "use_caliber" in copilot_context.LOOK_WHEN_ANSWER_PROMPT and "含退款" in copilot_context.LOOK_WHEN_ANSWER_PROMPT
async with SessionLocal() as session:
composing = LookService([await CaliberKind.load(session, compose=True)])
plain = LookService([await CaliberKind.load(session)])
assert copilot_context.look_when(composing) == copilot_context.LOOK_WHEN_ANSWER_PROMPT
assert copilot_context.look_when(plain) == copilot_context.LOOK_WHEN_PROMPT
assert copilot_context.composer(plain) is None and copilot_context.composer(composing) is not None
# --------------------------------------------------------------------------
# 落地:GraphApplier
# --------------------------------------------------------------------------
async def test_the_applier_lands_the_nodes_wires_the_entry_and_reports_refusals():
wf = await S.publish()
nodes: dict = {}
edges: list = []
applier = GraphApplier(nodes, edges, composer=await composer())
op = use(wf)
applied = applier.apply(op)
assert applied.changed and applied.forward and op["result"]["status"] == "ok"
assert set(nodes) == {"c1_fetch", "c1"} and edges == [{"source": "c1_fetch", "target": "c1", "sourceHandle": None}]
# 入口后加的:加上时接到第一个取数节点;模型再发一条同样的连线不重复
applier.apply({"op": "add_node", "node": {"id": "start", "type": "input", "label": "入口", "config": {}}})
assert {"source": "start", "target": "c1_fetch", "sourceHandle": None} in edges
assert not applier.apply({"op": "add_edge", "edge": {"source": "start", "target": "c1_fetch"}}).changed
assert len([e for e in edges if e["target"] == "c1_fetch"]) == 1
# 接着写时重发同一个选用:不再落地
applier.continuing()
assert not applier.apply(use(wf)).changed
assert "c1" in applier.landed()
bad = use(wf, id="c2", inputs={"start": "2026-09-01"})
refused = applier.apply(bad)
assert not refused.changed and refused.forward and bad["result"]["status"] == "refused"
checker = GraphChecker(applier)
first = await checker.issues()
assert [i["code"] for i in first if i["code"] == compose.REFUSED] == [compose.REFUSED]
assert "请填写「end」" in first[0]["message"] and "重新输出 use_caliber" in first[0]["for_model"]
# 交回过一次就不再交
assert not [i for i in await checker.issues() if i["code"] == compose.REFUSED]
# 兜底重放到上一轮的图上:拷进来的节点和接线照样在
applier.replay_onto({"nodes": [{"id": "start", "type": "input", "data": {"config": {}}}], "edges": []})
assert {"c1_fetch", "c1"} <= set(nodes) and any(e["target"] == "c1_fetch" for e in edges)
async def test_without_a_composer_use_caliber_is_dropped_like_any_unknown_op():
nodes: dict = {}
applier = GraphApplier(nodes, [])
applied = applier.apply({"op": "use_caliber", "id": "c1", "workflow_id": "w", "version": 1})
assert not applied.changed and not applied.forward and nodes == {}
# --------------------------------------------------------------------------
# 问数据走一遍:生成 → 运行 → 比对标 composed
# --------------------------------------------------------------------------
def _answer_script(wf: str) -> list[dict]:
from tests.test_copilot_answer_ops import _edge, _node
return [
{"op": "plan", "summary": "选用周销售口径"},
_node("start", "input", {}),
use(wf),
_node("write", "report", {"instructions": "回答:{{ input.question }}"}),
_edge("c1", "write"),
_node("out", "output", {"fields": [{"name": "答案", "value": "{{ nodes.write.text }}"}]}),
_edge("write", "out"),
{"op": "done", "explanation": "按口径周销售回答", "run": True},
]
async def test_an_answer_built_with_use_caliber_is_labelled_composed(client, monkeypatch):
from tests.test_copilot_answer_ops import BODY, _Scripted, _sse, _use
wf = await S.publish()
_use(monkeypatch, _Scripted(_answer_script(wf), []))
ops = await _sse(client, {**BODY, "instruction": "九月第一周卖了多少"})
[chosen] = [o for o in ops if o["op"] == "use_caliber"]
assert chosen["result"]["status"] == "ok" and chosen["result"]["workflow_name"] == "周销售"
assert not [o for o in ops if o["op"] == "add_node" and o["node"]["id"] in ("c1", "c1_fetch")], \
"拷进来的节点不另外转给界面:过程里只有 use_caliber 那一行"
[final] = [o for o in ops if o["op"] == "final"]
assert final["next"] == "run", final["issues"]
graph = final["graph"]
assert {n["id"] for n in graph["nodes"]} == {"start", "c1_fetch", "c1", "write", "out"}
assert not [i for i in validate_graph(GraphSpec.model_validate(graph)).issues if i.level == "error"]
S.script(monkeypatch, "九月第一周销售额 [[m:gmv]]。")
run = await run_manager.start(graph=graph, input_payload={"question": "九月第一周卖了多少"}, origin_kind="chat")
row = await S.wait(run.id)
assert row.status == "succeeded", row.error
await ledger_match.drain()
recorded = await store.latest(row.id)
content = recorded.content
assert content["algo"] == store.ALGO == "2"
[query] = content["queries"]
assert query["status"] == "composed" and query["node_id"] == "c1_fetch"
[cand] = query["candidates"]
assert cand["status"] == "proven" and cand["how"] == "composed" and cand["caliber"] == f"{wf}@1"
[label] = content["reports"]["write"]["labels"].values()
assert label["kind"] == "caliber" and label["how"] == "composed" and label["queries"] == [query["artifact"]]
assert content["whole"]["equivalent"] is True and content["whole"]["how"] == "composed"
assert recorded.content and (await store.latest(row.id)).trigger == "sealed"
async with SessionLocal() as session:
from app.db.models import Artifact
ref = await session.get(Artifact, (recorded.artifact, row.id))
assert ref.meta["calibers"] == [f"{wf}@1"] and ref.meta["counts"]["composed"] == 1
async def test_users_do_not_get_calibers_in_the_answer_look(monkeypatch):
"""使用者(开登录后的 user 角色):问数据里不看口径、不能选用(拷进来的取数 SQL 在他的图里看得到)。"""
import app.api.copilot as copilot
from app.api.copilot import GenerateIn
seen: dict = {}
async def spy(session, payload, sources, **kw):
seen["kinds"] = kw.get("kinds")
raise RuntimeError("到这里就够了")
monkeypatch.setattr(copilot, "_look_service", spy)
monkeypatch.setattr("app.api.deps.restricted", lambda: True)
with pytest.raises(RuntimeError):
await copilot.answer_ops(GenerateIn(instruction="x", intent="answer")).__anext__()
assert seen["kinds"] == ("table", "capability")
monkeypatch.setattr("app.api.deps.restricted", lambda: False)
with pytest.raises(RuntimeError):
await copilot.answer_ops(GenerateIn(instruction="x", intent="answer")).__anext__()
assert seen["kinds"] == ("table", "capability", "caliber")
with pytest.raises(RuntimeError):
await copilot.answer_ops(GenerateIn(instruction="x", intent="build")).__anext__()
assert seen["kinds"] == ("table", "capability")
+371
View File
@@ -0,0 +1,371 @@
"""按口径正式出具(next-round 02 做法 5,拍板 L1、L5,app/ledger/issue.py)。
真的跑探索运行(虚构店铺库、mock 模型写报告),等后台比对做完,看自动出具、一键出具、拒绝原因、去重:
- 问数据的答案整题独立等价于受管口径:比对完自动发起一次正式运行(来源 ledger_issue),origin 记来源运行、口径、绑定
哈希,参数是绑定 + 缺省值,按 graph_hash 钉住;GET 账本的 actions 写能自动、issued 列出它;再点一次交回同一次;
- 已发布级别、助手选用的、不是问数据的(画布试运行等):不自动,只给一键(写明为什么),一键照常发起;设置关了自动出具
也不自动;
- 拒绝写明原因:整题不等价、点名别的口径、版本退役、图里有人工节点、缺入口字段、超过每日上限、使用者没有应用授权;
使用者有授权(应用固定着这一版)就能出具,origin 记靠的是哪个应用;
- 设置 ledger 一组:缺省值、写坏了拒收。
数据是虚构店铺库。
"""
from __future__ import annotations
import asyncio
from datetime import datetime, timezone
import pytest
from httpx import ASGITransport, AsyncClient
from sqlalchemy import select
from app.api.deps import Principal
from app.db.base import SessionLocal
from app.db.models import Run, Setting, WorkflowVersion
from app.engine.runner import run_manager
from app.ledger import issue as ledger_issue
from app.ledger import match as ledger_match
from app.main import app
from tests import ledger_samples as S
ADMIN = Principal(username=None, role="admin", mode="off", via="off")
@pytest.fixture(autouse=True)
async def engine_up(tmp_path):
await run_manager.setup()
await S.fresh_source(tmp_path / "shop.db")
# 测试库按进程共用:每日上限数的是今天全部的按口径出具,别的用例发起的也算。每个用例从缺省设置起步、上限放宽,
# 要测上限的用例自己改
await reset_ledger(issue_daily_limit=100_000)
yield
await ledger_match.drain()
await run_manager.shutdown()
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
async def reset_ledger(**values) -> None:
async with SessionLocal() as session:
row = await session.get(Setting, "ledger")
if row is not None:
await session.delete(row)
if values:
session.add(Setting(key="ledger", value=values))
await session.commit()
async def set_ledger(**values) -> None:
async with SessionLocal() as session:
row = await session.get(Setting, "ledger")
merged = {**(row.value if row else {}), **values}
if row:
row.value = merged
else:
session.add(Setting(key="ledger", value=merged))
await session.commit()
async def issued(run_id: str) -> list[Run]:
async with SessionLocal() as session:
return await ledger_issue.issued_runs(session, run_id)
async def settle(run_id: str) -> list[Run]:
"""等自动出具发起的正式运行跑完。"""
# 自动出具在比对里就发起了(ledger/match.match_run 等它),drain 之后有没有已经定了:只等发起了的跑完
for _ in range(200):
found = await issued(run_id)
if all(r.status in ("succeeded", "failed", "cancelled") for r in found):
return found
await asyncio.sleep(0.05)
return await issued(run_id)
def two_writers(monkeypatch, answer: str = "本周销售额 [[v:Q1.r0.销售额]]。") -> None:
"""mock 模型:口径那张图的报告(指令「写周销售」)引用口径卡的指标,问数据那张图的报告照 answer 写。"""
from langchain_core.messages import AIMessage
from app.providers import mock_model
def decide(self, messages):
text = "\n".join(str(getattr(m, "content", "")) for m in messages)
return AIMessage(content="周销售 [[m:gmv]]。" if "写周销售" in text else answer)
monkeypatch.setattr(mock_model.MockChatModel, "_decide", decide)
async def equivalent_run(monkeypatch, origin: str = "chat"):
"""一次整题独立等价于「周销售」v1 的探索运行(缺省是问数据的答案),等它封存、比对完。"""
from app.engine.runner import run_manager as manager
two_writers(monkeypatch)
run = await manager.start(graph=S.explore_graph(S.EQUIVALENT), input_payload={}, origin_kind=origin)
row = await S.wait(run.id)
assert row.status == "succeeded", row.error
await ledger_match.drain()
return row
# --------------------------------------------------------------------------
# 自动出具
# --------------------------------------------------------------------------
async def test_an_independent_equivalent_answer_to_a_governed_caliber_is_issued_automatically(client, monkeypatch):
wf = await S.publish()
row = await equivalent_run(monkeypatch)
[formal] = await settle(row.id)
assert formal.run_class == "formal" and formal.origin_kind == "ledger_issue" and formal.status == "succeeded"
assert (formal.workflow_id, formal.version) == (wf, 1)
assert formal.input == {"start": "2026-09-01", "end": "2026-09-08", "store": "all"}
async with SessionLocal() as session:
version = (await session.execute(select(WorkflowVersion).where(
WorkflowVersion.workflow_id == wf, WorkflowVersion.version == 1))).scalar_one()
assert formal.version_hash == version.graph_hash
origin = formal.origin
assert origin["source_run"] == row.id and origin["caliber"] == f"{wf}@1" and origin["how"] == "independent"
assert origin["auto"] is True and origin["binding_hash"] and origin["match"]
body = (await client.get(f"/api/runs/{row.id}/ledger")).json()
[action] = [a for a in body["actions"] if a["kind"] == "issue"]
assert action["available"] is True and action["auto"] is True and action["issued_run"] == formal.id
assert action["level_text"] == "受管" and action["workflow_name"] == "周销售"
[item] = body["issued"]
assert item["run_id"] == formal.id and item["auto"] is True and item["status"] == "succeeded"
# 再点一次「按口径出具」:同一运行、同一口径、同一组参数,交回已经发起的那一次
again = await client.post(f"/api/ledger/answers/{row.id}/issue", json={"workflow_id": wf, "version": 1})
assert again.status_code == 200 and again.json()["created"] is False and again.json()["run_id"] == formal.id
assert len(await issued(row.id)) == 1
async def test_only_answers_in_chat_are_issued_automatically(client, monkeypatch):
"""自动出具只给问数据的答案(拍板 L1、做法 5):画布上反复试跑一份和受管口径等价的草图,每次都是新的来源运行,
自动的话每试一次就多一次正式运行(含报告撰写的模型费用)。画布试运行只给一键,写明为什么,一键照常发起。"""
wf = await S.publish()
row = await equivalent_run(monkeypatch, origin="canvas")
assert await settle(row.id) == []
[action] = [a for a in (await client.get(f"/api/runs/{row.id}/ledger")).json()["actions"] if a["kind"] == "issue"]
assert action["available"] is True and action["auto"] is False
assert action["auto_reason"] == ledger_issue.AUTO_NOT_CHAT.format(what="画布试运行")
assert await ledger_issue.auto_issue(row.id) is None and await issued(row.id) == []
res = await client.post(f"/api/ledger/answers/{row.id}/issue", json={"workflow_id": wf, "version": 1})
assert res.status_code == 200 and res.json()["created"] is True
[formal] = await settle(row.id)
assert formal.origin["auto"] is False and formal.status == "succeeded"
async def test_published_level_calibers_only_get_one_click(client, monkeypatch):
wf = await S.publish(level="published")
row = await equivalent_run(monkeypatch)
assert await settle(row.id) == []
body = (await client.get(f"/api/runs/{row.id}/ledger")).json()
[action] = [a for a in body["actions"] if a["kind"] == "issue"]
assert action["available"] is True and action["auto"] is False and action["auto_reason"] == ledger_issue.AUTO_LEVEL
res = await client.post(f"/api/ledger/answers/{row.id}/issue", json={"workflow_id": wf, "version": 1})
assert res.status_code == 200 and res.json()["created"] is True
[formal] = await settle(row.id)
assert formal.origin["auto"] is False and formal.status == "succeeded"
assert res.json()["issued"]["run_id"] == formal.id
async def test_switching_auto_issue_off_keeps_one_click(client, monkeypatch):
await S.publish()
await set_ledger(auto_issue=False)
row = await equivalent_run(monkeypatch)
assert await settle(row.id) == []
[action] = (await client.get(f"/api/runs/{row.id}/ledger")).json()["actions"]
assert action["auto"] is False and action["auto_reason"] == ledger_issue.AUTO_OFF
async def test_composed_answers_are_never_issued_automatically(monkeypatch):
from app.ledger import compose
from app.look.calibers import CaliberKind
wf = await S.publish()
async with SessionLocal() as session:
kind = await CaliberKind.load(session, compose=True)
made = compose.Composer(list(kind.options)).compose(
{"op": "use_caliber", "id": "c1", "workflow_id": wf, "version": 1, "metrics": ["gmv"],
"inputs": {"start": "2026-09-01", "end": "2026-09-08"}}, taken={"start"})
graph = S.chain(S.node("start", "input"), *made.nodes, S.node("write", "report", instructions="写答案"),
S.node("out", "output", fields=[{"name": "答案", "value": "{{ nodes.write.text }}"}]))
two_writers(monkeypatch, "本周销售额 [[m:gmv]]。")
row = await S.wait((await run_manager.start(graph=graph, input_payload={}, origin_kind="chat")).id)
await ledger_match.drain()
assert await settle(row.id) == []
async with SessionLocal() as session:
recorded = await ledger_match.store.latest(row.id)
plan = await ledger_issue.plan_issue(session, await session.get(Run, row.id), recorded.content, wf, 1,
principal=ADMIN)
assert plan.available and plan.how == "composed" and plan.auto_reason == ledger_issue.AUTO_COMPOSED
# --------------------------------------------------------------------------
# 拒绝
# --------------------------------------------------------------------------
async def test_refusals_say_why(client, monkeypatch):
wf = await S.publish(level="published")
other = await S.publish(name="另一份口径", level="published")
unrelated = await S.explore(monkeypatch, S.WITH_REFUNDS)
res = await client.post(f"/api/ledger/answers/{unrelated.id}/issue", json={"workflow_id": wf, "version": 1})
assert res.status_code == 409 and res.json()["code"] == ledger_issue.REFUSED
assert res.json()["detail"].startswith("整题没有等价于这份口径")
row = await equivalent_run(monkeypatch)
whole_on = (await ledger_match.store.latest(row.id)).content["whole"]["workflow_id"]
named = other if whole_on == wf else wf
res = await client.post(f"/api/ledger/answers/{row.id}/issue", json={"workflow_id": named, "version": 1})
assert res.status_code == 409 and "整题等价的是另一份口径" in res.json()["detail"]
# 版本退役
async with SessionLocal() as session:
version = (await session.execute(select(WorkflowVersion).where(
WorkflowVersion.workflow_id == whole_on, WorkflowVersion.version == 1))).scalar_one()
version.retired_at = datetime.now(timezone.utc)
await session.commit()
res = await client.post(f"/api/ledger/answers/{row.id}/issue", json={"workflow_id": whole_on, "version": 1})
assert res.status_code == 409 and "已退役" in res.json()["detail"]
async def test_a_caliber_with_a_human_step_is_not_issued_on_anyones_behalf(client, monkeypatch):
graph = S.caliber_graph()
graph["nodes"].insert(3, S.node("ok", "human", mode="approve", prompt="确认"))
graph["edges"] = [{"source": a["id"], "target": b["id"]} for a, b in zip(graph["nodes"], graph["nodes"][1:])]
wf = await S.publish(graph=graph)
row = await equivalent_run(monkeypatch)
assert await settle(row.id) == []
res = await client.post(f"/api/ledger/answers/{row.id}/issue", json={"workflow_id": wf, "version": 1})
assert res.status_code == 409 and "人工节点" in res.json()["detail"] and "不代人做这些事" in res.json()["detail"]
async def test_missing_entry_fields_are_named(client, monkeypatch):
graph = S.caliber_graph()
graph["nodes"][0]["data"]["config"]["fields"].append({"name": "region", "required": True})
wf = await S.publish(graph=graph)
row = await equivalent_run(monkeypatch)
[action] = (await client.get(f"/api/runs/{row.id}/ledger")).json()["actions"]
assert action["available"] is False and "请填写「region」" in action["reason"]
res = await client.post(f"/api/ledger/answers/{row.id}/issue", json={"workflow_id": wf, "version": 1})
assert res.status_code == 409 and "入口字段还不齐" in res.json()["detail"]
async def test_the_daily_limit_counts_every_issue(client, monkeypatch):
wf = await S.publish(level="published")
await set_ledger(issue_daily_limit=0)
row = await equivalent_run(monkeypatch)
res = await client.post(f"/api/ledger/answers/{row.id}/issue", json={"workflow_id": wf, "version": 1})
assert res.status_code == 429 and res.json()["code"] == ledger_issue.DAILY_LIMIT and "上限(0 次)" in res.json()["detail"]
async def test_users_issue_only_versions_pinned_by_apps_they_can_use(monkeypatch):
from app.db.models_apps import App, AppGrant, AppVersion
wf = await S.publish(level="published")
row = await equivalent_run(monkeypatch)
async with SessionLocal() as session:
run = await session.get(Run, row.id)
run.started_by = "carol"
await session.commit()
carol = Principal(username="carol", role="user", verified=True, mode="local")
with pytest.raises(ledger_issue.IssueRefused) as e:
await ledger_issue.issue(row.id, wf, 1, principal=carol)
assert e.value.status_code == 403 and e.value.message == ledger_issue.ASK_BUILDER
# 别人的运行读不到:先说读不到
dave = Principal(username="dave", role="user", verified=True, mode="local")
with pytest.raises(ledger_issue.IssueRefused) as e:
await ledger_issue.issue(row.id, wf, 1, principal=dave)
assert e.value.status_code == 403 and e.value.message == ledger_issue.NO_RUN_ACCESS
async with SessionLocal() as session:
board = App(name="周销售看板", published_version=1, created_by="bob")
session.add(board)
await session.flush()
session.add(AppVersion(app_id=board.id, version=1, spec={}, spec_hash="x",
pins=[{"alias": "week", "workflow_id": wf, "version": 1}]))
session.add(AppGrant(app_id=board.id, principal_kind="user", principal="carol", role="use"))
await session.commit()
board_id = board.id
formal, created = await ledger_issue.issue(row.id, wf, 1, principal=carol)
assert created and formal.started_by == "carol" and formal.origin["granted_by_app"] == board_id
async def test_concurrent_requests_issue_once(monkeypatch):
wf = await S.publish(level="published")
row = await equivalent_run(monkeypatch)
results = await asyncio.gather(*(ledger_issue.issue(row.id, wf, 1, principal=ADMIN) for _ in range(3)))
assert sorted(created for _, created in results) == [False, False, True]
assert len({r.id for r, _ in results}) == 1
# --------------------------------------------------------------------------
# 设置
# --------------------------------------------------------------------------
async def test_ledger_settings_have_defaults_and_refuse_bad_values(client):
await reset_ledger()
body = (await client.get("/api/settings")).json()
assert body["ledger"] == {"auto_issue": True, "issue_daily_limit": 20, "match_timeout_s": 8}
bad = await client.put("/api/settings", json={"values": {"ledger": {"auto_issue": "yes"}}})
assert bad.status_code == 422 and "自动按口径正式出具" in bad.json()["detail"]
bad = await client.put("/api/settings", json={"values": {"ledger": {"match_timeout_s": 600}}})
assert bad.status_code == 422 and "口径比对时限" in bad.json()["detail"]
ok = await client.put("/api/settings", json={"values": {"ledger": {"auto_issue": False, "issue_daily_limit": 5,
"match_timeout_s": 3}}})
assert ok.status_code == 200 and ok.json()["ledger"] == {"auto_issue": False, "issue_daily_limit": 5,
"match_timeout_s": 3}
async def test_auto_issue_goes_by_the_askers_permissions_when_login_is_on(monkeypatch):
"""开登录时自动出具按这次运行的发起人判(拍板 L5):使用者没有应用授权就不自动出具,有了才出具;账号不在了的按使用者算。"""
from app.db.models import User
from app.db.models_apps import App, AppGrant, AppVersion
monkeypatch.setattr("app.auth.current_mode", lambda: "local")
await set_ledger(auto_issue=False)
wf = await S.publish()
row = await equivalent_run(monkeypatch)
await set_ledger(auto_issue=True)
async with SessionLocal() as session:
session.add(User(username=f"erin_{row.id[:6]}", role="user", password_hash="x"))
(await session.get(Run, row.id)).started_by = f"erin_{row.id[:6]}"
await session.commit()
assert await ledger_issue.auto_issue(row.id) is None and await issued(row.id) == []
async with SessionLocal() as session:
board = App(name="周销售看板", published_version=1, created_by="bob")
session.add(board)
await session.flush()
session.add(AppVersion(app_id=board.id, version=1, spec={}, spec_hash="x",
pins=[{"alias": "week", "workflow_id": wf, "version": 1}]))
session.add(AppGrant(app_id=board.id, principal_kind="user", principal=f"erin_{row.id[:6]}", role="use"))
await session.commit()
launched = await ledger_issue.auto_issue(row.id)
[formal] = await settle(row.id)
assert launched == formal.id and formal.started_by == f"erin_{row.id[:6]}" and formal.origin["auto"] is True
async def test_binding_values_that_would_break_the_sql_are_not_filled_in(client, monkeypatch):
"""绑定出来的值要渲染进口径的 SQL 模板:写在引号里的值带着引号(这次的 SQL 里写的是转义过的 '',绑定照原样取出)就不代填
出具——和组合时校验助手给的参数是同一个判断,宁可少出具。"""
template = "SELECT SUM(amount) AS gmv FROM orders WHERE status = '{{ input.status }}'"
graph = S.caliber_graph(template)
graph["nodes"][0]["data"]["config"]["fields"] = [{"name": "status", "required": True}]
wf = await S.publish(graph=graph, level="published")
two_writers(monkeypatch, "本周销售额 [[v:Q1.r0.gmv]]。")
run = await run_manager.start(graph=S.explore_graph("SELECT SUM(amount) AS gmv FROM orders WHERE status = 'pa''id'"),
input_payload={}, origin_kind="canvas")
row = await S.wait(run.id)
await ledger_match.drain()
whole = (await ledger_match.store.latest(row.id)).content["whole"]
assert whole["equivalent"] is True and whole["binding"] == {"status": "pa''id"}
res = await client.post(f"/api/ledger/answers/{row.id}/issue", json={"workflow_id": wf, "version": 1})
assert res.status_code == 409 and "不能有引号" in res.json()["detail"] and "不代填出具" in res.json()["detail"]
+250
View File
@@ -0,0 +1,250 @@
"""口径对账:差多少(next-round 02 做法 6,app/ledger/reconcile.py、ledger/derive.reconcile_graph)。
真的跑探索运行(虚构店铺库、mock 模型写报告)、等后台比对做完,再点开对账:
- 动作:有相关口径的查询给「看差多少」;参数是按「同一列、同一运算符」猜的,要先核对(needs_check);
- 不核对直接发起回 409(要核对参数);核对过的参数发起:派生图是确定的、过 validate、出口契约 metrics_from 指向对账卡、
不加报告;跑出来的这次、口径、差值、相对差都对,差值的出处链点得开;GET 账本列出对过的账;统计 E2 数到它;
- 认不出哪一列对哪个指标(口径的指标式不是取数的一格):要人选,选了照记进 origin;
- 多行结果:被引用的行按维度列经合并查询对齐,每行一组对账指标;维度是遮罩列、合并键不唯一时照实拒绝。
数据是虚构店铺库。
"""
from __future__ import annotations
import pytest
from httpx import ASGITransport, AsyncClient
from app.db.base import SessionLocal
from app.db.models import Run
from app.engine.runner import run_manager
from app.engine.schema import GraphSpec, validate_graph
from app.ledger import match as ledger_match
from app.ledger import reconcile as ledger_reconcile
from app.main import app
from tests import ledger_samples as S
BY_REGION = ("SELECT region AS 区域, SUM(amount) AS gmv FROM orders WHERE status = 'paid' "
"AND paid_at >= '{{ input.start }}' AND paid_at < '{{ input.end }}' GROUP BY region ORDER BY region")
BY_REGION_ALL = ("SELECT region AS 区域, SUM(amount) AS gmv FROM orders WHERE paid_at >= '2026-09-01' "
"AND paid_at < '2026-09-08' GROUP BY region ORDER BY region")
CHECKED = {"start": "2026-09-01", "end": "2026-09-08"}
@pytest.fixture(autouse=True)
async def engine_up(tmp_path):
await run_manager.setup()
path = tmp_path / "shop.db"
await S.fresh_source(path)
yield path
await ledger_match.drain()
await run_manager.shutdown()
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
async def ledger_of(client, run_id: str) -> dict:
res = await client.get(f"/api/runs/{run_id}/ledger")
assert res.status_code == 200, res.text
return res.json()
async def card_of(run_id: str, node_id: str = "reconcile") -> dict:
from app.api.evidence import _sealed, _sealed_cards
sealed = await _sealed(run_id)
[(_, card)] = [(a, c) for n, a, c in _sealed_cards(sealed) if n == node_id]
return {m["id"]: m for m in card["metrics"]}
async def reconcile(client, run_id: str, body: dict) -> Run:
res = await client.post(f"/api/ledger/answers/{run_id}/reconcile", json=body)
assert res.status_code == 201, res.text
return await S.wait(res.json()["run_id"])
async def test_reconcile_the_refunds_answer_against_the_weekly_caliber(client, monkeypatch):
wf = await S.publish()
row = await S.explore(monkeypatch, S.WITH_REFUNDS)
body = await ledger_of(client, row.id)
[action] = [a for a in body["actions"] if a["kind"] == "reconcile"]
assert action["workflow_id"] == wf and action["version"] == 1 and action["node_id"] == "fetch"
assert action["needs_check"] is True and action["check_reason"] == ledger_reconcile.CHECK_REASON
assert action["binding"] == [{"field": "end", "value": "2026-09-08"}, {"field": "start", "value": "2026-09-01"}]
query = action["query"]
# 参数是猜的:不核对就发起,照实拒绝
res = await client.post(f"/api/ledger/answers/{row.id}/reconcile", json={"query": query, "workflow_id": wf,
"version": 1})
assert res.status_code == 409 and res.json()["code"] == ledger_reconcile.CHECK_INPUTS
assert res.json()["detail"] == ledger_reconcile.CONFIRM_FIRST
done = await reconcile(client, row.id, {"query": query, "workflow_id": wf, "version": 1, "inputs": CHECKED,
"confirm": True})
assert done.status == "succeeded", done.error
assert done.origin_kind == "reconcile" and done.run_class == "exploratory" and done.workflow_id is None
origin = done.origin
assert origin["source_run"] == row.id and origin["query"] == query and origin["inputs_from"] == "checked"
assert origin["pairs"] == [{"ours": "gmv", "metric": "gmv"}] and origin["caliber"] == f"{wf}@1"
assert done.input == {"start": "2026-09-01", "end": "2026-09-08", "store": "all"}
graph = done.graph
ids = [n["id"] for n in graph["nodes"]]
assert ids == ["derived_in", "ours_q1", "fetch", "card", "reconcile", "derived_out"]
nodes = {n["id"]: n for n in graph["nodes"]}
assert nodes["ours_q1"]["data"]["config"]["args"]["sql"] == S.WITH_REFUNDS
assert "{{ input.start }}" in nodes["fetch"]["data"]["config"]["args"]["sql"], "口径的取数保留模板,运行时按入口渲染"
assert nodes["derived_out"]["data"]["config"]["contract"] == {"metrics_from": "reconcile"}
assert not [n for n in graph["nodes"] if n["type"] in ("report", "llm", "agent")]
assert not [i for i in validate_graph(GraphSpec.model_validate(graph)).issues if i.level == "error"]
metrics = await card_of(done.id)
assert metrics["ours_0"]["value"] == 221.0 and metrics["cal_0"]["value"] == 170.5
assert metrics["diff_0"]["value"] == 50.5 and metrics["rel_0"]["value"] == 0.2962
assert metrics["rel_0"]["rendered"] == "29.62%" and metrics["diff_0"]["rendered"] == "50.50元"
# 三个数各自点得开:差值的出处链追到两边的查询
res = await client.get(f"/api/runs/{done.id}/evidence/metrics/reconcile/diff_0")
assert res.status_code == 200, res.text
inputs = [s for s in res.json()["chain"] if s["step"] == "input"]
assert {i["node_id"] for i in inputs} == {"ours_q1", "card"}
listed = (await ledger_of(client, row.id))["reconciled"]
assert [r["run_id"] for r in listed] == [done.id] and listed[0]["inputs_from"] == "checked"
# 统计 E2:对账运行数到了(它不进使用统计,E2 专门数它)
stats = (await client.get("/api/ledger/stats")).json()["e2"]
assert stats["launched"] >= 1 and stats["succeeded"] >= 1
async def test_the_derived_graph_is_deterministic():
from app.ledger import derive
wf = await S.publish()
async with SessionLocal() as session:
pinned = await derive.load_pinned(session, wf, 1, "card")
plan = ledger_reconcile.Plan(pairs=[ledger_reconcile.Pair(ours="gmv", metric="gmv", index=0, name="销售额",
unit="元", decimals=2)])
a = ledger_reconcile.build(pinned, sql=S.WITH_REFUNDS, tool=S.tool(), plan=plan, inputs=CHECKED, fetch="fetch")
b = ledger_reconcile.build(pinned, sql=S.WITH_REFUNDS, tool=S.tool(), plan=plan, inputs=CHECKED, fetch="fetch")
assert a.graph == b.graph and a.kind == "reconcile"
assert a.origin["from"][0] == {"kind": "executed", "node_id": "ours_q1"} and a.origin["from"][1]["node_id"] == "card"
async def test_metrics_that_are_not_a_plain_cell_need_a_person_to_pick(client, monkeypatch):
wf = await S.publish(graph=S.caliber_graph(expression="cell(nodes.fetch, 0, 'gmv') * 1"))
row = await S.explore(monkeypatch, S.WITH_REFUNDS)
[action] = [a for a in (await ledger_of(client, row.id))["actions"] if a["kind"] == "reconcile"]
body = {"query": action["query"], "workflow_id": wf, "version": 1, "inputs": CHECKED, "confirm": True}
res = await client.post(f"/api/ledger/answers/{row.id}/reconcile", json=body)
assert res.status_code == 409 and res.json()["code"] == ledger_reconcile.NEEDS_PAIRS
bad = await client.post(f"/api/ledger/answers/{row.id}/reconcile",
json={**body, "pairs": [{"ours": "nope", "metric": "gmv"}]})
assert bad.status_code == 422 and "没有列「nope」" in bad.json()["detail"]
done = await reconcile(client, row.id, {**body, "pairs": [{"ours": "gmv", "metric": "gmv"}]})
assert done.status == "succeeded", done.error
assert done.origin["pairs_chosen"] is True
assert (await card_of(done.id))["diff_0"]["value"] == 50.5
async def test_multi_row_answers_are_aligned_by_their_dimensions(client, monkeypatch):
wf = await S.publish(graph=S.caliber_graph(BY_REGION))
row = await S.explore(monkeypatch, BY_REGION_ALL, text="东区 [[v:Q1.r0.gmv]],西区 [[v:Q1.r1.gmv]]。")
[action] = [a for a in (await ledger_of(client, row.id))["actions"] if a["kind"] == "reconcile"]
done = await reconcile(client, row.id, {"query": action["query"], "workflow_id": wf, "version": 1,
"inputs": CHECKED, "confirm": True})
assert done.status == "succeeded", done.error
assert done.origin["rows"] == [0, 1] and done.origin["pairs"] == [{"ours": "gmv", "caliber": "gmv"}]
align = next(n for n in done.graph["nodes"] if n["id"] == "align")
assert align["type"] == "merge" and align["data"]["config"]["inputs"] == {"o": "ours_q1", "c": "fetch"}
metrics = await card_of(done.id)
assert metrics["ours_0_0"]["name"] == "区域=east · gmv · 这次"
assert (metrics["ours_0_0"]["value"], metrics["cal_0_0"]["value"], metrics["diff_0_0"]["value"]) == (163.0, 150.5, 12.5)
assert (metrics["ours_1_0"]["value"], metrics["cal_1_0"]["value"], metrics["diff_1_0"]["value"]) == (58.0, 20.0, 38.0)
async def test_masked_dimensions_and_duplicate_keys_are_refused(client, monkeypatch, engine_up):
wf = await S.publish(graph=S.caliber_graph(BY_REGION))
duplicated = ("SELECT region AS 区域, amount AS gmv FROM orders WHERE paid_at >= '2026-09-01' "
"AND paid_at < '2026-09-08' ORDER BY id")
row = await S.explore(monkeypatch, duplicated, text="第一单 [[v:Q1.r0.gmv]],第二单 [[v:Q1.r1.gmv]]。")
[action] = [a for a in (await ledger_of(client, row.id))["actions"] if a["kind"] == "reconcile"]
res = await client.post(f"/api/ledger/answers/{row.id}/reconcile",
json={"query": action["query"], "workflow_id": wf, "version": 1, "inputs": CHECKED,
"confirm": True})
assert res.status_code == 409 and "有重复的行" in res.json()["detail"]
await S.set_source(engine_up, options={"mask_columns": ["region"]})
row = await S.explore(monkeypatch, BY_REGION_ALL, text="东区 [[v:Q1.r0.gmv]],西区 [[v:Q1.r1.gmv]]。")
[action] = [a for a in (await ledger_of(client, row.id))["actions"] if a["kind"] == "reconcile"]
res = await client.post(f"/api/ledger/answers/{row.id}/reconcile",
json={"query": action["query"], "workflow_id": wf, "version": 1, "inputs": CHECKED,
"confirm": True})
assert res.status_code == 409 and "设置了遮罩" in res.json()["detail"]
def test_align_sql_keeps_the_referenced_rows_in_order():
pairs = [ledger_reconcile.Pair(ours="gmv", column="销售额")]
sql = ledger_reconcile.align_sql([("区域", "区域")], pairs, [("west",), ("o'east",)])
assert sql == ('SELECT o."区域" AS "区域", o."gmv" AS "ours_0", c."销售额" AS "cal_0" FROM o LEFT JOIN c ON '
'o."区域" = c."区域" WHERE (o."区域" = \'west\') OR (o."区域" = \'o\'\'east\') '
'ORDER BY CASE WHEN (o."区域" = \'west\') THEN 0 WHEN (o."区域" = \'o\'\'east\') THEN 1 END')
async def test_users_reconcile_their_own_answers_read_only_and_masked(client, monkeypatch):
"""拍板 L5:对账(只读的探索运行)使用者都可以,带上问数据同一套 MASK_RESULTS / READ_ONLY;别人的答案读不到就不给对。"""
from app.api.deps import Principal
from app.tools.datasource import MASK_RESULTS_OPTION, READ_ONLY_OPTION
wf = await S.publish()
row = await S.explore(monkeypatch, S.WITH_REFUNDS)
async with SessionLocal() as session:
(await session.get(Run, row.id)).started_by = "carol"
await session.commit()
[action] = [a for a in (await ledger_of(client, row.id))["actions"] if a["kind"] == "reconcile"]
req = ledger_reconcile.Request(query=action["query"], workflow_id=wf, version=1, inputs=CHECKED, confirm=True)
dave = Principal(username="dave", role="user", verified=True, mode="local")
with pytest.raises(ledger_reconcile.ReconcileRefused) as e:
await ledger_reconcile.reconcile(row.id, req, principal=dave)
assert e.value.status_code == 403
carol = Principal(username="carol", role="user", verified=True, mode="local")
run = await ledger_reconcile.reconcile(row.id, req, principal=carol)
assert run.started_by == "carol"
assert run.options[MASK_RESULTS_OPTION] is True and run.options[READ_ONLY_OPTION] is True
done = await S.wait(run.id)
assert done.status == "succeeded", done.error
async def test_users_read_their_reconcile_runs_only_through_the_evidence(client, monkeypatch):
"""使用者发起的对账运行带 MASK_RESULTS、记在他名下,但派生图里有口径取数节点的 SQL 模板(口径的 SQL 可能写着遮罩值,
所以问数据里使用者不看口径、不能 use_caliber):运行的原始记录、事件、运行时的图(/api/runs/{id}、/events、/graph)只给
他自己的问数据运行,对账运行经证据接口读(那里按遮罩渲染)。"""
from app.tools.datasource import MASK_RESULTS_OPTION
from tests.fakes import auth_env
wf = await S.publish()
row = await S.explore(monkeypatch, S.WITH_REFUNDS, origin="chat")
# 当作开登录后使用者 carol 问的:问数据运行在工具边界遮
async with SessionLocal() as session:
source = await session.get(Run, row.id)
source.started_by, source.options = "carol", {MASK_RESULTS_OPTION: True}
await session.commit()
[action] = [a for a in (await ledger_of(client, row.id))["actions"] if a["kind"] == "reconcile"]
await auth_env.reset()
try:
admin = await auth_env.turn_on_local()
await auth_env.make_user("carol", "user")
assert (await admin.put("/api/auth/settings", json={"user_can_chat": True})).status_code == 200
carol = await auth_env.login("carol")
res = await carol.post(f"/api/ledger/answers/{row.id}/reconcile", json={
"query": action["query"], "workflow_id": wf, "version": 1, "inputs": CHECKED, "confirm": True})
assert res.status_code == 201, res.text
done = await S.wait(res.json()["run_id"])
assert done.status == "succeeded" and done.started_by == "carol" and done.options[MASK_RESULTS_OPTION] is True
for rest in ("", "/events", "/graph"):
assert (await carol.get(f"/api/runs/{done.id}{rest}")).status_code == 403, rest
assert (await carol.get(f"/api/runs/{row.id}{rest}")).status_code == 200, f"自己的问数据运行照旧:{rest}"
assert (await carol.get(f"/api/runs/{done.id}/evidence")).status_code == 200
finally:
await auth_env.reset()