mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-20 17:02:01 +00:00
合入 W3-b 口径账本闭环:caliber_from.nodes 改名映射;问数据的助手 use_caliber 选用已发布口径(服务端确定性内联实例化);按口径正式出具(自动只给问数据的答案)与口径对账;认证队列、固化成口径草稿、发布后补比;账本统计补上对账与认证转化;使用者的原始运行记录只给问数据的运行
This commit is contained in:
@@ -0,0 +1,247 @@
|
||||
"""口径卡钉住别处口径卡时的节点改名映射(caliber_from.nodes,next-round 02 做法 4 ③)。
|
||||
|
||||
那一版的口径卡读 cell(nodes.fetch, 0, 'gmv');拷进这张图时取数节点改名成 c1_fetch(组合式问数、对账、召回同一张图里放
|
||||
两份口径会撞 id),caliber_from 带 nodes={fetch: c1_fetch}。要一起认这个映射的几处:
|
||||
|
||||
- 运行时(metrics._pinned_caliber):式子按语法树把 nodes.fetch 的根换成 c1_fetch 再算,指标集的 source 照记映射;
|
||||
- 画布校验(schema._caliber_from_issues):映射的形状、改名后的节点在不在图里、在不在口径卡上游;
|
||||
- 升版检查(governance.unresolved_caliber_upgrades):照常挡、照常记事件,事件带上映射;
|
||||
- 证据(指标步骤的 source、输入链到改名后的那条查询)、推演(按改名后的式子原样重算);
|
||||
- 运行对比(compare):同一份口径换了节点名不算口径变了。
|
||||
|
||||
没写 nodes 的老图行为一字不差:式子原文、指标集、升版事件都和以前一样(单独测)。数据是虚构店铺库。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core import artifact_store
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import RunEvent
|
||||
from app.engine.nodes.metrics import rename_expression, rename_problem, source_expression
|
||||
from app.engine.runner import run_manager
|
||||
from app.engine.schema import GraphSpec, validate_graph
|
||||
from app.main import app
|
||||
from tests import ledger_samples as S
|
||||
|
||||
RENDERED = ("SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' "
|
||||
"AND paid_at >= '2026-09-01' AND paid_at < '2026-09-08'")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def engine_up(tmp_path):
|
||||
from app.ledger import match as ledger_match
|
||||
|
||||
await run_manager.setup()
|
||||
await S.fresh_source(tmp_path / "shop.db")
|
||||
yield
|
||||
await ledger_match.drain()
|
||||
await run_manager.shutdown()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
def composed(wf: str, *, rename: dict | None = None, fetch_id: str = "c1_fetch", card_id: str = "c1",
|
||||
version: int = 1, **card) -> dict:
|
||||
ref = {"workflow_id": wf, "workflow_version": version, "node_id": "card"}
|
||||
if rename is not None:
|
||||
ref["nodes"] = rename
|
||||
return S.chain(S.node("start", "input"), S.node(fetch_id, "tool", tool=S.tool(), args={"sql": RENDERED}),
|
||||
S.node(card_id, "metrics", caliber_from=ref, **card),
|
||||
S.node("out", "output", fields=[{"name": "r", "value": f"{{{{ nodes.{card_id}.text }}}}"}]))
|
||||
|
||||
|
||||
async def finished(run_id: str, node_id: str) -> RunEvent:
|
||||
async with SessionLocal() as session:
|
||||
return (await session.execute(select(RunEvent).where(
|
||||
RunEvent.run_id == run_id, RunEvent.type == "node.finished", RunEvent.node_id == node_id))).scalars().one()
|
||||
|
||||
|
||||
async def run(graph: dict):
|
||||
return await S.wait((await run_manager.start(graph=graph, input_payload={})).id)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 改写式子
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_renaming_rewrites_only_the_roots_of_node_chains():
|
||||
rename = {"fetch": "c1_fetch"}
|
||||
assert rename_expression("cell(nodes.fetch, 0, 'gmv')", rename) == "cell(nodes.c1_fetch, 0, 'gmv')"
|
||||
assert rename_expression("nodes.fetch.rows[0]['gmv'] / 2", rename) == "nodes.c1_fetch.rows[0]['gmv'] / 2"
|
||||
assert rename_expression("nodes['fetch'].row_count", rename) == "nodes['c1_fetch'].row_count"
|
||||
# 字符串里的字、别的根、别的节点不动
|
||||
assert rename_expression("cell(nodes.other, 0, 'nodes.fetch') + vars.fetch", rename) == \
|
||||
"cell(nodes.other, 0, 'nodes.fetch') + vars.fetch"
|
||||
# 一处都没换到的原样交回(不按 unparse 改写空白)
|
||||
assert rename_expression("cell(nodes.other,0,'gmv')", rename) == "cell(nodes.other,0,'gmv')"
|
||||
assert rename_expression("cell(nodes.fetch,", rename) == "cell(nodes.fetch,"
|
||||
# 反着换回那一版里的写法:对比口径按它比
|
||||
assert source_expression("cell(nodes.c1_fetch, 0, 'gmv')", {"nodes": rename}) == "cell(nodes.fetch, 0, 'gmv')"
|
||||
assert source_expression("cell(nodes.c1_fetch, 0, 'gmv')", {"workflow_id": "w"}) == "cell(nodes.c1_fetch, 0, 'gmv')"
|
||||
|
||||
|
||||
def test_mapping_shape_problems_are_named():
|
||||
assert rename_problem({"fetch": "c1_fetch"}) is None
|
||||
assert rename_problem([]) and rename_problem({})
|
||||
assert "节点 id" in rename_problem({"fetch": "c1 fetch"})
|
||||
assert "两个节点" in rename_problem({"a": "x", "b": "x"})
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 运行时、证据、推演
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_renamed_fetch_feeds_the_pinned_card(client):
|
||||
wf = await S.publish()
|
||||
row = await run(composed(wf, rename={"fetch": "c1_fetch"}))
|
||||
assert row.status == "succeeded", row.error
|
||||
event = await finished(row.id, "c1")
|
||||
card = artifact_store.load(event.data["artifact"])
|
||||
assert (card["caliber"], card["caliber_version"]) == ("周销售", "v1")
|
||||
[metric] = card["metrics"]
|
||||
assert metric["value"] == 170.5 and metric["rendered"] == "170.50元"
|
||||
# 卡上记的是这张图里实际算的式子;来源照记映射
|
||||
assert metric["expression"] == "cell(nodes.c1_fetch, 0, 'gmv')"
|
||||
assert card["source"] == {"workflow_id": wf, "workflow_version": 1, "node_id": "card",
|
||||
"nodes": {"fetch": "c1_fetch"}}
|
||||
[entry] = event.data["evidence"]
|
||||
assert entry["source"]["nodes"] == {"fetch": "c1_fetch"}
|
||||
[item] = metric["inputs"]
|
||||
assert item["via"] == "tool_cell" and item["node_id"] == "c1_fetch" and item["artifact"]
|
||||
|
||||
# 证据:指标步骤的来源带映射,输入链到改名后的那条查询的那一格
|
||||
resp = await client.get(f"/api/runs/{row.id}/evidence/metrics/c1/gmv")
|
||||
assert resp.status_code == 200, resp.text
|
||||
body = resp.json()
|
||||
[step] = [s for s in body["chain"] if s["step"] == "metric"]
|
||||
assert step["source"]["nodes"] == {"fetch": "c1_fetch"} and step["source"]["workflow_name"] == "周销售"
|
||||
[inp] = [s for s in body["chain"] if s["step"] == "input"]
|
||||
assert inp["node_id"] == "c1_fetch" and inp["artifact"] == item["artifact"]
|
||||
|
||||
# 推演:按改名后的式子原样重算,改一个输入照常算
|
||||
resp = await client.post(f"/api/runs/{row.id}/evidence/whatif",
|
||||
json={"card": "c1", "overrides": {"inputs": {"cell(nodes.c1_fetch, 0, 'gmv')": 200}}})
|
||||
assert resp.status_code == 200, resp.text
|
||||
[m] = resp.json()["metrics"]
|
||||
assert m["old"] == 170.5 and m["new"] == 200 and m["rendered_new"] == "200.00元"
|
||||
|
||||
|
||||
async def test_without_the_mapping_the_renamed_fetch_is_not_found():
|
||||
"""没写映射:式子照那一版读 nodes.fetch,这张图里没有它——缺输入,照实失败,不会算错。"""
|
||||
wf = await S.publish()
|
||||
row = await run(composed(wf))
|
||||
assert row.status == "failed" and "口径卡计算失败" in (row.error or "")
|
||||
|
||||
|
||||
async def test_old_graphs_without_the_mapping_are_unchanged():
|
||||
"""没写 nodes:式子原文一个字不改(不经 unparse),来源只有那三项。"""
|
||||
graph = S.caliber_graph(expression="cell(nodes.fetch,0,'gmv')")
|
||||
wf = await S.publish(graph=graph)
|
||||
pinned = {"workflow_id": wf, "workflow_version": 1, "node_id": "card"}
|
||||
row = await run(S.chain(S.node("start", "input"), S.node("fetch", "tool", tool=S.tool(), args={"sql": RENDERED}),
|
||||
S.node("mine", "metrics", caliber_from=pinned),
|
||||
S.node("out", "output", fields=[{"name": "r", "value": "{{ nodes.mine.text }}"}])))
|
||||
assert row.status == "succeeded", row.error
|
||||
[entry] = (await finished(row.id, "mine")).data["evidence"]
|
||||
card = artifact_store.load(entry["artifact"])
|
||||
assert card["source"] == pinned and entry["source"] == pinned
|
||||
assert [m["expression"] for m in card["metrics"]] == ["cell(nodes.fetch,0,'gmv')"]
|
||||
assert set(card) == {"kind", "caliber", "caliber_version", "metrics", "source"}
|
||||
|
||||
|
||||
async def test_a_malformed_mapping_is_refused_before_running():
|
||||
wf = await S.publish()
|
||||
with pytest.raises(ValueError, match="节点改名映射有误"):
|
||||
await run_manager.start(graph=composed(wf, rename={"fetch": "c1 fetch"}), input_payload={})
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 画布校验、升版检查
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _codes(graph: dict) -> list[tuple[str | None, str]]:
|
||||
return [(i.field, i.message) for i in validate_graph(GraphSpec.model_validate(graph)).issues if i.level == "error"]
|
||||
|
||||
|
||||
def test_validation_checks_the_mapping():
|
||||
good = composed("wf1", rename={"fetch": "c1_fetch"})
|
||||
assert not [f for f, _ in _codes(good) if (f or "").startswith("caliber_from")]
|
||||
shape = _codes(composed("wf1", rename={"fetch": ""}))
|
||||
assert any(f == "caliber_from.nodes" and "映射有误" in m for f, m in shape)
|
||||
missing = _codes(composed("wf1", rename={"fetch": "nowhere"}))
|
||||
assert any(f == "caliber_from.nodes" and "没有节点「nowhere」" in m for f, m in missing)
|
||||
# 改名后的节点在口径卡下游:算卡的时候它还没执行
|
||||
late = composed("wf1", rename={"fetch": "c1_fetch"})
|
||||
late["edges"] = [{"source": "start", "target": "c1"}, {"source": "c1", "target": "c1_fetch"},
|
||||
{"source": "c1_fetch", "target": "out"}]
|
||||
assert any(f == "caliber_from.nodes" and "不在口径卡的上游" in m for f, m in _codes(late))
|
||||
|
||||
|
||||
async def test_upgrade_checks_treat_renamed_cards_like_any_pinned_card():
|
||||
from app.engine.governance import unresolved_caliber_upgrades
|
||||
|
||||
wf = await S.publish()
|
||||
await S.publish(version=2, workflow_id=wf)
|
||||
async with SessionLocal() as session:
|
||||
errors, _ = await unresolved_caliber_upgrades(
|
||||
session, GraphSpec.model_validate(composed(wf, rename={"fetch": "c1_fetch"})))
|
||||
assert errors and "上游已有 v2" in errors[0]
|
||||
_, [event] = await unresolved_caliber_upgrades(session, GraphSpec.model_validate(
|
||||
composed(wf, rename={"fetch": "c1_fetch"}, upgrade_policy="dual")))
|
||||
assert event["caliber_nodes"] == {"fetch": "c1_fetch"} and event["pinned"] == 1 and event["latest"] == 2
|
||||
# 没改名的卡:事件和以前一样,没有这个键
|
||||
plain = composed(wf, fetch_id="fetch", card_id="mine", upgrade_policy="dual")
|
||||
_, [old] = await unresolved_caliber_upgrades(session, GraphSpec.model_validate(plain))
|
||||
assert "caliber_nodes" not in old
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 运行对比
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_compare_sees_the_same_caliber_under_different_node_names():
|
||||
from app.engine.compare import _caliber_reasons
|
||||
|
||||
def side(prefix: str) -> tuple[dict, dict]:
|
||||
metric = {"id": "gmv", "expression": f"cell(nodes.{prefix}fetch, 0, 'gmv')", "decimals": 2,
|
||||
"format": "thousands", "unit": "元"}
|
||||
card = {"caliber": "周销售", "caliber_version": "v1", "metrics": [metric],
|
||||
"source": {"workflow_id": "w", "workflow_version": 1, "node_id": "card",
|
||||
**({"nodes": {"fetch": f"{prefix}fetch"}} if prefix else {})}}
|
||||
return metric, card
|
||||
|
||||
(bm, bc), (tm, tc) = side(""), side("c1_")
|
||||
assert _caliber_reasons(bm, tm, bc, tc) == []
|
||||
# 映射反着换回来之后式子还是不同的,照旧说表达式改了
|
||||
tm2 = {**tm, "expression": "cell(nodes.c1_fetch, 0, 'gmv') * 2"}
|
||||
assert any("表达式由" in r for r in _caliber_reasons(bm, tm2, bc, tc))
|
||||
|
||||
|
||||
async def test_runs_of_the_same_composed_graph_compare_unchanged():
|
||||
from app.db.models import Workflow
|
||||
|
||||
wf = await S.publish()
|
||||
graph = composed(wf, rename={"fetch": "c1_fetch"})
|
||||
async with SessionLocal() as session:
|
||||
saved = Workflow(name="组合的答案", graph=graph)
|
||||
session.add(saved)
|
||||
await session.commit()
|
||||
saved_id = saved.id
|
||||
a, b = [await S.wait((await run_manager.start(graph=graph, workflow_id=saved_id, input_payload={})).id)
|
||||
for _ in range(2)]
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||||
resp = await c.get(f"/api/runs/{b.id}/compare", params={"base": a.id})
|
||||
assert resp.status_code == 200, resp.text
|
||||
[metric] = resp.json()["metrics"]
|
||||
assert metric["classification"] == "unchanged", metric
|
||||
@@ -61,7 +61,8 @@ async def test_a_matched_run_gives_labels_and_the_whole_judgement(client, monkey
|
||||
body = await ledger_of(client, row.id)
|
||||
assert body["state"] == "matched" and body["note"] == "封存后比对,不在封存范围内" and body["post_seal"] is True
|
||||
assert body["trigger"] == "sealed" and body["retro"] is False and "retro_note" not in body
|
||||
assert body["actions"] == [] and body["primary"] == "write"
|
||||
# 第 3 波起整题等价时有「按口径出具」这一项(详细的在 tests/test_ledger_issue.py)
|
||||
assert [a["kind"] for a in body["actions"]] == ["issue"] and body["primary"] == "write"
|
||||
[caliber] = body["calibers"]
|
||||
assert caliber["workflow_name"] == "周销售" and caliber["level_text"] == "受管" and caliber["caliber"] == f"{wf}@1"
|
||||
[label] = body["reports"]["write"]["labels"].values()
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
"""沉淀:可沉淀的问题(认证队列)和「固化成口径草稿」(next-round 02 做法 8,app/ledger/certify.py)。
|
||||
|
||||
真的跑三次问数据(两段对话、同一个 SQL 形状、日期不同,虚构店铺库),等后台比对做完:
|
||||
- 队列:确定性重复进队(3 次运行、2 段对话),变了的字面量是参数候选、不变的是常量,形状里不带取值;同一个问题问了 3 次
|
||||
进问题聚类;导出过两次的、做成过应用的答案进「被反复用的答案」;点「不是口径」写审计后不再进队;
|
||||
- 固化:报告引用的查询变成固定 SQL 的取数节点,跨轮变化的日期成为入口参数(缺省值取代表运行的),常量 'paid' 照写;单行
|
||||
结果的数值列成口径卡指标;出口契约 report_from + metrics_from;过 validate_graph 和受管级别的发布门禁;origin.sources 记
|
||||
三个来源问题;
|
||||
- 按受管级别发布之后:来源运行在后台按它补比(trigger=certified),查询变成等价已证明、出了队;再问一次同样的形状,标
|
||||
「等价已证明」;
|
||||
- 统计 E3:认证队列的转化(固化、发布、不是口径)有数了。
|
||||
数据是虚构店铺库。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Conversation, ConversationTurn, Run, Workflow
|
||||
from app.engine.runner import run_manager
|
||||
from app.engine.schema import GraphSpec, validate_graph
|
||||
from app.ledger import certify
|
||||
from app.ledger import match as ledger_match
|
||||
from app.ledger import store
|
||||
from app.main import app
|
||||
from tests import ledger_samples as S
|
||||
|
||||
QUESTION = "九月第一周卖了多少"
|
||||
WEEKS = [("2026-09-01", "2026-09-08"), ("2026-09-02", "2026-09-09"), ("2026-09-03", "2026-09-10")]
|
||||
|
||||
|
||||
def weekly(start: str, end: str) -> str:
|
||||
return (f"SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' AND paid_at >= '{start}' "
|
||||
f"AND paid_at < '{end}'")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def engine_up(tmp_path):
|
||||
await run_manager.setup()
|
||||
await S.fresh_source(tmp_path / "shop.db")
|
||||
yield
|
||||
await ledger_match.drain()
|
||||
await run_manager.shutdown()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
async def conversation() -> str:
|
||||
async with SessionLocal() as session:
|
||||
conv = Conversation(title="问销售", kind="chat")
|
||||
session.add(conv)
|
||||
await session.commit()
|
||||
return conv.id
|
||||
|
||||
|
||||
async def ask(monkeypatch, conversation_id: str, sql: str, question: str = QUESTION) -> Run:
|
||||
"""问数据的一轮:建轮次、跑一次问数据运行(来源记着对话和轮次)、挂回轮次、等比对做完。"""
|
||||
async with SessionLocal() as session:
|
||||
turn = ConversationTurn(conversation_id=conversation_id, question=question)
|
||||
session.add(turn)
|
||||
await session.commit()
|
||||
turn_id = turn.id
|
||||
S.script(monkeypatch, "这一周销售额 [[v:Q1.r0.gmv]]。")
|
||||
run = await run_manager.start(graph=S.explore_graph(sql), input_payload={"question": question}, origin_kind="chat",
|
||||
origin={"conversation_id": conversation_id, "turn_id": turn_id})
|
||||
row = await S.wait(run.id)
|
||||
assert row.status == "succeeded", row.error
|
||||
async with SessionLocal() as session:
|
||||
(await session.get(ConversationTurn, turn_id)).run_id = row.id
|
||||
await session.commit()
|
||||
await ledger_match.drain()
|
||||
return row
|
||||
|
||||
|
||||
async def three_asks(monkeypatch) -> list[Run]:
|
||||
a, b = await conversation(), await conversation()
|
||||
return [await ask(monkeypatch, conv, weekly(*week)) for conv, week in zip((a, a, b), WEEKS)]
|
||||
|
||||
|
||||
def shape_items(body: dict) -> list[dict]:
|
||||
return [i for i in body["items"] if i["kind"] == "shape" and i["source"] == S.CURRENT["source"]]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 队列
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_varying_literals_are_parameters_and_the_template_puts_them_back():
|
||||
shape = certify.Shape("shop", "SELECT SUM(amount) AS gmv FROM orders WHERE status = ? AND paid_at >= ? AND x = ?",
|
||||
("'paid'", "'2026-09-01'", "3"))
|
||||
assert certify.varying([shape.literals, ("'paid'", "'2026-09-02'", "4")]) == [1, 2]
|
||||
template, fields = certify.template_of(shape, [1, 2], "sqlite", start=3)
|
||||
assert template == ("SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' AND paid_at >= '{{ input.p3 }}' "
|
||||
"AND x = {{ input.p4 }}")
|
||||
assert fields == [{"name": "p3", "type": "string", "required": True, "default": "2026-09-01"},
|
||||
{"name": "p4", "type": "number", "required": True, "default": 3}]
|
||||
|
||||
|
||||
async def test_repeated_shapes_questions_and_reused_answers_queue_up(client, monkeypatch):
|
||||
runs = await three_asks(monkeypatch)
|
||||
body = (await client.get("/api/ledger/certify-queue")).json()
|
||||
[item] = shape_items(body)
|
||||
assert item["runs"] == 3 and item["conversations"] == 2 and item["question"] == QUESTION
|
||||
assert item["params"] == [{"index": 1, "distinct": 3}, {"index": 2, "distinct": 3}] and item["constants"] == 1
|
||||
assert "2026-09" not in item["shape"] and "'paid'" not in item["shape"] and "?" in item["shape"]
|
||||
assert {t["run_id"] for t in item["turns"]} == {r.id for r in runs}
|
||||
assert all(t["turn_id"] and t["conversation_id"] for t in item["turns"])
|
||||
assert item["representative"]["run_id"] in {r.id for r in runs} and len(item["peers"]) == 2
|
||||
assert item["related"] == [] and item["frozen_into"] == []
|
||||
clusters = [i for i in body["items"] if i["kind"] == "cluster" and i["question"] == QUESTION]
|
||||
assert clusters and clusters[0]["count"] >= 3
|
||||
|
||||
# 被反复用的答案:导出过两次的、做成过应用的
|
||||
from app.db.models_blueprints import Blueprint
|
||||
|
||||
async with SessionLocal() as session:
|
||||
from app.auth.audit import record
|
||||
|
||||
for _ in range(2):
|
||||
await record(session, "report.exported", "run", runs[0].id, data={"format": "html"})
|
||||
session.add(Blueprint(name="周销售应用", source={"kind": "run", "id": runs[1].id}, spec={}))
|
||||
await session.commit()
|
||||
body = (await client.get("/api/ledger/certify-queue")).json()
|
||||
reused = {i["representative"]["run_id"]: i for i in body["items"] if i["kind"] == "reused"}
|
||||
assert reused[runs[0].id]["uses"]["exports"] == 2 and reused[runs[1].id]["uses"]["app"] is True
|
||||
assert runs[2].id not in reused
|
||||
|
||||
# 「不是口径」:记审计,之后不再进队
|
||||
res = await client.post("/api/ledger/certify-queue/dismiss", json={"item_id": item["id"], "kind": "shape"})
|
||||
assert res.status_code == 200
|
||||
assert shape_items((await client.get("/api/ledger/certify-queue")).json()) == []
|
||||
stats = (await client.get("/api/ledger/stats")).json()["e3"]["certify"]
|
||||
assert stats["available"] is True and stats["dismissed"] >= 1
|
||||
|
||||
|
||||
async def test_two_runs_or_one_conversation_are_not_enough(client, monkeypatch):
|
||||
conv = await conversation()
|
||||
for week in WEEKS:
|
||||
await ask(monkeypatch, conv, weekly(*week))
|
||||
assert shape_items((await client.get("/api/ledger/certify-queue")).json()) == [], "只来自一段对话"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 固化、发布、再问
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_freezing_makes_a_fixed_sql_draft_that_passes_the_governed_gate(client, monkeypatch):
|
||||
from app.engine.governance import publish_issues
|
||||
|
||||
runs = await three_asks(monkeypatch)
|
||||
[item] = shape_items((await client.get("/api/ledger/certify-queue")).json())
|
||||
rep = item["representative"]["run_id"]
|
||||
res = await client.post("/api/copilot/from-run", json={"run_id": rep, "freeze": True, "peers": item["peers"]})
|
||||
assert res.status_code == 200, res.text
|
||||
made = res.json()
|
||||
assert made["frozen"] is True and made["params"] == ["p1", "p2"] and made["metrics"] == ["m1"]
|
||||
async with SessionLocal() as session:
|
||||
wf = await session.get(Workflow, made["workflow_id"])
|
||||
assert wf.status == "draft" and "certified" in wf.tags
|
||||
nodes = {n["id"]: n for n in wf.graph["nodes"]}
|
||||
assert set(nodes) == {"start", "fetch", "card", "write", "out"}
|
||||
rep_run = next(r for r in runs if r.id == rep)
|
||||
start, end = WEEKS[[r.id for r in runs].index(rep)]
|
||||
assert nodes["start"]["data"]["config"]["fields"] == [
|
||||
{"name": "p1", "type": "string", "required": True, "default": start},
|
||||
{"name": "p2", "type": "string", "required": True, "default": end}]
|
||||
sql = nodes["fetch"]["data"]["config"]["args"]["sql"]
|
||||
assert "'{{ input.p1 }}'" in sql and "'{{ input.p2 }}'" in sql and "'paid'" in sql
|
||||
assert nodes["fetch"]["data"]["config"]["tool"] == S.tool()
|
||||
assert nodes["card"]["data"]["config"]["metrics"] == [
|
||||
{"id": "m1", "name": "gmv", "expression": "cell(nodes.fetch, 0, 'gmv')"}]
|
||||
assert nodes["out"]["data"]["config"]["contract"] == {"report_from": "write", "metrics_from": "card",
|
||||
"required": ["m1"]}
|
||||
assert {k: nodes["write"]["data"]["config"].get(k) for k in ("numbers", "on_violation", "claims")} == {
|
||||
"numbers": "strict", "on_violation": "fail", "claims": "require_citation"}
|
||||
assert nodes["write"]["data"]["config"]["instructions"] == f"回答:{QUESTION}"
|
||||
spec = GraphSpec.model_validate(wf.graph)
|
||||
assert not [i for i in validate_graph(spec).issues if i.level == "error"]
|
||||
assert not [i for i in publish_issues(spec, level="governed") if i.level == "error"], \
|
||||
[i.message for i in publish_issues(spec, level="governed") if i.level == "error"]
|
||||
assert [s["run_id"] for s in wf.origin["sources"]][0] == rep and len(wf.origin["sources"]) == 3
|
||||
assert all(s["question"] == QUESTION for s in wf.origin["sources"])
|
||||
assert wf.origin["run_id"] == rep_run.id and wf.origin["freeze"] is True
|
||||
|
||||
# 队列上标着已经固化过
|
||||
[again] = shape_items((await client.get("/api/ledger/certify-queue")).json())
|
||||
assert {w["workflow_id"] for w in again["frozen_into"]} == {wf.id}
|
||||
|
||||
# 按受管级别发布:来源运行在后台补比,变成等价已证明、出了队
|
||||
res = await client.post(f"/api/workflows/{wf.id}/publish", json={"level": "governed"})
|
||||
assert res.status_code == 200, res.text
|
||||
await ledger_match.drain()
|
||||
for run in runs:
|
||||
recorded = await store.latest(run.id)
|
||||
assert recorded.trigger == "certified" and recorded.retro is True
|
||||
[query] = recorded.content["queries"]
|
||||
assert query["status"] == "proven" and query["candidates"][0]["caliber"] == f"{wf.id}@1"
|
||||
body = (await client.get(f"/api/runs/{runs[0].id}/ledger")).json()
|
||||
assert body["retro_note"] == "按之后发布的口径回看"
|
||||
assert shape_items((await client.get("/api/ledger/certify-queue")).json()) == []
|
||||
|
||||
# 再问一次同样的形状:标等价已证明
|
||||
later = await ask(monkeypatch, await conversation(), weekly("2026-09-05", "2026-09-12"))
|
||||
[label] = (await store.latest(later.id)).content["reports"]["write"]["labels"].values()
|
||||
assert label["kind"] == "caliber" and label["how"] == "independent" and label["caliber"] == f"{wf.id}@1"
|
||||
assert label["binding"] == {"p1": "2026-09-05", "p2": "2026-09-12"}
|
||||
|
||||
stats = (await client.get("/api/ledger/stats")).json()["e3"]["certify"]
|
||||
assert stats["frozen"] >= 1 and stats["published"] >= 1
|
||||
|
||||
|
||||
async def test_freezing_needs_a_matched_successful_run(client, monkeypatch):
|
||||
res = await client.post("/api/copilot/from-run", json={"run_id": "nope", "freeze": True})
|
||||
assert res.status_code == 404
|
||||
row = await S.explore(monkeypatch, S.ELSEWHERE, text="一共几家店,没有引用。")
|
||||
res = await client.post("/api/copilot/from-run", json={"run_id": row.id, "freeze": True})
|
||||
assert res.status_code == 409 and res.json()["detail"] == certify.NO_QUERIES
|
||||
@@ -0,0 +1,294 @@
|
||||
"""组合式问数:use_caliber 选用已发布口径,服务端按那一版确定性内联实例化(next-round 02 做法 4,ledger/compose.py)。
|
||||
|
||||
- 实例化是确定的:同一版、同一个操作得到一模一样的节点;取数节点照抄、id 加前缀、模板按参数渲染成字面 SQL,口径卡 id
|
||||
就是操作 id、caliber_from 带改名映射;取数首尾相接再接口径卡,入口连到第一个取数节点;
|
||||
- 参数按口径入口的字段校验(未知字段、缺必填、类型、引号里的值带引号、裸写的值不是数),不合规拒收、原因交回模型;
|
||||
只能选这一轮查看到的那一版、能组合的卡、卡上的指标,id 不能撞;
|
||||
- 问数据的查看对象多了口径:给入口字段和「能不能组合」,不给 SQL;「什么时候该看」换成问数据那一份;使用者不给;
|
||||
- 走一遍问数据的生成(剧本模型)再真跑出来:比对器把这些数标成 composed「助手选用」,整题 how=composed。
|
||||
数据是虚构店铺库。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from app.api import copilot_context
|
||||
from app.api.copilot_gen import GraphApplier, GraphChecker
|
||||
from app.db.base import SessionLocal
|
||||
from app.engine.runner import run_manager
|
||||
from app.engine.schema import GraphSpec, validate_graph
|
||||
from app.ledger import compose
|
||||
from app.ledger import match as ledger_match
|
||||
from app.ledger import store
|
||||
from app.look.calibers import CaliberKind
|
||||
from app.main import app
|
||||
from tests import ledger_samples as S
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def engine_up(tmp_path):
|
||||
await run_manager.setup()
|
||||
await S.fresh_source(tmp_path / "shop.db")
|
||||
yield
|
||||
await ledger_match.drain()
|
||||
await run_manager.shutdown()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
def use(wf: str, **extra) -> dict:
|
||||
return {"op": "use_caliber", "id": "c1", "workflow_id": wf, "version": 1, "metrics": ["gmv"],
|
||||
"inputs": {"start": "2026-09-01", "end": "2026-09-08"}, **extra}
|
||||
|
||||
|
||||
async def composer() -> compose.Composer:
|
||||
async with SessionLocal() as session:
|
||||
kind = await CaliberKind.load(session, compose=True)
|
||||
return compose.Composer(list(kind.options))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 实例化
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_instantiation_is_deterministic_and_copies_the_pinned_fetch():
|
||||
wf = await S.publish()
|
||||
made = (await composer()).compose(use(wf), taken={"start"})
|
||||
again = (await composer()).compose(use(wf), taken={"start"})
|
||||
assert made.nodes == again.nodes and made.edges == again.edges
|
||||
fetch, card = made.nodes
|
||||
assert fetch["id"] == "c1_fetch" and fetch["type"] == "tool"
|
||||
assert fetch["data"]["config"]["tool"] == S.tool()
|
||||
assert fetch["data"]["config"]["args"]["sql"] == S.TEMPLATE.replace("{{ input.start }}", "2026-09-01").replace(
|
||||
"{{ input.end }}", "2026-09-08")
|
||||
assert card["id"] == "c1" and card["type"] == "metrics" and card["data"]["label"] == "card"
|
||||
assert card["data"]["config"]["caliber_from"] == {"workflow_id": wf, "workflow_version": 1, "node_id": "card",
|
||||
"nodes": {"fetch": "c1_fetch"}}
|
||||
assert made.edges == [{"source": "c1_fetch", "target": "c1"}]
|
||||
# 缺省值补上了(store 有缺省 all),不进结果
|
||||
assert made.inputs == {"start": "2026-09-01", "end": "2026-09-08", "store": "all"}
|
||||
result = made.result()
|
||||
assert result["status"] == "ok" and result["card_id"] == "c1" and result["fetches"] == ["c1_fetch"]
|
||||
assert result["level_text"] == "受管" and result["metrics"] == [{"id": "gmv", "name": "销售额"}]
|
||||
assert "sql" not in json.dumps(result).lower()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("change", "words"), [
|
||||
({"workflow_id": "nope"}, "只能选用这一轮查看到的已发布口径"),
|
||||
({"version": 2}, "能选用的是 v1"),
|
||||
({"card": "other"}, "没有口径卡「other」"),
|
||||
({"id": "C 1"}, "简短的英文小写"),
|
||||
({"metrics": []}, "metrics 要写"),
|
||||
({"metrics": ["profit"]}, "没有指标「profit」"),
|
||||
({"inputs": {"start": "2026-09-01", "end": "2026-09-08", "city": "x"}}, "入口没有字段「city」"),
|
||||
({"inputs": {"start": "2026-09-01"}}, "请填写「end」"),
|
||||
({"inputs": {"start": "2026-09-01' OR '1'='1", "end": "2026-09-08"}}, "引号里,值里不能有引号"),
|
||||
({"inputs": "start=2026-09-01"}, "inputs 要写成"),
|
||||
])
|
||||
async def test_bad_choices_are_refused_with_a_reason(change, words):
|
||||
wf = await S.publish()
|
||||
op = use(wf, **change) if change.get("workflow_id") != "nope" else {**use(wf), **change}
|
||||
with pytest.raises(compose.ComposeRefused) as e:
|
||||
(await composer()).compose(op, taken={"start"})
|
||||
assert words in e.value.message, e.value.message
|
||||
|
||||
|
||||
async def test_bare_slots_take_numbers_only_and_ids_must_not_clash():
|
||||
graph = S.caliber_graph(sql="SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' AND id > {{ input.after }}")
|
||||
# 字段没写类型(按文字收):表单校验放行,裸写进 SQL 的只能是数
|
||||
graph["nodes"][0]["data"]["config"]["fields"] = [{"name": "after", "default": "0"}]
|
||||
wf = await S.publish(graph=graph)
|
||||
pool = await composer()
|
||||
with pytest.raises(compose.ComposeRefused, match="只能是数"):
|
||||
pool.compose({**use(wf), "inputs": {"after": "1 OR 1=1"}}, taken=set())
|
||||
made = pool.compose({**use(wf), "inputs": {"after": "3"}}, taken=set())
|
||||
assert made.nodes[0]["data"]["config"]["args"]["sql"].endswith("id > 3")
|
||||
with pytest.raises(compose.ComposeRefused, match="撞了"):
|
||||
pool.compose({**use(wf), "inputs": {"after": "3"}}, taken={"c1"})
|
||||
|
||||
|
||||
async def test_a_card_that_reads_model_output_is_listed_but_not_composable():
|
||||
graph = S.chain(S.node("start", "input"), S.node("ask", "agent", tools=[S.tool()], assign_to="kpi"),
|
||||
S.node("card", "metrics", caliber="周销售", metrics=[{"id": "gmv", "expression": "vars.kpi.gmv"}]),
|
||||
S.node("out", "output", fields=[{"name": "r", "value": "{{ nodes.card.text }}"}]))
|
||||
wf = await S.publish(graph=graph)
|
||||
async with SessionLocal() as session:
|
||||
kind = await CaliberKind.load(session, compose=True)
|
||||
[option] = [o for o in kind.options if o.workflow_id == wf]
|
||||
assert not option.composable and "不能复用" in option.reason
|
||||
text, _ = await kind.detail([(i, "named") for i in kind.items if i.workflow_id == wf], 4000)
|
||||
assert "不能组合" in text
|
||||
with pytest.raises(compose.ComposeRefused, match="不能组合"):
|
||||
compose.Composer(list(kind.options)).compose(use(wf), taken=set())
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 查看:入口字段、能不能组合,不给 SQL
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_the_answer_look_shows_fields_and_how_to_compose_but_no_sql():
|
||||
wf = await S.publish()
|
||||
async with SessionLocal() as session:
|
||||
kind = await CaliberKind.load(session, compose=True)
|
||||
plain = await CaliberKind.load(session)
|
||||
objects = [(i, "named") for i in kind.items if i.workflow_id == wf]
|
||||
text, [record] = await kind.detail(objects, 4000)
|
||||
assert "入口字段:start(string,必填);end(string,必填);store(string,缺省 all)" in text
|
||||
assert '"op": "use_caliber"' in text and f'"workflow_id": "{wf}"' in text and '"card": "card"' in text
|
||||
assert "SELECT" not in text.upper() and "orders" not in text
|
||||
assert record["kind"] == "caliber" and record["named"]
|
||||
# 蓝图助手那一支(不组合)照旧给复用写法,没有组合写法
|
||||
plain_text, _ = await plain.detail([(i, "named") for i in plain.items if i.workflow_id == wf], 4000)
|
||||
assert "复用写法" in plain_text and "use_caliber" not in plain_text
|
||||
|
||||
|
||||
async def test_the_answer_prompt_switches_only_when_calibers_can_be_composed():
|
||||
from app.look.service import LookService
|
||||
|
||||
assert copilot_context.LOOK_WHEN_ANSWER_PROMPT != copilot_context.LOOK_WHEN_PROMPT
|
||||
assert "use_caliber" in copilot_context.LOOK_WHEN_ANSWER_PROMPT and "含退款" in copilot_context.LOOK_WHEN_ANSWER_PROMPT
|
||||
async with SessionLocal() as session:
|
||||
composing = LookService([await CaliberKind.load(session, compose=True)])
|
||||
plain = LookService([await CaliberKind.load(session)])
|
||||
assert copilot_context.look_when(composing) == copilot_context.LOOK_WHEN_ANSWER_PROMPT
|
||||
assert copilot_context.look_when(plain) == copilot_context.LOOK_WHEN_PROMPT
|
||||
assert copilot_context.composer(plain) is None and copilot_context.composer(composing) is not None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 落地:GraphApplier
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_the_applier_lands_the_nodes_wires_the_entry_and_reports_refusals():
|
||||
wf = await S.publish()
|
||||
nodes: dict = {}
|
||||
edges: list = []
|
||||
applier = GraphApplier(nodes, edges, composer=await composer())
|
||||
op = use(wf)
|
||||
applied = applier.apply(op)
|
||||
assert applied.changed and applied.forward and op["result"]["status"] == "ok"
|
||||
assert set(nodes) == {"c1_fetch", "c1"} and edges == [{"source": "c1_fetch", "target": "c1", "sourceHandle": None}]
|
||||
# 入口后加的:加上时接到第一个取数节点;模型再发一条同样的连线不重复
|
||||
applier.apply({"op": "add_node", "node": {"id": "start", "type": "input", "label": "入口", "config": {}}})
|
||||
assert {"source": "start", "target": "c1_fetch", "sourceHandle": None} in edges
|
||||
assert not applier.apply({"op": "add_edge", "edge": {"source": "start", "target": "c1_fetch"}}).changed
|
||||
assert len([e for e in edges if e["target"] == "c1_fetch"]) == 1
|
||||
# 接着写时重发同一个选用:不再落地
|
||||
applier.continuing()
|
||||
assert not applier.apply(use(wf)).changed
|
||||
assert "c1" in applier.landed()
|
||||
|
||||
bad = use(wf, id="c2", inputs={"start": "2026-09-01"})
|
||||
refused = applier.apply(bad)
|
||||
assert not refused.changed and refused.forward and bad["result"]["status"] == "refused"
|
||||
checker = GraphChecker(applier)
|
||||
first = await checker.issues()
|
||||
assert [i["code"] for i in first if i["code"] == compose.REFUSED] == [compose.REFUSED]
|
||||
assert "请填写「end」" in first[0]["message"] and "重新输出 use_caliber" in first[0]["for_model"]
|
||||
# 交回过一次就不再交
|
||||
assert not [i for i in await checker.issues() if i["code"] == compose.REFUSED]
|
||||
|
||||
# 兜底重放到上一轮的图上:拷进来的节点和接线照样在
|
||||
applier.replay_onto({"nodes": [{"id": "start", "type": "input", "data": {"config": {}}}], "edges": []})
|
||||
assert {"c1_fetch", "c1"} <= set(nodes) and any(e["target"] == "c1_fetch" for e in edges)
|
||||
|
||||
|
||||
async def test_without_a_composer_use_caliber_is_dropped_like_any_unknown_op():
|
||||
nodes: dict = {}
|
||||
applier = GraphApplier(nodes, [])
|
||||
applied = applier.apply({"op": "use_caliber", "id": "c1", "workflow_id": "w", "version": 1})
|
||||
assert not applied.changed and not applied.forward and nodes == {}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 问数据走一遍:生成 → 运行 → 比对标 composed
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _answer_script(wf: str) -> list[dict]:
|
||||
from tests.test_copilot_answer_ops import _edge, _node
|
||||
|
||||
return [
|
||||
{"op": "plan", "summary": "选用周销售口径"},
|
||||
_node("start", "input", {}),
|
||||
use(wf),
|
||||
_node("write", "report", {"instructions": "回答:{{ input.question }}"}),
|
||||
_edge("c1", "write"),
|
||||
_node("out", "output", {"fields": [{"name": "答案", "value": "{{ nodes.write.text }}"}]}),
|
||||
_edge("write", "out"),
|
||||
{"op": "done", "explanation": "按口径周销售回答", "run": True},
|
||||
]
|
||||
|
||||
|
||||
async def test_an_answer_built_with_use_caliber_is_labelled_composed(client, monkeypatch):
|
||||
from tests.test_copilot_answer_ops import BODY, _Scripted, _sse, _use
|
||||
|
||||
wf = await S.publish()
|
||||
_use(monkeypatch, _Scripted(_answer_script(wf), []))
|
||||
ops = await _sse(client, {**BODY, "instruction": "九月第一周卖了多少"})
|
||||
[chosen] = [o for o in ops if o["op"] == "use_caliber"]
|
||||
assert chosen["result"]["status"] == "ok" and chosen["result"]["workflow_name"] == "周销售"
|
||||
assert not [o for o in ops if o["op"] == "add_node" and o["node"]["id"] in ("c1", "c1_fetch")], \
|
||||
"拷进来的节点不另外转给界面:过程里只有 use_caliber 那一行"
|
||||
[final] = [o for o in ops if o["op"] == "final"]
|
||||
assert final["next"] == "run", final["issues"]
|
||||
graph = final["graph"]
|
||||
assert {n["id"] for n in graph["nodes"]} == {"start", "c1_fetch", "c1", "write", "out"}
|
||||
assert not [i for i in validate_graph(GraphSpec.model_validate(graph)).issues if i.level == "error"]
|
||||
|
||||
S.script(monkeypatch, "九月第一周销售额 [[m:gmv]]。")
|
||||
run = await run_manager.start(graph=graph, input_payload={"question": "九月第一周卖了多少"}, origin_kind="chat")
|
||||
row = await S.wait(run.id)
|
||||
assert row.status == "succeeded", row.error
|
||||
await ledger_match.drain()
|
||||
recorded = await store.latest(row.id)
|
||||
content = recorded.content
|
||||
assert content["algo"] == store.ALGO == "2"
|
||||
[query] = content["queries"]
|
||||
assert query["status"] == "composed" and query["node_id"] == "c1_fetch"
|
||||
[cand] = query["candidates"]
|
||||
assert cand["status"] == "proven" and cand["how"] == "composed" and cand["caliber"] == f"{wf}@1"
|
||||
[label] = content["reports"]["write"]["labels"].values()
|
||||
assert label["kind"] == "caliber" and label["how"] == "composed" and label["queries"] == [query["artifact"]]
|
||||
assert content["whole"]["equivalent"] is True and content["whole"]["how"] == "composed"
|
||||
assert recorded.content and (await store.latest(row.id)).trigger == "sealed"
|
||||
async with SessionLocal() as session:
|
||||
from app.db.models import Artifact
|
||||
|
||||
ref = await session.get(Artifact, (recorded.artifact, row.id))
|
||||
assert ref.meta["calibers"] == [f"{wf}@1"] and ref.meta["counts"]["composed"] == 1
|
||||
|
||||
|
||||
async def test_users_do_not_get_calibers_in_the_answer_look(monkeypatch):
|
||||
"""使用者(开登录后的 user 角色):问数据里不看口径、不能选用(拷进来的取数 SQL 在他的图里看得到)。"""
|
||||
import app.api.copilot as copilot
|
||||
from app.api.copilot import GenerateIn
|
||||
|
||||
seen: dict = {}
|
||||
|
||||
async def spy(session, payload, sources, **kw):
|
||||
seen["kinds"] = kw.get("kinds")
|
||||
raise RuntimeError("到这里就够了")
|
||||
|
||||
monkeypatch.setattr(copilot, "_look_service", spy)
|
||||
monkeypatch.setattr("app.api.deps.restricted", lambda: True)
|
||||
with pytest.raises(RuntimeError):
|
||||
await copilot.answer_ops(GenerateIn(instruction="x", intent="answer")).__anext__()
|
||||
assert seen["kinds"] == ("table", "capability")
|
||||
monkeypatch.setattr("app.api.deps.restricted", lambda: False)
|
||||
with pytest.raises(RuntimeError):
|
||||
await copilot.answer_ops(GenerateIn(instruction="x", intent="answer")).__anext__()
|
||||
assert seen["kinds"] == ("table", "capability", "caliber")
|
||||
with pytest.raises(RuntimeError):
|
||||
await copilot.answer_ops(GenerateIn(instruction="x", intent="build")).__anext__()
|
||||
assert seen["kinds"] == ("table", "capability")
|
||||
@@ -0,0 +1,371 @@
|
||||
"""按口径正式出具(next-round 02 做法 5,拍板 L1、L5,app/ledger/issue.py)。
|
||||
|
||||
真的跑探索运行(虚构店铺库、mock 模型写报告),等后台比对做完,看自动出具、一键出具、拒绝原因、去重:
|
||||
- 问数据的答案整题独立等价于受管口径:比对完自动发起一次正式运行(来源 ledger_issue),origin 记来源运行、口径、绑定
|
||||
哈希,参数是绑定 + 缺省值,按 graph_hash 钉住;GET 账本的 actions 写能自动、issued 列出它;再点一次交回同一次;
|
||||
- 已发布级别、助手选用的、不是问数据的(画布试运行等):不自动,只给一键(写明为什么),一键照常发起;设置关了自动出具
|
||||
也不自动;
|
||||
- 拒绝写明原因:整题不等价、点名别的口径、版本退役、图里有人工节点、缺入口字段、超过每日上限、使用者没有应用授权;
|
||||
使用者有授权(应用固定着这一版)就能出具,origin 记靠的是哪个应用;
|
||||
- 设置 ledger 一组:缺省值、写坏了拒收。
|
||||
数据是虚构店铺库。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.api.deps import Principal
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Run, Setting, WorkflowVersion
|
||||
from app.engine.runner import run_manager
|
||||
from app.ledger import issue as ledger_issue
|
||||
from app.ledger import match as ledger_match
|
||||
from app.main import app
|
||||
from tests import ledger_samples as S
|
||||
|
||||
ADMIN = Principal(username=None, role="admin", mode="off", via="off")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def engine_up(tmp_path):
|
||||
await run_manager.setup()
|
||||
await S.fresh_source(tmp_path / "shop.db")
|
||||
# 测试库按进程共用:每日上限数的是今天全部的按口径出具,别的用例发起的也算。每个用例从缺省设置起步、上限放宽,
|
||||
# 要测上限的用例自己改
|
||||
await reset_ledger(issue_daily_limit=100_000)
|
||||
yield
|
||||
await ledger_match.drain()
|
||||
await run_manager.shutdown()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
async def reset_ledger(**values) -> None:
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(Setting, "ledger")
|
||||
if row is not None:
|
||||
await session.delete(row)
|
||||
if values:
|
||||
session.add(Setting(key="ledger", value=values))
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def set_ledger(**values) -> None:
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(Setting, "ledger")
|
||||
merged = {**(row.value if row else {}), **values}
|
||||
if row:
|
||||
row.value = merged
|
||||
else:
|
||||
session.add(Setting(key="ledger", value=merged))
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def issued(run_id: str) -> list[Run]:
|
||||
async with SessionLocal() as session:
|
||||
return await ledger_issue.issued_runs(session, run_id)
|
||||
|
||||
|
||||
async def settle(run_id: str) -> list[Run]:
|
||||
"""等自动出具发起的正式运行跑完。"""
|
||||
# 自动出具在比对里就发起了(ledger/match.match_run 等它),drain 之后有没有已经定了:只等发起了的跑完
|
||||
for _ in range(200):
|
||||
found = await issued(run_id)
|
||||
if all(r.status in ("succeeded", "failed", "cancelled") for r in found):
|
||||
return found
|
||||
await asyncio.sleep(0.05)
|
||||
return await issued(run_id)
|
||||
|
||||
|
||||
def two_writers(monkeypatch, answer: str = "本周销售额 [[v:Q1.r0.销售额]]。") -> None:
|
||||
"""mock 模型:口径那张图的报告(指令「写周销售」)引用口径卡的指标,问数据那张图的报告照 answer 写。"""
|
||||
from langchain_core.messages import AIMessage
|
||||
|
||||
from app.providers import mock_model
|
||||
|
||||
def decide(self, messages):
|
||||
text = "\n".join(str(getattr(m, "content", "")) for m in messages)
|
||||
return AIMessage(content="周销售 [[m:gmv]]。" if "写周销售" in text else answer)
|
||||
|
||||
monkeypatch.setattr(mock_model.MockChatModel, "_decide", decide)
|
||||
|
||||
|
||||
async def equivalent_run(monkeypatch, origin: str = "chat"):
|
||||
"""一次整题独立等价于「周销售」v1 的探索运行(缺省是问数据的答案),等它封存、比对完。"""
|
||||
from app.engine.runner import run_manager as manager
|
||||
|
||||
two_writers(monkeypatch)
|
||||
run = await manager.start(graph=S.explore_graph(S.EQUIVALENT), input_payload={}, origin_kind=origin)
|
||||
row = await S.wait(run.id)
|
||||
assert row.status == "succeeded", row.error
|
||||
await ledger_match.drain()
|
||||
return row
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 自动出具
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_an_independent_equivalent_answer_to_a_governed_caliber_is_issued_automatically(client, monkeypatch):
|
||||
wf = await S.publish()
|
||||
row = await equivalent_run(monkeypatch)
|
||||
[formal] = await settle(row.id)
|
||||
assert formal.run_class == "formal" and formal.origin_kind == "ledger_issue" and formal.status == "succeeded"
|
||||
assert (formal.workflow_id, formal.version) == (wf, 1)
|
||||
assert formal.input == {"start": "2026-09-01", "end": "2026-09-08", "store": "all"}
|
||||
async with SessionLocal() as session:
|
||||
version = (await session.execute(select(WorkflowVersion).where(
|
||||
WorkflowVersion.workflow_id == wf, WorkflowVersion.version == 1))).scalar_one()
|
||||
assert formal.version_hash == version.graph_hash
|
||||
origin = formal.origin
|
||||
assert origin["source_run"] == row.id and origin["caliber"] == f"{wf}@1" and origin["how"] == "independent"
|
||||
assert origin["auto"] is True and origin["binding_hash"] and origin["match"]
|
||||
|
||||
body = (await client.get(f"/api/runs/{row.id}/ledger")).json()
|
||||
[action] = [a for a in body["actions"] if a["kind"] == "issue"]
|
||||
assert action["available"] is True and action["auto"] is True and action["issued_run"] == formal.id
|
||||
assert action["level_text"] == "受管" and action["workflow_name"] == "周销售"
|
||||
[item] = body["issued"]
|
||||
assert item["run_id"] == formal.id and item["auto"] is True and item["status"] == "succeeded"
|
||||
|
||||
# 再点一次「按口径出具」:同一运行、同一口径、同一组参数,交回已经发起的那一次
|
||||
again = await client.post(f"/api/ledger/answers/{row.id}/issue", json={"workflow_id": wf, "version": 1})
|
||||
assert again.status_code == 200 and again.json()["created"] is False and again.json()["run_id"] == formal.id
|
||||
assert len(await issued(row.id)) == 1
|
||||
|
||||
|
||||
async def test_only_answers_in_chat_are_issued_automatically(client, monkeypatch):
|
||||
"""自动出具只给问数据的答案(拍板 L1、做法 5):画布上反复试跑一份和受管口径等价的草图,每次都是新的来源运行,
|
||||
自动的话每试一次就多一次正式运行(含报告撰写的模型费用)。画布试运行只给一键,写明为什么,一键照常发起。"""
|
||||
wf = await S.publish()
|
||||
row = await equivalent_run(monkeypatch, origin="canvas")
|
||||
assert await settle(row.id) == []
|
||||
[action] = [a for a in (await client.get(f"/api/runs/{row.id}/ledger")).json()["actions"] if a["kind"] == "issue"]
|
||||
assert action["available"] is True and action["auto"] is False
|
||||
assert action["auto_reason"] == ledger_issue.AUTO_NOT_CHAT.format(what="画布试运行")
|
||||
assert await ledger_issue.auto_issue(row.id) is None and await issued(row.id) == []
|
||||
res = await client.post(f"/api/ledger/answers/{row.id}/issue", json={"workflow_id": wf, "version": 1})
|
||||
assert res.status_code == 200 and res.json()["created"] is True
|
||||
[formal] = await settle(row.id)
|
||||
assert formal.origin["auto"] is False and formal.status == "succeeded"
|
||||
|
||||
|
||||
async def test_published_level_calibers_only_get_one_click(client, monkeypatch):
|
||||
wf = await S.publish(level="published")
|
||||
row = await equivalent_run(monkeypatch)
|
||||
assert await settle(row.id) == []
|
||||
body = (await client.get(f"/api/runs/{row.id}/ledger")).json()
|
||||
[action] = [a for a in body["actions"] if a["kind"] == "issue"]
|
||||
assert action["available"] is True and action["auto"] is False and action["auto_reason"] == ledger_issue.AUTO_LEVEL
|
||||
res = await client.post(f"/api/ledger/answers/{row.id}/issue", json={"workflow_id": wf, "version": 1})
|
||||
assert res.status_code == 200 and res.json()["created"] is True
|
||||
[formal] = await settle(row.id)
|
||||
assert formal.origin["auto"] is False and formal.status == "succeeded"
|
||||
assert res.json()["issued"]["run_id"] == formal.id
|
||||
|
||||
|
||||
async def test_switching_auto_issue_off_keeps_one_click(client, monkeypatch):
|
||||
await S.publish()
|
||||
await set_ledger(auto_issue=False)
|
||||
row = await equivalent_run(monkeypatch)
|
||||
assert await settle(row.id) == []
|
||||
[action] = (await client.get(f"/api/runs/{row.id}/ledger")).json()["actions"]
|
||||
assert action["auto"] is False and action["auto_reason"] == ledger_issue.AUTO_OFF
|
||||
|
||||
|
||||
async def test_composed_answers_are_never_issued_automatically(monkeypatch):
|
||||
from app.ledger import compose
|
||||
from app.look.calibers import CaliberKind
|
||||
|
||||
wf = await S.publish()
|
||||
async with SessionLocal() as session:
|
||||
kind = await CaliberKind.load(session, compose=True)
|
||||
made = compose.Composer(list(kind.options)).compose(
|
||||
{"op": "use_caliber", "id": "c1", "workflow_id": wf, "version": 1, "metrics": ["gmv"],
|
||||
"inputs": {"start": "2026-09-01", "end": "2026-09-08"}}, taken={"start"})
|
||||
graph = S.chain(S.node("start", "input"), *made.nodes, S.node("write", "report", instructions="写答案"),
|
||||
S.node("out", "output", fields=[{"name": "答案", "value": "{{ nodes.write.text }}"}]))
|
||||
two_writers(monkeypatch, "本周销售额 [[m:gmv]]。")
|
||||
row = await S.wait((await run_manager.start(graph=graph, input_payload={}, origin_kind="chat")).id)
|
||||
await ledger_match.drain()
|
||||
assert await settle(row.id) == []
|
||||
async with SessionLocal() as session:
|
||||
recorded = await ledger_match.store.latest(row.id)
|
||||
plan = await ledger_issue.plan_issue(session, await session.get(Run, row.id), recorded.content, wf, 1,
|
||||
principal=ADMIN)
|
||||
assert plan.available and plan.how == "composed" and plan.auto_reason == ledger_issue.AUTO_COMPOSED
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 拒绝
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_refusals_say_why(client, monkeypatch):
|
||||
wf = await S.publish(level="published")
|
||||
other = await S.publish(name="另一份口径", level="published")
|
||||
unrelated = await S.explore(monkeypatch, S.WITH_REFUNDS)
|
||||
res = await client.post(f"/api/ledger/answers/{unrelated.id}/issue", json={"workflow_id": wf, "version": 1})
|
||||
assert res.status_code == 409 and res.json()["code"] == ledger_issue.REFUSED
|
||||
assert res.json()["detail"].startswith("整题没有等价于这份口径")
|
||||
|
||||
row = await equivalent_run(monkeypatch)
|
||||
whole_on = (await ledger_match.store.latest(row.id)).content["whole"]["workflow_id"]
|
||||
named = other if whole_on == wf else wf
|
||||
res = await client.post(f"/api/ledger/answers/{row.id}/issue", json={"workflow_id": named, "version": 1})
|
||||
assert res.status_code == 409 and "整题等价的是另一份口径" in res.json()["detail"]
|
||||
|
||||
# 版本退役
|
||||
async with SessionLocal() as session:
|
||||
version = (await session.execute(select(WorkflowVersion).where(
|
||||
WorkflowVersion.workflow_id == whole_on, WorkflowVersion.version == 1))).scalar_one()
|
||||
version.retired_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
res = await client.post(f"/api/ledger/answers/{row.id}/issue", json={"workflow_id": whole_on, "version": 1})
|
||||
assert res.status_code == 409 and "已退役" in res.json()["detail"]
|
||||
|
||||
|
||||
async def test_a_caliber_with_a_human_step_is_not_issued_on_anyones_behalf(client, monkeypatch):
|
||||
graph = S.caliber_graph()
|
||||
graph["nodes"].insert(3, S.node("ok", "human", mode="approve", prompt="确认"))
|
||||
graph["edges"] = [{"source": a["id"], "target": b["id"]} for a, b in zip(graph["nodes"], graph["nodes"][1:])]
|
||||
wf = await S.publish(graph=graph)
|
||||
row = await equivalent_run(monkeypatch)
|
||||
assert await settle(row.id) == []
|
||||
res = await client.post(f"/api/ledger/answers/{row.id}/issue", json={"workflow_id": wf, "version": 1})
|
||||
assert res.status_code == 409 and "人工节点" in res.json()["detail"] and "不代人做这些事" in res.json()["detail"]
|
||||
|
||||
|
||||
async def test_missing_entry_fields_are_named(client, monkeypatch):
|
||||
graph = S.caliber_graph()
|
||||
graph["nodes"][0]["data"]["config"]["fields"].append({"name": "region", "required": True})
|
||||
wf = await S.publish(graph=graph)
|
||||
row = await equivalent_run(monkeypatch)
|
||||
[action] = (await client.get(f"/api/runs/{row.id}/ledger")).json()["actions"]
|
||||
assert action["available"] is False and "请填写「region」" in action["reason"]
|
||||
res = await client.post(f"/api/ledger/answers/{row.id}/issue", json={"workflow_id": wf, "version": 1})
|
||||
assert res.status_code == 409 and "入口字段还不齐" in res.json()["detail"]
|
||||
|
||||
|
||||
async def test_the_daily_limit_counts_every_issue(client, monkeypatch):
|
||||
wf = await S.publish(level="published")
|
||||
await set_ledger(issue_daily_limit=0)
|
||||
row = await equivalent_run(monkeypatch)
|
||||
res = await client.post(f"/api/ledger/answers/{row.id}/issue", json={"workflow_id": wf, "version": 1})
|
||||
assert res.status_code == 429 and res.json()["code"] == ledger_issue.DAILY_LIMIT and "上限(0 次)" in res.json()["detail"]
|
||||
|
||||
|
||||
async def test_users_issue_only_versions_pinned_by_apps_they_can_use(monkeypatch):
|
||||
from app.db.models_apps import App, AppGrant, AppVersion
|
||||
|
||||
wf = await S.publish(level="published")
|
||||
row = await equivalent_run(monkeypatch)
|
||||
async with SessionLocal() as session:
|
||||
run = await session.get(Run, row.id)
|
||||
run.started_by = "carol"
|
||||
await session.commit()
|
||||
carol = Principal(username="carol", role="user", verified=True, mode="local")
|
||||
with pytest.raises(ledger_issue.IssueRefused) as e:
|
||||
await ledger_issue.issue(row.id, wf, 1, principal=carol)
|
||||
assert e.value.status_code == 403 and e.value.message == ledger_issue.ASK_BUILDER
|
||||
# 别人的运行读不到:先说读不到
|
||||
dave = Principal(username="dave", role="user", verified=True, mode="local")
|
||||
with pytest.raises(ledger_issue.IssueRefused) as e:
|
||||
await ledger_issue.issue(row.id, wf, 1, principal=dave)
|
||||
assert e.value.status_code == 403 and e.value.message == ledger_issue.NO_RUN_ACCESS
|
||||
|
||||
async with SessionLocal() as session:
|
||||
board = App(name="周销售看板", published_version=1, created_by="bob")
|
||||
session.add(board)
|
||||
await session.flush()
|
||||
session.add(AppVersion(app_id=board.id, version=1, spec={}, spec_hash="x",
|
||||
pins=[{"alias": "week", "workflow_id": wf, "version": 1}]))
|
||||
session.add(AppGrant(app_id=board.id, principal_kind="user", principal="carol", role="use"))
|
||||
await session.commit()
|
||||
board_id = board.id
|
||||
formal, created = await ledger_issue.issue(row.id, wf, 1, principal=carol)
|
||||
assert created and formal.started_by == "carol" and formal.origin["granted_by_app"] == board_id
|
||||
|
||||
|
||||
async def test_concurrent_requests_issue_once(monkeypatch):
|
||||
wf = await S.publish(level="published")
|
||||
row = await equivalent_run(monkeypatch)
|
||||
results = await asyncio.gather(*(ledger_issue.issue(row.id, wf, 1, principal=ADMIN) for _ in range(3)))
|
||||
assert sorted(created for _, created in results) == [False, False, True]
|
||||
assert len({r.id for r, _ in results}) == 1
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 设置
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_ledger_settings_have_defaults_and_refuse_bad_values(client):
|
||||
await reset_ledger()
|
||||
body = (await client.get("/api/settings")).json()
|
||||
assert body["ledger"] == {"auto_issue": True, "issue_daily_limit": 20, "match_timeout_s": 8}
|
||||
bad = await client.put("/api/settings", json={"values": {"ledger": {"auto_issue": "yes"}}})
|
||||
assert bad.status_code == 422 and "自动按口径正式出具" in bad.json()["detail"]
|
||||
bad = await client.put("/api/settings", json={"values": {"ledger": {"match_timeout_s": 600}}})
|
||||
assert bad.status_code == 422 and "口径比对时限" in bad.json()["detail"]
|
||||
ok = await client.put("/api/settings", json={"values": {"ledger": {"auto_issue": False, "issue_daily_limit": 5,
|
||||
"match_timeout_s": 3}}})
|
||||
assert ok.status_code == 200 and ok.json()["ledger"] == {"auto_issue": False, "issue_daily_limit": 5,
|
||||
"match_timeout_s": 3}
|
||||
|
||||
|
||||
async def test_auto_issue_goes_by_the_askers_permissions_when_login_is_on(monkeypatch):
|
||||
"""开登录时自动出具按这次运行的发起人判(拍板 L5):使用者没有应用授权就不自动出具,有了才出具;账号不在了的按使用者算。"""
|
||||
from app.db.models import User
|
||||
from app.db.models_apps import App, AppGrant, AppVersion
|
||||
|
||||
monkeypatch.setattr("app.auth.current_mode", lambda: "local")
|
||||
await set_ledger(auto_issue=False)
|
||||
wf = await S.publish()
|
||||
row = await equivalent_run(monkeypatch)
|
||||
await set_ledger(auto_issue=True)
|
||||
async with SessionLocal() as session:
|
||||
session.add(User(username=f"erin_{row.id[:6]}", role="user", password_hash="x"))
|
||||
(await session.get(Run, row.id)).started_by = f"erin_{row.id[:6]}"
|
||||
await session.commit()
|
||||
assert await ledger_issue.auto_issue(row.id) is None and await issued(row.id) == []
|
||||
|
||||
async with SessionLocal() as session:
|
||||
board = App(name="周销售看板", published_version=1, created_by="bob")
|
||||
session.add(board)
|
||||
await session.flush()
|
||||
session.add(AppVersion(app_id=board.id, version=1, spec={}, spec_hash="x",
|
||||
pins=[{"alias": "week", "workflow_id": wf, "version": 1}]))
|
||||
session.add(AppGrant(app_id=board.id, principal_kind="user", principal=f"erin_{row.id[:6]}", role="use"))
|
||||
await session.commit()
|
||||
launched = await ledger_issue.auto_issue(row.id)
|
||||
[formal] = await settle(row.id)
|
||||
assert launched == formal.id and formal.started_by == f"erin_{row.id[:6]}" and formal.origin["auto"] is True
|
||||
|
||||
|
||||
async def test_binding_values_that_would_break_the_sql_are_not_filled_in(client, monkeypatch):
|
||||
"""绑定出来的值要渲染进口径的 SQL 模板:写在引号里的值带着引号(这次的 SQL 里写的是转义过的 '',绑定照原样取出)就不代填
|
||||
出具——和组合时校验助手给的参数是同一个判断,宁可少出具。"""
|
||||
template = "SELECT SUM(amount) AS gmv FROM orders WHERE status = '{{ input.status }}'"
|
||||
graph = S.caliber_graph(template)
|
||||
graph["nodes"][0]["data"]["config"]["fields"] = [{"name": "status", "required": True}]
|
||||
wf = await S.publish(graph=graph, level="published")
|
||||
two_writers(monkeypatch, "本周销售额 [[v:Q1.r0.gmv]]。")
|
||||
run = await run_manager.start(graph=S.explore_graph("SELECT SUM(amount) AS gmv FROM orders WHERE status = 'pa''id'"),
|
||||
input_payload={}, origin_kind="canvas")
|
||||
row = await S.wait(run.id)
|
||||
await ledger_match.drain()
|
||||
whole = (await ledger_match.store.latest(row.id)).content["whole"]
|
||||
assert whole["equivalent"] is True and whole["binding"] == {"status": "pa''id"}
|
||||
res = await client.post(f"/api/ledger/answers/{row.id}/issue", json={"workflow_id": wf, "version": 1})
|
||||
assert res.status_code == 409 and "不能有引号" in res.json()["detail"] and "不代填出具" in res.json()["detail"]
|
||||
@@ -0,0 +1,250 @@
|
||||
"""口径对账:差多少(next-round 02 做法 6,app/ledger/reconcile.py、ledger/derive.reconcile_graph)。
|
||||
|
||||
真的跑探索运行(虚构店铺库、mock 模型写报告)、等后台比对做完,再点开对账:
|
||||
- 动作:有相关口径的查询给「看差多少」;参数是按「同一列、同一运算符」猜的,要先核对(needs_check);
|
||||
- 不核对直接发起回 409(要核对参数);核对过的参数发起:派生图是确定的、过 validate、出口契约 metrics_from 指向对账卡、
|
||||
不加报告;跑出来的这次、口径、差值、相对差都对,差值的出处链点得开;GET 账本列出对过的账;统计 E2 数到它;
|
||||
- 认不出哪一列对哪个指标(口径的指标式不是取数的一格):要人选,选了照记进 origin;
|
||||
- 多行结果:被引用的行按维度列经合并查询对齐,每行一组对账指标;维度是遮罩列、合并键不唯一时照实拒绝。
|
||||
数据是虚构店铺库。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Run
|
||||
from app.engine.runner import run_manager
|
||||
from app.engine.schema import GraphSpec, validate_graph
|
||||
from app.ledger import match as ledger_match
|
||||
from app.ledger import reconcile as ledger_reconcile
|
||||
from app.main import app
|
||||
from tests import ledger_samples as S
|
||||
|
||||
BY_REGION = ("SELECT region AS 区域, SUM(amount) AS gmv FROM orders WHERE status = 'paid' "
|
||||
"AND paid_at >= '{{ input.start }}' AND paid_at < '{{ input.end }}' GROUP BY region ORDER BY region")
|
||||
BY_REGION_ALL = ("SELECT region AS 区域, SUM(amount) AS gmv FROM orders WHERE paid_at >= '2026-09-01' "
|
||||
"AND paid_at < '2026-09-08' GROUP BY region ORDER BY region")
|
||||
CHECKED = {"start": "2026-09-01", "end": "2026-09-08"}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def engine_up(tmp_path):
|
||||
await run_manager.setup()
|
||||
path = tmp_path / "shop.db"
|
||||
await S.fresh_source(path)
|
||||
yield path
|
||||
await ledger_match.drain()
|
||||
await run_manager.shutdown()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
async def ledger_of(client, run_id: str) -> dict:
|
||||
res = await client.get(f"/api/runs/{run_id}/ledger")
|
||||
assert res.status_code == 200, res.text
|
||||
return res.json()
|
||||
|
||||
|
||||
async def card_of(run_id: str, node_id: str = "reconcile") -> dict:
|
||||
from app.api.evidence import _sealed, _sealed_cards
|
||||
|
||||
sealed = await _sealed(run_id)
|
||||
[(_, card)] = [(a, c) for n, a, c in _sealed_cards(sealed) if n == node_id]
|
||||
return {m["id"]: m for m in card["metrics"]}
|
||||
|
||||
|
||||
async def reconcile(client, run_id: str, body: dict) -> Run:
|
||||
res = await client.post(f"/api/ledger/answers/{run_id}/reconcile", json=body)
|
||||
assert res.status_code == 201, res.text
|
||||
return await S.wait(res.json()["run_id"])
|
||||
|
||||
|
||||
async def test_reconcile_the_refunds_answer_against_the_weekly_caliber(client, monkeypatch):
|
||||
wf = await S.publish()
|
||||
row = await S.explore(monkeypatch, S.WITH_REFUNDS)
|
||||
body = await ledger_of(client, row.id)
|
||||
[action] = [a for a in body["actions"] if a["kind"] == "reconcile"]
|
||||
assert action["workflow_id"] == wf and action["version"] == 1 and action["node_id"] == "fetch"
|
||||
assert action["needs_check"] is True and action["check_reason"] == ledger_reconcile.CHECK_REASON
|
||||
assert action["binding"] == [{"field": "end", "value": "2026-09-08"}, {"field": "start", "value": "2026-09-01"}]
|
||||
query = action["query"]
|
||||
|
||||
# 参数是猜的:不核对就发起,照实拒绝
|
||||
res = await client.post(f"/api/ledger/answers/{row.id}/reconcile", json={"query": query, "workflow_id": wf,
|
||||
"version": 1})
|
||||
assert res.status_code == 409 and res.json()["code"] == ledger_reconcile.CHECK_INPUTS
|
||||
assert res.json()["detail"] == ledger_reconcile.CONFIRM_FIRST
|
||||
|
||||
done = await reconcile(client, row.id, {"query": query, "workflow_id": wf, "version": 1, "inputs": CHECKED,
|
||||
"confirm": True})
|
||||
assert done.status == "succeeded", done.error
|
||||
assert done.origin_kind == "reconcile" and done.run_class == "exploratory" and done.workflow_id is None
|
||||
origin = done.origin
|
||||
assert origin["source_run"] == row.id and origin["query"] == query and origin["inputs_from"] == "checked"
|
||||
assert origin["pairs"] == [{"ours": "gmv", "metric": "gmv"}] and origin["caliber"] == f"{wf}@1"
|
||||
assert done.input == {"start": "2026-09-01", "end": "2026-09-08", "store": "all"}
|
||||
|
||||
graph = done.graph
|
||||
ids = [n["id"] for n in graph["nodes"]]
|
||||
assert ids == ["derived_in", "ours_q1", "fetch", "card", "reconcile", "derived_out"]
|
||||
nodes = {n["id"]: n for n in graph["nodes"]}
|
||||
assert nodes["ours_q1"]["data"]["config"]["args"]["sql"] == S.WITH_REFUNDS
|
||||
assert "{{ input.start }}" in nodes["fetch"]["data"]["config"]["args"]["sql"], "口径的取数保留模板,运行时按入口渲染"
|
||||
assert nodes["derived_out"]["data"]["config"]["contract"] == {"metrics_from": "reconcile"}
|
||||
assert not [n for n in graph["nodes"] if n["type"] in ("report", "llm", "agent")]
|
||||
assert not [i for i in validate_graph(GraphSpec.model_validate(graph)).issues if i.level == "error"]
|
||||
|
||||
metrics = await card_of(done.id)
|
||||
assert metrics["ours_0"]["value"] == 221.0 and metrics["cal_0"]["value"] == 170.5
|
||||
assert metrics["diff_0"]["value"] == 50.5 and metrics["rel_0"]["value"] == 0.2962
|
||||
assert metrics["rel_0"]["rendered"] == "29.62%" and metrics["diff_0"]["rendered"] == "50.50元"
|
||||
# 三个数各自点得开:差值的出处链追到两边的查询
|
||||
res = await client.get(f"/api/runs/{done.id}/evidence/metrics/reconcile/diff_0")
|
||||
assert res.status_code == 200, res.text
|
||||
inputs = [s for s in res.json()["chain"] if s["step"] == "input"]
|
||||
assert {i["node_id"] for i in inputs} == {"ours_q1", "card"}
|
||||
|
||||
listed = (await ledger_of(client, row.id))["reconciled"]
|
||||
assert [r["run_id"] for r in listed] == [done.id] and listed[0]["inputs_from"] == "checked"
|
||||
|
||||
# 统计 E2:对账运行数到了(它不进使用统计,E2 专门数它)
|
||||
stats = (await client.get("/api/ledger/stats")).json()["e2"]
|
||||
assert stats["launched"] >= 1 and stats["succeeded"] >= 1
|
||||
|
||||
|
||||
async def test_the_derived_graph_is_deterministic():
|
||||
from app.ledger import derive
|
||||
|
||||
wf = await S.publish()
|
||||
async with SessionLocal() as session:
|
||||
pinned = await derive.load_pinned(session, wf, 1, "card")
|
||||
plan = ledger_reconcile.Plan(pairs=[ledger_reconcile.Pair(ours="gmv", metric="gmv", index=0, name="销售额",
|
||||
unit="元", decimals=2)])
|
||||
a = ledger_reconcile.build(pinned, sql=S.WITH_REFUNDS, tool=S.tool(), plan=plan, inputs=CHECKED, fetch="fetch")
|
||||
b = ledger_reconcile.build(pinned, sql=S.WITH_REFUNDS, tool=S.tool(), plan=plan, inputs=CHECKED, fetch="fetch")
|
||||
assert a.graph == b.graph and a.kind == "reconcile"
|
||||
assert a.origin["from"][0] == {"kind": "executed", "node_id": "ours_q1"} and a.origin["from"][1]["node_id"] == "card"
|
||||
|
||||
|
||||
async def test_metrics_that_are_not_a_plain_cell_need_a_person_to_pick(client, monkeypatch):
|
||||
wf = await S.publish(graph=S.caliber_graph(expression="cell(nodes.fetch, 0, 'gmv') * 1"))
|
||||
row = await S.explore(monkeypatch, S.WITH_REFUNDS)
|
||||
[action] = [a for a in (await ledger_of(client, row.id))["actions"] if a["kind"] == "reconcile"]
|
||||
body = {"query": action["query"], "workflow_id": wf, "version": 1, "inputs": CHECKED, "confirm": True}
|
||||
res = await client.post(f"/api/ledger/answers/{row.id}/reconcile", json=body)
|
||||
assert res.status_code == 409 and res.json()["code"] == ledger_reconcile.NEEDS_PAIRS
|
||||
bad = await client.post(f"/api/ledger/answers/{row.id}/reconcile",
|
||||
json={**body, "pairs": [{"ours": "nope", "metric": "gmv"}]})
|
||||
assert bad.status_code == 422 and "没有列「nope」" in bad.json()["detail"]
|
||||
done = await reconcile(client, row.id, {**body, "pairs": [{"ours": "gmv", "metric": "gmv"}]})
|
||||
assert done.status == "succeeded", done.error
|
||||
assert done.origin["pairs_chosen"] is True
|
||||
assert (await card_of(done.id))["diff_0"]["value"] == 50.5
|
||||
|
||||
|
||||
async def test_multi_row_answers_are_aligned_by_their_dimensions(client, monkeypatch):
|
||||
wf = await S.publish(graph=S.caliber_graph(BY_REGION))
|
||||
row = await S.explore(monkeypatch, BY_REGION_ALL, text="东区 [[v:Q1.r0.gmv]],西区 [[v:Q1.r1.gmv]]。")
|
||||
[action] = [a for a in (await ledger_of(client, row.id))["actions"] if a["kind"] == "reconcile"]
|
||||
done = await reconcile(client, row.id, {"query": action["query"], "workflow_id": wf, "version": 1,
|
||||
"inputs": CHECKED, "confirm": True})
|
||||
assert done.status == "succeeded", done.error
|
||||
assert done.origin["rows"] == [0, 1] and done.origin["pairs"] == [{"ours": "gmv", "caliber": "gmv"}]
|
||||
align = next(n for n in done.graph["nodes"] if n["id"] == "align")
|
||||
assert align["type"] == "merge" and align["data"]["config"]["inputs"] == {"o": "ours_q1", "c": "fetch"}
|
||||
metrics = await card_of(done.id)
|
||||
assert metrics["ours_0_0"]["name"] == "区域=east · gmv · 这次"
|
||||
assert (metrics["ours_0_0"]["value"], metrics["cal_0_0"]["value"], metrics["diff_0_0"]["value"]) == (163.0, 150.5, 12.5)
|
||||
assert (metrics["ours_1_0"]["value"], metrics["cal_1_0"]["value"], metrics["diff_1_0"]["value"]) == (58.0, 20.0, 38.0)
|
||||
|
||||
|
||||
async def test_masked_dimensions_and_duplicate_keys_are_refused(client, monkeypatch, engine_up):
|
||||
wf = await S.publish(graph=S.caliber_graph(BY_REGION))
|
||||
duplicated = ("SELECT region AS 区域, amount AS gmv FROM orders WHERE paid_at >= '2026-09-01' "
|
||||
"AND paid_at < '2026-09-08' ORDER BY id")
|
||||
row = await S.explore(monkeypatch, duplicated, text="第一单 [[v:Q1.r0.gmv]],第二单 [[v:Q1.r1.gmv]]。")
|
||||
[action] = [a for a in (await ledger_of(client, row.id))["actions"] if a["kind"] == "reconcile"]
|
||||
res = await client.post(f"/api/ledger/answers/{row.id}/reconcile",
|
||||
json={"query": action["query"], "workflow_id": wf, "version": 1, "inputs": CHECKED,
|
||||
"confirm": True})
|
||||
assert res.status_code == 409 and "有重复的行" in res.json()["detail"]
|
||||
|
||||
await S.set_source(engine_up, options={"mask_columns": ["region"]})
|
||||
row = await S.explore(monkeypatch, BY_REGION_ALL, text="东区 [[v:Q1.r0.gmv]],西区 [[v:Q1.r1.gmv]]。")
|
||||
[action] = [a for a in (await ledger_of(client, row.id))["actions"] if a["kind"] == "reconcile"]
|
||||
res = await client.post(f"/api/ledger/answers/{row.id}/reconcile",
|
||||
json={"query": action["query"], "workflow_id": wf, "version": 1, "inputs": CHECKED,
|
||||
"confirm": True})
|
||||
assert res.status_code == 409 and "设置了遮罩" in res.json()["detail"]
|
||||
|
||||
|
||||
def test_align_sql_keeps_the_referenced_rows_in_order():
|
||||
pairs = [ledger_reconcile.Pair(ours="gmv", column="销售额")]
|
||||
sql = ledger_reconcile.align_sql([("区域", "区域")], pairs, [("west",), ("o'east",)])
|
||||
assert sql == ('SELECT o."区域" AS "区域", o."gmv" AS "ours_0", c."销售额" AS "cal_0" FROM o LEFT JOIN c ON '
|
||||
'o."区域" = c."区域" WHERE (o."区域" = \'west\') OR (o."区域" = \'o\'\'east\') '
|
||||
'ORDER BY CASE WHEN (o."区域" = \'west\') THEN 0 WHEN (o."区域" = \'o\'\'east\') THEN 1 END')
|
||||
|
||||
|
||||
async def test_users_reconcile_their_own_answers_read_only_and_masked(client, monkeypatch):
|
||||
"""拍板 L5:对账(只读的探索运行)使用者都可以,带上问数据同一套 MASK_RESULTS / READ_ONLY;别人的答案读不到就不给对。"""
|
||||
from app.api.deps import Principal
|
||||
from app.tools.datasource import MASK_RESULTS_OPTION, READ_ONLY_OPTION
|
||||
|
||||
wf = await S.publish()
|
||||
row = await S.explore(monkeypatch, S.WITH_REFUNDS)
|
||||
async with SessionLocal() as session:
|
||||
(await session.get(Run, row.id)).started_by = "carol"
|
||||
await session.commit()
|
||||
[action] = [a for a in (await ledger_of(client, row.id))["actions"] if a["kind"] == "reconcile"]
|
||||
req = ledger_reconcile.Request(query=action["query"], workflow_id=wf, version=1, inputs=CHECKED, confirm=True)
|
||||
dave = Principal(username="dave", role="user", verified=True, mode="local")
|
||||
with pytest.raises(ledger_reconcile.ReconcileRefused) as e:
|
||||
await ledger_reconcile.reconcile(row.id, req, principal=dave)
|
||||
assert e.value.status_code == 403
|
||||
carol = Principal(username="carol", role="user", verified=True, mode="local")
|
||||
run = await ledger_reconcile.reconcile(row.id, req, principal=carol)
|
||||
assert run.started_by == "carol"
|
||||
assert run.options[MASK_RESULTS_OPTION] is True and run.options[READ_ONLY_OPTION] is True
|
||||
done = await S.wait(run.id)
|
||||
assert done.status == "succeeded", done.error
|
||||
|
||||
|
||||
async def test_users_read_their_reconcile_runs_only_through_the_evidence(client, monkeypatch):
|
||||
"""使用者发起的对账运行带 MASK_RESULTS、记在他名下,但派生图里有口径取数节点的 SQL 模板(口径的 SQL 可能写着遮罩值,
|
||||
所以问数据里使用者不看口径、不能 use_caliber):运行的原始记录、事件、运行时的图(/api/runs/{id}、/events、/graph)只给
|
||||
他自己的问数据运行,对账运行经证据接口读(那里按遮罩渲染)。"""
|
||||
from app.tools.datasource import MASK_RESULTS_OPTION
|
||||
from tests.fakes import auth_env
|
||||
|
||||
wf = await S.publish()
|
||||
row = await S.explore(monkeypatch, S.WITH_REFUNDS, origin="chat")
|
||||
# 当作开登录后使用者 carol 问的:问数据运行在工具边界遮
|
||||
async with SessionLocal() as session:
|
||||
source = await session.get(Run, row.id)
|
||||
source.started_by, source.options = "carol", {MASK_RESULTS_OPTION: True}
|
||||
await session.commit()
|
||||
[action] = [a for a in (await ledger_of(client, row.id))["actions"] if a["kind"] == "reconcile"]
|
||||
await auth_env.reset()
|
||||
try:
|
||||
admin = await auth_env.turn_on_local()
|
||||
await auth_env.make_user("carol", "user")
|
||||
assert (await admin.put("/api/auth/settings", json={"user_can_chat": True})).status_code == 200
|
||||
carol = await auth_env.login("carol")
|
||||
res = await carol.post(f"/api/ledger/answers/{row.id}/reconcile", json={
|
||||
"query": action["query"], "workflow_id": wf, "version": 1, "inputs": CHECKED, "confirm": True})
|
||||
assert res.status_code == 201, res.text
|
||||
done = await S.wait(res.json()["run_id"])
|
||||
assert done.status == "succeeded" and done.started_by == "carol" and done.options[MASK_RESULTS_OPTION] is True
|
||||
for rest in ("", "/events", "/graph"):
|
||||
assert (await carol.get(f"/api/runs/{done.id}{rest}")).status_code == 403, rest
|
||||
assert (await carol.get(f"/api/runs/{row.id}{rest}")).status_code == 200, f"自己的问数据运行照旧:{rest}"
|
||||
assert (await carol.get(f"/api/runs/{done.id}/evidence")).status_code == 200
|
||||
finally:
|
||||
await auth_env.reset()
|
||||
Reference in New Issue
Block a user