Files
AgentLab/backend/tests/test_ledger_reconcile.py
T
ThearchyHelios 906ec072ce 按口径正式出具与口径对账(口径账本做法 5、6,拍板 L1、L5)
为什么:问数据的答案整题等价于已发布口径时,要能按那一版发起一次完整走过门禁的正式运行(路径一、二a);只是相关、不等价时,要能看「差多少」(路径二b)。两条路都不能让客户端冒充来源、自己传参数,也不能在 Python 里算差值。

正式出具(新增 ledger/issue.py):
- POST /api/ledger/answers/{run_id}/issue {workflow_id, version}:参数只从比对工件的绑定取,补入口缺省值再过 apps/forms 校验;经 launch_run 发起正式运行,来源 ledger_issue,origin 记 source_run、conversation_id、turn_id、binding_hash、口径和认法、是否自动;按 expect_hash 钉 graph_hash。
- 不能出具照实写原因:整题不等价、点名别的口径、版本退役或没发布、图里有人工节点或写入(apps/backfill._side_effect 抽成共用的 graph_side_effect)、缺入口字段、超过每日上限、权限(L5:使用者只能按他有权使用的应用固定着的版本出具,复用应用授权,origin 记 granted_by_app;别的写「请搭建者出具」)。
- 自动出具(L1):比对完(只在运行一封存就比的那一次)整题独立等价、版本受管、设置开着、当天没超上限才自动;助手选用的只给一键。不自动的原因写进 actions 的 auto_reason。
- 同一运行、同一口径、同一组参数只发起一次(input_key 去重),进程内按(运行、工作流、版本)串行。
- 设置加 ledger 一组:自动出具开关、每日上限、比对时限(ledger/match 读它)。

口径对账(新增 ledger/reconcile.py,derive 扩 reconcile_graph / ours_block / assemble 的出口契约):
- POST /api/ledger/answers/{run_id}/reconcile:按那一版确定性派生只读图(这次的取数 ours_q1 → 口径的取数 → 口径卡 → 对账卡,出口契约 metrics_from,不加报告、不调模型),来源 reconcile 的探索运行;对账卡算这次、口径、差值、相对差。
- 参数是启发得来的、不全、含遮罩值时先让人核对(needs_check,没核对的 409);指标式不是取数一格、差异有兜底时要人选对照哪个指标,选择记进 origin。
- 多行结果:被引用的行按维度列经合并查询 LEFT JOIN 对齐、按引用先后排,最多 10 行;维度是遮罩列、合并键不唯一照实拒绝。使用者发起带 MASK_RESULTS / READ_ONLY,搭建者也带 READ_ONLY。
- GET /api/runs/{id}/ledger 的 actions 补上出具、对账两类,issued、reconciled 列出发起过的运行。

测试:tests/test_ledger_issue.py(自动出具、只给一键的几种、拒绝原因、使用者的应用授权、并发只发一次、设置)、tests/test_ledger_reconcile.py(核对参数、派生图确定与契约、差值和相对差、出处链、要人选列、多行对齐、遮罩维度和重复键拒绝、使用者只读遮罩);第 2 波 test_ledger_api 的 actions 断言按这一波改。
2026-10-16 16:55:06 +08:00

217 lines
13 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""口径对账:差多少(next-round 02 做法 6,app/ledger/reconcile.py、ledger/derive.reconcile_graph)。
真的跑探索运行(虚构店铺库、mock 模型写报告)、等后台比对做完,再点开对账:
- 动作:有相关口径的查询给「看差多少」;参数是按「同一列、同一运算符」猜的,要先核对(needs_check);
- 不核对直接发起回 409(要核对参数);核对过的参数发起:派生图是确定的、过 validate、出口契约 metrics_from 指向对账卡、
不加报告;跑出来的这次、口径、差值、相对差都对,差值的出处链点得开;GET 账本列出对过的账;统计 E2 数到它;
- 认不出哪一列对哪个指标(口径的指标式不是取数的一格):要人选,选了照记进 origin;
- 多行结果:被引用的行按维度列经合并查询对齐,每行一组对账指标;维度是遮罩列、合并键不唯一时照实拒绝。
数据是虚构店铺库。
"""
from __future__ import annotations
import pytest
from httpx import ASGITransport, AsyncClient
from app.db.base import SessionLocal
from app.db.models import Run
from app.engine.runner import run_manager
from app.engine.schema import GraphSpec, validate_graph
from app.ledger import match as ledger_match
from app.ledger import reconcile as ledger_reconcile
from app.main import app
from tests import ledger_samples as S
BY_REGION = ("SELECT region AS 区域, SUM(amount) AS gmv FROM orders WHERE status = 'paid' "
"AND paid_at >= '{{ input.start }}' AND paid_at < '{{ input.end }}' GROUP BY region ORDER BY region")
BY_REGION_ALL = ("SELECT region AS 区域, SUM(amount) AS gmv FROM orders WHERE paid_at >= '2026-09-01' "
"AND paid_at < '2026-09-08' GROUP BY region ORDER BY region")
CHECKED = {"start": "2026-09-01", "end": "2026-09-08"}
@pytest.fixture(autouse=True)
async def engine_up(tmp_path):
await run_manager.setup()
path = tmp_path / "shop.db"
await S.fresh_source(path)
yield path
await ledger_match.drain()
await run_manager.shutdown()
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
async def ledger_of(client, run_id: str) -> dict:
res = await client.get(f"/api/runs/{run_id}/ledger")
assert res.status_code == 200, res.text
return res.json()
async def card_of(run_id: str, node_id: str = "reconcile") -> dict:
from app.api.evidence import _sealed, _sealed_cards
sealed = await _sealed(run_id)
[(_, card)] = [(a, c) for n, a, c in _sealed_cards(sealed) if n == node_id]
return {m["id"]: m for m in card["metrics"]}
async def reconcile(client, run_id: str, body: dict) -> Run:
res = await client.post(f"/api/ledger/answers/{run_id}/reconcile", json=body)
assert res.status_code == 201, res.text
return await S.wait(res.json()["run_id"])
async def test_reconcile_the_refunds_answer_against_the_weekly_caliber(client, monkeypatch):
wf = await S.publish()
row = await S.explore(monkeypatch, S.WITH_REFUNDS)
body = await ledger_of(client, row.id)
[action] = [a for a in body["actions"] if a["kind"] == "reconcile"]
assert action["workflow_id"] == wf and action["version"] == 1 and action["node_id"] == "fetch"
assert action["needs_check"] is True and action["check_reason"] == ledger_reconcile.CHECK_REASON
assert action["binding"] == [{"field": "end", "value": "2026-09-08"}, {"field": "start", "value": "2026-09-01"}]
query = action["query"]
# 参数是猜的:不核对就发起,照实拒绝
res = await client.post(f"/api/ledger/answers/{row.id}/reconcile", json={"query": query, "workflow_id": wf,
"version": 1})
assert res.status_code == 409 and res.json()["code"] == ledger_reconcile.CHECK_INPUTS
assert res.json()["detail"] == ledger_reconcile.CONFIRM_FIRST
done = await reconcile(client, row.id, {"query": query, "workflow_id": wf, "version": 1, "inputs": CHECKED,
"confirm": True})
assert done.status == "succeeded", done.error
assert done.origin_kind == "reconcile" and done.run_class == "exploratory" and done.workflow_id is None
origin = done.origin
assert origin["source_run"] == row.id and origin["query"] == query and origin["inputs_from"] == "checked"
assert origin["pairs"] == [{"ours": "gmv", "metric": "gmv"}] and origin["caliber"] == f"{wf}@1"
assert done.input == {"start": "2026-09-01", "end": "2026-09-08", "store": "all"}
graph = done.graph
ids = [n["id"] for n in graph["nodes"]]
assert ids == ["derived_in", "ours_q1", "fetch", "card", "reconcile", "derived_out"]
nodes = {n["id"]: n for n in graph["nodes"]}
assert nodes["ours_q1"]["data"]["config"]["args"]["sql"] == S.WITH_REFUNDS
assert "{{ input.start }}" in nodes["fetch"]["data"]["config"]["args"]["sql"], "口径的取数保留模板,运行时按入口渲染"
assert nodes["derived_out"]["data"]["config"]["contract"] == {"metrics_from": "reconcile"}
assert not [n for n in graph["nodes"] if n["type"] in ("report", "llm", "agent")]
assert not [i for i in validate_graph(GraphSpec.model_validate(graph)).issues if i.level == "error"]
metrics = await card_of(done.id)
assert metrics["ours_0"]["value"] == 221.0 and metrics["cal_0"]["value"] == 170.5
assert metrics["diff_0"]["value"] == 50.5 and metrics["rel_0"]["value"] == 0.2962
assert metrics["rel_0"]["rendered"] == "29.62%" and metrics["diff_0"]["rendered"] == "50.50元"
# 三个数各自点得开:差值的出处链追到两边的查询
res = await client.get(f"/api/runs/{done.id}/evidence/metrics/reconcile/diff_0")
assert res.status_code == 200, res.text
inputs = [s for s in res.json()["chain"] if s["step"] == "input"]
assert {i["node_id"] for i in inputs} == {"ours_q1", "card"}
listed = (await ledger_of(client, row.id))["reconciled"]
assert [r["run_id"] for r in listed] == [done.id] and listed[0]["inputs_from"] == "checked"
# 统计 E2:对账运行数到了(它不进使用统计,E2 专门数它)
stats = (await client.get("/api/ledger/stats")).json()["e2"]
assert stats["launched"] >= 1 and stats["succeeded"] >= 1
async def test_the_derived_graph_is_deterministic():
from app.ledger import derive
wf = await S.publish()
async with SessionLocal() as session:
pinned = await derive.load_pinned(session, wf, 1, "card")
plan = ledger_reconcile.Plan(pairs=[ledger_reconcile.Pair(ours="gmv", metric="gmv", index=0, name="销售额",
unit="元", decimals=2)])
a = ledger_reconcile.build(pinned, sql=S.WITH_REFUNDS, tool=S.tool(), plan=plan, inputs=CHECKED, fetch="fetch")
b = ledger_reconcile.build(pinned, sql=S.WITH_REFUNDS, tool=S.tool(), plan=plan, inputs=CHECKED, fetch="fetch")
assert a.graph == b.graph and a.kind == "reconcile"
assert a.origin["from"][0] == {"kind": "executed", "node_id": "ours_q1"} and a.origin["from"][1]["node_id"] == "card"
async def test_metrics_that_are_not_a_plain_cell_need_a_person_to_pick(client, monkeypatch):
wf = await S.publish(graph=S.caliber_graph(expression="cell(nodes.fetch, 0, 'gmv') * 1"))
row = await S.explore(monkeypatch, S.WITH_REFUNDS)
[action] = [a for a in (await ledger_of(client, row.id))["actions"] if a["kind"] == "reconcile"]
body = {"query": action["query"], "workflow_id": wf, "version": 1, "inputs": CHECKED, "confirm": True}
res = await client.post(f"/api/ledger/answers/{row.id}/reconcile", json=body)
assert res.status_code == 409 and res.json()["code"] == ledger_reconcile.NEEDS_PAIRS
bad = await client.post(f"/api/ledger/answers/{row.id}/reconcile",
json={**body, "pairs": [{"ours": "nope", "metric": "gmv"}]})
assert bad.status_code == 422 and "没有列「nope」" in bad.json()["detail"]
done = await reconcile(client, row.id, {**body, "pairs": [{"ours": "gmv", "metric": "gmv"}]})
assert done.status == "succeeded", done.error
assert done.origin["pairs_chosen"] is True
assert (await card_of(done.id))["diff_0"]["value"] == 50.5
async def test_multi_row_answers_are_aligned_by_their_dimensions(client, monkeypatch):
wf = await S.publish(graph=S.caliber_graph(BY_REGION))
row = await S.explore(monkeypatch, BY_REGION_ALL, text="东区 [[v:Q1.r0.gmv]],西区 [[v:Q1.r1.gmv]]。")
[action] = [a for a in (await ledger_of(client, row.id))["actions"] if a["kind"] == "reconcile"]
done = await reconcile(client, row.id, {"query": action["query"], "workflow_id": wf, "version": 1,
"inputs": CHECKED, "confirm": True})
assert done.status == "succeeded", done.error
assert done.origin["rows"] == [0, 1] and done.origin["pairs"] == [{"ours": "gmv", "caliber": "gmv"}]
align = next(n for n in done.graph["nodes"] if n["id"] == "align")
assert align["type"] == "merge" and align["data"]["config"]["inputs"] == {"o": "ours_q1", "c": "fetch"}
metrics = await card_of(done.id)
assert metrics["ours_0_0"]["name"] == "区域=east · gmv · 这次"
assert (metrics["ours_0_0"]["value"], metrics["cal_0_0"]["value"], metrics["diff_0_0"]["value"]) == (163.0, 150.5, 12.5)
assert (metrics["ours_1_0"]["value"], metrics["cal_1_0"]["value"], metrics["diff_1_0"]["value"]) == (58.0, 20.0, 38.0)
async def test_masked_dimensions_and_duplicate_keys_are_refused(client, monkeypatch, engine_up):
wf = await S.publish(graph=S.caliber_graph(BY_REGION))
duplicated = ("SELECT region AS 区域, amount AS gmv FROM orders WHERE paid_at >= '2026-09-01' "
"AND paid_at < '2026-09-08' ORDER BY id")
row = await S.explore(monkeypatch, duplicated, text="第一单 [[v:Q1.r0.gmv]],第二单 [[v:Q1.r1.gmv]]。")
[action] = [a for a in (await ledger_of(client, row.id))["actions"] if a["kind"] == "reconcile"]
res = await client.post(f"/api/ledger/answers/{row.id}/reconcile",
json={"query": action["query"], "workflow_id": wf, "version": 1, "inputs": CHECKED,
"confirm": True})
assert res.status_code == 409 and "有重复的行" in res.json()["detail"]
await S.set_source(engine_up, options={"mask_columns": ["region"]})
row = await S.explore(monkeypatch, BY_REGION_ALL, text="东区 [[v:Q1.r0.gmv]],西区 [[v:Q1.r1.gmv]]。")
[action] = [a for a in (await ledger_of(client, row.id))["actions"] if a["kind"] == "reconcile"]
res = await client.post(f"/api/ledger/answers/{row.id}/reconcile",
json={"query": action["query"], "workflow_id": wf, "version": 1, "inputs": CHECKED,
"confirm": True})
assert res.status_code == 409 and "设置了遮罩" in res.json()["detail"]
def test_align_sql_keeps_the_referenced_rows_in_order():
pairs = [ledger_reconcile.Pair(ours="gmv", column="销售额")]
sql = ledger_reconcile.align_sql([("区域", "区域")], pairs, [("west",), ("o'east",)])
assert sql == ('SELECT o."区域" AS "区域", o."gmv" AS "ours_0", c."销售额" AS "cal_0" FROM o LEFT JOIN c ON '
'o."区域" = c."区域" WHERE (o."区域" = \'west\') OR (o."区域" = \'o\'\'east\') '
'ORDER BY CASE WHEN (o."区域" = \'west\') THEN 0 WHEN (o."区域" = \'o\'\'east\') THEN 1 END')
async def test_users_reconcile_their_own_answers_read_only_and_masked(client, monkeypatch):
"""拍板 L5:对账(只读的探索运行)使用者都可以,带上问数据同一套 MASK_RESULTS / READ_ONLY;别人的答案读不到就不给对。"""
from app.api.deps import Principal
from app.tools.datasource import MASK_RESULTS_OPTION, READ_ONLY_OPTION
wf = await S.publish()
row = await S.explore(monkeypatch, S.WITH_REFUNDS)
async with SessionLocal() as session:
(await session.get(Run, row.id)).started_by = "carol"
await session.commit()
[action] = [a for a in (await ledger_of(client, row.id))["actions"] if a["kind"] == "reconcile"]
req = ledger_reconcile.Request(query=action["query"], workflow_id=wf, version=1, inputs=CHECKED, confirm=True)
dave = Principal(username="dave", role="user", verified=True, mode="local")
with pytest.raises(ledger_reconcile.ReconcileRefused) as e:
await ledger_reconcile.reconcile(row.id, req, principal=dave)
assert e.value.status_code == 403
carol = Principal(username="carol", role="user", verified=True, mode="local")
run = await ledger_reconcile.reconcile(row.id, req, principal=carol)
assert run.started_by == "carol"
assert run.options[MASK_RESULTS_OPTION] is True and run.options[READ_ONLY_OPTION] is True
done = await S.wait(run.id)
assert done.status == "succeeded", done.error