Files
AgentLab/backend/tests/test_caliber_validation_real_chain.py
T
ThearchyHelios 2f90f0eb3a 口径台账:固定执行、验证审阅发布与增量重导入
- QueryCore 固定执行(K09):按已发布的不可变口径跑一次,交回类型化标量、发布/manifest 引用与逐查询证据
- 依赖闭包(K08):按选中的指标算出要拷的取数节点、要的入口字段,接进 derive / compose / metrics
- SQL 模板参数完整性(K01):试跑、组合问数、派生图、运行时取数共用一套语义,缺参数在渲染前就拒
- 受限字段变更服务(K03)与口径领域边界(K01):改动打在整图深拷贝上,身份与证据归属不可改
- 版本化受控模板绑定(K04):业务口径按显式绑定落到 SQL,对不上就报 unsupported / needs_update
- 验证记录(K05)与最小审阅:样例匹配不等于正式批准,审批人只从服务端身份取
- 发布依据清单(K07):发布时按不可变图物化内容寻址清单,正式用途按它判
- 增量重导入(K11)与 B/L/R 三方合并、受限自然语言修改提案(K12):只产候选,落图仍走既有写路径
- 接口与前端:口径草稿页接上字段变更、绑定状态、验证与重导入,新增 caliber-drafts / studio 检查脚本

验证:前端 tsc -b 通过。
2026-11-10 15:05:18 +08:00

200 lines
11 KiB
Python

"""K05 的真实链路:真引擎、真 SQLite 数据源、真 HTTP —— 试运行 → matched → 认证审阅 → verified → 发布。
**这份文件不换 `_trial_metrics` / `_trial_queries` 的桩**:走的是「起草出来的那种口径卡 + 真跑一次探针图」的完整
路径,考的就是「真实运行的证据能不能被验证记录认出来」。以前的阻断点正是这里:界面那份指标投影里没有取数节点,
而验证记录按节点把指标对到查询快照上——对不上就一律判不了形状,matched / verified / 发布永远到不了。
覆盖的形状(都是真跑出来的):恰好一行(matched)、多行(标量取数有歧义)、空结果(没有可取的数)。
发布那一步证明的是:**认证审阅过的这条记录,绑的是被发布的那一版定义与绑定**(发布不改图,所以记录照旧覆盖)。
"""
from __future__ import annotations
import copy
import uuid
import pytest
from sqlalchemy import delete, select
from app.db.base import SessionLocal
from app.db.models import CaliberDraft, Run, Workflow, WorkflowVersion
from app.engine.runner import run_manager
from app.ledger import validation
from tests import ledger_samples as S
from tests.test_caliber_draft_reads import attached_draft # noqa: F401 - 帮手共用
from tests.test_ledger_docdraft_api import client # noqa: F401 - fixture 和别的口径用例共用
#: 这次起草的口径:九月一号到八号,契约数据集里 SUM(amount) = 170.5(独立期望照这个写,不照这次运行的结果抄)
EXPECTED = 170.5
WINDOW = {"start": "2026-09-01", "end": "2026-09-08"}
#: 这一份文件自己建的工作流(收尾时连草稿、版本、记录一起清掉)
OWN_PREFIX = "K05 真实链路 "
def real_graph(sql: str, *, fields: list[dict] | None = None) -> dict:
"""一份像**起草产物**那样已接管的图:入口 → 取数 → 口径卡(标量契约)。
契约是照着 app/ledger/draftplan 建图时写的来(`contract: {"shape": "scalar"}`):起草出来的指标都声明标量,
多行和空结果才会被引擎和验证记录判成形状不符。
"""
return S.chain(
S.node("start", "input", fields=fields if fields is not None else [
{"name": "start", "required": True}, {"name": "end", "required": True}]),
S.node("fetch", "tool", tool=S.tool(), args={"sql": sql}),
S.node("card", "metrics", caliber="周销售", caliber_version="v1",
metrics=[{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 2,
"expression": "cell(nodes.fetch, 0, 'gmv')", "contract": {"shape": "scalar"}}]))
@pytest.fixture(autouse=True)
async def engine_and_source(tmp_path):
await run_manager.setup()
await S.fresh_source(tmp_path / "shop.db")
yield
await run_manager.shutdown()
async with SessionLocal() as session:
ids = list((await session.execute(
select(Workflow.id).where(Workflow.name.like(f"{OWN_PREFIX}%")))).scalars())
# 试运行留下的运行行:工作流删掉之后 workflow_id 会被置空,按当时记下的工作流名清掉
await session.execute(delete(Run).where(Run.workflow_name.like(f"{OWN_PREFIX}%")))
if ids:
await session.execute(delete(CaliberDraft).where(CaliberDraft.workflow_id.in_(ids)))
await session.execute(delete(WorkflowVersion).where(WorkflowVersion.workflow_id.in_(ids)))
await session.execute(delete(Workflow).where(Workflow.id.in_(ids)))
await session.commit()
async def real_draft(*, sql: str | None = None) -> dict:
"""一份像起草产物那样已接管、指向真实数据源的草稿(图里就一张口径卡 + 一次取数)。"""
made = await attached_draft(graph=real_graph(sql or S.TEMPLATE),
payload={"sql": S.TEMPLATE, "entry_fields": [{"name": "start"}, {"name": "end"}]})
async with SessionLocal() as session:
workflow = await session.get(Workflow, made["workflow_id"])
workflow.name = f"{OWN_PREFIX}{uuid.uuid4().hex[:6]}"
await session.commit()
return made
async def validations(client, draft_id: str) -> dict:
return (await client.get(f"/api/ledger/drafts/{draft_id}/validations")).json()
async def repoint(client, body: dict, sql: str) -> int:
"""把这份草稿的取数换成另一句(画布那种改法),返回新版本号。"""
saved = await client.patch(f"/api/workflows/{body['workflow_id']}",
json={"graph": real_graph(sql), "expected_revision": body["revision"]})
assert saved.status_code == 200, saved.text
return saved.json()["version"]
async def test_a_real_trial_matches_confirms_verifies_and_survives_publication(client):
"""真跑一次:独立期望对得上 → matched;认证审阅(全指标)→ verified;发布的那一版和记录绑的是同一份依据。"""
body = await real_draft()
done = await client.post(f"/api/ledger/drafts/{body['id']}/trial",
json={"inputs": WINDOW,
"expect": {"metrics": {"gmv": EXPECTED}, "source": "fixture",
"sample_id": "sept-2026", "sample_version": "1"}})
assert done.status_code == 200, done.text
trial = done.json()
assert trial["status"] == "succeeded", trial.get("error")
# 真实运行交回的指标值就是契约数据集里那一份(不是桩)
assert float(trial["metrics"][0]["value"]) == EXPECTED
assert "node_id" not in trial["metrics"][0] # 关联网不放给用户看的响应
record = trial["validation"]["record"]
assert record["status"] == "matched", record["problems"]
assert record["problems"] == []
assert record["metric_ids"] == ["gmv"]
assert record["expected"]["metrics"] == {"gmv": EXPECTED}
assert record["actual"]["gmv"]["value"] == EXPECTED
assert record["actual"]["gmv"]["rows"] == 1 and record["actual"]["gmv"]["shape_ok"] is True
assert record["actual"]["gmv"]["ok"] is True
assert record["covers_current"] is True
# 对得上不等于批准:还没有审阅
listed = await validations(client, body["id"])
assert listed["summary"]["status"] == "matched" and listed["summary"]["verdict"] == "unverified"
confirmed = await client.post(
f"/api/ledger/drafts/{body['id']}/validations/{record['id']}/review",
json={"decision": "confirmed", "note": "按独立算出来的九月预期核过"}, headers={"X-Actor": "alice"})
assert confirmed.status_code == 201, confirmed.text
review = confirmed.json()["record"]["review"]
assert review["decision"] == "confirmed" and review["scope"] == ["gmv"]
listed = await validations(client, body["id"])
assert listed["summary"]["verdict"] == "verified" and listed["summary"]["unconfirmed"] == []
# 发布:这一版就是被审阅过的那一份定义与绑定;发布不改图,所以记录照旧覆盖
published = await client.post(f"/api/workflows/{body['workflow_id']}/publish",
json={"level": "published"})
assert published.status_code == 200 and published.json()["ok"], published.text
detail = (await client.get(f"/api/ledger/drafts/{body['id']}")).json()
assert detail["validation"]["verdict"] == "verified"
assert detail["validation"]["covers_current"] is True
async with SessionLocal() as session:
row = (await session.execute(select(WorkflowVersion).where(
WorkflowVersion.workflow_id == body["workflow_id"],
WorkflowVersion.version == body["revision"]))).scalar_one()
assert row.level == "published"
# 被发布的这份图和记录里的定义依据是同一份(发布依据和验证依据对得上)
assert validation.definition_hash(row.graph, "card") == record["definition_hash"]
assert (await validations(client, body["id"]))["records"][0]["id"] == record["id"]
async def test_a_real_multi_row_result_does_not_pass_on_the_first_row(client):
"""多行形状(真跑):标量指标取回两行时不能只看第一行,记 mismatched 并把「恰好一行」说清楚。"""
body = await real_draft()
sql = ("SELECT o.region AS gmv FROM orders o WHERE o.status = 'paid' "
"AND o.paid_at >= '{{ input.start }}' AND o.paid_at < '{{ input.end }}'")
version = await repoint(client, body, sql)
done = await client.post(f"/api/ledger/drafts/{body['id']}/trial",
json={"inputs": WINDOW, "expect": {"metrics": {"gmv": EXPECTED}}})
assert done.status_code == 200, done.text
trial = done.json()
assert trial["status"] == "succeeded", trial.get("error")
[query] = trial["queries"]
assert query["total_rows"] > 1
record = trial["validation"]["record"]
assert record["version"] == version
assert record["status"] == "mismatched", record["problems"]
assert record["actual"]["gmv"]["rows"] > 1 and record["actual"]["gmv"]["shape_ok"] is False
assert any("恰好一行" in problem for problem in record["problems"])
async def test_a_real_empty_result_never_counts_as_a_pass(client):
"""空结果形状(真跑):查询一行都没有时(0 行)不能当作取到了值,记录不会是 matched。"""
body = await real_draft()
# 非聚合取数:窗口里没有行时结果就是 0 行(聚合在 0 行上仍交回一行 NULL,另按「没有值」判)
await repoint(client, body, "SELECT o.region AS gmv FROM orders o WHERE 1 = 0 "
"AND o.paid_at >= '{{ input.start }}' AND o.paid_at < '{{ input.end }}'")
done = await client.post(f"/api/ledger/drafts/{body['id']}/trial",
json={"inputs": WINDOW, "expect": {"metrics": {"gmv": EXPECTED}}})
assert done.status_code == 200, done.text
trial = done.json()
record = trial["validation"]["record"]
assert record is not None
assert record["status"] != "matched"
actual = record["actual"].get("gmv") or {}
if record["status"] == "mismatched":
assert actual.get("rows") == 0 and actual.get("shape_ok") is False
assert any("空结果" in problem or "0 行" in problem for problem in record["problems"])
else:
# 这条取数让口径卡算不出值(取不到第一格)时,整个试运行就是 failed:同样不能算通过
assert record["status"] == "failed"
assert any("没有成功" in problem or "没有结果" in problem for problem in record["problems"])
async def test_a_real_aggregate_over_no_rows_is_not_a_pass(client):
"""聚合查询在空窗口上仍交回一行 NULL:有行、没有值,也不算通过(不能拿 NULL 当 0)。"""
body = await real_draft()
await repoint(client, body, "SELECT SUM(amount) AS gmv FROM orders WHERE 1 = 0 "
"AND paid_at >= '{{ input.start }}' AND paid_at < '{{ input.end }}'")
done = await client.post(f"/api/ledger/drafts/{body['id']}/trial",
json={"inputs": WINDOW, "expect": {"metrics": {"gmv": EXPECTED}}})
assert done.status_code == 200, done.text
record = done.json()["validation"]["record"]
assert record["status"] == "mismatched", record["problems"]
assert record["actual"]["gmv"]["value"] is None
assert any("没有取到值" in problem for problem in record["problems"])