Files
AgentLab/backend/tests/test_caliber_validation_records.py
T
ThearchyHelios 2f90f0eb3a 口径台账:固定执行、验证审阅发布与增量重导入
- QueryCore 固定执行(K09):按已发布的不可变口径跑一次,交回类型化标量、发布/manifest 引用与逐查询证据
- 依赖闭包(K08):按选中的指标算出要拷的取数节点、要的入口字段,接进 derive / compose / metrics
- SQL 模板参数完整性(K01):试跑、组合问数、派生图、运行时取数共用一套语义,缺参数在渲染前就拒
- 受限字段变更服务(K03)与口径领域边界(K01):改动打在整图深拷贝上,身份与证据归属不可改
- 版本化受控模板绑定(K04):业务口径按显式绑定落到 SQL,对不上就报 unsupported / needs_update
- 验证记录(K05)与最小审阅:样例匹配不等于正式批准,审批人只从服务端身份取
- 发布依据清单(K07):发布时按不可变图物化内容寻址清单,正式用途按它判
- 增量重导入(K11)与 B/L/R 三方合并、受限自然语言修改提案(K12):只产候选,落图仍走既有写路径
- 接口与前端:口径草稿页接上字段变更、绑定状态、验证与重导入,新增 caliber-drafts / studio 检查脚本

验证:前端 tsc -b 通过。
2026-11-10 15:05:18 +08:00

772 lines
46 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""K05 的验证记录与最小审阅:追加存储、绑定定义/绑定依据、独立期望、覆盖判断、认证审阅。
这份文件盯住任务书 4.8 与 E05/E07/E15/R04/R05/R06/R07 的硬要求:
1. **没有期望只能记 observed,不能 pass。** 试跑成功不等于验证通过:没有独立期望时状态是 observed,证据不足是
unverified,只有「有期望、逐条对得上」才是 matched——而 matched 也只是「这一次对得上」,不等于批准。
2. **独立期望能做全指标、全约定形状的比较。** 期望要逐条给全卡指标(漏一条拒);声明成标量的指标必须恰好一行,
空结果与多行**不能只看第一行**(R04/R05)。比较是原值比较、不做隐式舍入(R06)。
3. **覆盖是读的时候按指纹现算的。** 定义、绑定或取数语义(含 SQL)变了,旧记录不再覆盖当前版本(covers_current
false);把一段没验证过的业务描述删掉不会让旧的验证/审阅升级成通过(E15/R07)。
4. **交错结束的两次试跑各留一条记录。** 晚到的旧试跑替换不了当前 trial 指针,但它自己那一行一定要落下(E07)。
5. **审阅绑定认证主体。** 审批人只来自服务端身份:请求体里写 actor 回 422,没有可记的主体回 409;决定绑定所审
记录的依据,样例匹配不会自动变成批准。
模型一律注入假模型、试跑换成假的(结果证据用桩替代);走的是真实 HTTP 接口,不连真实业务库。
"""
from __future__ import annotations
import copy
import json
import re
import uuid
import pytest
from sqlalchemy import delete, select
from app.api import ledger as api_ledger
from app.db.base import SessionLocal
from app.db.models import CaliberDraft, CaliberReviewDecision, CaliberValidationRecord, Workflow
from app.ledger import validation
from tests.test_caliber_field_baseline import accepted_draft # noqa: F401 - 起草帮手和夹具共用
from tests.test_ledger_docdraft_api import ( # noqa: F401 - fixture 和起草用例共用
client, datasource, documents, extract, fake_model, fake_trial, groups_reply, plan_reply,
)
def plant_evidence(monkeypatch, *, value=2, values=None, rows=1, masked=False, status="ok", semantic=False,
metric_ids=("m1",), node="fetch", nodes=None):
"""把这次试运行交回的证据换成桩:指标 + 每个取数节点一条查询快照。
值、行数从这里给,用来构造空结果、多行、遮罩这些反例;`values` 可以逐条指标给不同的值,`nodes` 给
metric_id → 取数节点(多指标卡的两个节点各查一次)。semantic=True 模拟引擎判出的
「业务描述和取数条件的关系还没被证明」(semantic_unverified)。"""
per_metric = dict(values or {})
per_node = dict(nodes or {})
async def metrics(run_id: str) -> list[dict]:
out: list[dict] = []
for metric_id in metric_ids:
reported = None if (masked or rows == 0) else per_metric.get(metric_id, value)
item = {"id": metric_id, "name": f"指标 {metric_id}", "unit": "%", "value": reported,
"status": status, "rendered": reported, "node_id": per_node.get(metric_id, node)}
if masked:
item.update(masked=True, quality={"status": "unknown", "checks": [
{"code": "masked", "status": "unknown", "detail": "指标值已隐藏,不能在此验证"}]})
else:
checks = []
if semantic:
checks.append({"code": "semantic_definition", "status": "unknown",
"detail": "业务描述与取数条件的一致性仍需审阅"})
item["quality"] = {"status": "unknown" if checks else "pass", "checks": checks}
out.append(item)
return out
async def queries(run_id: str) -> list[dict]:
wanted = list(dict.fromkeys(per_node.get(mid, node) for mid in metric_ids))
return [{"node_id": node_id, "tool": "db_query", "source": "shop", "columns": ["rate"],
"rows": [], "total_rows": rows, "truncated": False, "masked": [],
"source_truncated": False, "preview_truncated": False} for node_id in wanted]
async def links(run_id: str) -> tuple[dict[str, str], list[dict]]:
"""指标 → 取数快照的关联(真实运行里由 _trial_metric_links 从封存口径卡认出来):桩按 nodes 给。"""
wanted = list(dict.fromkeys(per_node.get(mid, node) for mid in metric_ids))
return ({mid: per_node.get(mid, node) for mid in metric_ids},
[{"key": node_id, "node_id": node_id, "total_rows": rows, "truncated": False}
for node_id in wanted])
monkeypatch.setattr(api_ledger, "_trial_metrics", metrics)
monkeypatch.setattr(api_ledger, "_trial_queries", queries)
monkeypatch.setattr(api_ledger, "_trial_metric_links", links)
def nodes_of(metrics: list[dict]) -> dict[str, str]:
"""每条指标读哪个取数节点(从表达式里取):桩证据按它对到查询上,多指标卡两个节点各一条快照。"""
out: dict[str, str] = {}
for metric in metrics if isinstance(metrics, list) else []:
found = re.search(r"nodes\.([\w-]+)", str((metric or {}).get("expression") or ""))
if isinstance(metric, dict) and metric.get("id") and found:
out[str(metric["id"])] = found.group(1)
return out
async def run_trial(client, draft_id: str, *, expect=None, inputs=None) -> dict:
body: dict = {"inputs": inputs or {}}
if expect is not None:
body["expect"] = expect
return (await client.post(f"/api/ledger/drafts/{draft_id}/trial", json=body)).json()
@pytest.fixture(autouse=True)
def _fake_trial(fake_trial):
"""本模块的草稿都走接口起草:试跑必须换成假的,不然会去打夹具里不存在的库。"""
return fake_trial
async def validations(client, draft_id: str) -> dict:
return (await client.get(f"/api/ledger/drafts/{draft_id}/validations")).json()
def baseline_metrics(one: dict, *, name: str = "订单金额", mid: str = "m2", column: str = "gmv") -> dict:
"""第二组指标:改 id / 名字 / 表达式 / SQL,用来构造「一张卡两条指标」的全指标比较。"""
two = copy.deepcopy(one)
two["metrics"][0].update(id=mid, name=name, expression=f"cell(nodes.fetch, 0, '{column}')")
two["sql"] = f"SELECT SUM(amount) AS {column} FROM orders WHERE store = '{{{{ input.store }}}}'"
return two
# ==========================================================================
# 1. 没有期望只记 observed
# ==========================================================================
async def test_a_trial_without_an_expectation_is_only_observed(client, documents, datasource, monkeypatch):
"""自动试跑没有独立期望:记录照落,但状态是 observed——跑了、取了数,一个字也不能说「通过」。"""
made = await accepted_draft(client, documents, datasource, monkeypatch)
listed = await validations(client, made["id"])
assert len(listed["records"]) == 1 # 同一次试跑保存两次也只落一行
[record] = listed["records"]
assert record["status"] == "observed" and record["expected"] is None
assert record["covers_current"] is True
assert record["definition_hash"] and record["semantic_hash"]
assert record["method"]["version"] == validation.METHOD_VERSION
assert any("没有独立期望" in problem for problem in record["problems"])
assert listed["summary"]["verdict"] == "observed" and listed["summary"]["review"] is None
assert "不能算验证通过" in listed["summary"]["reason"]
# 试跑接口里也带上这条记录,并且同样不说通过
trial = (await client.get(f"/api/ledger/drafts/{made['id']}/trial")).json()
assert trial["validation"]["record"]["id"] == record["id"]
assert trial["validation"]["covers_current"] is True
detail = (await client.get(f"/api/ledger/drafts/{made['id']}")).json()
assert detail["validation"]["verdict"] == "observed"
# ==========================================================================
# 2. 固定期望:对得上 / 对不上 / 空结果 / 多行
# ==========================================================================
async def test_a_fixed_fixture_expectation_decides_matched_or_mismatched(client, documents, datasource,
monkeypatch):
"""显式固定 fixture 期望:对得上记 matched,对不上记 mismatched;但它只是「这一次对得上」,不是批准。"""
made = await accepted_draft(client, documents, datasource, monkeypatch)
plant_evidence(monkeypatch, value=2, rows=1)
ok = await run_trial(client, made["id"], expect={"metrics": {"m1": 2}, "source": "fixture",
"sample_id": "oct-2026", "sample_version": "1",
"note": "独立算出来的十月预期"})
assert ok["status"] == "succeeded", ok
record = ok["validation"]["record"]
assert record["status"] == "matched" and record["problems"] == []
assert record["expected"]["source"] == "fixture"
assert record["expected"]["metrics"] == {"m1": 2}
assert record["actual"]["m1"]["value"] == 2 and record["actual"]["m1"]["ok"] is True
assert record["actual"]["m1"]["rows"] == 1 and record["actual"]["m1"]["shape"] == "scalar"
assert record["sample"] == {"id": "oct-2026", "version": "1"}
assert record["covers_current"] is True and record["inputs_hash"]
# 对得上也不等于批准:摘要仍是 unverified,并且这么说
listed = await validations(client, made["id"])
assert listed["summary"]["status"] == "matched"
assert listed["summary"]["verdict"] == "unverified"
assert "不等于正式批准" in listed["summary"]["reason"]
bad = await run_trial(client, made["id"], expect={"metrics": {"m1": 3}, "source": "fixture"})
record = bad["validation"]["record"]
assert record["status"] == "mismatched"
assert record["actual"]["m1"]["ok"] is False
assert any("对不上" in problem and "3" in problem for problem in record["problems"])
assert (await validations(client, made["id"]))["summary"]["verdict"] == "mismatched"
async def test_an_empty_result_is_not_a_pass_and_multi_row_does_not_read_the_first_row(
client, documents, datasource, monkeypatch):
"""空结果与多行都不是「取第一行就对」:声明成标量的指标要恰好一行(R04/R05)。"""
made = await accepted_draft(client, documents, datasource, monkeypatch)
plant_evidence(monkeypatch, value=None, rows=0)
empty = await run_trial(client, made["id"], expect={"metrics": {"m1": 2}})
record = empty["validation"]["record"]
assert record["status"] == "mismatched" and record["actual"]["m1"]["rows"] == 0
assert any("空结果" in problem or "0 行" in problem for problem in record["problems"])
# 第一行正好等于期望,但查询回来 3 行:标量取数有歧义,不能算通过
plant_evidence(monkeypatch, value=2, rows=3)
multi = await run_trial(client, made["id"], expect={"metrics": {"m1": 2}})
record = multi["validation"]["record"]
assert record["status"] == "mismatched" and record["actual"]["m1"]["value"] == 2
assert record["actual"]["m1"]["rows"] == 3
assert any("恰好一行" in problem for problem in record["problems"])
# 值被遮罩:判不了,记 unverified(不是 matched、也不是 mismatched)
plant_evidence(monkeypatch, value=2, rows=1, masked=True)
hidden = await run_trial(client, made["id"], expect={"metrics": {"m1": 2}})
record = hidden["validation"]["record"]
assert record["status"] == "unverified" and record["actual"]["m1"]["masked"] is True
assert any("遮罩" in problem for problem in record["problems"])
async def test_a_semantic_gap_is_recorded_and_never_becomes_a_pass_by_itself(client, documents, datasource,
monkeypatch):
"""R07:语义说明没被证明是一处待确认的缺口,照实记在记录里;它自己不会让任何东西算「通过」。"""
made = await accepted_draft(client, documents, datasource, monkeypatch)
plant_evidence(monkeypatch, value=2, rows=1, semantic=True)
trial = await run_trial(client, made["id"], expect={"metrics": {"m1": 2}, "source": "fixture"})
record = trial["validation"]["record"]
assert record["status"] == "matched" and record["actual"]["m1"]["ok"] is True
assert record["actual"]["m1"]["semantic_unverified"] is True
assert not any("语义" in problem for problem in record["problems"]) # 不是比较结论,不并进 problems
listed = await validations(client, made["id"])
assert listed["summary"]["semantic_gaps"] == ["m1"]
assert listed["summary"]["status"] == "matched"
assert listed["summary"]["verdict"] == "unverified" # 没有认证审阅就还是不算通过
# 人工确认之后才算 verified,但语义缺口仍旧写在结论里(「已确认」不等于「这段描述被证明」)
confirmed = await client.post(f"/api/ledger/drafts/{made['id']}/validations/{record['id']}/review",
json={"decision": "confirmed"}, headers={"X-Actor": "alice"})
assert confirmed.status_code == 201, confirmed.text
listed = await validations(client, made["id"])
assert listed["summary"]["verdict"] == "verified"
assert listed["summary"]["semantic_gaps"] == ["m1"]
assert "语义说明仍待人工确认" in listed["summary"]["reason"]
async def test_a_trial_that_never_ran_is_recorded_as_unverified(client, documents, datasource, monkeypatch):
"""停在 needs_input 的试跑(一条 SQL 都没发):记录照落,状态是 unverified,并明说没有可比较的实际结果。"""
made = await accepted_draft(client, documents, datasource, monkeypatch)
async def waiting(*, draft_id, workflow_id, graph, values, started_by, entry_fields=(), version=1):
return {**api_ledger._trial_binding(version, graph, values), "trial_id": "wait-1", "run_id": "",
"status": "needs_input", "error": "缺少必填输入:start"}
monkeypatch.setattr(api_ledger, "_run_trial", waiting)
await run_trial(client, made["id"], expect={"metrics": {"m1": 2}})
listed = await validations(client, made["id"])
newest = listed["records"][0]
assert newest["status"] == "unverified" and newest["run_id"] == "" and newest["trial_id"] == "wait-1"
assert newest["expected"]["metrics"] == {"m1": 2}
assert any("没有成功" in problem for problem in newest["problems"])
assert listed["summary"]["verdict"] == "unverified"
async def test_the_expectation_must_cover_the_whole_card_and_use_a_known_source(client, documents, datasource,
monkeypatch):
"""全指标比较:期望要逐条给全卡指标;来源只能是三种之一。不合规在发起运行之前回 422。"""
doc = await documents("val-multi-" + uuid.uuid4().hex[:8])
source = await datasource()
one = json.loads(plan_reply(f"{doc}-2"))
fake_model(monkeypatch, groups_reply(one, baseline_metrics(one)))
body = await extract(client, doc, source)
draft_id = body["draft_id"]
made = (await client.get(f"/api/ledger/drafts/{draft_id}")).json()
assert sorted(m["id"] for m in made["metrics"]) == ["m1", "m2"]
partial = await client.post(f"/api/ledger/drafts/{draft_id}/trial",
json={"inputs": {}, "expect": {"metrics": {"m1": 2}}})
assert partial.status_code == 422 and "m2" in partial.text
stranger = await client.post(f"/api/ledger/drafts/{draft_id}/trial",
json={"inputs": {}, "expect": {"metrics": {"nope": 2}}})
assert stranger.status_code == 422 and "nope" in stranger.text
invented = await client.post(f"/api/ledger/drafts/{draft_id}/trial",
json={"inputs": {}, "expect": {"metrics": {"m1": 2, "m2": 3},
"source": "model_said_so"}})
assert invented.status_code == 422 and "human_confirmed" in invented.text
plant_evidence(monkeypatch, value=2, rows=1)
whole = await client.post(f"/api/ledger/drafts/{draft_id}/trial",
json={"inputs": {}, "expect": {"metrics": {"m1": 2, "m2": 3},
"source": "independent_computation"}})
assert whole.status_code == 200, whole.text
record = whole.json()["validation"]["record"]
assert record["metric_ids"] == ["m1", "m2"]
assert record["actual"]["m1"]["ok"] is True
# m2 这一次没有交回结果(桩只造了 m1):判不了,不能算通过
assert record["actual"]["m2"]["status"] == "missing" and record["actual"]["m2"]["ok"] is None
assert record["status"] == "unverified"
# ==========================================================================
# 3. 覆盖判断与 E15/R07:删掉没验证过的描述不会升级成通过
# ==========================================================================
async def test_deleting_a_semantic_field_does_not_keep_an_old_verification_verified(
client, documents, datasource, monkeypatch):
"""确认过的一次验证,在定义被改(删掉一段业务描述)之后不再覆盖当前版本,也不能拿它当新定义的批准。"""
made = await accepted_draft(client, documents, datasource, monkeypatch)
draft_id = made["id"]
plant_evidence(monkeypatch, value=2, rows=1)
trial = await run_trial(client, draft_id, expect={"metrics": {"m1": 2}, "source": "fixture"})
record_id = trial["validation"]["record"]["id"]
confirmed = await client.post(f"/api/ledger/drafts/{draft_id}/validations/{record_id}/review",
json={"decision": "confirmed", "note": "看过独立样例"},
headers={"X-Actor": "alice"})
assert confirmed.status_code == 201, confirmed.text
listed = await validations(client, draft_id)
assert listed["summary"]["verdict"] == "verified"
# 删掉「口径说明」这段没被取数证明过的业务描述:定义指纹变了
revision = (await client.get(f"/api/ledger/drafts/{draft_id}")).json()["revision"]
removed = await client.patch(f"/api/ledger/drafts/{draft_id}", json={
"operations": [{"op": "remove", "metric_id": "m1", "path": "/definition/text"}],
"expected_revision": revision, "request_id": "drop-semantic-1"})
assert removed.status_code == 200, removed.text
after = await validations(client, draft_id)
assert after["summary"]["covers_current"] is False and after["summary"]["verdict"] == "stale"
# 旧记录和旧审阅都还在(历史可追),但都明说不再覆盖当前版本
old = next(item for item in after["records"] if item["id"] == record_id)
assert old["status"] == "matched" and old["covers_current"] is False
assert old["review"]["decision"] == "confirmed" and old["review"]["covers_current"] is False
assert old["review"]["actor"] == "alice"
# 摘要里带上的是「覆盖不了当前版本」的结论,不是拿旧的 confirmed 当通过
assert after["summary"]["verdict"] != "verified"
assert after["summary"]["covers_current"] is False
assert (await client.get(f"/api/ledger/drafts/{draft_id}")).json()["validation"]["verdict"] == "stale"
async def test_a_sql_change_alone_stops_the_old_record_from_covering(client, documents, datasource, monkeypatch):
"""取数 SQL 改了(定义没动)也算对新版本不覆盖:覆盖看的不只是定义指纹;画布编辑也不推进 B。"""
made = await accepted_draft(client, documents, datasource, monkeypatch)
listed = await validations(client, made["id"])
assert listed["summary"]["covers_current"] is True
before_baseline = listed["field_baseline"]
assert before_baseline["status"] == "known"
async with SessionLocal() as session:
workflow = await session.get(Workflow, made["workflow_id"])
graph = copy.deepcopy(workflow.graph)
revision = workflow.version
fetch = next(node for node in graph["nodes"] if node["id"] == "fetch")
fetch["data"]["config"]["args"]["sql"] = "SELECT COUNT(*) * 2.0 AS refund_rate FROM orders " \
"WHERE status = 'paid' AND store = '{{ input.store }}'"
saved = await client.patch(f"/api/workflows/{made['workflow_id']}",
json={"graph": graph, "expected_revision": revision})
assert saved.status_code == 200, saved.text
after = await validations(client, made["id"])
assert after["summary"]["covers_current"] is False and after["summary"]["verdict"] == "stale"
# 画布上的改动只动 L:字段基线 B 的引用和值一个字没变
assert after["field_baseline"] == before_baseline
# ==========================================================================
# 4. 交错结束的两次试跑:两条记录都留下(E07)
# ==========================================================================
async def test_interleaved_trials_each_leave_their_own_record(client, documents, datasource, monkeypatch):
"""晚到的旧试跑替换不了当前 trial 指针,但它自己那一行一定要落下:新旧两条都可追踪。"""
made = await accepted_draft(client, documents, datasource, monkeypatch)
draft_id = made["id"]
async with SessionLocal() as session:
workflow = await session.get(Workflow, made["workflow_id"])
graph_v1 = copy.deepcopy(workflow.graph)
metrics = copy.deepcopy(made["metrics"])
metrics[0]["unit"] = "%" # 定义变了一处 → 新版本
changed = await client.patch(f"/api/ledger/drafts/{draft_id}",
json={"metrics": metrics, "expected_revision": made["revision"]})
assert changed.status_code == 200, changed.text
plant_evidence(monkeypatch, value=2, rows=1)
newest = await run_trial(client, draft_id)
# 旧那一次(还对着 v1 的图)晚到:指针不换,记录照落
late = {**api_ledger._trial_binding(1, graph_v1, {}), "trial_id": "late-1", "run_id": "",
"status": "succeeded", "error": ""}
assert await api_ledger._save_trial(draft_id, late) is False
detail = (await client.get(f"/api/ledger/drafts/{draft_id}")).json()
assert detail["trial"]["trial_id"] == newest["trial_id"] # 当前指针还是新的那一次
listed = await validations(client, draft_id)
by_trial = {record["trial_id"]: record for record in listed["records"]}
assert {"late-1", newest["trial_id"]} <= set(by_trial)
late_record, new_record = by_trial["late-1"], by_trial[newest["trial_id"]]
assert late_record["definition_hash"] != new_record["definition_hash"]
assert late_record["covers_current"] is False and new_record["covers_current"] is True
assert listed["summary"]["covers_current"] is True and listed["summary"]["record_id"] == new_record["id"]
# 晚到那一条照实说它不对着当前版本跑
assert any("找不到对应的定义版本" in problem for problem in late_record["problems"]) is False
# ==========================================================================
# 5. 审阅:认证主体、只追加、样例匹配不是批准
# ==========================================================================
async def test_a_review_binds_the_authenticated_subject_and_the_record_basis(client, documents, datasource,
monkeypatch):
"""审阅的审批人只来自服务端身份:请求体写 actor 回 422,没有主体回 409;决定绑定所审记录的依据。"""
made = await accepted_draft(client, documents, datasource, monkeypatch)
draft_id = made["id"]
plant_evidence(monkeypatch, value=2, rows=1)
trial = await run_trial(client, draft_id, expect={"metrics": {"m1": 2}, "source": "fixture"})
record = trial["validation"]["record"]
record_id = record["id"]
forged = await client.post(f"/api/ledger/drafts/{draft_id}/validations/{record_id}/review",
json={"decision": "confirmed", "actor": "模型自己写的审批人"},
headers={"X-Actor": "alice"})
assert forged.status_code == 422
anonymous = await client.post(f"/api/ledger/drafts/{draft_id}/validations/{record_id}/review",
json={"decision": "confirmed"})
assert anonymous.status_code == 409 and anonymous.json()["code"] == "review_actor_required"
unknown = await client.post(f"/api/ledger/drafts/{draft_id}/validations/{record_id}/review",
json={"decision": "approved"}, headers={"X-Actor": "alice"})
assert unknown.status_code == 422 and unknown.json()["code"] == "review_decision"
outside = await client.post(f"/api/ledger/drafts/{draft_id}/validations/{record_id}/review",
json={"decision": "confirmed", "metrics": ["m9"]},
headers={"X-Actor": "alice"})
assert outside.status_code == 422 and outside.json()["code"] == "review_scope"
made_review = await client.post(f"/api/ledger/drafts/{draft_id}/validations/{record_id}/review",
json={"decision": "confirmed", "note": "看过独立样例"},
headers={"X-Actor": "alice"})
assert made_review.status_code == 201, made_review.text
review = made_review.json()["record"]["review"]
assert review["decision"] == "confirmed" and review["actor"] == "alice" and review["verified"] is False
assert review["scope"] == ["m1"] and review["covers_current"] is True
assert review["basis"]["definition_hash"] == record["definition_hash"]
assert review["basis"]["method_version"] == validation.METHOD_VERSION
listed = await validations(client, draft_id)
assert listed["summary"]["verdict"] == "verified"
assert listed["summary"]["review"]["actor"] == "alice"
# 再审一次(另一名主体驳回):追加一条,显示用最新的一条,旧的一条还在
again = await client.post(f"/api/ledger/drafts/{draft_id}/validations/{record_id}/review",
json={"decision": "rejected", "note": "样例覆盖不全"},
headers={"X-Actor": "bob"})
assert again.status_code == 201
listed = await validations(client, draft_id)
assert listed["summary"]["verdict"] == "rejected"
assert listed["records"][0]["review"]["actor"] == "bob"
async with SessionLocal() as session:
rows = list((await session.execute(select(CaliberReviewDecision).where(
CaliberReviewDecision.record_id == record_id)
.order_by(CaliberReviewDecision.created_at.asc()))).scalars())
assert [row.decision for row in rows] == ["confirmed", "rejected"]
assert [row.actor for row in rows] == ["alice", "bob"]
# 记录不存在时 404;跨草稿也读不到
missing = await client.post(f"/api/ledger/drafts/{draft_id}/validations/nope/review",
json={"decision": "confirmed"}, headers={"X-Actor": "alice"})
assert missing.status_code == 404
# ==========================================================================
# 6. 比较方法与覆盖判断本身(纯函数,R06 的原值口径)
# ==========================================================================
def fake_record(**over) -> CaliberValidationRecord:
"""一个不落库的记录对象:只用来算覆盖、摘要和读形状。"""
fields = {"id": "rec-1", "draft_id": "d-1", "workflow_id": "wf-1", "version": 1, "card_id": "card",
"metric_ids": ["m1"], "definition_hash": "a" * 64, "binding_fingerprints": {"fetch": "b" * 64},
"semantic_hash": "c" * 64, "execution_hash": "d" * 64, "expectation": None,
"expectation_hash": "", "expectation_source": "", "sample_id": "", "sample_version": "",
"inputs": {}, "inputs_hash": "", "method_version": validation.METHOD_VERSION,
"actual": {}, "status": validation.MATCHED, "problems": [], "trial_id": "t-1", "run_id": "r-1",
"actor": "alice", "origin": "trial"}
fields.update(over)
return CaliberValidationRecord(**fields)
CARD = [{"id": "m1", "name": "退款率", "contract": {"shape": "scalar"}}]
def actual_of_one(*, value=2, rows=1, masked=False, status="ok") -> dict:
return validation.actual_of(
metrics=[{"id": "m1", "value": None if (masked or rows == 0) else value, "status": status,
"node_id": "fetch", "masked": masked,
"quality": {"status": "pass", "checks": []}}],
queries=[{"node_id": "fetch", "total_rows": rows, "truncated": False}],
card_metrics_=CARD, metric_ids=["m1"])
def test_the_method_compares_raw_values_over_the_whole_card_and_every_declared_shape():
"""方法版本 1:原值比较(不做隐式舍入)、全指标、全约定形状;没有期望只记 observed。"""
# 原值:2 与 2.0 同,字符串 "2" 与数字 2 不同(不做隐式转换)
assert validation.same_value(2, 2.0) is True
assert validation.same_value(2, "2") is False
assert validation.same_value(True, 1) is False
assert validation.same_value(2.25, 2.2500001) is False
observed = validation.compare(expected=None, actual=actual_of_one(), metric_ids=["m1"],
card_metrics_=CARD)
assert observed["status"] == validation.OBSERVED
assert any("没有独立期望" in problem for problem in observed["problems"])
matched = validation.compare(expected={"m1": 2.0}, actual=actual_of_one(value=2), metric_ids=["m1"],
card_metrics_=CARD)
assert matched["status"] == validation.MATCHED and matched["actual"]["m1"]["ok"] is True
# 期望漏了一条:全指标比较,算 mismatched 而不是「只比了几条就通过」
partial = validation.compare(expected={"other": 1}, actual=actual_of_one(),
metric_ids=["m1", "other"], card_metrics_=CARD, problems=())
assert partial["status"] == validation.MISMATCHED
assert any("期望没有覆盖指标" in problem for problem in partial["problems"])
# 卡上没有可比对的指标:不能是 matched(缺证据不得升级)
empty = validation.compare(expected={}, actual={}, metric_ids=[], card_metrics_=[])
assert empty["status"] == validation.UNVERIFIED
# 运行没成功:failed,且不问期望
failed = validation.compare(expected={"m1": 2}, actual=actual_of_one(),
metric_ids=["m1"], card_metrics_=CARD, run_status="failed")
assert failed["status"] == validation.FAILED
def test_coverage_tracks_definition_binding_and_semantics_not_layout():
"""覆盖看得是定义/绑定/取数语义三样指纹:布局变更(版本号变了)不作废,删语义字段作废。"""
record = fake_record()
current = {"workflow_id": "wf-1", "card_id": "card", "definition_hash": "a" * 64,
"binding_fingerprints": {"fetch": "b" * 64}, "semantic_hash": "c" * 64}
assert validation.covers_current(record, current=current) is True
# 版本号(布局变更带来的)不在判断里
assert validation.covers_current(fake_record(version=9), current=current) is True
assert validation.covers_current(record, current={**current, "definition_hash": "z" * 64}) is False
assert validation.covers_current(record, current={**current, "binding_fingerprints": {}}) is False
assert validation.covers_current(record, current={**current, "semantic_hash": "z" * 64}) is False
assert validation.covers_current(fake_record(definition_hash=""), current=current) is False
# 取数语义指纹**任一侧缺失都算不覆盖**(不得跳过这一项)
assert validation.covers_current(fake_record(semantic_hash=""), current=current) is False
assert validation.covers_current(record, current={**current, "semantic_hash": ""}) is False
# 摘要:只有「覆盖 + matched + 认证确认**全部指标**」才是 verified;missing/stale/部分确认一律不算
assert validation.summarize([record], current=current)["verdict"] == "unverified"
confirmed = {"decision": "confirmed", "covers_current": True, "actor": "alice", "scope": ["m1"]}
assert validation.summarize([record], current=current,
reviews={record.id: confirmed})["verdict"] == "verified"
# 空范围 / 只确认一部分:不产生卡级 verified,未确认的指标要暴露出来
partial = validation.summarize([record], current=current,
reviews={record.id: {**confirmed, "scope": []}})
assert partial["verdict"] == "unverified" and partial["confirmed"] == []
assert partial["unconfirmed"] == ["m1"] and "没被确认" in partial["reason"]
assert validation.summarize([record], current=current,
reviews={record.id: {**confirmed, "scope": ["m2"]}})["verdict"] == "unverified"
assert validation.summarize([record], current={**current, "definition_hash": "z" * 64},
reviews={record.id: {**confirmed, "covers_current": False}})["verdict"] == "stale"
assert validation.summarize([], current=current)["verdict"] == "none"
observed = validation.summarize([fake_record(status=validation.OBSERVED)], current=current)
assert observed["verdict"] == "observed" and observed["unconfirmed"] == ["m1"]
rejected = validation.summarize([record], current=current,
reviews={record.id: {"decision": "rejected", "covers_current": True,
"scope": ["m1"]}})
assert rejected["verdict"] == "rejected"
def test_a_review_decision_needs_its_own_semantic_fingerprint_to_cover_current():
"""审阅决定同样按定义/绑定/取数语义三样指纹判覆盖:缺取数语义(任一侧)都算不覆盖。"""
from app.db.models import CaliberReviewDecision
from app.ledger import review
current = {"workflow_id": "wf-1", "card_id": "card", "definition_hash": "a" * 64,
"binding_fingerprints": {"fetch": "b" * 64}, "semantic_hash": "c" * 64}
basis = {"version": 1, "definition_hash": "a" * 64, "binding_fingerprints": {"fetch": "b" * 64},
"semantic_hash": "c" * 64, "method_version": 1}
row = CaliberReviewDecision(id="rev-1", draft_id="d-1", record_id="rec-1", workflow_id="wf-1",
decision="confirmed", actor="alice", verified=False, scope=["m1"],
basis=dict(basis))
assert review.covers_current(row, current=current) is True
assert review.covers_current(row, current={**current, "semantic_hash": ""}) is False
assert review.covers_current(row, current={**current, "semantic_hash": "z" * 64}) is False
missing = CaliberReviewDecision(id="rev-2", draft_id="d-1", record_id="rec-1", workflow_id="wf-1",
decision="confirmed", scope=["m1"],
basis={**basis, "semantic_hash": ""})
assert review.covers_current(missing, current=current) is False
async def test_a_failed_trial_keeps_a_stable_identity_and_reading_never_grows_the_ledger(
client, documents, datasource, monkeypatch):
"""反例(K05 复核):试运行没起来那一条没有身份时,每读一次就多插一行。
新试运行(含起草时试运行出错这一条)必须带稳定 trial_id;老草稿里没有身份的试运行,读入口不补记。"""
async def boom(**kwargs):
raise RuntimeError("试运行没有起来")
monkeypatch.setattr(api_ledger, "_run_trial", boom)
doc = await documents("val-stable-" + uuid.uuid4().hex[:8])
source_id = await datasource()
fake_model(monkeypatch, plan_reply(f"{doc}-2"))
body = await extract(client, doc, source_id)
draft_id = body["draft_id"]
detail = (await client.get(f"/api/ledger/drafts/{draft_id}")).json()
assert detail["trial"]["status"] == "failed" and detail["trial"]["trial_id"] # 稳定身份
assert detail["trial"]["error"].startswith("试运行出错")
first = await validations(client, draft_id)
assert len(first["records"]) == 1
[record] = first["records"]
assert record["trial_id"] == detail["trial"]["trial_id"] and record["status"] == "failed"
# 反复读:草稿详情 / 试运行 / 验证列表各读几遍,行数不增长(以前每次生成一个随机身份 → 每次多一行)
for _ in range(3):
await client.get(f"/api/ledger/drafts/{draft_id}")
await client.get(f"/api/ledger/drafts/{draft_id}/trial")
await validations(client, draft_id)
assert len((await validations(client, draft_id))["records"]) == 1
async def test_a_trial_without_a_stable_identity_is_never_appended_by_reads(client):
"""反例(K05 复核):老草稿 payload 里那一条没有 trial_id / run_id 的试运行,读入口不得补记。"""
draft_id = "legacy-trial-" + uuid.uuid4().hex[:8]
stale_trial = {"version": 1, "graph_hash": "g" * 64, "execution_hash": "e" * 64, "inputs_hash": "i" * 64,
"run_id": "", "status": "needs_input", "error": "缺少必填输入:start"}
async with SessionLocal() as session:
session.add(CaliberDraft(id=draft_id, title="老草稿的试运行", status="questions_open",
payload={"metrics": [{"id": "m1", "name": "旧指标"}], "trial": stale_trial}))
await session.commit()
try:
first = (await client.get(f"/api/ledger/drafts/{draft_id}")).json()
assert first["trial"] == stale_trial
# 三个读入口各读两遍:一条都不补(没有身份就认不出「是不是同一次」)
for _ in range(2):
assert (await client.get(f"/api/ledger/drafts/{draft_id}/trial")).status_code == 200
await validations(client, draft_id)
listed = await validations(client, draft_id)
assert listed["records"] == [] and listed["summary"]["verdict"] == "none"
# 试运行响应里这一段形状照旧,只是没有本次记录:covers_current 是 null(判不了),不是 false,也不顶别人的记录
trial = (await client.get(f"/api/ledger/drafts/{draft_id}/trial")).json()
assert trial["validation"]["record"] is None and trial["validation"]["covers_current"] is None
assert "不能算验证通过" in trial["validation"]["note"]
finally:
async with SessionLocal() as session:
row = await session.get(CaliberDraft, draft_id)
if row is not None:
await session.delete(row)
await session.commit()
async def test_the_trial_validation_block_has_one_shape_and_never_borrows_another_record(
client, documents, datasource, monkeypatch):
"""反例(K05 复核):GET 试运行的 validation 各分支形状不一,还会拿别的最新记录冒充本次。
每个分支都给同一组键;本次没有记录时 record 是 null、covers_current 是 null,绝不退回最新一条。"""
made = await accepted_draft(client, documents, datasource, monkeypatch)
draft_id = made["id"]
keys = {"schema_version", "execution", "status", "checks", "record", "covers_current", "note"}
ok = (await client.get(f"/api/ledger/drafts/{draft_id}/trial")).json()
assert set(ok["validation"]) == keys and ok["validation"]["record"]["trial_id"]
assert ok["validation"]["covers_current"] is True
# 把 payload 里的试运行换成一条**没有稳定身份**的老写法(run_id 也空):本次没有记录,
# 库里明明还有别的记录,这一段也不能拿别的顶上
async with SessionLocal() as session:
row = await session.get(CaliberDraft, draft_id)
keyless = {k: v for k, v in row.payload["trial"].items() if k != "trial_id"}
keyless.update(run_id="", status="needs_input", error="缺少必填输入:start")
row.payload = {**row.payload, "trial": keyless}
await session.commit()
before = (await validations(client, draft_id))["records"]
assert before # 库里确实有别的记录
alone = (await client.get(f"/api/ledger/drafts/{draft_id}/trial")).json()
assert set(alone["validation"]) == keys
assert alone["validation"]["record"] is None and alone["validation"]["covers_current"] is None
assert alone["validation"]["status"] == "unknown"
assert len((await validations(client, draft_id))["records"]) == len(before) # 没有补记
# 换成一个这次还没记过、但有稳定身份的身份:本请求里补记一条,读回来就是它的记录
async with SessionLocal() as session:
row = await session.get(CaliberDraft, draft_id)
row.payload = {**row.payload, "trial": {**keyless, "trial_id": "another-trial"}}
await session.commit()
after = (await client.get(f"/api/ledger/drafts/{draft_id}/trial")).json()
assert set(after["validation"]) == keys
assert after["validation"]["record"]["trial_id"] == "another-trial"
assert after["validation"]["record"]["status"] == "unverified" # 没发起运行,没有可比较的结果
# 它对着的还是当前这张图,所以覆盖判断是 true——记录里的依据说话,不看它是哪一次
assert after["validation"]["covers_current"] is True
async def test_a_partial_review_never_makes_the_whole_card_verified(client, documents, datasource, monkeypatch):
"""反例(K05 复核):空范围 / 只确认一部分时不能给卡级 verified,未确认的指标要说出来。"""
doc = await documents("val-scope-" + uuid.uuid4().hex[:8])
source = await datasource()
one = json.loads(plan_reply(f"{doc}-2"))
fake_model(monkeypatch, groups_reply(one, baseline_metrics(one)))
body = await extract(client, doc, source)
draft_id = body["draft_id"]
metrics = (await client.get(f"/api/ledger/drafts/{draft_id}")).json()["metrics"]
plant_evidence(monkeypatch, values={"m1": 2, "m2": 3}, metric_ids=[m["id"] for m in metrics],
nodes=nodes_of(metrics))
trial = await run_trial(client, draft_id, expect={"metrics": {"m1": 2, "m2": 3}, "source": "fixture"})
record = trial["validation"]["record"]
assert record["status"] == "matched" and record["metric_ids"] == ["m1", "m2"]
empty = await client.post(f"/api/ledger/drafts/{draft_id}/validations/{record['id']}/review",
json={"decision": "confirmed", "metrics": []}, headers={"X-Actor": "alice"})
assert empty.status_code == 422 and empty.json()["code"] == "review_scope"
partial = await client.post(f"/api/ledger/drafts/{draft_id}/validations/{record['id']}/review",
json={"decision": "confirmed", "metrics": ["m1"]}, headers={"X-Actor": "alice"})
assert partial.status_code == 201 and partial.json()["record"]["review"]["scope"] == ["m1"]
listed = await validations(client, draft_id)
assert listed["summary"]["verdict"] == "unverified"
assert listed["summary"]["confirmed"] == ["m1"] and listed["summary"]["unconfirmed"] == ["m2"]
assert "没被确认" in listed["summary"]["reason"]
whole = await client.post(f"/api/ledger/drafts/{draft_id}/validations/{record['id']}/review",
json={"decision": "confirmed"}, headers={"X-Actor": "bob"})
assert whole.status_code == 201 and sorted(whole.json()["record"]["review"]["scope"]) == ["m1", "m2"]
listed = await validations(client, draft_id)
assert listed["summary"]["verdict"] == "verified" and listed["summary"]["unconfirmed"] == []
async def test_an_unreadable_baseline_artifact_is_never_reported_as_known(client, documents, datasource,
monkeypatch):
"""反例(K05 复核):基线工件被删掉或改过时,读草稿不能报 known,也不能抛出去把页面弄崩。"""
from app.core.config import settings
made = await accepted_draft(client, documents, datasource, monkeypatch)
baseline = made["field_baseline"]
assert baseline["status"] == "known" and baseline["readable"] is True
artifact = baseline["artifact_id"]
path = settings.data_dir / "artifacts" / artifact[:2] / f"{artifact}.json"
original = path.read_bytes()
try:
path.write_text('{"kind": "caliber_field_baseline", "tampered": true}', encoding="utf-8")
tampered = (await client.get(f"/api/ledger/drafts/{made['id']}")).json()
assert tampered["field_baseline"]["status"] == "baseline_unknown"
assert tampered["field_baseline"]["readable"] is False
assert tampered["field_baseline"]["artifact_id"] == artifact
assert "读不回来" in tampered["field_baseline"]["reason"]
# 验证列表和试运行接口照常给(读页面不崩)
listed = (await client.get(f"/api/ledger/drafts/{made['id']}/validations")).json()
assert listed["field_baseline"]["status"] == "baseline_unknown"
assert (await client.get(f"/api/ledger/drafts/{made['id']}/trial")).status_code == 200
path.unlink()
missing = (await client.get(f"/api/ledger/drafts/{made['id']}")).json()
assert missing["field_baseline"]["status"] == "baseline_unknown"
assert missing["field_baseline"]["readable"] is False
finally:
path.write_bytes(original)
assert (await client.get(f"/api/ledger/drafts/{made['id']}")).json()["field_baseline"]["status"] == "known"
# ==========================================================================
# 收尾:本模块造的行(草稿由夹具清,记录/审阅跟着草稿级联,这里兜一下)
# ==========================================================================
@pytest.fixture(autouse=True)
async def _cleanup_records():
yield
async with SessionLocal() as session:
drafts = list((await session.execute(select(CaliberDraft.id))).scalars())
if drafts:
await session.execute(delete(CaliberReviewDecision).where(
CaliberReviewDecision.draft_id.in_(drafts)))
await session.execute(delete(CaliberValidationRecord).where(
CaliberValidationRecord.draft_id.in_(drafts)))
await session.commit()