Files
AgentLab/backend/tests/test_caliber_source_snapshot.py
T
ThearchyHelios 2f90f0eb3a 口径台账:固定执行、验证审阅发布与增量重导入
- QueryCore 固定执行(K09):按已发布的不可变口径跑一次,交回类型化标量、发布/manifest 引用与逐查询证据
- 依赖闭包(K08):按选中的指标算出要拷的取数节点、要的入口字段,接进 derive / compose / metrics
- SQL 模板参数完整性(K01):试跑、组合问数、派生图、运行时取数共用一套语义,缺参数在渲染前就拒
- 受限字段变更服务(K03)与口径领域边界(K01):改动打在整图深拷贝上,身份与证据归属不可改
- 版本化受控模板绑定(K04):业务口径按显式绑定落到 SQL,对不上就报 unsupported / needs_update
- 验证记录(K05)与最小审阅:样例匹配不等于正式批准,审批人只从服务端身份取
- 发布依据清单(K07):发布时按不可变图物化内容寻址清单,正式用途按它判
- 增量重导入(K11)与 B/L/R 三方合并、受限自然语言修改提案(K12):只产候选,落图仍走既有写路径
- 接口与前端:口径草稿页接上字段变更、绑定状态、验证与重导入,新增 caliber-drafts / studio 检查脚本

验证:前端 tsc -b 通过。
2026-11-10 15:05:18 +08:00

434 lines
24 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""K05 的来源侧前置切片:提取时把来源存成不可篡改的工件,引用只认实际进提示的材料。
这份文件盯住几件 K05 明确要求、而其它测试没覆盖的事:
1. **快照可还原,不只是哈希。** 提取时把逐份原文、逐段原文、目录文本按内容寻址存进工件库;草稿 payload 只记引用。
读的时候按 id 回读,原文一模一样,哈希对得上(app/core/artifact_store.load 会复验)。
2. **chunk id 换了,快照照旧读得回来。** 文档被重新切块、段落 id 换了一批,旧草稿记的引用仍指向当初那一份材料。
3. **可见的引用才核得上。** 材料超限时 fit_material 从后往前丢段落;被丢掉的段落不在提示里,模型引用它也核不上。
同一段话在「全量 chunks」下核得上、在「实际进提示的可见 chunks」下核不上。
4. **每次提示的可见范围有记录。** payload.prompts 逐次记下 visible_chunk_ids、提示词/解析版本、模型、截断情况。
5. **checkpoint 只是任务恢复,不冒充 B。** checkpoint 里只有「哪些组起草好了」,把它当 payload 读,来源基线仍
是 baseline_unknown;来源的还原只走工件。
6. **旧草稿标记 baseline_unknown。** 没有快照的老草稿不拿当前内容伪造历史基线。
模型一律注入假模型(monkeypatch resolve_draft_model),试跑换成假的,测试不发真实请求、不连真实业务库。
"""
from __future__ import annotations
import copy
import json
import uuid
import pytest
from sqlalchemy import delete, select
from app.core.artifact_store import canonical_json, content_hash, load
from app.db.base import SessionLocal
from app.db.models import CaliberDraft, Chunk, Document, Workflow, WorkflowVersion
from app.ledger import authoring, docdraft, draftplan
from tests.test_ledger_docdraft_api import client, datasource, fake_model, fake_trial # noqa: F401 (fixtures)
SMALL_ONE = "退款率按已支付订单统计,退款订单不计入。"
SMALL_TWO = "门店维度只保留已支付订单,未支付不进分子分母。"
@pytest.fixture(autouse=True)
def _fake_trial(fake_trial):
"""本模块的草稿都走接口起草:试跑必须换成假的,不然会去打夹具里不存在的库。
(fake_trial 是从别的模块 import 进来的普通夹具,它自带的 autouse 在本模块不生效;这里替本模块的每个用例把它拉起来。)"""
return fake_trial
# ==========================================================================
# 夹具:可自定义段落的文档 + 收尾清理
# ==========================================================================
@pytest.fixture
async def docs():
"""按给定的分段建一份文档;收尾删掉它、它的段落、以及测试期间建的草稿/工作流/运行。"""
made: list[str] = []
async def make(doc_id: str, chunks: list[tuple[int, str]], *, title: str = "口径.md") -> str:
async with SessionLocal() as session:
session.add(Document(id=doc_id, collection="default", title=title, source=title,
content="".join(content for _, content in chunks), chunk_count=len(chunks)))
for ordinal, content in chunks:
session.add(Chunk(id=f"{doc_id}-{ordinal}", document_id=doc_id, ordinal=ordinal,
content=content))
await session.commit()
made.append(doc_id)
return doc_id
yield make
async with SessionLocal() as session:
drafts = (await session.execute(select(CaliberDraft))).scalars().all()
workflow_ids = [draft.workflow_id for draft in drafts if draft.workflow_id]
workflow_ids += [draft.payload.get("workflow_id") for draft in drafts
if isinstance(draft.payload, dict) and draft.payload.get("workflow_id")]
workflow_ids = list(dict.fromkeys(wid for wid in workflow_ids if wid))
for draft in drafts:
await session.delete(draft)
if workflow_ids:
await session.execute(delete(WorkflowVersion).where(WorkflowVersion.workflow_id.in_(workflow_ids)))
await session.execute(delete(Workflow).where(Workflow.id.in_(workflow_ids)))
for doc_id in made:
await session.execute(delete(Chunk).where(Chunk.document_id == doc_id))
row = await session.get(Document, doc_id)
if row is not None:
await session.delete(row)
await session.commit()
async def _extract(client, doc: str, source_id: str, **extra) -> dict:
body = (await client.post("/api/ledger/drafts/extract",
json={"documents": [doc], "datasource_id": source_id, **extra})).json()
return body
async def _draft(client, draft_id: str) -> dict:
return (await client.get(f"/api/ledger/drafts/{draft_id}")).json()
def _reply(chunk_id: str, *, quote: str, name: str = "退款率", sql: str | None = None) -> dict:
"""一条指标的回复:引用逐字抄原文。"""
return {
"entry_fields": [],
"sql": sql or "SELECT COUNT(*) AS refund_rate FROM orders",
"metrics": [{
"id": "m1", "name": name, "synonyms": [], "unit": "%", "decimals": 2,
"expression": "cell(nodes.fetch, 0, 'refund_rate')",
"definition": {"text": "已支付订单里发生退款的比例",
"citations": [{"chunk": chunk_id, "quote": quote}]},
"open_questions": [],
}],
"open_questions": [],
}
# ==========================================================================
# 1. 快照可还原、不可变
# ==========================================================================
async def test_the_first_extraction_persists_a_restorable_source_snapshot(client, docs, datasource,
monkeypatch):
"""提取时把原文存进工件库:payload 只有引用,按引用回读,逐份、逐段原文一模一样,哈希对得上。"""
doc = await docs("snap-1", [(0, SMALL_ONE), (1, SMALL_TWO)], title="退款口径.md")
source_id = await datasource()
fake_model(monkeypatch, json.dumps(_reply(f"{doc}-0", quote="退款率按已支付订单统计"), ensure_ascii=False))
made = await _draft(client, (await _extract(client, doc, source_id))["draft_id"])
ref = made["source_snapshot"]
assert ref["version"] == docdraft.SNAPSHOT_VERSION and ref["kind"] == docdraft.SNAPSHOT_KIND
assert ref["chunk_ids"] == [f"{doc}-0", f"{doc}-1"] and ref["documents"] == [doc]
# 引用里的 id 就是规范化内容的 sha256:内容不变,id 不变;内容变了,读的时候复验哈希会当场报错
assert ref["content_hash"] == ref["artifact_id"]
snapshot = load(ref["artifact_id"])
assert snapshot is not None
assert content_hash(canonical_json(snapshot)) == ref["artifact_id"]
assert snapshot["chunks"][f"{doc}-0"]["content"] == SMALL_ONE
assert snapshot["chunks"][f"{doc}-1"]["content"] == SMALL_TWO
assert snapshot["documents"][0]["content"] == SMALL_ONE + SMALL_TWO
assert snapshot["documents"][0]["content_hash"] == content_hash(SMALL_ONE + SMALL_TWO)
assert snapshot["catalog"].strip(), "目录文本也要存:它改了,模型看到的材料和写出的 SQL 就变了"
# 基线状态:有可还原快照 → known(只说明材料还原得出来,不等于字段基线,更不等于业务验证通过)
assert made["source_baseline"] == {"status": "known", "readable": True, "artifact_id": ref["artifact_id"],
"version": docdraft.SNAPSHOT_VERSION,
"chunk_ids": [f"{doc}-0", f"{doc}-1"]}
assert docdraft.source_baseline(made) == made["source_baseline"]
async def test_the_snapshot_still_reads_after_the_document_is_rechunked(client, docs, datasource,
monkeypatch):
"""重导入会重新切块、换一批 chunk id;旧草稿记的引用仍指向当初那一份材料(按内容寻址,不按名字)。"""
doc = await docs("snap-2", [(0, SMALL_ONE), (1, SMALL_TWO)])
source_id = await datasource()
fake_model(monkeypatch, json.dumps(_reply(f"{doc}-0", quote="退款率按已支付订单统计"), ensure_ascii=False))
made = await _draft(client, (await _extract(client, doc, source_id))["draft_id"])
artifact_id = made["source_snapshot"]["artifact_id"]
# 文档被重新切块:旧段落删掉,换成三个新 id、顺序和内容也变了
async with SessionLocal() as session:
await session.execute(delete(Chunk).where(Chunk.document_id == doc))
for ordinal, content in enumerate([SMALL_TWO, "新加的一段说明。", SMALL_ONE]):
session.add(Chunk(id=f"re-{doc}-{ordinal}", document_id=doc, ordinal=ordinal, content=content))
await session.commit()
snapshot = load(artifact_id)
assert snapshot is not None, "旧引用必须还读得回来"
assert set(snapshot["chunks"]) == {f"{doc}-0", f"{doc}-1"}, "当初收进去的段落一个都不能少"
assert snapshot["chunks"][f"{doc}-0"]["content"] == SMALL_ONE
assert snapshot["chunks"][f"{doc}-1"]["content"] == SMALL_TWO
# 草稿里那条指标仍指着当初的段落 id 和原文,不受重新切块影响
assert made["metrics"][0]["definition"]["citations"][0]["chunk"] == f"{doc}-0"
# ==========================================================================
# 2. 引用核对只认实际可见的材料
# ==========================================================================
async def test_a_citation_to_a_truncated_away_chunk_is_not_accepted(client, docs, datasource, monkeypatch):
"""材料超限时从后往前丢段落:被丢掉的段落没进提示,模型引用它也核不上,只留能核对的那条指标。"""
tail = "这一段很长,超限时会被整段丢掉。" * 3000 # 六万字,稳稳超过一次提示的上限
doc = await docs("snap-3", [(0, SMALL_ONE), (1, tail)])
source_id = await datasource()
reply = _reply(f"{doc}-0", quote="退款率按已支付订单统计", name="退款率")
reply["metrics"].append({**reply["metrics"][0], "id": "m2", "name": "长文里的指标",
"definition": {"text": "引用被丢段的原文",
"citations": [{"chunk": f"{doc}-1",
"quote": "这一段很长,超限时会被整段丢掉"}]}})
fake_model(monkeypatch, json.dumps(reply, ensure_ascii=False))
made = await _draft(client, (await _extract(client, doc, source_id))["draft_id"])
assert made["doc_truncated"] is True
assert [m["name"] for m in made["metrics"]] == ["退款率"], "引用不到可见材料的指标不该留下"
assert any(item["name"] == "长文里的指标" and "引用" in item["reason"] for item in made["dropped"])
# 提示记录里也如实写着:只看得见第一段
[trace] = [t for t in made["prompts"] if t["purpose"] == "plan"]
assert trace["visible_chunk_ids"] == [f"{doc}-0"]
assert trace["doc_truncated"] is True
assert set(trace["visible_chars"]) == {f"{doc}-0"}
def test_normalize_plan_checks_citations_only_against_what_the_model_saw():
"""同一段话:在「全量 chunks」下核得上,在「实际进提示的可见 chunks」下核不上——引用核对以可见范围为准。"""
full = {"c1": {"ordinal": 0, "content": "退款率按已支付订单统计", "document_id": "d"},
"c2": {"ordinal": 1, "content": "门店维度只保留已支付订单", "document_id": "d"}}
visible = {"c1": full["c1"]} # c2 被截断丢掉了
raw = {"entry_fields": [], "sql": "SELECT COUNT(*) AS n FROM orders",
"metrics": [{"id": "m1", "name": "退款率", "expression": "cell(nodes.fetch, 0, 'n')",
"definition": {"text": "口径", "citations": [{"chunk": "c2",
"quote": "门店维度只保留已支付订单"}]}}],
"open_questions": []}
with_visibility, dropped = draftplan.normalize_plan(raw, visible, allowed_tables={"orders"})
assert with_visibility["metrics"] == [] and dropped and "引用" in dropped[0]["reason"]
without_visibility, dropped = draftplan.normalize_plan(raw, full, allowed_tables={"orders"})
assert dropped == [] and [m["name"] for m in without_visibility["metrics"]] == ["退款率"]
async def test_plan_groups_reports_the_same_visible_chunks_it_sends_to_the_model():
"""draftplan 自己算材料时,报出来的 visible_chunk_ids 就是真正进提示的那几段(与提示文本一致)。"""
from types import SimpleNamespace
class Model:
model_name = "visible-model"
def __init__(self) -> None:
self.messages: list = []
async def ainvoke(self, messages):
self.messages = messages
return SimpleNamespace(content='{"groups": [], "open_questions": [], "uncovered": []}')
first = "第一段:" + "退款率按已支付订单统计。" * 3000
second = "第二段:" + "门店维度只保留已支付订单。" * 3000
chunks = {"c1": {"ordinal": 0, "content": first, "document_id": "d"},
"c2": {"ordinal": 1, "content": second, "document_id": "d"}}
model = Model()
result = await draftplan.plan_groups(None, docs=[], chunks=chunks, tables_text="表 orders", model=model)
sent = "\n".join(str(getattr(message, "content", "")) for message in model.messages)
assert result["visible_chunk_ids"] == ["c1"] and result["doc_truncated"] is True
assert "第一段" in sent and "第二段" not in sent
assert result["prompt_version"] == draftplan.PROMPT_VERSION
assert result["parse_version"] == docdraft.PARSE_VERSION
# ==========================================================================
# 3. 提示记录
# ==========================================================================
async def test_every_model_prompt_records_what_it_saw_and_which_versions(client, docs, datasource,
monkeypatch):
"""payload.prompts 逐次记下 visible_chunk_ids、模型、提示词/解析版本:历史结果才知道是按哪一版抽的。"""
doc = await docs("snap-4", [(0, SMALL_ONE), (1, SMALL_TWO)])
source_id = await datasource()
fake_model(monkeypatch, json.dumps(_reply(f"{doc}-0", quote="退款率按已支付订单统计"), ensure_ascii=False))
made = await _draft(client, (await _extract(client, doc, source_id))["draft_id"])
[trace] = made["prompts"]
assert trace["purpose"] == "plan" and trace["round"] == 1
assert trace["visible_chunk_ids"] == [f"{doc}-0", f"{doc}-1"]
assert trace["model_id"] == "fake-model"
assert trace["prompt_version"] == draftplan.PROMPT_VERSION
assert trace["parse_version"] == docdraft.PARSE_VERSION
assert trace["doc_truncated"] is False and trace["catalog_truncated"] is False
assert trace["visible_chars"][f"{doc}-0"] == len(SMALL_ONE)
async def test_the_auxiliary_extract_path_reports_visibility_and_versions():
"""docdraft.extract(辅助路径)也交回可见段落与版本,和主路径一套说法。"""
from types import SimpleNamespace
class Model:
model_name = "aux-model"
async def ainvoke(self, _messages):
return SimpleNamespace(content=json.dumps(
{"metrics": [{"name": "退款率", "definition": {"text": "口径", "citations": [
{"chunk": "c1", "quote": "退款率按已支付订单统计"}]}}]}, ensure_ascii=False))
result = await docdraft.extract(None, docs=[{"document_id": "d", "title": "口径.md"}],
chunks={"c1": {"ordinal": 0, "content": SMALL_ONE, "document_id": "d"}},
model=Model())
assert result["visible_chunk_ids"] == ["c1"]
assert result["prompt_version"] == docdraft.PROMPT_VERSION
assert result["parse_version"] == docdraft.PARSE_VERSION
assert [m["name"] for m in result["metrics"]] == ["退款率"]
# ==========================================================================
# 4. checkpoint 只是任务恢复,不冒充 B
# ==========================================================================
async def _run_extract(draft_id: str) -> dict:
"""直接调后台起草,试跑换成不启动引擎的桩(不经 HTTP、不建任务)。"""
trial: dict = {}
async def run_trial(**_kwargs):
return {"run_id": "", "status": "succeeded", "error": ""}
async def save_trial(_draft_id, value):
trial.update(value or {})
def binding(*_args):
return {}
await authoring.extract(draft_id, run_trial=run_trial, save_trial=save_trial, trial_binding=binding)
return trial
async def test_the_checkpoint_is_recovery_only_and_never_masquerades_as_a_baseline(client, docs, datasource,
monkeypatch):
"""材料指纹相同的重试接续 checkpoint;checkpoint 只记「哪些组起草好了」,把它当 payload 读仍是 baseline_unknown。"""
doc = await docs("snap-5", [(0, SMALL_ONE), (1, SMALL_TWO)])
source_id = await datasource()
group = _reply(f"{doc}-0", quote="退款率按已支付订单统计")
calls: list[list[str]] = []
async def interrupted(_session, **kwargs):
calls.append(list(kwargs["covered"]))
if len(calls) == 1:
return {"groups": [group], "uncovered": ["门店数"]}
raise RuntimeError("这一轮被打断")
monkeypatch.setattr(draftplan, "plan_groups", interrupted)
extracted = await _extract(client, doc, source_id)
draft_id = extracted["draft_id"]
await _run_extract(draft_id)
async with SessionLocal() as session:
row = await session.get(CaliberDraft, draft_id)
payload = copy.deepcopy(row.payload)
# 被打断的那一轮之前,第一组和来源快照都已经写下来了
assert payload["checkpoint"]["groups"][0]["metrics"][0]["name"] == "退款率"
assert payload["source_snapshot"]["artifact_id"] and payload["source_baseline"]["status"] == "known"
# checkpoint 里只有恢复用的东西:把它当成一份 payload 去读来源基线,只会得到 baseline_unknown
assert set(payload["checkpoint"]) == {"groups"}
assert docdraft.source_baseline(payload["checkpoint"])["status"] == "baseline_unknown"
assert "source_snapshot" not in payload["checkpoint"]
# 重试:材料没变,checkpoint 里的组接续下去;来源快照仍从这次读到的材料现取(不来自 checkpoint)
second = copy.deepcopy(group)
second["metrics"][0].update(id="m2", name="门店数")
async def resume(_session, **kwargs):
assert kwargs["covered"] == ["退款率"], "重试要接着上次起草过的地方,不重头再来"
return {"groups": [second], "uncovered": []}
monkeypatch.setattr(draftplan, "plan_groups", resume)
async with SessionLocal() as session: # 重试会先把草稿置回 drafting(api/ledger 的 retry 就是这样)
row = await session.get(CaliberDraft, draft_id)
row.status = "drafting"
await session.commit()
await _run_extract(draft_id)
async with SessionLocal() as session:
row = await session.get(CaliberDraft, draft_id)
resumed = copy.deepcopy(row.payload)
assert [m["name"] for m in resumed["metrics"]] == ["退款率", "门店数"]
assert resumed["source_snapshot"]["artifact_id"] == payload["source_snapshot"]["artifact_id"]
assert load(resumed["source_snapshot"]["artifact_id"])["chunks"][f"{doc}-1"]["content"] == SMALL_TWO
# ==========================================================================
# 5. 旧草稿:baseline_unknown,不拿当前值伪造历史
# ==========================================================================
async def test_an_old_draft_without_a_snapshot_reads_as_baseline_unknown(client, docs):
"""保存来源快照之前起草的草稿没有引用:如实标记 baseline_unknown,不能拿它现在的内容当历史基线。"""
draft_id = f"old-{uuid.uuid4().hex[:10]}"
old_payload = {"metrics": [{"id": "m1", "name": "旧指标", "expression": "cell(nodes.fetch, 0, 'n')"}],
"sql": "SELECT 1 AS n", "version": 1, "workflow_id": None}
async with SessionLocal() as session:
session.add(CaliberDraft(id=draft_id, title="老草稿", status="path_pending",
payload=copy.deepcopy(old_payload)))
await session.commit()
async with SessionLocal() as session:
stored = (await session.get(CaliberDraft, draft_id)).payload
baseline = docdraft.source_baseline(stored)
assert baseline["status"] == "baseline_unknown" and "无法还原" in baseline["reason"]
assert "artifact_id" not in baseline
body = await _draft(client, draft_id)
assert body["status"] == "path_pending" and body["metrics"][0]["name"] == "旧指标"
# 老草稿里根本没有快照,也没有凭空补一条:当前内容照旧显示,但它不是「上次被接受的导入值」。
# 读接口如实给出 baseline_unknown(没有快照本身就是要说的话,不能让客户端从「缺字段」去猜)
assert "source_snapshot" not in body
assert body["source_baseline"]["status"] == "baseline_unknown"
assert "无法还原" in body["source_baseline"]["reason"] and "artifact_id" not in body["source_baseline"]
assert docdraft.source_baseline(body)["status"] == "baseline_unknown"
async def test_a_snapshot_is_not_invented_for_a_draft_without_one(client, docs, datasource, monkeypatch):
"""新草稿照常记快照;同一批材料被再次读取时快照不变(按内容寻址,同一份材料只存一份、id 相同)。"""
doc = await docs("snap-6", [(0, SMALL_ONE)])
source_id = await datasource()
fake_model(monkeypatch, json.dumps(_reply(f"{doc}-0", quote="退款率按已支付订单统计"), ensure_ascii=False))
first = await _draft(client, (await _extract(client, doc, source_id))["draft_id"])
fake_model(monkeypatch, json.dumps(_reply(f"{doc}-0", quote="退款率按已支付订单统计",
name="退款率(第二份)"), ensure_ascii=False))
second = await _draft(client, (await _extract(client, doc, source_id))["draft_id"])
assert first["source_snapshot"]["artifact_id"] == second["source_snapshot"]["artifact_id"]
assert docdraft.source_baseline(second)["status"] == "known"
# ==========================================================================
# 6. 装快照失败时如实说「没有基线」,不假装有
# ==========================================================================
async def test_a_snapshot_that_cannot_be_stored_is_reported_as_baseline_unknown(client, docs, datasource,
monkeypatch):
"""落工件失败(磁盘/库出错)不让整份草稿失败,但也绝不假装有基线:source_snapshot 为空、状态 unknown。"""
import app.core.artifact_store as store
doc = await docs("snap-7", [(0, SMALL_ONE)])
source_id = await datasource()
fake_model(monkeypatch, json.dumps(_reply(f"{doc}-0", quote="退款率按已支付订单统计"), ensure_ascii=False))
async def broken(*_args, **_kwargs):
raise OSError("磁盘满了")
# _capture_source 在函数里 `from app.core.artifact_store import put_json`,换掉模块属性就换掉了它
monkeypatch.setattr(store, "put_json", broken)
made = await _draft(client, (await _extract(client, doc, source_id))["draft_id"])
assert made["status"] in ("ready", "questions_open"), made.get("error")
assert made["workflow_id"], "快照存不下不该把草稿判死:图和试跑照旧"
assert made["source_snapshot"] is None
assert made["source_baseline"]["status"] == "baseline_unknown"