mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-16 22:12:01 +00:00
5b4bd91476
为什么:问数据的答案要和已治理口径接起来(next-round 02 做法 2、3、9、10;总方案 X1、X2、X3、X12,拍板 L2、L4、L8)。
- 比对(ledger/match.py):探索运行(chat、canvas、trial、personal_app)成功结束、封存完、有查询快照时,在 runner 封存之后和
notify_run 同一个位置放到后台;候选口径是每个工作流最新的已发布未退役版本里查库的调用工具节点(受管、已发布都算),按数据源
和表预筛;逐个用 match_template 比,schema 用查询当时冻结的表结构、列名取同一份快照里冻结的数据目录。每次运行限时 8 秒、在
线程里、同时最多 2 个;超时、快照被改过或读不出、SQL 解析不了照实记 skipped。启动时(run_manager.setup,和调度循环同一个开关)
补比最近 7 天封存了却没比过的运行,只读工件和版本表。
- 记法(ledger/store.py,拍板 L8):内容寻址工件 kind=caliber_match(agentlab.caliber_match/1)+ run_append 的 sealed 方式追加
ledger.matched(core/events.py 登记);工件里不放时间和工作流名,同一份封存再比一次哈希一样;meta 记 calibers / related / counts。
- 标签(ledger/labels.py):单元格按查询的比对状态;指标要每个输入都等价于同一份口径同一版、绑定一致,式子按「哪个取数节点、
第几行、哪一列」换完和口径卡上的一字不差。caliber(how:independent 等价已证明;composed 助手选用留给第 3 波)/ related /
reasoning / unknown;整题判断附还缺的入口字段。
- 读(ledger/view.py):GET /api/runs/{id}/ledger(ledger_api.runs_router,和证据同一道门),没比对过、能比的按需排上;证据
片段接口加 ledger 键(没比对过的运行没有这个键,返回一字不差);报告导出的面板最上面多「口径」一步并写明「封存后比对,不在
封存范围内」。遮罩按读的那一刻重滤:绑定的值是这次运行查到过的遮罩值时不给值,差异条目和逐行对照走 _Shown。
- 来源链:POST /api/runs 加 origin_ref(只在 origin=chat 时收,服务端核对对话访问权和轮次归属,写进 Run.origin 和封存的
run.origin);WorkflowIn 加 origin(run_id、conversation_id、turn_id,核对规则同 from-run);报告导出和可验证报告下载各记一条
审计 report.exported。可验证报告列封存后追加的事件时给 ledger.matched 一句只有条数的说明。
- 统计(ledger/stats.py):E1–E4 从工件、运行、查询快照、审计现算,GET /api/ledger/stats(搭建者);E3 用 app/chat/questions
的聚类加按(数据源、normalize_sql 文本)的确定性重复;评测直接调 compute_stats(可点名 run_ids)。对账反馈、认证转化、按发布
的召回在后面几波,照实写没有数据。
测试:test_ledger_match.py(真跑探索运行:等价、相关、没有相关口径、写死的码值、非探索运行、超时、快照被改、解析失败、预筛、
指标两种、级别、只认最新版、补比)、test_ledger_api.py(账本接口、按需比、遮罩、证据片段、导出、审计、统计)、
test_run_origin_ref.py(origin_ref 与画布来源的核对)。
259 lines
14 KiB
Python
259 lines
14 KiB
Python
"""口径账本的读接口(next-round 02 做法 3、9、10):GET /api/runs/{id}/ledger、证据片段的 ledger 键、报告导出的「口径」一步、
|
||
导出留痕、GET /api/ledger/stats。
|
||
|
||
- 比对过的运行:标签、整题判断、每条查询的比对,写明封存后比对、不在封存范围内;工作流名按现在的取;
|
||
- 没比对过的:能比的排上按需比对(pending → 比完 trigger=on_demand、写明按现在的口径回看),不比的照实说为什么;
|
||
- 遮罩:绑定的值是这次运行查到过的遮罩值时不给值,差异条目里的 SQL 片段换成「已遮罩」,整份答复里找不到那个号码;
|
||
- 证据片段:比对过的带 ledger 键,没比对过的没有这个键;
|
||
- 报告导出:数据块带同一份标签和那句说明;没比对过的没有这个键;每次导出(报告、可验证报告)各记一条审计 report.exported;
|
||
- 统计:E1–E4 的口径和夹具上手算的一致,skipped 单独计。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
|
||
import pytest
|
||
from httpx import ASGITransport, AsyncClient
|
||
from sqlalchemy import select
|
||
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import AuditEvent, Workflow, WorkflowVersion
|
||
from app.engine.runner import run_manager
|
||
from app.ledger import match as ledger_match
|
||
from app.main import app
|
||
from tests import ledger_samples as S
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
async def engine_up(tmp_path):
|
||
from app.db.base import utcnow
|
||
|
||
await run_manager.setup()
|
||
path = tmp_path / "shop.db"
|
||
S.build_shop(path)
|
||
await S.set_source(path)
|
||
async with SessionLocal() as session:
|
||
for row in (await session.execute(select(WorkflowVersion).where(WorkflowVersion.retired_at.is_(None)))).scalars():
|
||
row.retired_at = utcnow()
|
||
await session.commit()
|
||
yield path
|
||
await ledger_match.drain()
|
||
await run_manager.shutdown()
|
||
|
||
|
||
@pytest.fixture
|
||
async def client():
|
||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||
yield c
|
||
|
||
|
||
async def ledger_of(client, run_id: str) -> dict:
|
||
res = await client.get(f"/api/runs/{run_id}/ledger")
|
||
assert res.status_code == 200, res.text
|
||
return res.json()
|
||
|
||
|
||
def first_segment(row, text: str) -> str:
|
||
from app.core.artifact_store import load
|
||
from app.engine.evidence import iter_segments
|
||
|
||
doc = load(row.output["_evidence"]["doc_artifact"])
|
||
return next(s["id"] for s in iter_segments(doc) if s.get("text") == text and s.get("cite"))
|
||
|
||
|
||
async def test_a_matched_run_gives_labels_and_the_whole_judgement(client, monkeypatch):
|
||
wf = await S.publish()
|
||
row = await S.explore(monkeypatch, S.EQUIVALENT, text="本周销售额 [[v:Q1.r0.销售额]]。")
|
||
body = await ledger_of(client, row.id)
|
||
assert body["state"] == "matched" and body["note"] == "封存后比对,不在封存范围内" and body["post_seal"] is True
|
||
assert body["trigger"] == "sealed" and body["retro"] is False and "retro_note" not in body
|
||
assert body["actions"] == [] and body["primary"] == "write"
|
||
[caliber] = body["calibers"]
|
||
assert caliber["workflow_name"] == "周销售" and caliber["level_text"] == "受管" and caliber["caliber"] == f"{wf}@1"
|
||
[label] = body["reports"]["write"]["labels"].values()
|
||
assert label["kind"] == "caliber" and label["how"] == "independent" and label["workflow_name"] == "周销售"
|
||
# 工件是规范 JSON(键排好序),绑定按字段名的先后给
|
||
assert label["binding"] == [{"field": "end", "value": "2026-09-08"}, {"field": "start", "value": "2026-09-01"}]
|
||
assert body["whole"]["equivalent"] is True and body["whole"]["missing"] == ["store"]
|
||
[query] = body["queries"]
|
||
assert query["status"] == "proven" and query["candidates"][0]["status"] == "proven"
|
||
# 工作流改了名:按现在的名字给
|
||
async with SessionLocal() as session:
|
||
(await session.get(Workflow, wf)).name = "周销售(新)"
|
||
await session.commit()
|
||
assert (await ledger_of(client, row.id))["calibers"][0]["workflow_name"] == "周销售(新)"
|
||
|
||
|
||
async def test_an_unmatched_run_is_matched_on_demand_and_says_it_looks_back(client, monkeypatch):
|
||
await S.publish()
|
||
real = ledger_match.schedule
|
||
monkeypatch.setattr(ledger_match, "schedule", lambda run_id, **kw: False)
|
||
row = await S.explore(monkeypatch, S.EQUIVALENT, text="本周销售额 [[v:Q1.r0.销售额]]。")
|
||
monkeypatch.setattr(ledger_match, "schedule", real)
|
||
body = await ledger_of(client, row.id)
|
||
assert body["state"] == "pending"
|
||
await ledger_match.drain()
|
||
body = await ledger_of(client, row.id)
|
||
assert body["state"] == "matched" and body["trigger"] == "on_demand" and body["retro"] is True
|
||
assert body["retro_note"] == "按现在的口径回看"
|
||
|
||
|
||
async def test_runs_that_are_not_compared_say_why(client, monkeypatch):
|
||
row = await S.explore(monkeypatch, S.EQUIVALENT, origin="api", text="本周销售额 [[v:Q1.r0.销售额]]。")
|
||
body = await ledger_of(client, row.id)
|
||
assert body["state"] == "ineligible" and body["reason"] == ledger_match.NOT_EXPLORE
|
||
res = await client.get("/api/runs/nope/ledger")
|
||
assert res.status_code == 404 and res.json()["code"] == "run_not_found"
|
||
|
||
|
||
async def test_masked_binding_values_and_sql_fragments_are_withheld(client, monkeypatch, engine_up):
|
||
"""口径按手机号筛({{ input.phone }});这次运行先查过手机号(数据源设了遮罩),再按其中一个号码筛:绑定的值是遮罩值,
|
||
不给值;差异条目、逐行对照里的那个号码也换成「已遮罩」。"""
|
||
await S.set_source(engine_up, options={"mask_columns": ["phone"]})
|
||
template = "SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' AND phone = '{{ input.phone }}'"
|
||
await S.publish(graph=S.caliber_graph(template))
|
||
graph = S.chain(
|
||
S.node("start", "input"),
|
||
S.node("p", "tool", tool="db_query__shop", args={"sql": "SELECT phone FROM orders ORDER BY id"}),
|
||
S.node("q", "tool", tool="db_query__shop",
|
||
args={"sql": "SELECT SUM(amount) AS gmv FROM orders WHERE phone = '13800000001'"}),
|
||
S.node("write", "report", instructions="写答案"),
|
||
S.node("out", "output", fields=[{"name": "答案", "value": "{{ nodes.write.text }}"}]))
|
||
S.script(monkeypatch, "这位会员的消费 [[v:Q2.r0.gmv]]。")
|
||
run = await run_manager.start(graph=graph, input_payload={}, origin_kind="canvas")
|
||
row = await S.wait(run.id)
|
||
assert row.status == "succeeded", row.error
|
||
await ledger_match.drain()
|
||
body = await ledger_of(client, row.id)
|
||
assert body["state"] == "matched"
|
||
[label] = body["reports"]["write"]["labels"].values()
|
||
assert label["kind"] == "related"
|
||
[related] = label["related"]
|
||
assert related["binding"] == [{"field": "phone", "masked": True, "value": "已遮罩"}]
|
||
assert "13800000001" not in json.dumps(body, ensure_ascii=False)
|
||
assert related["diff"]["items"] and "13800000001" not in json.dumps(related["diff"], ensure_ascii=False)
|
||
|
||
|
||
def test_diff_fragments_go_through_the_same_mask_as_the_evidence_panel():
|
||
"""差异条目、逐行对照里的 SQL 片段走证据面板同一套换法(api/evidence._Shown):这次运行查到过的遮罩值换成「已遮罩」。"""
|
||
from app.ledger.labels import view_label
|
||
from app.ledger.view import _Hide
|
||
|
||
class Secrets:
|
||
def hit(self, value):
|
||
return str(value) == "13800000001"
|
||
|
||
def weak_hit(self, text):
|
||
return text == "13800000001"
|
||
|
||
def scrub(self, text, *, prose=False, loose=None):
|
||
return text.replace("13800000001", "已遮罩")
|
||
|
||
hider = _Hide(Secrets())
|
||
label = {"kind": "related", "related": [{
|
||
"caliber": "wf@1", "workflow_id": "wf", "version": 1, "level": "published", "binding": {"phone": "13800000001"},
|
||
"diff": {"items": [{"aspect": "filter", "a": "phone = '13800000001'", "b": None,
|
||
"text": "这次只保留 phone = '13800000001' 的行,口径没有这个条件"}],
|
||
"fallback": False, "text_diff": ["- WHERE phone = '13800000001'", "+ WHERE TRUE"]}}]}
|
||
shown = view_label(label, hide=hider.hide, shown=hider.shown, names={"wf": "周销售"})
|
||
text = json.dumps(shown, ensure_ascii=False)
|
||
assert "13800000001" not in text and text.count("已遮罩") >= 4
|
||
assert shown["related"][0]["workflow_name"] == "周销售" and shown["related"][0]["level_text"] == "已发布"
|
||
|
||
|
||
async def test_the_evidence_segment_carries_the_label_only_when_matched(client, monkeypatch):
|
||
await S.publish()
|
||
row = await S.explore(monkeypatch, S.EQUIVALENT, text="本周销售额 [[v:Q1.r0.销售额]]。")
|
||
seg = first_segment(row, "170.5")
|
||
res = await client.get(f"/api/runs/{row.id}/evidence/segments/{seg}")
|
||
assert res.status_code == 200, res.text
|
||
ledger = res.json()["ledger"]
|
||
assert ledger["label"]["kind"] == "caliber" and ledger["note"] == "封存后比对,不在封存范围内"
|
||
assert ledger["post_seal"] is True and ledger["artifact"]
|
||
|
||
monkeypatch.setattr(ledger_match, "schedule", lambda run_id, **kw: False)
|
||
other = await S.explore(monkeypatch, S.EQUIVALENT, text="本周销售额 [[v:Q1.r0.销售额]]。")
|
||
res = await client.get(f"/api/runs/{other.id}/evidence/segments/{first_segment(other, '170.5')}")
|
||
assert res.status_code == 200 and "ledger" not in res.json()
|
||
|
||
|
||
def _data_block(page: str) -> dict:
|
||
found = re.search(r'<script type="application/json" id="report-data">(.*?)</script>', page, re.S)
|
||
assert found, "页面里没有数据块"
|
||
return json.loads(found.group(1))
|
||
|
||
|
||
async def exported(run_id: str) -> list[AuditEvent]:
|
||
async with SessionLocal() as session:
|
||
return list((await session.execute(select(AuditEvent).where(
|
||
AuditEvent.action == "report.exported", AuditEvent.subject_id == run_id).order_by(AuditEvent.at))).scalars())
|
||
|
||
|
||
async def test_report_export_shows_the_label_and_records_the_export(client, monkeypatch):
|
||
await S.publish()
|
||
row = await S.explore(monkeypatch, S.EQUIVALENT, text="本周销售额 [[v:Q1.r0.销售额]]。")
|
||
res = await client.get(f"/api/runs/{row.id}/report.html")
|
||
assert res.status_code == 200, res.text
|
||
data = _data_block(res.text)
|
||
assert data["ledger"]["note"] == "封存后比对,不在封存范围内"
|
||
[label] = data["ledger"]["labels"].values()
|
||
assert label["kind"] == "caliber"
|
||
[audit] = await exported(row.id)
|
||
assert audit.subject_kind == "run" and audit.data["format"] == "html"
|
||
assert audit.data["report_node"] == "write" and audit.data["doc_artifact"] == row.output["_evidence"]["doc_artifact"]
|
||
|
||
res = await client.get(f"/api/runs/{row.id}/bundle.html")
|
||
assert res.status_code == 200, res.text
|
||
audits = await exported(row.id)
|
||
assert [a.data["format"] for a in audits] == ["html", "bundle"]
|
||
assert audits[1].data["reports"] == [{"report_node": "write", "doc_artifact": audit.data["doc_artifact"]}]
|
||
# 可验证报告把封存之后追加的比对列出来,只说几条,不带绑定
|
||
assert "封存后比对已发布口径:1 条等价已证明" in res.text
|
||
|
||
|
||
async def test_an_unmatched_report_export_has_no_ledger_block(client, monkeypatch):
|
||
monkeypatch.setattr(ledger_match, "schedule", lambda run_id, **kw: False)
|
||
row = await S.explore(monkeypatch, S.EQUIVALENT, text="本周销售额 [[v:Q1.r0.销售额]]。")
|
||
res = await client.get(f"/api/runs/{row.id}/report.html")
|
||
assert res.status_code == 200 and "ledger" not in _data_block(res.text)
|
||
|
||
|
||
async def test_stats_count_what_the_fixtures_did(client, monkeypatch):
|
||
from datetime import datetime, timezone
|
||
|
||
since = datetime.now(timezone.utc)
|
||
wf = await S.publish()
|
||
await S.explore(monkeypatch, S.EQUIVALENT, text="本周销售额 [[v:Q1.r0.销售额]]。")
|
||
await S.explore(monkeypatch, S.WITH_REFUNDS, text="含退款 [[v:Q1.r0.gmv]]。")
|
||
await S.explore(monkeypatch, S.ELSEWHERE, text="一共 [[v:Q1.r0.n]] 家店。")
|
||
monkeypatch.setattr(ledger_match, "schedule", lambda run_id, **kw: False)
|
||
timed = await S.explore(monkeypatch, S.EQUIVALENT + " -- 超时用", text="本周销售额 [[v:Q1.r0.销售额]]。")
|
||
await ledger_match.match_run(timed.id, timeout=0)
|
||
res = await client.get("/api/ledger/stats", params={"since": since.isoformat()})
|
||
assert res.status_code == 200, res.text
|
||
body = res.json()
|
||
e1 = body["e1"]
|
||
assert e1["runs"] == 4 and e1["queries"] == 4
|
||
assert e1["status"] == {"proven": 1, "composed": 0, "related": 1, "none": 1, "skipped": 1}
|
||
assert e1["with_candidates"] == 3 and e1["rates"]["proven"] == round(1 / 3, 4)
|
||
assert e1["diff_aspects"]["filter"] == 1 and e1["labels"]["caliber"] == 1 and e1["labels"]["unknown"] == 1
|
||
assert e1["whole_equivalent"] == 1
|
||
assert body["e2"]["launched"] == 0 and body["e2"]["feedback"]["available"] is False
|
||
assert body["e3"]["chat_runs"] == 0 and body["e3"]["threshold"] == {"runs": 3, "conversations": 2}
|
||
usage = {u["caliber"]: u for u in body["e4"]["usage"]}
|
||
assert usage[f"{wf}@1"]["proven_runs"] == 1
|
||
assert body["e4"]["per_publish"]["available"] is False
|
||
|
||
|
||
async def test_stats_take_the_run_ids_the_caller_names(monkeypatch):
|
||
"""评测(第 3 波)直接调 compute_stats,点名自己的运行:只算这几次。"""
|
||
from app.ledger.stats import compute_stats
|
||
|
||
await S.publish()
|
||
row = await S.explore(monkeypatch, S.EQUIVALENT, text="本周销售额 [[v:Q1.r0.销售额]]。")
|
||
await S.explore(monkeypatch, S.WITH_REFUNDS, text="含退款 [[v:Q1.r0.gmv]]。")
|
||
async with SessionLocal() as session:
|
||
body = await compute_stats(session, run_ids=[row.id])
|
||
assert body["e1"]["runs"] == 1 and body["e1"]["status"]["proven"] == 1 and body["window"]["runs"] == [row.id]
|