mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-16 19:11:58 +00:00
fcdcc30d4e
一、可点击证据第二期
取数链路
- 智能体、工具节点、代码节点、知识检索取数时都记进证据台账;报告目录按台账编出 Q1… / K1…
- 报告可以引用快照里的一格([[v:Q1.r0.gmv]]),也可以由系统从快照生成整张表([[table:Q1 cols=… rows=0-4]]),
表里每一格都是可点的片段;取值一律以快照为准,取回快照时复验哈希
- 口径卡表达式新增 cell(x, row, column),lineage 精确到单元格
- 智能体开 cite_fields(配合 output_schema)后,循环结束做一次结构化抽取,每个字段标出取自哪次查询的哪一格,
再到快照里核对:一致记 verified;不一致以快照为准并告警;查不到的记为空值,永远不兜底成 0
- 代码节点新增 evidence_role(source / compute)和 code_sha;沙箱算出来的数不能被报告直接引用,要进口径卡
- 查询快照记下列类型,文本列按文本渲染(年份 "2026" 不再显示成「2,026」)
- 升级前发起的运行(停在审批上的)一律走旧逻辑:新的 task 只追加在循环之后,恢复时不会错号
证据接口与面板
- 片段链接上查询步骤:SQL、被引用的行加前后各 2 行、高亮到格;数据源可设 mask_columns 遮罩列,
遮罩在查询时记进快照,数据源事后改名或删除也照样遮罩(在有身份体系之前,遮罩只减少暴露,不是安全边界)
- 输入来源写明核对结果(与快照一致 / 模型报的值和快照不一致 / 没查到记为空);口径卡缺输入时也给出完整的链
- 受管出具要在契约里声明 cells 才能引用单元格;受管与否记在所钉版本上,改画布不再让受管版本的正式运行降级
- 口径卡可以用 caliber_from 钉住另一个已发布工作流里的口径卡,上游有新版本时按 upgrade_policy 处置,
和子工作流同一套规则;子工作流节点也补上了升版处置的配置项
Copilot、校验与报错
- 问数据的答案里有数字时,搭 input → 智能体 → 报告 → 出口,报告直接引用查过的单元格
- 写 SQL 前核对数据源的表结构,SQL 里出现表结构里没有的列会被打回
- 不再搭「智能体文字 → 转换节点解析 JSON」这种图;转换节点的 JSON 报错分清是上游模型写坏了、
模板里裸插了模型文字(给出 | json 的改法),还是模板本身写错了
- 工具节点遇到「查询失败」「SQL 被拒绝」判为失败;发起运行前检查绑定的工具是否存在,不存在时直接拒绝并给出入口
二、发布前自动修复
- 问题带上机器编号(code)和修复 id;新增只读接口 POST /api/workflows/{id}/publish-check 和 /autofix
- 修复分三类:答案唯一、不降低要求的一键修复;要人拿主意的给选项(不替人选);结构性问题交给 Copilot
- 一律先预览,不自动保存、不自动发布;删节点、删契约、放宽审批、关掉 strict、改级别等降低要求的改动一律拒绝,
Copilot 的方案也要过这一关;应用后重跑检查,错误必须变少且不能冒出新的
- 发布弹窗打开就先检查;问题面板新增「发布前检查」,平时就能看到、就地修掉
- 门禁修正:契约有 report_from 时不再要求 narrative(模板⑨可以发成受管);全图默认审批为「全部放行」时同样拦下
- 修掉画布保存会抹掉全图默认(defaults)的老问题:以前任何一次保存都会把它清空,审批 always 因此被悄悄放宽
验证(沙箱):tsc 干净,生产构建通过,pytest 2268 passed(两遍),check-all 17 项检查、3915 项全过
(新增 check-publish);关键行为做了变异验证、全部抓到。
329 lines
14 KiB
Python
329 lines
14 KiB
Python
"""报告直接引用查询单元格 [[v:Qn.r<行>.<列>]]:受管级别的正式运行要在契约里显式声明 cells。
|
|
|
|
用户拍板:探索运行和已发布级别允许直接引用单元格;受管级别的正式运行,契约里没写
|
|
"cells": true 的,单元格引用按 unresolved 计,原因写「受管出具要在契约里声明 cells」。
|
|
|
|
- 出口复核(io.py)按这条规则独立判,不信报告节点的自查
|
|
- 报告节点写作时也照这条规则建目录:不允许时提示里只给 [[see:Qn]],写了单元格当场就被打回,
|
|
不必等到出口才发现
|
|
- 旧 narrative 契约的 matched 里,单元格引用的数字记成 cell: "Q1.r0.gmv",不是 input: None
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import sqlite3
|
|
|
|
import pytest
|
|
from httpx import ASGITransport, AsyncClient
|
|
from langchain_core.messages import AIMessage
|
|
from sqlalchemy import select
|
|
|
|
from app.data.engine import engines
|
|
from app.db.base import SessionLocal
|
|
from app.db.models import DataSource, Run, RunEvent, Workflow, WorkflowVersion
|
|
from app.engine.evidence import CELLS_REASON
|
|
from app.engine.runner import run_manager
|
|
from app.main import app
|
|
|
|
CELL_TEXT = "本周销售额 [[v:Q1.r0.gmv]],按口径卡是 [[m:gmv]]。"
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
async def engine_up():
|
|
await run_manager.setup()
|
|
yield
|
|
await run_manager.shutdown()
|
|
|
|
|
|
@pytest.fixture
|
|
async def client():
|
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
|
yield c
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
async def shop(tmp_path):
|
|
path = tmp_path / "shop.db"
|
|
db = sqlite3.connect(path)
|
|
db.executescript("CREATE TABLE orders (id INTEGER PRIMARY KEY, week TEXT, amount REAL);")
|
|
db.executemany("INSERT INTO orders VALUES (?, ?, ?)", [(1, "2026-W37", 100.5), (2, "2026-W37", 200.0)])
|
|
db.commit()
|
|
db.close()
|
|
async with SessionLocal() as session:
|
|
row = (await session.execute(select(DataSource).where(DataSource.name == "shop"))).scalar_one_or_none()
|
|
if row is None:
|
|
row = DataSource(name="shop", kind="sqlite", database=str(path), readonly=True, enabled=True)
|
|
session.add(row)
|
|
else:
|
|
row.database, row.kind, row.enabled, row.readonly, row.options = str(path), "sqlite", True, True, {}
|
|
await session.commit()
|
|
source_id = row.id
|
|
await engines.invalidate(source_id)
|
|
yield
|
|
await engines.invalidate(source_id)
|
|
|
|
|
|
def node(nid, ntype, **config):
|
|
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
|
|
|
|
|
def weekly(*, cells=None, on_violation="flag"):
|
|
contract = {"report_from": "write", "metrics_from": ["card"], "required": ["gmv"], "strict": True}
|
|
if cells is not None:
|
|
contract["cells"] = cells
|
|
nodes = [
|
|
node("start", "input"),
|
|
node("fetch", "tool", tool="db_query__shop",
|
|
args={"sql": "SELECT SUM(amount) AS gmv, COUNT(*) AS orders FROM orders"}),
|
|
node("card", "metrics", caliber="周报口径", caliber_version="v2", metrics=[
|
|
{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "format": "thousands",
|
|
"expression": "cell(nodes.fetch, 0, 'gmv')"}]),
|
|
node("write", "report", instructions="写周报", on_violation=on_violation),
|
|
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}], contract=contract),
|
|
]
|
|
return {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
|
|
|
|
|
class Writer:
|
|
"""报告写作者:每次都照 text 写;记下每次看到的提示。"""
|
|
|
|
def __init__(self, monkeypatch, text=CELL_TEXT):
|
|
from app.providers import mock_model
|
|
|
|
self.prompts: list[str] = []
|
|
|
|
def decide(model, messages):
|
|
self.prompts.append("\n".join(str(m.content) for m in messages))
|
|
return AIMessage(content=text)
|
|
|
|
monkeypatch.setattr(mock_model.MockChatModel, "_decide", decide)
|
|
|
|
|
|
async def wait(run_id: str) -> Run:
|
|
for _ in range(400):
|
|
await asyncio.sleep(0.05)
|
|
async with SessionLocal() as session:
|
|
row = await session.get(Run, run_id)
|
|
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
|
|
return row
|
|
raise AssertionError(f"run {run_id} 没跑完:{row.status}")
|
|
|
|
|
|
async def formal(client, graph, *, status: str) -> Run:
|
|
"""把这张图发布成 status 级别,再从发布版本发起正式运行。"""
|
|
async with SessionLocal() as session:
|
|
wf = Workflow(name=f"周报-{status}", graph=graph, status=status, published_version=1)
|
|
session.add(wf)
|
|
await session.flush()
|
|
session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=graph))
|
|
await session.commit()
|
|
wf_id = wf.id
|
|
res = await client.post("/api/runs", json={"workflow_id": wf_id, "run_class": "formal"})
|
|
assert res.status_code == 201, res.text
|
|
return await wait(res.json()["id"])
|
|
|
|
|
|
async def checked(run_id: str) -> dict:
|
|
async with SessionLocal() as session:
|
|
[event] = (await session.execute(select(RunEvent).where(
|
|
RunEvent.run_id == run_id, RunEvent.type == "report.checked"))).scalars()
|
|
return event.data
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 出口复核
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def test_governed_formal_run_without_cells_leaves_cell_refs_unresolved(client, monkeypatch):
|
|
Writer(monkeypatch)
|
|
row = await formal(client, weekly(), status="governed")
|
|
assert row.status == "succeeded", row.error
|
|
issuance = row.output["_issuance"]
|
|
assert issuance["mode"] == "citations" and issuance["tier"] == "withheld", issuance
|
|
[unresolved] = issuance["unresolved"]
|
|
assert unresolved["ref"] == "v:Q1.r0.gmv" and CELLS_REASON in unresolved["message"], unresolved
|
|
# 指标照常核对过;单元格那个数不算「有出处」
|
|
assert [m.get("metric") for m in issuance["matched"]] == ["gmv"]
|
|
|
|
|
|
async def test_governed_formal_run_with_cells_declared_passes(client, monkeypatch):
|
|
Writer(monkeypatch)
|
|
row = await formal(client, weekly(cells=True), status="governed")
|
|
assert row.status == "succeeded", row.error
|
|
issuance = row.output["_issuance"]
|
|
assert issuance["tier"] == "formal" and issuance["unresolved"] == [], issuance
|
|
cell = next(m for m in issuance["matched"] if m.get("cell"))
|
|
assert cell["cell"] == "Q1.r0.gmv" and cell["token"] == "300.5" and cell["metric"] is None
|
|
assert "input" not in cell and cell["eid"].startswith("ev:cell:")
|
|
|
|
|
|
@pytest.mark.parametrize("status", ["published"])
|
|
async def test_published_formal_runs_allow_cells(client, monkeypatch, status):
|
|
Writer(monkeypatch)
|
|
row = await formal(client, weekly(), status=status)
|
|
assert row.output["_issuance"]["tier"] == "formal", row.output["_issuance"]
|
|
|
|
|
|
async def test_exploratory_runs_are_not_affected(monkeypatch):
|
|
Writer(monkeypatch)
|
|
run = await run_manager.start(graph=weekly(), input_payload={})
|
|
row = await wait(run.id)
|
|
issuance = row.output["_issuance"]
|
|
assert issuance["tier"] == "formal" and issuance["unresolved"] == [], issuance
|
|
assert any(m.get("cell") == "Q1.r0.gmv" for m in issuance["matched"])
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 报告节点写作时就照这条规则
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def test_report_node_refuses_cells_while_writing_for_governed_formal_runs(client, monkeypatch):
|
|
writer = Writer(monkeypatch)
|
|
row = await formal(client, weekly(), status="governed")
|
|
data = await checked(row.id)
|
|
assert data["ok"] is False and data["repairs"] == 1
|
|
assert any(CELLS_REASON in v["message"] for v in data["violations"]), data["violations"]
|
|
# 提示里不再教它写单元格,只给依据写法
|
|
assert "[[v:Q1.r0." not in writer.prompts[0] and "[[see:Q1]]" in writer.prompts[0]
|
|
|
|
|
|
async def test_report_node_with_on_violation_fail_stops_before_the_exit(client, monkeypatch):
|
|
Writer(monkeypatch)
|
|
row = await formal(client, weekly(on_violation="fail"), status="governed")
|
|
assert row.status == "failed" and CELLS_REASON in (row.error or ""), row.error
|
|
|
|
|
|
async def test_report_node_teaches_cells_when_they_are_allowed(client, monkeypatch):
|
|
writer = Writer(monkeypatch)
|
|
row = await formal(client, weekly(cells=True), status="governed")
|
|
data = await checked(row.id)
|
|
assert data["ok"] is True and data["repairs"] == 0, data
|
|
assert "[[v:Q1.r0." in writer.prompts[0]
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# report_no_evidence:有任何一种可引用的来源就不报,报的时候说清缺的是哪种
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def no_evidence_logs(run_id: str) -> list[dict]:
|
|
async with SessionLocal() as session:
|
|
rows = (await session.execute(select(RunEvent).where(
|
|
RunEvent.run_id == run_id, RunEvent.type == "log"))).scalars()
|
|
return [e.data for e in rows if e.data.get("code") == "report_no_evidence"]
|
|
|
|
|
|
def chain(*nodes):
|
|
return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
|
|
|
|
|
FETCH = node("fetch", "tool", tool="db_query__shop", args={"sql": "SELECT SUM(amount) AS gmv FROM orders"})
|
|
WRITE = node("write", "report", instructions="回答问题", on_violation="flag")
|
|
OUT = node("out", "output", fields=[{"name": "答案", "value": "{{ nodes.write.text }}"}])
|
|
|
|
|
|
async def test_a_report_over_queries_only_is_not_warned(monkeypatch):
|
|
"""问数据的图(input → 取数 → report)只有查询条目、没有口径卡:单元格就是它的出处,不该报没有证据。"""
|
|
Writer(monkeypatch, "销售额 [[v:Q1.r0.gmv]]。")
|
|
row = await wait((await run_manager.start(graph=chain(node("start", "input"), FETCH, WRITE, OUT),
|
|
input_payload={})).id)
|
|
assert row.status == "succeeded", row.error
|
|
assert await no_evidence_logs(row.id) == []
|
|
|
|
|
|
async def test_a_report_with_nothing_upstream_names_every_missing_kind(monkeypatch):
|
|
Writer(monkeypatch, "什么都没有。")
|
|
row = await wait((await run_manager.start(graph=chain(node("start", "input"), WRITE, OUT),
|
|
input_payload={})).id)
|
|
[log] = await no_evidence_logs(row.id)
|
|
assert all(kind in log["message"] for kind in ("口径卡指标", "查询结果", "知识库检索", "运行输入")), log
|
|
|
|
|
|
async def test_queries_without_cells_in_a_governed_formal_run_are_named(client, monkeypatch):
|
|
Writer(monkeypatch, "销售额 [[v:Q1.r0.gmv]]。")
|
|
card = node("card", "metrics", caliber="周报口径", metrics=[
|
|
{"id": "gmv", "name": "销售额", "expression": "cell(nodes.fetch, 0, 'gmv')"}])
|
|
out = node("out", "output", fields=[{"name": "答案", "value": "{{ nodes.write.text }}"}],
|
|
contract={"report_from": "write", "metrics_from": ["card"], "required": ["gmv"]})
|
|
row = await formal(client, chain(node("start", "input"), FETCH, WRITE, card, out), status="governed")
|
|
[log] = await no_evidence_logs(row.id)
|
|
assert "只有查询结果" in log["message"] and CELLS_REASON in log["message"], log
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 受管与否在发起时定下来:之后改图、改级别,这次运行都按发起时的算
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def governance_notes(run_id: str) -> list[RunEvent]:
|
|
async with SessionLocal() as session:
|
|
rows = (await session.execute(select(RunEvent).where(
|
|
RunEvent.run_id == run_id, RunEvent.type == "log").order_by(RunEvent.seq))).scalars()
|
|
return [e for e in rows if e.data.get("code") == "run_governance"]
|
|
|
|
|
|
async def set_status(run_id: str, status: str) -> None:
|
|
async with SessionLocal() as session:
|
|
run = await session.get(Run, run_id)
|
|
wf = await session.get(Workflow, run.workflow_id)
|
|
wf.status = status
|
|
await session.commit()
|
|
|
|
|
|
@pytest.mark.parametrize("status, governed, later", [("governed", True, "draft"), ("published", False, "governed")])
|
|
async def test_the_governed_decision_is_recorded_at_start_and_does_not_follow_later_edits(
|
|
client, monkeypatch, status, governed, later):
|
|
from app.engine.governance import governed_formal
|
|
|
|
Writer(monkeypatch)
|
|
row = await formal(client, weekly(cells=True), status=status)
|
|
[note] = await governance_notes(row.id)
|
|
assert note.data["governed"] is governed and note.data["level"] == "info" and note.node_id is None
|
|
assert note.seq <= row.manifest_seq # 在封存范围里
|
|
assert await governed_formal(row.id) is governed
|
|
await set_status(row.id, later) # 改图退回草稿 / 事后升成受管
|
|
assert await governed_formal(row.id) is governed
|
|
|
|
|
|
async def test_exploratory_runs_record_nothing(monkeypatch):
|
|
Writer(monkeypatch)
|
|
row = await wait((await run_manager.start(graph=weekly(), input_payload={})).id)
|
|
assert await governance_notes(row.id) == []
|
|
|
|
|
|
async def test_runs_started_before_the_record_existed_follow_the_current_status(client, monkeypatch):
|
|
from app.engine.governance import governed_formal
|
|
|
|
Writer(monkeypatch)
|
|
row = await formal(client, weekly(cells=True), status="governed")
|
|
async with SessionLocal() as session:
|
|
for note in await governance_notes(row.id):
|
|
await session.delete(await session.get(RunEvent, note.id))
|
|
await session.commit()
|
|
assert await governed_formal(row.id) is True
|
|
await set_status(row.id, "draft")
|
|
assert await governed_formal(row.id) is False
|
|
|
|
|
|
async def test_the_report_node_and_the_exit_check_agree_when_the_status_changes_mid_run(client, monkeypatch):
|
|
"""报告节点写作时按受管拒了单元格,跑到出口前工作流被改回草稿:出口仍按受管判,两边说法一致。"""
|
|
from app.engine import governance
|
|
|
|
Writer(monkeypatch)
|
|
real, seen = governance.governed_formal, []
|
|
|
|
async def edit_after_first_look(run_id):
|
|
result = await real(run_id)
|
|
seen.append(result)
|
|
if len(seen) == 1:
|
|
await set_status(run_id, "draft")
|
|
return result
|
|
|
|
monkeypatch.setattr(governance, "governed_formal", edit_after_first_look)
|
|
row = await formal(client, weekly(), status="governed")
|
|
assert seen == [True, True], seen
|
|
issuance = row.output["_issuance"]
|
|
assert issuance["tier"] == "withheld" and CELLS_REASON in issuance["unresolved"][0]["message"], issuance
|