mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-22 19:11:59 +00:00
050bea4b72
一、一键升级(只给预览,不自动保存)
- engine/upgrade.py:确定性改写,纯函数、重复执行结果不变
- R1 契约的叙述指向模型调用节点 → 换成报告撰写节点(system 和 prompt 合成写作要求,模型配置照搬),契约改成 report_from
- R2 没有契约、但模型写的文字进了出口,而且上游有取数 → 同样换成报告撰写
- R3 input → 智能体 → 出口(问数据的典型图)→ 中间插入报告撰写节点,出口字段接报告的正文;
只改直接相连的出口,经过别的节点读它的出口原样保留并写明原因
- R4 配了 output_schema 的智能体 → 打开 cite_fields
- R5 代码节点的产出被口径卡读 → 不改,只建议标成取数,等人确认
- 一个叙述里混了几个写作者的,不猜,整步不做并写明原因
- POST /api/copilot/upgrade-evidence:返回改动、说明和改完的检查结果;assist 时再交给 Copilot 把纯算术的代码节点
改写成口径卡表达式,不是纯算术的保留并给警告
- 升级和发布前修复走同一道「不许降低要求」的检查,只多两个由升级点名的例外:模型调用换成报告撰写、R3 改接的那条连线;
改完不能更糟
- 校验对符合 R1–R4 的旧结构给一条建议「可以升级为可追溯结构」,不算提示、不挡发布
- Copilot 自查:报告撰写节点上游没有任何能引用数字的来源时打回
二、界面
- 问题面板的快速修复「升级为可追溯结构(预览改动)」:逐项写「节点 · 字段:原值 → 新值」,节点类型的变化写成
「模型调用 → 报告撰写」,新插的节点和改接的连线另外写明;「应用」后走现有的保存,全图默认不丢
- 画布改过、存失败、画布被锁都有明确状态;保存失败后手动存成功会清掉「没存上」的提示,
画布和预览对不上时「重试保存」不再盲存;校验接口失败时升级块照样在
- 记录页里没有逐段证据的运行给横幅「这次的报告没有逐段证据 → 升级这张图」,跳到编排页并打开升级预览
- 修掉一处旧缺陷:应用修复时只换配置、不换节点类型,模型调用换成报告撰写后存下去的还是模型调用
三、受管门禁:cells 一键选
- 图里没有口径卡、契约又没写 cells: true 的受管图,以前能过门禁、正式运行却每次失败;现在门禁拦下,
发布弹窗给两个选项:写 cells: true,或交给 Copilot 改成口径卡指标(人亲手选的不算替人打开,Copilot 自己打开照样拦)
- 「修完要更好」的复核不再把改完才冒出来、只能由人拍板的这一处算作改坏:Copilot 补上契约、把 cells 留给人问,
这一步会被采用,不再被判「冒出新的问题」作废
四、测试
- 开发依赖加上 pytest-xdist(锁文件只多了 pytest-xdist 和 execnet):全量后端测试 -n auto 约 16 秒,原来串行约 2 分钟
- 修掉并行下暴露的一处测试竞态:运行先落终态、再另一个事务封存,测试改成等封存完成再核对
266 lines
14 KiB
Python
266 lines
14 KiB
Python
"""Copilot 搭图自查拦得下配错的报告撰写节点(证据五期)。
|
||
|
||
- metrics_from 指向的不是口径卡:validate 本来就报 error,自查交回去改
|
||
- 报告撰写的上游没有任何证据来源(没有口径卡、查库的 agent / 调用工具、知识检索):运行时只会给一条警告、
|
||
报告里每个数都被判成没有出处——搭图时就打回去,按 error 交给模型改
|
||
- 配对了的报告撰写节点不多花一次自查调用
|
||
- 旧结构「可以升级为可追溯结构」的建议不混进这一轮的问题清单
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
|
||
import pytest
|
||
from httpx import ASGITransport, AsyncClient
|
||
|
||
from app.api.copilot import authored_issues
|
||
from app.engine.schema import GraphSpec
|
||
from app.main import app
|
||
|
||
|
||
@pytest.fixture
|
||
async def client():
|
||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||
yield c
|
||
|
||
|
||
def add(nid, ntype, label, **config):
|
||
return {"op": "add_node", "node": {"id": nid, "type": ntype, "label": label, "config": config}}
|
||
|
||
|
||
def link(a, b):
|
||
return {"op": "add_edge", "edge": {"source": a, "target": b}}
|
||
|
||
|
||
class Scripted:
|
||
"""第一轮吐 first;收到自查请求(human 里有「上线前自查」)时吐 fix。"""
|
||
|
||
def __init__(self, first: list[dict], fix: list[dict] | None = None) -> None:
|
||
self.first = [json.dumps(o, ensure_ascii=False) for o in first]
|
||
self.fix = [json.dumps(o, ensure_ascii=False) for o in fix or []]
|
||
self.calls: list[str] = []
|
||
|
||
async def astream(self, messages):
|
||
from langchain_core.messages import AIMessageChunk
|
||
|
||
human = next(text for role, text in messages if role == "human")
|
||
self.calls.append(human)
|
||
for line in (self.fix if "上线前自查" in human else self.first):
|
||
yield AIMessageChunk(content=line + "\n")
|
||
|
||
|
||
async def generate(client, monkeypatch, model: Scripted) -> list[dict]:
|
||
import app.api.copilot as copilot
|
||
|
||
async def _model(*_a, **_k):
|
||
return model, "mock-fast"
|
||
|
||
monkeypatch.setattr(copilot, "get_chat_model", _model)
|
||
events: list[dict] = []
|
||
async with client.stream("POST", "/api/copilot/generate-stream", json={"instruction": "写一份订单周报"}) as r:
|
||
async for line in r.aiter_lines():
|
||
if line.startswith("data: "):
|
||
events.append(json.loads(line[6:]))
|
||
return events
|
||
|
||
|
||
def wrap(*ops) -> list[dict]:
|
||
return [{"op": "plan", "summary": "周报"}, *ops, {"op": "done", "explanation": "周报", "run": False}]
|
||
|
||
|
||
FETCH = add("fetch", "tool", "查库", tool="db_query__shop", args={"sql": "SELECT COUNT(*) AS n FROM orders"})
|
||
CARD = add("card", "metrics", "口径卡", metrics=[{"id": "n", "name": "订单数", "expression": "cell(nodes.fetch, 0, 'n')"}])
|
||
OUT = add("out", "output", "成果", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}])
|
||
|
||
|
||
def issues_of(events, status):
|
||
return [i for e in events if e["op"] == "check" and e.get("status") == status for i in e["issues"]]
|
||
|
||
|
||
async def test_metrics_from_pointing_at_a_non_card_is_sent_back(client, monkeypatch):
|
||
first = wrap(add("start", "input", "输入"), FETCH, link("start", "fetch"), CARD, link("fetch", "card"),
|
||
add("write", "report", "报告撰写", instructions="写周报", metrics_from=["fetch"]),
|
||
link("card", "write"), OUT, link("write", "out"))
|
||
fix = [{"op": "update_node", "id": "write", "config": {"metrics_from": ["card"]}}, {"op": "done"}]
|
||
model = Scripted(first, fix)
|
||
events = await generate(client, monkeypatch, model)
|
||
sent = issues_of(events, "repairing")
|
||
assert any(i["code"] == "report.metrics_from_invalid" and i["node_id"] == "write" for i in sent), sent
|
||
assert "不是「口径卡」" in model.calls[1]
|
||
assert [e["status"] for e in events if e["op"] == "check"] == ["repairing", "passed"]
|
||
|
||
|
||
async def test_a_report_with_nothing_to_cite_upstream_is_sent_back(client, monkeypatch):
|
||
first = wrap(add("start", "input", "输入"), add("write", "report", "报告撰写", instructions="写周报"),
|
||
link("start", "write"), OUT, link("write", "out"))
|
||
fix = [FETCH, link("start", "fetch"), link("fetch", "write"), {"op": "done"}]
|
||
model = Scripted(first, fix)
|
||
events = await generate(client, monkeypatch, model)
|
||
[sent] = [i for i in issues_of(events, "repairing") if i["code"] == "report_no_source"]
|
||
assert sent["level"] == "error" and sent["node_id"] == "write"
|
||
assert "没有能引用数字的证据来源" in model.calls[1]
|
||
assert [e["status"] for e in events if e["op"] == "check"] == ["repairing", "passed"]
|
||
final = events[-1]
|
||
assert final["op"] == "final" and not [i for i in final["issues"] if i.get("code") == "report_no_source"]
|
||
|
||
|
||
async def test_a_well_wired_report_costs_no_extra_call(client, monkeypatch):
|
||
first = wrap(add("start", "input", "输入"),
|
||
add("ask", "agent", "查数", tools=["db_query__shop"], prompt="查本周订单数"),
|
||
link("start", "ask"), add("write", "report", "报告撰写", instructions="写周报"),
|
||
link("ask", "write"), OUT, link("write", "out"))
|
||
model = Scripted(first)
|
||
events = await generate(client, monkeypatch, model)
|
||
assert len(model.calls) == 1 and [e["status"] for e in events if e["op"] == "check"] == ["passed"]
|
||
|
||
|
||
async def test_the_upgrade_hint_is_not_part_of_the_copilot_issue_list(client, monkeypatch):
|
||
# 只问清单的问数据图就是 agent → output:validate 会建议升级,但这一轮的问题清单里不该有它
|
||
first = wrap(add("start", "input", "输入"),
|
||
add("ask", "agent", "查数", tools=["db_query__shop"], prompt="列出订单表里的店铺"),
|
||
link("start", "ask"),
|
||
add("out", "output", "成果", fields=[{"name": "answer", "value": "{{ nodes.ask.text }}"}]),
|
||
link("ask", "out"))
|
||
events = await generate(client, monkeypatch, Scripted(first))
|
||
final = events[-1]
|
||
assert final["op"] == "final"
|
||
assert not [i for i in final["issues"] if i.get("code") == "evidence.upgrade_available" or i["level"] == "info"]
|
||
|
||
|
||
def spec_of(nodes, edges):
|
||
return GraphSpec.model_validate({"nodes": nodes, "edges": edges})
|
||
|
||
|
||
def n(nid, ntype, **config):
|
||
return {"id": nid, "type": ntype, "data": {"label": nid, "config": config}}
|
||
|
||
|
||
@pytest.mark.parametrize("source", [
|
||
n("fetch", "tool", tool="db_query__shop"),
|
||
n("fetch", "agent", tools=["db_query__shop"]),
|
||
n("fetch", "retrieve", collection="docs"),
|
||
n("fetch", "metrics", metrics=[{"id": "a", "expression": "1"}]),
|
||
n("fetch", "subgraph", workflow_id="w1"),
|
||
n("fetch", "supervisor", goal="查清楚", agents=[{"name": "analyst", "tools": ["db_query__shop"]}]),
|
||
# 运行输入也是能引用的来源([[i:字段]]):声明了数值字段的入口算
|
||
n("fetch", "input", fields=[{"name": "target", "default": 100}]),
|
||
n("fetch", "input", fields=[{"name": "week"}, {"name": "rate", "default": "0.35"}]),
|
||
n("fetch", "input", fields=[{"name": "budget", "type": "number"}]),
|
||
])
|
||
def test_any_evidence_source_upstream_is_enough(source):
|
||
spec = spec_of([n("start", "input"), source, n("write", "report"), n("out", "output")],
|
||
[{"source": "start", "target": "fetch"}, {"source": "fetch", "target": "write"},
|
||
{"source": "write", "target": "out"}])
|
||
assert not [i for i in authored_issues(spec, []) if i["code"] == "report_no_source"]
|
||
|
||
|
||
@pytest.mark.parametrize("fields", [[], [{"name": "question", "required": True}], [{"name": "topic", "default": "本周"}]])
|
||
def test_an_input_without_numeric_fields_is_not_a_source(fields):
|
||
# 只声明了文字字段的入口:报告只能照引问题本身,要写的数照样没有出处
|
||
spec = spec_of([n("start", "input", fields=fields), n("write", "report"), n("out", "output")],
|
||
[{"source": "start", "target": "write"}, {"source": "write", "target": "out"}])
|
||
[issue] = [i for i in authored_issues(spec, []) if i["code"] == "report_no_source"]
|
||
assert issue["node_id"] == "write" and issue["level"] == "error"
|
||
assert "运行输入" in issue["message"] and "[[i:" in issue["message"]
|
||
|
||
|
||
def test_a_numeric_input_upstream_is_enough_for_the_copilot_self_check():
|
||
spec = spec_of([n("start", "input", fields=[{"name": "question"}, {"name": "gmv", "default": 1200.5}]),
|
||
n("write", "report"), n("out", "output")],
|
||
[{"source": "start", "target": "write"}, {"source": "write", "target": "out"}])
|
||
assert not [i for i in authored_issues(spec, []) if i["code"] == "report_no_source"]
|
||
|
||
|
||
def test_a_tool_less_agent_or_a_downstream_source_does_not_count():
|
||
spec = spec_of([n("start", "input"), n("chat", "agent", prompt="随便聊"), n("write", "report"),
|
||
n("late", "tool", tool="db_query__shop"), n("out", "output")],
|
||
[{"source": "start", "target": "chat"}, {"source": "chat", "target": "write"},
|
||
{"source": "write", "target": "late"}, {"source": "late", "target": "out"}])
|
||
[issue] = [i for i in authored_issues(spec, []) if i["code"] == "report_no_source"]
|
||
assert issue["node_id"] == "write" and issue["level"] == "error"
|
||
|
||
|
||
def test_reports_this_round_did_not_touch_are_left_alone():
|
||
nodes = [n("start", "input"), n("write", "report", instructions="写"), n("out", "output")]
|
||
spec = spec_of(nodes, [{"source": "start", "target": "write"}, {"source": "write", "target": "out"}])
|
||
assert not [i for i in authored_issues(spec, [], baseline=nodes) if i["code"] == "report_no_source"]
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 问数据的图发成受管:Copilot 补契约(用户报过的那一幕)
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def ask_report_graph() -> dict:
|
||
"""input → agent(查库)→ 报告撰写(受管三项都写了、正文引单元格)→ 出口,出口还没有契约。"""
|
||
def g(nid, ntype, label, **config):
|
||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label, "config": config}}
|
||
|
||
return {
|
||
"nodes": [
|
||
g("start", "input", "问题"),
|
||
g("ask", "agent", "查数", tools=["db_query__shop"], prompt="查活跃用户数和用户总数", approval="dangerous"),
|
||
g("report", "report", "写答案", instructions="回答:{{ input.question }}", numbers="strict",
|
||
on_violation="fail", claims="require_citation"),
|
||
g("out", "output", "输出答案", fields=[{"name": "answer", "value": "{{ nodes.report.text }}"}]),
|
||
],
|
||
"edges": [{"id": f"e{i}", "source": a, "target": b}
|
||
for i, (a, b) in enumerate([("start", "ask"), ("ask", "report"), ("report", "out")])],
|
||
}
|
||
|
||
|
||
class Assist:
|
||
def __init__(self, ops: list[dict]) -> None:
|
||
self.lines = [json.dumps(o, ensure_ascii=False) for o in ops]
|
||
|
||
async def astream(self, messages):
|
||
from langchain_core.messages import AIMessageChunk
|
||
|
||
for line in self.lines:
|
||
yield AIMessageChunk(content=line + "\n")
|
||
|
||
|
||
async def governed_assist(client, monkeypatch, ops: list[dict]) -> dict:
|
||
import app.api.copilot as copilot
|
||
|
||
async def _model(*_a, **_k):
|
||
return Assist(ops), "mock-fast"
|
||
|
||
monkeypatch.setattr(copilot, "get_chat_model", _model)
|
||
wf = (await client.post("/api/workflows", json={"name": "问数据-受管", "graph": ask_report_graph()})).json()["id"]
|
||
r = await client.post(f"/api/workflows/{wf}/autofix", json={"level": "governed", "apply": [], "assist": True})
|
||
assert r.status_code == 200, r.text
|
||
return r.json()
|
||
|
||
|
||
CELLS_QUESTION = {"op": "question", "node_id": "out",
|
||
"text": "报告正文直接引用了查询单元格:契约里写 cells: true,还是改成口径卡指标?"}
|
||
|
||
|
||
async def test_the_copilots_contract_for_a_cells_only_graph_is_adopted(client, monkeypatch):
|
||
contract = {"report_from": "report", "strict": True}
|
||
out = await governed_assist(client, monkeypatch, [
|
||
{"op": "plan", "summary": "补出具契约"},
|
||
{"op": "update_node", "id": "out", "config": {"contract": contract}}, CELLS_QUESTION,
|
||
{"op": "done", "explanation": "给出口加上契约,cells 要你拿主意"}])
|
||
assert out["assist"]["ok"] is True and "assist" in out["applied"], out["rejected"]
|
||
assert not [r for r in out["rejected"] if "metrics_from" in r["reason"]]
|
||
assert next(n for n in out["graph"]["nodes"] if n["id"] == "out")["data"]["config"]["contract"] == contract
|
||
assert out["assist"]["questions"] == ["「输出答案」:" + CELLS_QUESTION["text"]]
|
||
# 没有契约那一处没了;cells 要人拍板(Copilot 不许替人打开),改完冒出来的这一处不算 Copilot 改坏的,
|
||
# 发布弹窗里给一键选项:写 cells: true,或交给 Copilot 改成口径卡
|
||
assert out["ok"] is False
|
||
assert [i["code"] for i in out["remaining"] if i["level"] == "error"] == ["contract.cells_undeclared"]
|
||
[fix] = [f for f in out["fixes"] if f["code"] == "contract.cells_undeclared"]
|
||
assert fix["kind"] == "choice" and [o["value"] for o in fix["options"]] == [True, "copilot"]
|
||
|
||
|
||
async def test_the_copilot_still_may_not_switch_cells_on_by_itself(client, monkeypatch):
|
||
out = await governed_assist(client, monkeypatch, [
|
||
{"op": "plan", "summary": "补出具契约"},
|
||
{"op": "update_node", "id": "out", "config": {"contract": {"report_from": "report", "strict": True,
|
||
"cells": True}}},
|
||
{"op": "done", "explanation": "补上契约并打开 cells"}])
|
||
[rejected] = [r for r in out["rejected"] if r["fix_id"] == "assist"]
|
||
assert "cells" in rejected["reason"] and out["ok"] is False
|