Files
AgentLab/backend/tests/test_copilot_prompt_evidence.py
T
ThearchyHelios 03c18c4d78 界面文案整改:口语改书面、术语统一、不再露出实现细节;修掉几处说错的提示;新增文案检查防回潮
一、文案(前端约 1600 处,后端传到界面的约 880 处)
- 术语统一:工作流层面用「运行」(继续运行、重新运行、试运行),节点和工具层面用「执行」,不再用「跑」;
  总时长 / 执行时长 / 等待审批;服务端;助手;Agent;token;工作流;固定版本;模型接入;系统提示;批准 / 驳回;受管级别
- 口语改书面:取不到、没取回来、读不懂、连不上、多半、照样、兜底、揉成这类说法改成「无法 / 未能 / 失败 / 请」;去掉拟人写法
- 不再露出实现细节:界面上不写 checkpoint、同一拍、哈希(凭证处除外),不写改版历史(「以前…」「老数据」);
  报错、校验、门禁、自动修复里裸写的配置键名和枚举值换成界面上的中文标签,由新的 backend/app/engine/labels.py 统一提供;
  发布弹窗里结论句的选项不再是 off / require_citation
- 同一件事只留一种说法:生成阶段文案、出具档位说明、画布锁定提示各合成一份
- 去掉界面上的工业场景示例(MES、ERP、「张工」「王工」)和真实产品名,换成通用示例

二、说错的提示(都有测试证明)
- 数据库测连接失败时的专用提示以前从不出现(按报错文字全等比较,永远比不上):改为按错误类型判断,不再拼出重复的前缀
- 「设置 → 供应商」改指「设置 → 模型接入」;「在设置里建数据源」改指「数据」页
- 缺少 API Key、模型接入已停用的报错不再夹英文,写明去哪里填
- 两处原因说错(「这种引用在后续版本支持」「这一版还不支持」)改成真实原因
- 节点说明里不准确的说法改正:代码节点在 microVM 或系统沙箱中执行;报告撰写、口径卡的说明不再说过头
- 导出的证据 CSV 里两列同名,第三列改为「片段编号」

三、前后端对齐
- 前端按后端原文识别日志、报错的地方,新旧两种写法都认得(历史运行里存的是旧文字),逐条补了新旧原文的核对
- 后端交回模型的原指令放进单独的 for_model 字段,前端在解析响应时去掉,不会显示到界面上

四、检查
- 检查脚本里按原文匹配的断言改成新文案;改完会变成永远成立的反向断言逐条修正,并做了变异验证
- 新增 check-copy(静态扫描前端界面字符串和后端上界面的文案):再出现「跑」「后端」「多半」「令牌」、
  界面上的 Copilot、裸写的配置键名、半角标点等就报错;确需保留的写进 scripts/copy-allow.json 并注明理由
2026-09-30 08:05:15 +08:00

266 lines
14 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Copilot 搭图自查拦得下配错的报告撰写节点(证据五期)。
- metrics_from 指向的不是口径卡:validate 本来就报 error,自查交回去改
- 报告撰写的上游没有任何证据来源(没有口径卡、查库的 agent / 调用工具、知识检索):运行时只会给一条警告、
报告里每个数都被判成没有出处——搭图时就打回去,按 error 交给模型改
- 配对了的报告撰写节点不多花一次自查调用
- 旧结构「可以升级为可追溯结构」的建议不混进这一轮的问题清单
"""
from __future__ import annotations
import json
import pytest
from httpx import ASGITransport, AsyncClient
from app.api.copilot import authored_issues
from app.engine.schema import GraphSpec
from app.main import app
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
def add(nid, ntype, label, **config):
return {"op": "add_node", "node": {"id": nid, "type": ntype, "label": label, "config": config}}
def link(a, b):
return {"op": "add_edge", "edge": {"source": a, "target": b}}
class Scripted:
"""第一轮吐 first;收到自查请求(human 里有「上线前自查」)时吐 fix。"""
def __init__(self, first: list[dict], fix: list[dict] | None = None) -> None:
self.first = [json.dumps(o, ensure_ascii=False) for o in first]
self.fix = [json.dumps(o, ensure_ascii=False) for o in fix or []]
self.calls: list[str] = []
async def astream(self, messages):
from langchain_core.messages import AIMessageChunk
human = next(text for role, text in messages if role == "human")
self.calls.append(human)
for line in (self.fix if "上线前自查" in human else self.first):
yield AIMessageChunk(content=line + "\n")
async def generate(client, monkeypatch, model: Scripted) -> list[dict]:
import app.api.copilot as copilot
async def _model(*_a, **_k):
return model, "mock-fast"
monkeypatch.setattr(copilot, "get_chat_model", _model)
events: list[dict] = []
async with client.stream("POST", "/api/copilot/generate-stream", json={"instruction": "写一份订单周报"}) as r:
async for line in r.aiter_lines():
if line.startswith("data: "):
events.append(json.loads(line[6:]))
return events
def wrap(*ops) -> list[dict]:
return [{"op": "plan", "summary": "周报"}, *ops, {"op": "done", "explanation": "周报", "run": False}]
FETCH = add("fetch", "tool", "查库", tool="db_query__shop", args={"sql": "SELECT COUNT(*) AS n FROM orders"})
CARD = add("card", "metrics", "口径卡", metrics=[{"id": "n", "name": "订单数", "expression": "cell(nodes.fetch, 0, 'n')"}])
OUT = add("out", "output", "成果", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}])
def issues_of(events, status):
return [i for e in events if e["op"] == "check" and e.get("status") == status for i in e["issues"]]
async def test_metrics_from_pointing_at_a_non_card_is_sent_back(client, monkeypatch):
first = wrap(add("start", "input", "输入"), FETCH, link("start", "fetch"), CARD, link("fetch", "card"),
add("write", "report", "报告撰写", instructions="写周报", metrics_from=["fetch"]),
link("card", "write"), OUT, link("write", "out"))
fix = [{"op": "update_node", "id": "write", "config": {"metrics_from": ["card"]}}, {"op": "done"}]
model = Scripted(first, fix)
events = await generate(client, monkeypatch, model)
sent = issues_of(events, "repairing")
assert any(i["code"] == "report.metrics_from_invalid" and i["node_id"] == "write" for i in sent), sent
assert "不是口径卡" in model.calls[1]
assert [e["status"] for e in events if e["op"] == "check"] == ["repairing", "passed"]
async def test_a_report_with_nothing_to_cite_upstream_is_sent_back(client, monkeypatch):
first = wrap(add("start", "input", "输入"), add("write", "report", "报告撰写", instructions="写周报"),
link("start", "write"), OUT, link("write", "out"))
fix = [FETCH, link("start", "fetch"), link("fetch", "write"), {"op": "done"}]
model = Scripted(first, fix)
events = await generate(client, monkeypatch, model)
[sent] = [i for i in issues_of(events, "repairing") if i["code"] == "report_no_source"]
assert sent["level"] == "error" and sent["node_id"] == "write"
assert "没有能引用数字的证据来源" in model.calls[1]
assert [e["status"] for e in events if e["op"] == "check"] == ["repairing", "passed"]
final = events[-1]
assert final["op"] == "final" and not [i for i in final["issues"] if i.get("code") == "report_no_source"]
async def test_a_well_wired_report_costs_no_extra_call(client, monkeypatch):
first = wrap(add("start", "input", "输入"),
add("ask", "agent", "查数", tools=["db_query__shop"], prompt="查本周订单数"),
link("start", "ask"), add("write", "report", "报告撰写", instructions="写周报"),
link("ask", "write"), OUT, link("write", "out"))
model = Scripted(first)
events = await generate(client, monkeypatch, model)
assert len(model.calls) == 1 and [e["status"] for e in events if e["op"] == "check"] == ["passed"]
async def test_the_upgrade_hint_is_not_part_of_the_copilot_issue_list(client, monkeypatch):
# 只问清单的问数据图就是 agent → output:validate 会建议升级,但这一轮的问题清单里不该有它
first = wrap(add("start", "input", "输入"),
add("ask", "agent", "查数", tools=["db_query__shop"], prompt="列出订单表里的店铺"),
link("start", "ask"),
add("out", "output", "成果", fields=[{"name": "answer", "value": "{{ nodes.ask.text }}"}]),
link("ask", "out"))
events = await generate(client, monkeypatch, Scripted(first))
final = events[-1]
assert final["op"] == "final"
assert not [i for i in final["issues"] if i.get("code") == "evidence.upgrade_available" or i["level"] == "info"]
def spec_of(nodes, edges):
return GraphSpec.model_validate({"nodes": nodes, "edges": edges})
def n(nid, ntype, **config):
return {"id": nid, "type": ntype, "data": {"label": nid, "config": config}}
@pytest.mark.parametrize("source", [
n("fetch", "tool", tool="db_query__shop"),
n("fetch", "agent", tools=["db_query__shop"]),
n("fetch", "retrieve", collection="docs"),
n("fetch", "metrics", metrics=[{"id": "a", "expression": "1"}]),
n("fetch", "subgraph", workflow_id="w1"),
n("fetch", "supervisor", goal="查清楚", agents=[{"name": "analyst", "tools": ["db_query__shop"]}]),
# 运行输入也是能引用的来源([[i:字段]]):声明了数值字段的入口算
n("fetch", "input", fields=[{"name": "target", "default": 100}]),
n("fetch", "input", fields=[{"name": "week"}, {"name": "rate", "default": "0.35"}]),
n("fetch", "input", fields=[{"name": "budget", "type": "number"}]),
])
def test_any_evidence_source_upstream_is_enough(source):
spec = spec_of([n("start", "input"), source, n("write", "report"), n("out", "output")],
[{"source": "start", "target": "fetch"}, {"source": "fetch", "target": "write"},
{"source": "write", "target": "out"}])
assert not [i for i in authored_issues(spec, []) if i["code"] == "report_no_source"]
@pytest.mark.parametrize("fields", [[], [{"name": "question", "required": True}], [{"name": "topic", "default": "本周"}]])
def test_an_input_without_numeric_fields_is_not_a_source(fields):
# 只声明了文字字段的入口:报告只能照引问题本身,要写的数照样没有出处
spec = spec_of([n("start", "input", fields=fields), n("write", "report"), n("out", "output")],
[{"source": "start", "target": "write"}, {"source": "write", "target": "out"}])
[issue] = [i for i in authored_issues(spec, []) if i["code"] == "report_no_source"]
assert issue["node_id"] == "write" and issue["level"] == "error"
assert "运行输入" in issue["message"] and "[[i:" in issue["message"]
def test_a_numeric_input_upstream_is_enough_for_the_copilot_self_check():
spec = spec_of([n("start", "input", fields=[{"name": "question"}, {"name": "gmv", "default": 1200.5}]),
n("write", "report"), n("out", "output")],
[{"source": "start", "target": "write"}, {"source": "write", "target": "out"}])
assert not [i for i in authored_issues(spec, []) if i["code"] == "report_no_source"]
def test_a_tool_less_agent_or_a_downstream_source_does_not_count():
spec = spec_of([n("start", "input"), n("chat", "agent", prompt="随便聊"), n("write", "report"),
n("late", "tool", tool="db_query__shop"), n("out", "output")],
[{"source": "start", "target": "chat"}, {"source": "chat", "target": "write"},
{"source": "write", "target": "late"}, {"source": "late", "target": "out"}])
[issue] = [i for i in authored_issues(spec, []) if i["code"] == "report_no_source"]
assert issue["node_id"] == "write" and issue["level"] == "error"
def test_reports_this_round_did_not_touch_are_left_alone():
nodes = [n("start", "input"), n("write", "report", instructions="写"), n("out", "output")]
spec = spec_of(nodes, [{"source": "start", "target": "write"}, {"source": "write", "target": "out"}])
assert not [i for i in authored_issues(spec, [], baseline=nodes) if i["code"] == "report_no_source"]
# --------------------------------------------------------------------------
# 问数据的图发成受管:Copilot 补契约(用户报过的那一幕)
# --------------------------------------------------------------------------
def ask_report_graph() -> dict:
"""input → agent(查库)→ 报告撰写(受管三项都写了、正文引单元格)→ 出口,出口还没有契约。"""
def g(nid, ntype, label, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label, "config": config}}
return {
"nodes": [
g("start", "input", "问题"),
g("ask", "agent", "查数", tools=["db_query__shop"], prompt="查活跃用户数和用户总数", approval="dangerous"),
g("report", "report", "写答案", instructions="回答:{{ input.question }}", numbers="strict",
on_violation="fail", claims="require_citation"),
g("out", "output", "输出答案", fields=[{"name": "answer", "value": "{{ nodes.report.text }}"}]),
],
"edges": [{"id": f"e{i}", "source": a, "target": b}
for i, (a, b) in enumerate([("start", "ask"), ("ask", "report"), ("report", "out")])],
}
class Assist:
def __init__(self, ops: list[dict]) -> None:
self.lines = [json.dumps(o, ensure_ascii=False) for o in ops]
async def astream(self, messages):
from langchain_core.messages import AIMessageChunk
for line in self.lines:
yield AIMessageChunk(content=line + "\n")
async def governed_assist(client, monkeypatch, ops: list[dict]) -> dict:
import app.api.copilot as copilot
async def _model(*_a, **_k):
return Assist(ops), "mock-fast"
monkeypatch.setattr(copilot, "get_chat_model", _model)
wf = (await client.post("/api/workflows", json={"name": "问数据-受管", "graph": ask_report_graph()})).json()["id"]
r = await client.post(f"/api/workflows/{wf}/autofix", json={"level": "governed", "apply": [], "assist": True})
assert r.status_code == 200, r.text
return r.json()
CELLS_QUESTION = {"op": "question", "node_id": "out",
"text": "报告正文直接引用了查询单元格:契约里写 cells: true,还是改成口径卡指标?"}
async def test_the_copilots_contract_for_a_cells_only_graph_is_adopted(client, monkeypatch):
contract = {"report_from": "report", "strict": True}
out = await governed_assist(client, monkeypatch, [
{"op": "plan", "summary": "补出具契约"},
{"op": "update_node", "id": "out", "config": {"contract": contract}}, CELLS_QUESTION,
{"op": "done", "explanation": "给出口加上契约,cells 要你拿主意"}])
assert out["assist"]["ok"] is True and "assist" in out["applied"], out["rejected"]
assert not [r for r in out["rejected"] if "metrics_from" in r["reason"]]
assert next(n for n in out["graph"]["nodes"] if n["id"] == "out")["data"]["config"]["contract"] == contract
assert out["assist"]["questions"] == ["「输出答案」:" + CELLS_QUESTION["text"]]
# 没有契约那一处没了;cells 要人拍板(Copilot 不许替人打开),改完冒出来的这一处不算 Copilot 改坏的,
# 发布弹窗里给一键选项:写 cells: true,或交给 Copilot 改成口径卡
assert out["ok"] is False
assert [i["code"] for i in out["remaining"] if i["level"] == "error"] == ["contract.cells_undeclared"]
[fix] = [f for f in out["fixes"] if f["code"] == "contract.cells_undeclared"]
assert fix["kind"] == "choice" and [o["value"] for o in fix["options"]] == [True, "copilot"]
async def test_the_copilot_still_may_not_switch_cells_on_by_itself(client, monkeypatch):
out = await governed_assist(client, monkeypatch, [
{"op": "plan", "summary": "补出具契约"},
{"op": "update_node", "id": "out", "config": {"contract": {"report_from": "report", "strict": True,
"cells": True}}},
{"op": "done", "explanation": "补上契约并打开 cells"}])
[rejected] = [r for r in out["rejected"] if r["fix_id"] == "assist"]
assert "单元格引用" in rejected["reason"] and out["ok"] is False