diff --git a/backend/app/api/copilot.py b/backend/app/api/copilot.py index 54732cfc..be9168a5 100644 --- a/backend/app/api/copilot.py +++ b/backend/app/api/copilot.py @@ -193,6 +193,10 @@ NODE_REFERENCE = """\ 写进去再也读不回来——真发生过:一轮写进 scope"user",下一轮没写 scope 落回 default,于是「我叫张三」存下来了,问「我是谁」却答不上来 - human:人工审批。config: {mode: approve|input|edit, title, message} + title、message 说批了会怎样时写「批准后输出」「批准后继续」,**不要写「正式出具」「批准后才会出具」**:出具只发生在 + 成果节点配了出具契约、并且从已发布版本发起的正式运行里;画布上试运行、问数据都是探索运行,批准后只是输出。 + 这次是探索运行还是正式运行,审批卡会自己标明。审的是报告时把 human 接在 report 后面(message 里可以引用 + {{ nodes.报告节点id.text }}),审批卡会附上这份报告的核对概况 - validate:JSON Schema 校验,可自动让模型修复。config: {schema, source, max_retries} - transform:数据整形。config: {mode: expression|template|json, expression/template, assign_to} - subgraph:嵌套另一个工作流。config: {workflow_id, input} @@ -1323,6 +1327,7 @@ def authored_issues(spec: GraphSpec, sources: list[Any], baseline: list[dict[str changed = _changed(spec, baseline) out: list[dict[str, Any]] = [] out += _unsourced_reports(spec, changed) + out += _issuance_promises(spec, changed) for issue in text_parse_issues(spec): if issue.node_id in changed: # 模型的文字放在模板引号里的那种,改法是 | json,不是换结构化输出 @@ -1356,6 +1361,36 @@ def authored_issues(spec: GraphSpec, sources: list[Any], baseline: list[dict[str return out +#: 人工审批的标题、说明里不该出现的说法:出具要成果节点配了出具契约、从已发布版本发起的正式运行才谈得上 +_ISSUANCE_PROMISE = "正式出具" + + +def _issuance_promises(spec: GraphSpec, changed: set[str]) -> list[dict[str, Any]]: + """这一轮写的人工审批节点在标题、说明里许诺「正式出具」(human_formal_issuance)。 + + 真实运行里一次探索运行的审批写着「批准后才会正式出具」,审批人以为自己在签发正式的东西。画布上试运行、问数据 + 都是探索运行,成果节点也常常没有出具契约;同一张图发布后发起的正式运行才谈得上出具。审批说明写「批准后输出」, + 这次是哪一类运行由审批卡标明。""" + out: list[dict[str, Any]] = [] + for node in spec.nodes: + if node.id not in changed or node.type != NodeType.HUMAN: + continue + field = next((key for key in ("title", "message") + if _ISSUANCE_PROMISE in str(node.config.get(key) or "")), None) + if field is None: + continue + out.append({ + "level": "error", "node_id": node.id, "edge_id": None, "field": field, "code": "human_formal_issuance", + "message": f"人工审批「{node.title}」的{'标题' if field == 'title' else '说明'}写了「{_ISSUANCE_PROMISE}」。" + "只有成果节点配了出具契约、从已发布版本发起的正式运行才会出具,画布上试运行和问数据都是探索运行," + "批准后只是输出。请改成「批准后输出」这类说法", + "for_model": f"人工审批的 {field} 里不要写「{_ISSUANCE_PROMISE}」:只有成果节点配了出具契约、从已发布版本发起的" + "正式运行才会出具,这张图在画布上试运行、问数据时都是探索运行,批准后只是输出。改成「批准后输出」" + "「批准后继续」这类说法,运行类别由审批卡标明", + }) + return out + + def _catalog_issues(node_id: str, sql: str, checker: Any) -> list[dict[str, Any]]: """一个调用工具节点的 SQL 对照数据目录查出来的问题,形状和别的自查问题一样(node_id、field 都在), 另带 table、column、relation_id、sql_excerpt 给界面定位。""" diff --git a/backend/tests/test_copilot_approval_wording.py b/backend/tests/test_copilot_approval_wording.py new file mode 100644 index 00000000..36e36f66 --- /dev/null +++ b/backend/tests/test_copilot_approval_wording.py @@ -0,0 +1,142 @@ +"""助手写人工审批的说明时不说「正式出具」。 + +真实运行里:助手搭了「几个 Agent 查库 → 报告 → 人工审批 → 成果」,审批节点的标题和说明写着「请审核后正式出具」 +「批准后才会正式出具」。可那是一次探索运行,成果节点也没有配出具契约——根本不存在「正式出具」。审批人照着这句 +话批准,以为自己在签发一份正式的东西。 + +出具只发生在成果节点配了出具契约、从已发布版本发起的正式运行里;画布上试运行、问数据都是探索运行。审批说明 +写「批准后输出」这类说法,这次是哪一类运行由审批卡自己标明。 + +这组守的是: +- 节点手册里 human 那一条写明这条规矩和替代说法 +- 助手写出来的审批标题、说明里有「正式出具」时,自查交回去改(给人看的那句不带键名,给模型的那句说怎么改) +- 这一轮没碰过的审批节点不替人改 +""" +from __future__ import annotations + +import json + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.api.copilot import NODE_REFERENCE, authored_issues +from app.engine.schema import GraphSpec +from app.main import app + + +def _human_rule() -> str: + start = NODE_REFERENCE.index("- human:人工审批") + return NODE_REFERENCE[start:NODE_REFERENCE.index("\n- validate:", start)] + + +def test_node_reference_tells_the_assistant_not_to_promise_formal_issuance(): + rule = _human_rule() + assert "不要写「正式出具」" in rule + assert "批准后输出" in rule + # 说清什么时候才谈得上出具:配了出具契约、从已发布版本发起的正式运行 + assert "出具契约" in rule and "已发布版本" in rule and "正式运行" in rule and "探索运行" in rule + + +def _node(nid, ntype, **config): + return {"id": nid, "type": ntype, "data": {"label": nid, "config": config}} + + +def _spec(*nodes): + return GraphSpec.model_validate({"nodes": list(nodes), "edges": [ + {"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}) + + +def _graph(**human): + return _spec(_node("start", "input"), + _node("write", "report", instructions="写周报"), + _node("gate", "human", mode="approve", **human), + _node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}])) + + +@pytest.mark.parametrize("human, field", [ + ({"title": "请审核后正式出具", "message": "请审核本周报告"}, "title"), + ({"title": "审批", "message": "请审核本周报告,批准后才会正式出具。\n\n{{ nodes.write.text }}"}, "message"), +]) +def test_promising_formal_issuance_is_sent_back(human, field): + issues = [i for i in authored_issues(_graph(**human), []) if i["code"] == "human_formal_issuance"] + assert len(issues) == 1, issues + [issue] = issues + assert issue["level"] == "error" and issue["node_id"] == "gate" and issue["field"] == field + assert "正式出具" in issue["message"] and "批准后输出" in issue["message"] + assert "出具契约" in issue["for_model"] and "批准后输出" in issue["for_model"] + + +def test_accurate_wording_passes(): + spec = _graph(title="请审核本周报告", message="批准后输出这份报告。\n\n{{ nodes.write.text }}") + assert not [i for i in authored_issues(spec, []) if i["code"] == "human_formal_issuance"] + + +def test_approvals_this_round_did_not_touch_are_left_alone(): + spec = _graph(title="请审核后正式出具") + gate = next(n for n in spec.model_dump()["nodes"] if n["id"] == "gate") + assert not [i for i in authored_issues(spec, [], baseline=[gate]) if i["code"] == "human_formal_issuance"] + + +class Scripted: + """第一轮吐 first;收到自查请求(human 里有「上线前自查」)时吐 fix。""" + + def __init__(self, first: list[dict], fix: list[dict]) -> None: + self.first = [json.dumps(o, ensure_ascii=False) for o in first] + self.fix = [json.dumps(o, ensure_ascii=False) for o in fix] + self.calls: list[str] = [] + + async def astream(self, messages): + from langchain_core.messages import AIMessageChunk + + human = next(text for role, text in messages if role == "human") + self.calls.append(human) + for line in (self.fix if "上线前自查" in human else self.first): + yield AIMessageChunk(content=line + "\n") + + +def _add(nid, ntype, label, **config): + return {"op": "add_node", "node": {"id": nid, "type": ntype, "label": label, "config": config}} + + +def _link(a, b, handle=None): + return {"op": "add_edge", "edge": {"source": a, "target": b, **({"sourceHandle": handle} if handle else {})}} + + +async def test_the_self_check_rewrites_the_approval_before_delivery(monkeypatch): + import app.api.copilot as copilot + + first = [ + _add("start", "input", "输入", fields=[{"name": "week"}]), + _add("fetch", "transform", "取数", mode="expression", expression="{'gmv': 1}", assign_to="kpi"), + _add("card", "metrics", "口径卡", metrics=[{"id": "gmv", "name": "销售额", "expression": "vars.kpi.gmv"}]), + _add("write", "report", "写周报", instructions="写周报"), + _add("gate", "human", "审批", mode="approve", title="请审核后正式出具", + message="批准后才会正式出具。\n\n{{ nodes.write.text }}"), + _add("out", "output", "成果", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]), + _link("start", "fetch"), _link("fetch", "card"), _link("card", "write"), _link("write", "gate"), + _link("gate", "out", "approved"), + {"op": "done", "explanation": "周报", "run": False}, + ] + fix = [{"op": "update_node", "id": "gate", "config": { + "mode": "approve", "title": "请审核本周报告", "message": "批准后输出这份报告。\n\n{{ nodes.write.text }}"}}, + {"op": "done"}] + model = Scripted(first, fix) + + async def _model(*_a, **_k): + return model, "mock-fast" + + monkeypatch.setattr(copilot, "get_chat_model", _model) + events: list[dict] = [] + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + async with client.stream("POST", "/api/copilot/generate-stream", json={"instruction": "写周报并审批"}) as r: + async for line in r.aiter_lines(): + if line.startswith("data: "): + events.append(json.loads(line[6:])) + checks = [e for e in events if e.get("op") == "check"] + assert checks[0]["status"] == "repairing" + assert any(i.get("code") == "human_formal_issuance" for i in checks[0]["issues"]) + assert "出具契约" in model.calls[-1], "交回模型的修正请求里写明为什么不能这么说" + assert checks[-1]["status"] == "passed" + final = next(e for e in events if e.get("op") == "final") + gate = next(n for n in final["graph"]["nodes"] if n["id"] == "gate") + assert "正式出具" not in json.dumps(gate, ensure_ascii=False)