mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 14:52:02 +00:00
ThearchyHelios
304 lines
15 KiB
Python
304 lines
15 KiB
Python
"""自动修复的 Copilot 兜底（autofix 的 assist: true）。
|
|
|
|
确定性修复修不了的 error 交给 Copilot：模型给剧本，看采纳规则——
|
|
- 正常的修复（error 变少、没有新 error、不降低要求）被采纳，只是预览，不落库
|
|
- 试图删契约的被拒
|
|
- 让错误变多的被拒
|
|
- 要人拿主意的放进 questions，不硬改
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import pytest
|
|
from httpx import ASGITransport, AsyncClient
|
|
from sqlalchemy import select
|
|
|
|
from app.db.base import SessionLocal
|
|
from app.db.models import Workflow, WorkflowVersion
|
|
from app.main import app
|
|
|
|
|
|
@pytest.fixture
|
|
async def client():
|
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
|
yield c
|
|
|
|
|
|
def node(nid, ntype, label=None, **config):
|
|
return {"id": nid, "type": ntype, "position": {"x": 40, "y": 80}, "data": {"label": label or nid, "config": config}}
|
|
|
|
|
|
CONTRACT = {"report_from": "write", "metrics_from": ["card"], "required": ["gmv"], "strict": True}
|
|
|
|
|
|
def two_exits() -> dict:
|
|
"""受管模板没有带契约的出口，而且出口有两个：该给哪个写契约说不准，确定性修复只能交给 Copilot。"""
|
|
nodes = [
|
|
node("start", "input", "周期"),
|
|
node("fetch", "tool", "取数", tool="db_query__shop", args={"sql": "SELECT SUM(amount) AS gmv FROM orders"}),
|
|
node("card", "metrics", "周报口径卡", caliber="周报口径", metrics=[
|
|
{"id": "gmv", "name": "销售额", "unit": "元", "expression": "cell(nodes.fetch, 0, 'gmv')"}]),
|
|
node("write", "report", "报告撰写", instructions="写周报",
|
|
numbers="strict", on_violation="fail", claims="require_citation"),
|
|
node("done", "output", "出具", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]),
|
|
node("raw", "output", "原始数", fields=[{"name": "数", "value": "{{ nodes.fetch }}"}]),
|
|
]
|
|
edges = [{"id": f"e{i}", "source": a, "target": b} for i, (a, b) in enumerate(
|
|
[("start", "fetch"), ("fetch", "card"), ("card", "write"), ("write", "done"), ("fetch", "raw")])]
|
|
return {"nodes": nodes, "edges": edges, "viewport": {"x": 3, "y": 4, "zoom": 1.2}}
|
|
|
|
|
|
class Scripted:
|
|
"""按剧本吐操作流的模型；记下每次收到的请求。"""
|
|
|
|
def __init__(self, ops: list[dict]) -> None:
|
|
self.lines = [json.dumps(o, ensure_ascii=False) for o in ops]
|
|
self.calls: list[list] = []
|
|
|
|
async def astream(self, messages):
|
|
from langchain_core.messages import AIMessageChunk
|
|
|
|
self.calls.append(messages)
|
|
for line in self.lines:
|
|
yield AIMessageChunk(content=line + "\n")
|
|
|
|
|
|
def script(monkeypatch, ops: list[dict]) -> Scripted:
|
|
import app.api.copilot as copilot
|
|
|
|
model = Scripted(ops)
|
|
|
|
async def _model(*_a, **_k):
|
|
return model, "mock-fast"
|
|
|
|
monkeypatch.setattr(copilot, "get_chat_model", _model)
|
|
return model
|
|
|
|
|
|
async def create(client, graph: dict) -> str:
|
|
return (await client.post("/api/workflows", json={"name": "周报-兜底", "graph": graph})).json()["id"]
|
|
|
|
|
|
async def rows(wf: str) -> list:
|
|
async with SessionLocal() as session:
|
|
w = await session.get(Workflow, wf)
|
|
vs = (await session.execute(select(WorkflowVersion).where(WorkflowVersion.workflow_id == wf))).scalars()
|
|
return [{c.name: getattr(w, c.name) for c in Workflow.__table__.columns},
|
|
[{c.name: getattr(v, c.name) for c in WorkflowVersion.__table__.columns} for v in vs]]
|
|
|
|
|
|
async def assist(client, wf: str, **extra) -> dict:
|
|
r = await client.post(f"/api/workflows/{wf}/autofix", json={"level": "governed", "apply": [], "assist": True,
|
|
**extra})
|
|
assert r.status_code == 200, r.text
|
|
return r.json()
|
|
|
|
|
|
def done(ops: list[dict]) -> list[dict]:
|
|
return [{"op": "plan", "summary": "给出具节点补契约"}, *ops, {"op": "done", "explanation": "补上了出具契约"}]
|
|
|
|
|
|
async def test_a_sound_fix_is_adopted_as_a_preview(client, monkeypatch):
|
|
model = script(monkeypatch, done([{"op": "update_node", "id": "done", "config": {"contract": CONTRACT}}]))
|
|
wf = await create(client, two_exits())
|
|
before = await rows(wf)
|
|
out = await assist(client, wf)
|
|
assert out["assist"]["ok"] is True and out["assist"]["summary"] == "补上了出具契约"
|
|
assert out["applied"] == ["assist"] and out["ok"] is True, out
|
|
fixed = out["graph"]
|
|
assert next(n for n in fixed["nodes"] if n["id"] == "done")["data"]["config"]["contract"] == CONTRACT
|
|
assert fixed["viewport"] == two_exits()["viewport"] and [e["id"] for e in fixed["edges"]] == [
|
|
e["id"] for e in two_exits()["edges"]]
|
|
assert out["ops"] == [{"op": "update_node", "id": "done", "config": {"contract": CONTRACT}}]
|
|
assert [(c["fix_id"], c["node_id"], c["field"]) for c in out["changes"]] == [("assist", "done", "contract")]
|
|
assert await rows(wf) == before # 只是预览
|
|
# 交给模型的请求里有剩下的问题和不许降低要求、不许替人选的约定
|
|
human = next(text for role, text in model.calls[0] if role == "human")
|
|
assert "受管级别要求至少一个「成果」节点声明出具契约" in human
|
|
assert "不许降低要求" in human and "不替人选" in human and "question" in human
|
|
|
|
|
|
async def test_deleting_the_contract_is_refused(client, monkeypatch):
|
|
graph = two_exits()
|
|
graph["nodes"][4]["data"]["config"]["contract"] = {**CONTRACT, "report_from": "fetch"}
|
|
graph["nodes"][3] = node("write", "llm", "写周报", prompt="写周报") # 没有报告撰写节点：只能交给 Copilot
|
|
script(monkeypatch, done([{"op": "update_node", "id": "done", "config": {"contract": None}}]))
|
|
wf = await create(client, graph)
|
|
out = await assist(client, wf)
|
|
assert out["assist"]["ok"] is False and "assist" not in out["applied"]
|
|
[rejected] = [r for r in out["rejected"] if r["fix_id"] == "assist"]
|
|
assert "契约" in rejected["reason"] and "降低" in rejected["reason"]
|
|
assert out["graph"] == graph and out["ok"] is False
|
|
|
|
|
|
async def test_a_fix_that_makes_errors_grow_is_refused(client, monkeypatch):
|
|
script(monkeypatch, done([
|
|
{"op": "update_node", "id": "done", "config": {"contract": CONTRACT}},
|
|
{"op": "update_node", "id": "card", "config": {"metrics": [{"id": "gmv"}]}}, # 口径卡的表达式被写丢了
|
|
]))
|
|
wf = await create(client, two_exits())
|
|
out = await assist(client, wf)
|
|
assert out["assist"]["ok"] is False
|
|
[rejected] = [r for r in out["rejected"] if r["fix_id"] == "assist"]
|
|
assert "新的问题" in rejected["reason"] and "表达式" in rejected["reason"]
|
|
assert out["graph"] == two_exits()
|
|
|
|
|
|
async def test_removing_a_node_is_refused_even_if_it_helps(client, monkeypatch):
|
|
script(monkeypatch, done([
|
|
{"op": "remove_node", "id": "raw"},
|
|
{"op": "update_node", "id": "done", "config": {"contract": CONTRACT}},
|
|
]))
|
|
wf = await create(client, two_exits())
|
|
out = await assist(client, wf)
|
|
[rejected] = [r for r in out["rejected"] if r["fix_id"] == "assist"]
|
|
assert "删" in rejected["reason"] and out["graph"] == two_exits()
|
|
|
|
|
|
async def test_questions_are_passed_on_without_changes(client, monkeypatch):
|
|
script(monkeypatch, [
|
|
{"op": "plan", "summary": "要先问清楚"},
|
|
{"op": "question", "node_id": "done", "text": "两个出口里，哪个是要出具的？「出具」还是「原始数」？"},
|
|
{"op": "question", "node_id": "card", "text": "required 要包括哪些指标？候选：gmv"},
|
|
{"op": "question", "text": " "},
|
|
{"op": "done", "explanation": "等你决定出具的是哪个出口"},
|
|
])
|
|
wf = await create(client, two_exits())
|
|
out = await assist(client, wf)
|
|
# 一句一条的文字，前面标上是哪个节点的事（句子里已经点了名的不重复标）
|
|
assert out["assist"] == {"ok": False, "summary": "等你决定出具的是哪个出口", "questions": [
|
|
"两个出口里，哪个是要出具的？「出具」还是「原始数」？", "「周报口径卡」：required 要包括哪些指标？候选：gmv"],
|
|
"looks": []}
|
|
assert out["rejected"] == [] and out["graph"] == two_exits()
|
|
|
|
|
|
async def test_deterministic_fixes_run_first_and_copilot_sees_what_is_left(client, monkeypatch):
|
|
graph = two_exits()
|
|
graph["nodes"].insert(2, node("ask", "agent", "查数员", tools=["db_query__shop"], approval="never",
|
|
prompt="用 db_query__shop 查"))
|
|
graph["edges"] += [{"source": "start", "target": "ask"}, {"source": "ask", "target": "card"}]
|
|
model = script(monkeypatch, done([{"op": "update_node", "id": "done", "config": {"contract": CONTRACT}}]))
|
|
wf = await create(client, graph)
|
|
out = await assist(client, wf, apply=["governed.agent_approval_never:ask"])
|
|
assert out["applied"] == ["governed.agent_approval_never:ask", "assist"] and out["ok"] is True
|
|
human = next(text for role, text in model.calls[0] if role == "human")
|
|
assert "的审批策略是「全部无需审批」" not in human and '"approval": "dangerous"' in human
|
|
|
|
|
|
async def test_nothing_left_means_no_model_call(client, monkeypatch):
|
|
model = script(monkeypatch, [])
|
|
graph = two_exits()
|
|
graph["nodes"][4]["data"]["config"]["contract"] = CONTRACT
|
|
wf = await create(client, graph)
|
|
out = await assist(client, wf)
|
|
assert out["assist"]["ok"] is True and model.calls == []
|
|
|
|
|
|
async def test_an_unconfigured_model_is_said_plainly(client, monkeypatch):
|
|
import app.api.copilot as copilot
|
|
from app.providers.factory import ProviderNotConfigured
|
|
|
|
async def _nothing(*_a, **_k):
|
|
raise ProviderNotConfigured("没有可用的 provider")
|
|
|
|
monkeypatch.setattr(copilot, "get_chat_model", _nothing)
|
|
wf = await create(client, two_exits())
|
|
out = await assist(client, wf)
|
|
assert out["assist"]["ok"] is False and "设置" in out["assist"]["summary"]
|
|
assert out["rejected"] == []
|
|
|
|
|
|
async def test_blocking_issues_only_add_the_gate_when_asked():
|
|
from app.api.copilot import _blocking_issues
|
|
|
|
graph = two_exits()
|
|
nodes = {n["id"]: n for n in graph["nodes"]}
|
|
assert _blocking_issues(nodes, graph["edges"]) == []
|
|
gated = _blocking_issues(nodes, graph["edges"], level="governed")
|
|
assert [i["code"] for i in gated] == ["governed.no_contract"]
|
|
assert _blocking_issues(nodes, graph["edges"], level="published") == []
|
|
|
|
|
|
def guarded() -> dict:
|
|
"""受管图里只剩一处 error：协作团队节点。团队后面还有一个「人工把关」节点。"""
|
|
nodes = [
|
|
node("start", "input", "周期"),
|
|
node("fetch", "tool", "取数", tool="db_query__shop", args={"sql": "SELECT SUM(amount) AS gmv FROM orders"}),
|
|
node("card", "metrics", "周报口径卡", caliber="周报口径", metrics=[
|
|
{"id": "gmv", "name": "销售额", "unit": "元", "expression": "cell(nodes.fetch, 0, 'gmv')"}]),
|
|
node("team", "supervisor", "复核团队", goal="复核", agents=[{"name": "checker", "tools": []}]),
|
|
node("gate", "human", "人工把关", prompt="确认数字"),
|
|
node("write", "report", "报告撰写", instructions="写周报",
|
|
numbers="strict", on_violation="fail", claims="require_citation"),
|
|
node("done", "output", "出具", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}], contract=CONTRACT),
|
|
]
|
|
edges = [{"id": f"e{i}", "source": a, "target": b} for i, (a, b) in enumerate(
|
|
[("start", "fetch"), ("fetch", "card"), ("card", "team"), ("team", "gate"), ("gate", "write"),
|
|
("write", "done")])]
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
@pytest.mark.parametrize("ops, word", [
|
|
# 同一个 id 换了类型：团队换成模型调用，顺手把人工把关换成整形——等于删了重建
|
|
([{"op": "add_node", "node": {"id": "team", "type": "llm", "label": "复核团队", "config": {"prompt": "复核"}}},
|
|
{"op": "add_node", "node": {"id": "gate", "type": "transform", "label": "人工把关",
|
|
"config": {"mode": "expression", "expression": "vars"}}}], "人工把关"),
|
|
# 类型没变，也是整个覆盖掉旧节点（配置、位置全丢）
|
|
([{"op": "add_node", "node": {"id": "gate", "type": "human", "label": "人工把关", "config": {"prompt": "随便"}}},
|
|
{"op": "add_node", "node": {"id": "team", "type": "llm", "label": "复核团队", "config": {"prompt": "复核"}}}],
|
|
"覆盖"),
|
|
])
|
|
async def test_overwriting_an_existing_node_with_add_node_is_refused(client, monkeypatch, ops, word):
|
|
graph = guarded()
|
|
script(monkeypatch, done(ops))
|
|
wf = await create(client, graph)
|
|
before = await rows(wf)
|
|
out = await assist(client, wf)
|
|
assert out["assist"]["ok"] is False and "assist" not in out["applied"] and out["ok"] is False
|
|
[rejected] = [r for r in out["rejected"] if r["fix_id"] == "assist"]
|
|
assert "降低" in rejected["reason"] and word in rejected["reason"], rejected
|
|
assert out["graph"] == graph and out["changes"] == [] and out["ops"] == []
|
|
assert [(n["id"], n["type"]) for n in out["graph"]["nodes"]] == [(n["id"], n["type"]) for n in graph["nodes"]]
|
|
assert await rows(wf) == before
|
|
|
|
|
|
async def test_renaming_a_node_while_fixing_another_error_is_adopted(client, monkeypatch):
|
|
"""改了名的节点上没修好的 error 文案跟着变了，但还是同一处问题：不算冒出新问题。"""
|
|
graph = guarded()
|
|
graph["nodes"][6]["data"]["config"].pop("contract") # 两处 error：协作团队、没有带契约的出口
|
|
script(monkeypatch, done([
|
|
{"op": "update_node", "id": "team", "label": "复核团队（待拆成固定步骤）"},
|
|
{"op": "update_node", "id": "done", "config": {"contract": CONTRACT}},
|
|
]))
|
|
wf = await create(client, graph)
|
|
out = await assist(client, wf)
|
|
assert out["applied"] == ["assist"] and out["assist"]["ok"] is True, out["rejected"]
|
|
assert [(i["code"], i["node_id"]) for i in out["remaining"] if i["level"] == "error"] == [
|
|
("governed.supervisor", "team")]
|
|
assert "复核团队（待拆成固定步骤）" in out["remaining"][0]["message"] or any(
|
|
"复核团队（待拆成固定步骤）" in i["message"] for i in out["remaining"])
|
|
assert {(c["node_id"], c["field"]) for c in out["changes"]} == {("team", "label"), ("done", "contract")}
|
|
|
|
|
|
async def test_a_fix_cut_off_at_the_output_limit_is_not_adopted(client, monkeypatch):
|
|
"""修正写到一半额度用完（finish_reason=length）：哪怕已经写出来的那几条能让 error 变少，也不采纳半截修改。"""
|
|
from langchain_core.messages import AIMessageChunk
|
|
|
|
model = script(monkeypatch, [{"op": "plan", "summary": "给出具节点补契约"},
|
|
{"op": "update_node", "id": "done", "config": {"contract": CONTRACT}}])
|
|
lines = model.lines
|
|
|
|
async def cut_off(messages):
|
|
model.calls.append(messages)
|
|
for line in lines:
|
|
yield AIMessageChunk(content=line + "\n")
|
|
yield AIMessageChunk(content="", response_metadata={"finish_reason": "length"})
|
|
|
|
model.astream = cut_off
|
|
wf = await create(client, two_exits())
|
|
out = await assist(client, wf)
|
|
assert out["assist"]["ok"] is False and "assist" not in out["applied"]
|
|
assert out["assist"]["summary"].startswith("助手本轮未完成：") and "长度上限" in out["assist"]["summary"]
|
|
assert out["graph"] == two_exits()
|