Files
AgentLab/backend/tests/test_chat_turn_policy.py
T
ThearchyHelios 2f90f0eb3a 口径台账:固定执行、验证审阅发布与增量重导入
- QueryCore 固定执行(K09):按已发布的不可变口径跑一次,交回类型化标量、发布/manifest 引用与逐查询证据
- 依赖闭包(K08):按选中的指标算出要拷的取数节点、要的入口字段,接进 derive / compose / metrics
- SQL 模板参数完整性(K01):试跑、组合问数、派生图、运行时取数共用一套语义,缺参数在渲染前就拒
- 受限字段变更服务(K03)与口径领域边界(K01):改动打在整图深拷贝上,身份与证据归属不可改
- 版本化受控模板绑定(K04):业务口径按显式绑定落到 SQL,对不上就报 unsupported / needs_update
- 验证记录(K05)与最小审阅:样例匹配不等于正式批准,审批人只从服务端身份取
- 发布依据清单(K07):发布时按不可变图物化内容寻址清单,正式用途按它判
- 增量重导入(K11)与 B/L/R 三方合并、受限自然语言修改提案(K12):只产候选,落图仍走既有写路径
- 接口与前端:口径草稿页接上字段变更、绑定状态、验证与重导入,新增 caliber-drafts / studio 检查脚本

验证:前端 tsc -b 通过。
2026-11-10 15:05:18 +08:00

283 lines
16 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""问数据一轮的「决定」(app/chat/turn.py;细案 01 做法 1)。
要不要运行、输入怎么填、要不要带着原图重来、重来时说什么,从问数据页下沉到后端:final 带 next,复核带 retry_instruction,
问数据的一轮后来整个搬到了服务端(chat/drive.py、chat/job.py),页面只发起、只播放。这里守:
- 各条决定的分支(next_step、fill_input、retry_after_review、retry_instruction 的两种措辞);
- 页面上不再留一份同样的决定(一轮整个搬到服务端之后,frontend/src/store/chat.ts 只发起、只播放);用户点重试本轮时的措辞;
- /copilot/review 按这一轮试过几次给 retry_instruction:可以重来时是那句话,不重来是 null。
问题、字段名都是虚构的。
"""
from __future__ import annotations
import re
import uuid
from pathlib import Path
import pytest
from httpx import ASGITransport, AsyncClient
from app.chat import turn
from app.main import app
CHAT_TS = Path(__file__).resolve().parents[2] / "frontend" / "src" / "store" / "chat.ts"
def _final(*, issues=(), autorun=True, **extra) -> dict:
return {"op": "final", "graph": {"nodes": [], "edges": []}, "issues": list(issues), "autorun": autorun, **extra}
ERROR = {"level": "error", "message": "图是空的"}
WARNING = {"level": "warning", "message": "多余的 {{ }}"}
# --------------------------------------------------------------------------
# next_step
# --------------------------------------------------------------------------
def test_next_step_branches():
assert turn.next_step(_final()) == "run"
assert turn.next_step(_final(issues=[WARNING])) == "run", "warning 不挡运行"
assert turn.next_step(_final(autorun=False)) == "ready", "模型说不运行:只建不跑"
assert turn.next_step(_final(autorun=False, asked={"question": "哪个月?", "options": []})) == "ready"
assert turn.next_step(_final(issues=[ERROR])) == "fail", "自查过还有 error:判失败,不再整轮重来"
assert turn.next_step(_final(issues=[ERROR]), self_checked=False) == "retry", "老服务端没自查:重来一次"
assert turn.next_step(_final(issues=[ERROR], autorun=False)) == "fail", "没过检查比只建不跑优先"
# 这一轮的运行已经在生成里跑完了(固定执行的 final 带 execution.run_id):运行已经发生,不能再按 ready / run 解释一遍
done = _final(autorun=False, execution={"via": "querycore", "run_id": "run-1", "status": "succeeded"})
assert turn.next_step(done) == "done"
assert turn.next_step(done, self_checked=False) == "done"
assert turn.next_step(_final(execution={"via": "querycore"})) == "run", "没有运行 id:不算已经跑过"
assert turn.next_step(_final(execution={"run_id": None})) == "run"
assert turn.next_step({"op": "reply", "text": "已经查过了"}) == "reply"
assert turn.next_step({"op": "error", "message": "x"}) == "fail"
empty = {"op": "error", "message": "助手本轮未完成:模型没有给出任何修改,也没有说明原因",
"code": turn.EMPTY_GENERATION, "reason": "模型没有给出任何修改,也没有说明原因"}
assert turn.next_step(empty) == "retry", "只写了思考、一个操作都没给:按重来一次的规矩自动重来"
assert turn.error_reason(empty) == "模型没有给出任何修改,也没有说明原因"
assert turn.error_reason({"op": "error", "message": "助手本轮未完成:接入返回了 502"}) == "接入返回了 502"
assert turn.next_step(None) == "asked", "流结束时只有反问"
assert set(turn.NEXT_STEPS) == {"run", "ready", "done", "fail"}
# --------------------------------------------------------------------------
# fill_input
# --------------------------------------------------------------------------
def _graph(*fields: dict) -> dict:
return {"nodes": [{"id": "start", "type": "input", "data": {"label": "输入", "config": {"fields": list(fields)}}},
{"id": "out", "type": "output", "data": {"label": "成果", "config": {}}}], "edges": []}
def test_fill_input_rules():
assert turn.fill_input(_graph(), "上周销售额", "问:…") == {"question": "上周销售额", "history": "问:…"}
assert turn.fill_input({"nodes": []}, "上周销售额") == {"question": "上周销售额", "history": ""}
# 名字像「问题」的优先,其余字段填默认值(没有就空串)
got = turn.fill_input(_graph({"name": "region", "default": "华东"}, {"name": "topic"}, {"name": "limit"}), "退款率")
assert got == {"history": "", "region": "华东", "topic": "退款率", "limit": ""}
got = turn.fill_input(_graph({"name": "门店"}, {"name": "需求说明"}), "退款率")
assert got["需求说明"] == "退款率" and got["门店"] == ""
# 没有像问题的:第一个必填的;再没有:第一个
got = turn.fill_input(_graph({"name": "a"}, {"name": "b", "required": True}), "退款率")
assert got["b"] == "退款率" and got["a"] == ""
got = turn.fill_input(_graph({"name": "a"}, {"name": "b"}), "退款率")
assert got["a"] == "退款率"
# 大小写不敏感
assert turn.fill_input(_graph({"name": "UserQuery"}), "退款率")["UserQuery"] == "退款率"
# --------------------------------------------------------------------------
# 复核之后:要不要重来、说什么
# --------------------------------------------------------------------------
BROKEN = {"retry": True, "severity": "broken", "note": "结论不可用",
"signals": [{"kind": "step_limit", "detail": "agent 用满了 8 步"}, {"kind": "tool_error", "detail": "表不存在"}]}
def test_retry_after_review():
assert turn.retry_after_review(BROKEN, 0) is True
assert turn.retry_after_review(BROKEN, 1) is False, "一轮最多自动再试一次"
assert turn.retry_after_review({**BROKEN, "severity": "degraded"}, 0) is False
assert turn.retry_after_review({**BROKEN, "retry": False}, 0) is False
assert turn.retry_after_review(None, 0) is False
assert turn.review_reason(BROKEN) == "agent 用满了 8 步;表不存在"
assert turn.review_reason({"signals": [], "note": "结论不可用"}) == "结论不可用"
def test_retry_instruction_two_wordings():
with_base = turn.retry_instruction("上周销售额", "表不存在", True)
assert with_base.startswith("这个工作流上一次运行没有成功:表不存在。\n\n请针对这个原因改它")
assert with_base.endswith("改完继续回答原来的问题:上周销售额")
scratch = turn.retry_instruction("上周销售额", "表不存在", False)
assert scratch == "上周销售额\n\n(上一次没有成功:表不存在。请针对这个原因调整,重新给出完整的工作流。)"
# --------------------------------------------------------------------------
# 页面不再留一份同样的决定
# --------------------------------------------------------------------------
def _ts() -> str:
return CHAT_TS.read_text(encoding="utf-8")
def test_the_page_no_longer_keeps_its_own_copy_of_the_decisions():
"""一轮整个搬到服务端之后(docs/design/orchestration-round §1),问数据页只发起、只播放:要不要运行、要不要重来、重来说
什么、入口怎么填都只在这里(和 chat/drive.py),页面上那份退路(MAX_ATTEMPTS、fillInput、retryInstruction)去掉了——
两份并存迟早走样,界面层也不放逻辑。以前这里逐字对照两边的常量和措辞,现在守的是页面上没有这几样。"""
ts = _ts()
for name in ("const MAX_ATTEMPTS", "function fillInput", "function retryInstruction", "streamCopilot(",
"api.copilot.review(", "api.runs.start(", "patchTurn("):
assert name not in ts, f"chat.ts 里又出现了 {name}:这一轮的决定、发起、落库应在服务端"
def test_user_retry_wordings():
"""用户点「重试本轮」时交给助手的话(以前在页面 retryTurn 里拼):有图时说「没有给出可用的结果」,没图的同从零重建。"""
with_base = turn.user_retry_instruction("上周销售额", "已手动停止", True)
assert with_base == ("这个工作流上一次没有给出可用的结果:已手动停止。\n\n请针对这个原因改它(比如调大最大步数、改正工具参数、"
"换个查询方式),改完继续回答原来的问题:上周销售额")
assert turn.user_retry_instruction("上周销售额", "已手动停止", False) == turn.retry_instruction(
"上周销售额", "已手动停止", False)
# --------------------------------------------------------------------------
# /copilot/review 给 retry_instruction
# --------------------------------------------------------------------------
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
async def _run(output: dict, graph: dict, code: str) -> str:
from app.core.events import EventType
from app.db.base import SessionLocal
from app.db.models import Run, RunEvent
async with SessionLocal() as session:
run = Run(id=uuid.uuid4().hex, status="succeeded", output=output, graph=graph)
session.add(run)
await session.flush()
session.add(RunEvent(run_id=run.id, seq=1, type=EventType.LOG, node_id="a", ts=0.0,
data={"level": "warn", "code": code, "message": "agent 用满了 8 步还没给出结论"}))
await session.commit()
return run.id
def test_the_page_says_the_server_is_retrying_an_empty_generation():
"""只写了思考、一个操作都没给(error.next=retry)时,重来一次是服务端定的(chat/drive.py 照 next_step 走,问数据的一轮记一条
turn.retry,kind=empty);页面照它把这一轮重置、写「助手上一次没有给出任何操作,正在重新生成工作流…」。读 chat.ts 里
turn.retry 那一支(行为另由 tests/test_chat_turns.py 和 scripts/check-chat.mjs 的 emptyretry 一节守)。"""
branch = re.search(r"function retryPatch\((.*?)\n\}", _ts(), re.S)
assert branch, "chat.ts 里找不到 retryPatch"
assert re.search(r"d\.kind === 'empty' \? CHAT_RETRY_TEXT\.empty", branch.group(1)), branch.group(1)
async def test_answer_ops_says_to_retry_an_empty_generation(client, monkeypatch):
"""生成过程一个操作都没给(deepseek 偶发只出思考):error 操作带机读码 copilot_empty、原因、next=retry。"""
import json
import app.api.copilot as copilot
from tests.test_usage_meter import Scripted
model = Scripted(model="claude-opus-5", replies=[[""]])
async def _model(*_a, **_k):
return model, "claude-opus-5"
monkeypatch.setattr(copilot, "get_chat_model", _model)
events = []
async with client.stream("POST", "/api/copilot/generate-stream",
json={"instruction": "上周各地区销售额", "intent": "answer"}) as r:
async for line in r.aiter_lines():
if line.startswith("data: "):
events.append(json.loads(line[6:]))
[err] = [e for e in events if e["op"] == "error"]
assert err["code"] == turn.EMPTY_GENERATION and err["next"] == "retry", err
# 原因是重来时交给助手的那句纠正(这次连思考都没有),诊断写在 detail(不含原文)
assert err["reason"].startswith("什么都没有输出") and "look" in err["reason"], err
# 说法按诊断照实写;起草时已经当场纠正过一次(第二次还是空的),整轮才交 error
assert err["message"] == "助手本轮未完成:模型什么都没有输出"
assert "正文 0 字,思考 0 字" in err["detail"] and err["diagnostics"]["ops"] == 0
assert err["diagnostics"]["nudged"] is True and model.calls == 2, err
async def test_the_eval_driver_retries_an_empty_generation_once(monkeypatch):
"""评测的作答(app/eval/answer.Driver)走同一个决定:第一次一个操作都没给就带着原因重来一次,原因照实记在 retries;
重来还不行才照实报建图失败(助手调用失败),不再试第三次。每一次生成里起草时已经当场纠正过一次(copilot_gen
FORMAT_NUDGE_PROMPT),所以一次生成算两次调用。"""
import json
import app.api.copilot as copilot
from app.eval import answer
from tests.test_usage_meter import Scripted
reply = json.dumps({"op": "reply", "text": "上周没有华中区的数据"}, ensure_ascii=False)
for replies, path, calls in (([[""], [""], [reply]], "reply", 3), ([[""]], "build_failed", 4)):
model = Scripted(model="claude-opus-5", replies=replies)
seen: list[str] = []
async def _model(*_a, _m=model, **_k):
return _m, "claude-opus-5"
monkeypatch.setattr(copilot, "get_chat_model", _model)
real = answer.Driver._generate
async def spy(self, instruction, base_graph, _seen=seen, _real=real):
_seen.append(instruction)
return await _real(self, instruction, base_graph)
monkeypatch.setattr(answer.Driver, "_generate", spy)
driver = answer.Driver("上周华中区销售额", datasource_ids=[], ctx=answer.TurnContext(
suite_id="s", suite_name="虚构题集", eval_run_id="e", case_id=None, case_seq=1, actor=None))
t = await driver.drive()
assert t.path == path and t.attempts == 1 and model.calls == calls, (t.path, t.attempts, model.calls, t.reason)
assert len(t.retries) == 1 and t.retries[0].startswith("什么都没有输出"), t.retries
# 重来时问题后面附那句纠正,不原样重发
assert seen[1] == turn.empty_retry_instruction("上周华中区销售额", t.retries[0]) != seen[0], seen
assert "什么都没有输出" in seen[1]
if path == "build_failed":
assert t.copilot_failed and t.reason.startswith("助手本轮未完成"), t.reason
assert t.usage["assistant"]["calls"] == calls, "两次生成(连同各自当场纠正的那一次)都是这一题花的"
async def test_review_says_whether_and_how_to_retry(client, monkeypatch):
import app.api.copilot as copilot
from app.providers.factory import ProviderNotConfigured
async def unconfigured(*_a, **_k):
raise ProviderNotConfigured("没有可用的 provider")
# 没配模型:规则层照样判得出不可用、该重来
monkeypatch.setattr(copilot, "get_chat_model", unconfigured)
graph = _graph({"name": "question"})
run_id = await _run({"answer": "大概 3 家"}, graph, "step_limit")
body = (await client.post("/api/copilot/review", json={"run_id": run_id, "question": "有几家门店"})).json()
assert body["severity"] == "broken" and body["retry"] is True
assert body["retry_instruction"] == turn.retry_instruction("有几家门店", turn.review_reason(body), True)
# 这一轮已经重来过一次:不再给
body = (await client.post("/api/copilot/review",
json={"run_id": run_id, "question": "有几家门店", "attempt": 1})).json()
assert body["retry_instruction"] is None
# 运行的图是空的:措辞换成从零再建
bare = await _run({"answer": "大概 3 家"}, {}, "step_limit")
body = (await client.post("/api/copilot/review", json={"run_id": bare, "question": "有几家门店"})).json()
assert body["retry_instruction"].startswith("有几家门店\n\n(上一次没有成功:")
# 干净的运行:不重来
from app.db.base import SessionLocal
from app.db.models import Run
async with SessionLocal() as session:
clean = Run(status="succeeded", output={"answer": "3 家"}, graph=graph)
session.add(clean)
await session.commit()
clean_id = clean.id
body = (await client.post("/api/copilot/review", json={"run_id": clean_id, "question": "有几家门店"})).json()
assert body["verdict"] == "ok" and body["retry_instruction"] is None