mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 05:31:59 +00:00
ThearchyHelios
805 lines
42 KiB
Python
805 lines
42 KiB
Python
"""审批质询（engine/inquiry.py、api/inquiry.py）：运行停在人工审批时，审批人针对报告里的一句话提问，一个只读 agent
|
||
现查现答；问答写进这次运行（approval.inquiry 事件），随运行结束一起封存，报告本身一个字不改。
|
||
|
||
- 停在审批时质询：事件 node_id 为空、数据按前后端约定的形状；质询的查询工件只在 inquiry_artifacts / queries 里，
|
||
不在 query_artifact 下（证据接口不把它们当成报告的证据）；批准、运行结束封存之后 verify 一致，封存范围包含这条事件；
|
||
用量计入运行用量
|
||
- 回答期间运行被批准（竞态）：回答照样交回，recorded 为假、note 说明，事件不落库
|
||
- 同一运行同时只答一条：第二条质询、审批回复（批准 / 驳回）、继续运行都回 409 inquiry_busy，拒掉的批准不留痕迹
|
||
- 封存之后：探索运行可以追问（post_seal，落在封存之后，不计运行用量），正式运行 409；设置里关了 409；
|
||
没停在那张审批卡上、还在执行、封存核对不通过都 409；输入不合法 422，指明是哪一项
|
||
- 回答：先给结论，数字只能引用自己查询的格子或报告已有的指标，裸数字被标出；遮罩列的格子渲染成「已遮罩」，
|
||
回答正文、查询清单、事件里都不出现遮罩列的值
|
||
- 触顶、失败照样记一条事件
|
||
- 读取：列表按报告、句子筛选，按时间排；质询工件只认质询引用过的，遮罩照证据面板
|
||
- 恢复后的重放不受影响：前面的节点不重复执行
|
||
|
||
模型一律是 MockChatModel 剧本，数据源是现造的 SQLite 文件，区域、金额、手机号都是虚构的。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import sqlite3
|
||
import uuid
|
||
from dataclasses import dataclass, field
|
||
|
||
import pytest
|
||
from httpx import ASGITransport, AsyncClient
|
||
from langchain_core.messages import AIMessage, ToolMessage
|
||
from sqlalchemy import delete, select
|
||
|
||
from app.core import artifact_store
|
||
from app.data import introspect
|
||
from app.data.engine import engines
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import Approval, DataSource, Run, RunEvent, Setting
|
||
from app.engine import inquiry as inquiry_mod
|
||
from app.engine import spend, verify_agent
|
||
from app.engine.evidence import iter_segments, iter_units
|
||
from app.engine.judge import SPEND_KEY as JUDGE_SPEND_KEY
|
||
from app.engine.nodes import report as report_mod
|
||
from app.engine.runner import run_manager, verify_manifest
|
||
from app.main import app
|
||
from app.providers.factory import ProviderNotConfigured
|
||
from app.providers.mock_model import MockChatModel
|
||
|
||
PRICED = "claude-sonnet-5"
|
||
#: 各区域销售额：华东 1500（其中 300 是测试账号下的单）、华南 900、华北 400
|
||
QUERY = "SELECT region AS 区域, SUM(amount) AS 销售额 FROM sales GROUP BY region ORDER BY 销售额 DESC"
|
||
TEXT = "华东销售额 [[v:Q1.r0.销售额]]，居各区域之首。[[see:Q1]]华南销售额 [[v:Q1.r1.销售额]]。[[see:Q1]]"
|
||
EAST, SOUTH = "华东销售额 1,500，居各区域之首。", "华南销售额 900。"
|
||
NO_TEST = ("SELECT region AS 区域, SUM(amount) AS 销售额 FROM sales WHERE is_test = 0 GROUP BY region "
|
||
"ORDER BY 销售额 DESC")
|
||
NO_TEST_ANSWER = "仍居各区域之首：排除测试账号后华东为 [[v:Q1.r0.销售额]]，华南为 [[v:Q1.r1.销售额]]。[[see:Q1]]"
|
||
NO_TEST_TEXT = "仍居各区域之首：排除测试账号后华东为 1,200，华南为 900。"
|
||
QUESTION = "排除测试账号会怎样？"
|
||
USAGE_KEYS = {"input_tokens", "output_tokens", "cost_usd", "calls", "unpriced_calls"}
|
||
ITEM_KEYS = {"seq", "at", "actor", "report", "unit", "quote", "question", "answer", "queries", "model", "usage",
|
||
"post_seal", "approval_id"}
|
||
EVENT_KEYS = {"question", "quote", "report", "unit", "approval_id", "actor", "answer_text", "answer_doc", "ok",
|
||
"limit", "error", "inquiry_artifacts", "queries", "model", "usage"}
|
||
QUERY_KEYS = {"alias", "artifact", "sql", "row_count", "truncated", "source"}
|
||
NOT_RECORDED = "运行状态已经变化，这次质询的问答没有写入运行记录"
|
||
BUSY_DECIDE = {"detail": "正在回答质询，回答完再批准或驳回", "code": "inquiry_busy"}
|
||
|
||
|
||
async def _clean_settings() -> None:
|
||
async with SessionLocal() as session:
|
||
await session.execute(delete(Setting).where(Setting.key.in_(
|
||
[spend.SPEND_KEY, JUDGE_SPEND_KEY, "verify", "judge", "copilot"])))
|
||
await session.commit()
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
async def engine_up():
|
||
await run_manager.setup()
|
||
await _clean_settings()
|
||
yield
|
||
await run_manager.shutdown()
|
||
# 这里写过的核查设置、花费账不留给同一进程里后面的测试文件
|
||
await _clean_settings()
|
||
|
||
|
||
@pytest.fixture
|
||
async def client():
|
||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||
yield c
|
||
|
||
|
||
async def _source(tmp_path, *, options=None) -> tuple[str, int]:
|
||
path = tmp_path / f"{uuid.uuid4().hex[:8]}.db"
|
||
db = sqlite3.connect(path)
|
||
db.executescript(
|
||
"CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, buyer_phone TEXT, amount INTEGER, is_test INTEGER);"
|
||
"INSERT INTO sales VALUES (1, '华东', '13900000001', 1200, 0), (2, '华东', '13900000002', 300, 1),"
|
||
" (3, '华南', '13900000003', 900, 0), (4, '华北', '13900000004', 400, 0);")
|
||
db.commit()
|
||
db.close()
|
||
name = f"shop_{uuid.uuid4().hex[:8]}"
|
||
async with SessionLocal() as session:
|
||
row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True,
|
||
options=options or {})
|
||
session.add(row)
|
||
await session.flush()
|
||
row.schema_cache = await introspect.introspect(row)
|
||
await session.commit()
|
||
return name, row.id
|
||
|
||
|
||
@pytest.fixture
|
||
async def shop(tmp_path):
|
||
"""只读的手工源：四笔订单，华东两笔（其中一笔 300 是测试账号）、华南一笔 900、华北一笔 400。返回源名。"""
|
||
name, source_id = await _source(tmp_path)
|
||
yield name
|
||
await engines.invalidate(source_id)
|
||
|
||
|
||
@pytest.fixture
|
||
async def masked_shop(tmp_path):
|
||
"""同样的四笔订单，买家手机号这一列设了遮罩。"""
|
||
name, source_id = await _source(tmp_path, options={"mask_columns": "buyer_phone"})
|
||
yield name
|
||
await engines.invalidate(source_id)
|
||
|
||
|
||
def node(nid, ntype, label=None, **config):
|
||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}}
|
||
|
||
|
||
def chain(*nodes):
|
||
return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
||
|
||
|
||
def gated(source: str, *, gate: bool = True) -> dict:
|
||
"""取数 → 报告 → 人工审批 → 成果。gate 为假时没有审批，一口气跑完、封存。"""
|
||
return chain(
|
||
node("start", "input"),
|
||
node("fetch", "tool", tool=f"db_query__{source}", args={"sql": QUERY}),
|
||
node("write", "report", "写周报", instructions="写周报"),
|
||
*([node("gate", "human", mode="approve", title="周报可以出具吗")] if gate else []),
|
||
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]),
|
||
)
|
||
|
||
|
||
@dataclass
|
||
class Play:
|
||
"""质询 agent 对某个问题的剧本：依次执行的查询，最后交的回答。"""
|
||
|
||
queries: list[str] = field(default_factory=list)
|
||
answer: str = "查不到判断所需的数据。"
|
||
|
||
|
||
PLAYS = {"排除测试账号": Play([NO_TEST], NO_TEST_ANSWER)}
|
||
|
||
|
||
class Models:
|
||
"""写作者和质询 agent 两份剧本。写作者回 writer；质询 agent 按任务里的问题认出剧本，照着查询、交回答。
|
||
|
||
before：质询 agent 每次调用模型之前先 await 它（测试在这里卡住回答、或者在回答途中批掉审批）。
|
||
tasks 记下质询 agent 收到的任务原文，finals 记下交回答那次调用收到的消息，specs 记下每次要的模型。"""
|
||
|
||
def __init__(self, monkeypatch, source: str | None = None, *, writer: str = TEXT, plays=None,
|
||
model_id: str = PRICED, broken: Exception | None = None, before=None):
|
||
self.tasks: list[str] = []
|
||
self.finals: list[list] = []
|
||
self.writes = 0
|
||
self.specs: list = []
|
||
plays = PLAYS if plays is None else plays
|
||
|
||
def which(task: str) -> Play:
|
||
return next((p for k, p in plays.items() if k in task), Play())
|
||
|
||
def decide(model, messages):
|
||
system = str(messages[0].content)
|
||
if inquiry_mod.INQUIRY_SYSTEM in system:
|
||
task = str(messages[1].content) if len(messages) > 1 else ""
|
||
play = which(task)
|
||
if model.response_format:
|
||
self.finals.append(list(messages))
|
||
return AIMessage(content=json.dumps({"rationale": play.answer}, ensure_ascii=False))
|
||
if not any(isinstance(m, ToolMessage) for m in messages):
|
||
self.tasks.append(task)
|
||
done = sum(1 for m in messages if isinstance(m, ToolMessage))
|
||
if done < len(play.queries):
|
||
call = {"name": f"db_query__{source}", "args": {"sql": play.queries[done]}, "id": f"iq-{done}"}
|
||
return AIMessage(content="查一下。", tool_calls=[call])
|
||
return AIMessage(content="查完了。")
|
||
if report_mod.ROLE in system:
|
||
self.writes += 1
|
||
return AIMessage(content=writer)
|
||
raise AssertionError(f"没想到的模型调用：{system[:60]}")
|
||
|
||
monkeypatch.setattr(MockChatModel, "_decide", decide)
|
||
original = MockChatModel._agenerate
|
||
|
||
async def held(chat, messages, stop=None, run_manager=None, **kwargs):
|
||
if before is not None and inquiry_mod.INQUIRY_SYSTEM in str(messages[0].content):
|
||
await before()
|
||
return await original(chat, messages, stop, run_manager, **kwargs)
|
||
|
||
monkeypatch.setattr(MockChatModel, "_agenerate", held)
|
||
|
||
async def verify_model(session, spec):
|
||
self.specs.append(spec)
|
||
if broken is not None:
|
||
raise broken
|
||
return MockChatModel(model_name=model_id), model_id
|
||
|
||
monkeypatch.setattr(verify_agent, "get_chat_model", verify_model)
|
||
|
||
|
||
async def _row(run_id: str) -> Run:
|
||
async with SessionLocal() as session:
|
||
return await session.get(Run, run_id)
|
||
|
||
|
||
async def wait(run_id: str, statuses=("succeeded", "failed")) -> Run:
|
||
for _ in range(600):
|
||
row = await _row(run_id)
|
||
if row.status in statuses and (row.status == "interrupted" or row.manifest_seq is not None):
|
||
await run_manager.wait_idle(run_id)
|
||
return await _row(run_id)
|
||
await asyncio.sleep(0.03)
|
||
raise AssertionError(f"运行 {run_id} 停在 {row.status}")
|
||
|
||
|
||
async def pause(source: str, *, run_class: str = "exploratory", graph: dict | None = None) -> Run:
|
||
run = await run_manager.start(graph=graph or gated(source), input_payload={}, run_class=run_class)
|
||
return await wait(run.id, ("interrupted",))
|
||
|
||
|
||
async def finish(source: str, *, run_class: str = "exploratory", graph: dict | None = None) -> Run:
|
||
run = await run_manager.start(graph=graph or gated(source, gate=False), input_payload={}, run_class=run_class)
|
||
return await wait(run.id)
|
||
|
||
|
||
async def pending(run_id: str) -> list[Approval]:
|
||
async with SessionLocal() as session:
|
||
return list((await session.execute(
|
||
select(Approval).where(Approval.run_id == run_id, Approval.status == "pending"))).scalars())
|
||
|
||
|
||
async def events(run_id: str, etype: str, node_id: str | None = None) -> list[RunEvent]:
|
||
async with SessionLocal() as session:
|
||
q = select(RunEvent).where(RunEvent.run_id == run_id, RunEvent.type == etype)
|
||
if node_id:
|
||
q = q.where(RunEvent.node_id == node_id)
|
||
return list((await session.execute(q.order_by(RunEvent.seq))).scalars())
|
||
|
||
|
||
async def report_doc(run_id: str) -> dict:
|
||
[checked] = await events(run_id, "report.checked", "write")
|
||
return artifact_store.load(checked.data["doc_artifact"])
|
||
|
||
|
||
def unit_of(doc: dict, text: str) -> str:
|
||
md = doc["markdown"]
|
||
return next(u["id"] for _, u in iter_units(doc) if md[u["span"][0]:u["span"][1]].strip() == text)
|
||
|
||
|
||
async def ask(client, run_id: str, **body):
|
||
return await client.post(f"/api/runs/{run_id}/inquiries", json=body)
|
||
|
||
|
||
async def set_verify(value: dict) -> None:
|
||
async with SessionLocal() as session:
|
||
session.add(Setting(key="verify", value=value))
|
||
await session.commit()
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 停在审批时质询 → 写进事件 → 批准 → 封存
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_asking_while_waiting_is_recorded_and_sealed_with_the_run(monkeypatch, shop, client):
|
||
models = Models(monkeypatch, shop)
|
||
paused = await pause(shop)
|
||
[approval] = await pending(paused.id)
|
||
doc = await report_doc(paused.id)
|
||
east = unit_of(doc, EAST)
|
||
|
||
r = await ask(client, paused.id, report="write", unit=east, quote="居各区域之首", question=QUESTION,
|
||
approval_id=approval.id)
|
||
assert r.status_code == 200, r.text
|
||
body = r.json()
|
||
assert body.keys() == {"inquiry", "recorded", "note"}
|
||
assert body["recorded"] is True and body["note"] is None
|
||
item = body["inquiry"]
|
||
assert item.keys() == ITEM_KEYS
|
||
assert (item["report"], item["unit"], item["quote"], item["question"], item["approval_id"]) == (
|
||
"write", east, "居各区域之首", QUESTION, approval.id)
|
||
assert item["post_seal"] is False and item["actor"] is None and isinstance(item["at"], str)
|
||
answer = item["answer"]
|
||
assert answer.keys() == {"text", "doc_artifact", "ok", "limit", "error"}
|
||
assert (answer["ok"], answer["limit"], answer["error"]) == (True, None, None)
|
||
assert answer["text"] == NO_TEST_TEXT, "回答先给结论，数字按查询结果换好"
|
||
answer_doc = artifact_store.load(answer["doc_artifact"])
|
||
assert answer_doc["markdown"] == answer["text"] and answer_doc["violations"] == []
|
||
cited = [s["cite"] for s in iter_segments(answer_doc) if s.get("cite")]
|
||
assert [(c["alias"], c["locator"]["row"], c["value"]) for c in cited] == [("Q1", 0, 1200), ("Q1", 1, 900)]
|
||
[query] = item["queries"]
|
||
assert query.keys() == QUERY_KEYS
|
||
assert (query["alias"], query["source"], query["sql"], query["row_count"], query["truncated"]) == (
|
||
"Q1", shop, NO_TEST, 3, False)
|
||
assert artifact_store.load(query["artifact"])["rows"] == [["华东", 1200], ["华南", 900], ["华北", 400]]
|
||
assert item["model"] == PRICED
|
||
assert item["usage"].keys() == USAGE_KEYS and item["usage"]["calls"] >= 2 and item["usage"]["cost_usd"] > 0
|
||
|
||
# 交给质询 agent 的：被问的那句、它挂的证据摘录（原报告的编号标明）、选中的原文、问题、可以查的数据源
|
||
[task] = models.tasks
|
||
assert EAST in task and "【原报告 Q1】" in task and "居各区域之首" in task and QUESTION in task and shop in task
|
||
|
||
# 事件：node_id 为空，数据按约定；查询工件不在 query_artifact 下
|
||
[event] = await events(paused.id, "approval.inquiry")
|
||
assert event.node_id is None and event.seq == item["seq"]
|
||
assert event.data.keys() == EVENT_KEYS
|
||
assert event.data["inquiry_artifacts"] == [query["artifact"]]
|
||
assert event.data["queries"] == item["queries"]
|
||
assert (event.data["answer_text"], event.data["answer_doc"]) == (answer["text"], answer["doc_artifact"])
|
||
assert "query_artifact" not in event.data
|
||
assert (await _row(paused.id)).status == "interrupted"
|
||
|
||
# 列表里是同一条
|
||
listed = await client.get(f"/api/runs/{paused.id}/inquiries")
|
||
assert listed.status_code == 200 and listed.json() == {"items": [item]}
|
||
|
||
# 批准 → 运行结束封存：这条事件在封存范围里，verify 一致
|
||
r = await client.post(f"/api/approvals/{approval.id}/decide", json={"approved": True})
|
||
assert r.status_code == 200, r.text
|
||
final = await wait(paused.id)
|
||
assert final.status == "succeeded", final.error
|
||
assert final.manifest_seq > event.seq
|
||
assert (await verify_manifest(paused.id))["ok"] is True
|
||
# 用量计进运行用量，明细在 inquiry 下
|
||
assert final.usage["inquiry"]["calls"] == item["usage"]["calls"]
|
||
assert final.usage["inquiry"]["input_tokens"] == item["usage"]["input_tokens"]
|
||
# 证据接口不把质询的查询当成报告的证据
|
||
from app.api.evidence import _sealed
|
||
|
||
assert query["artifact"] not in (await _sealed(paused.id)).queries
|
||
# 报告本身一个字没改
|
||
assert (await report_doc(paused.id)) == doc
|
||
# 改它一个字，封存核对就对不上：它确实在封存范围里
|
||
async with SessionLocal() as session:
|
||
stored = await session.get(RunEvent, event.id)
|
||
stored.data = {**stored.data, "answer_text": "改过的回答"}
|
||
await session.commit()
|
||
assert (await verify_manifest(paused.id))["ok"] is False
|
||
|
||
|
||
async def test_an_approval_that_lands_while_answering_leaves_no_record(monkeypatch, shop, client):
|
||
"""回答途中运行被批准了（另一个进程、或者抢在登记之前的批准）：回答照样交回给提问的人，但不写进运行记录。"""
|
||
seen: dict = {}
|
||
|
||
async def approve_meanwhile():
|
||
if not seen.get("done"):
|
||
seen["done"] = True
|
||
await run_manager.resume(seen["run"], {"approved": True}, approval_id=seen["approval"])
|
||
|
||
Models(monkeypatch, shop, before=approve_meanwhile)
|
||
paused = await pause(shop)
|
||
[approval] = await pending(paused.id)
|
||
seen.update(run=paused.id, approval=approval.id)
|
||
|
||
r = await ask(client, paused.id, report="write", unit=None, question=QUESTION, approval_id=approval.id)
|
||
assert r.status_code == 200, r.text
|
||
body = r.json()
|
||
assert body["recorded"] is False and body["note"] == NOT_RECORDED
|
||
assert body["inquiry"]["seq"] is None and body["inquiry"]["answer"]["ok"] is True
|
||
assert body["inquiry"]["answer"]["text"] == NO_TEST_TEXT
|
||
|
||
final = await wait(paused.id)
|
||
assert final.status == "succeeded", final.error
|
||
assert await events(paused.id, "approval.inquiry") == []
|
||
assert (await verify_manifest(paused.id))["ok"] is True
|
||
assert "inquiry" not in final.usage
|
||
assert (await client.get(f"/api/runs/{paused.id}/inquiries")).json() == {"items": []}
|
||
|
||
|
||
async def test_while_answering_approving_resuming_and_continuing_are_refused(monkeypatch, shop, client):
|
||
entered, release = asyncio.Event(), asyncio.Event()
|
||
|
||
async def hold():
|
||
entered.set()
|
||
await release.wait()
|
||
|
||
Models(monkeypatch, shop, before=hold)
|
||
paused = await pause(shop)
|
||
[approval] = await pending(paused.id)
|
||
asking = asyncio.create_task(ask(client, paused.id, report="write", unit=None, question=QUESTION,
|
||
approval_id=approval.id))
|
||
try:
|
||
await asyncio.wait_for(entered.wait(), 10)
|
||
refused = await client.post(f"/api/approvals/{approval.id}/decide", json={"approved": False, "note": "先别批"},
|
||
headers={"X-Actor": "reviewer"})
|
||
assert refused.status_code == 409 and refused.json() == BUSY_DECIDE
|
||
refused = await client.post(f"/api/runs/{paused.id}/resume",
|
||
json={"response": {"approved": True}, "approval_id": approval.id})
|
||
assert refused.status_code == 409 and refused.json() == BUSY_DECIDE
|
||
refused = await client.post(f"/api/runs/{paused.id}/continue", json={})
|
||
assert refused.status_code == 409
|
||
assert refused.json() == {"detail": "正在回答质询，回答完再继续运行", "code": "inquiry_busy"}
|
||
second = await ask(client, paused.id, report="write", unit=None, question="华南为什么低？",
|
||
approval_id=approval.id)
|
||
assert second.status_code == 409 and second.json()["code"] == "inquiry_busy"
|
||
# 拒掉的批准不留痕迹：卡仍待审批，没记下签批人
|
||
async with SessionLocal() as session:
|
||
card = await session.get(Approval, approval.id)
|
||
assert card.status == "pending" and card.resolved_by is None
|
||
assert (await _row(paused.id)).status == "interrupted"
|
||
finally:
|
||
release.set()
|
||
answered = await asyncio.wait_for(asking, 30)
|
||
assert answered.status_code == 200 and answered.json()["recorded"] is True
|
||
|
||
# 回答完了，照常批准
|
||
r = await client.post(f"/api/approvals/{approval.id}/decide", json={"approved": True})
|
||
assert r.status_code == 200, r.text
|
||
assert (await wait(paused.id)).status == "succeeded"
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 封存之后：探索运行可以追问，正式运行不行
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_after_the_seal_only_exploratory_runs_take_inquiries(monkeypatch, shop, client):
|
||
Models(monkeypatch, shop)
|
||
done = await finish(shop)
|
||
assert done.status == "succeeded", done.error
|
||
east = unit_of(await report_doc(done.id), EAST)
|
||
|
||
r = await ask(client, done.id, report="write", unit=east, question=QUESTION)
|
||
assert r.status_code == 200, r.text
|
||
body = r.json()
|
||
assert body["recorded"] is True and body["note"] is None
|
||
item = body["inquiry"]
|
||
assert item["post_seal"] is True and item["approval_id"] is None and item["answer"]["text"] == NO_TEST_TEXT
|
||
[event] = await events(done.id, "approval.inquiry")
|
||
assert event.data["post_seal"] is True and event.seq > done.manifest_seq
|
||
after = await _row(done.id)
|
||
assert (after.manifest_seq, after.manifest_hash) == (done.manifest_seq, done.manifest_hash)
|
||
assert (await verify_manifest(done.id))["ok"] is True
|
||
assert "inquiry" not in after.usage, "封存之后的问答是批注，不计进运行用量"
|
||
assert (await client.get(f"/api/runs/{done.id}/inquiries")).json()["items"] == [item]
|
||
|
||
# 带着审批卡来问：那张卡已经不在等了
|
||
r = await ask(client, done.id, report="write", unit=east, question=QUESTION, approval_id="no-such-card")
|
||
assert r.status_code == 409 and r.json()["code"] == "inquiry_not_open"
|
||
|
||
formal = await finish(shop, run_class="formal")
|
||
assert formal.status == "succeeded", formal.error
|
||
r = await ask(client, formal.id, report="write", unit=unit_of(await report_doc(formal.id), EAST),
|
||
question=QUESTION)
|
||
assert r.status_code == 409 and r.json()["code"] == "inquiry_not_open"
|
||
assert "正式运行" in r.json()["detail"]
|
||
assert await events(formal.id, "approval.inquiry") == []
|
||
|
||
|
||
async def test_a_formal_run_waiting_for_approval_takes_inquiries(monkeypatch, shop, client):
|
||
"""正式运行停在审批时可以质询：问答随运行一起封存，正式运行同样可以复核。"""
|
||
Models(monkeypatch, shop)
|
||
paused = await pause(shop, run_class="formal")
|
||
[approval] = await pending(paused.id)
|
||
r = await ask(client, paused.id, report="write", unit=None, question=QUESTION, approval_id=approval.id)
|
||
assert r.status_code == 200 and r.json()["recorded"] is True
|
||
await client.post(f"/api/approvals/{approval.id}/decide", json={"approved": True})
|
||
final = await wait(paused.id)
|
||
[event] = await events(paused.id, "approval.inquiry")
|
||
assert final.manifest_seq > event.seq and (await verify_manifest(paused.id))["ok"] is True
|
||
|
||
|
||
async def test_switched_off_in_settings_it_is_refused(monkeypatch, shop, client):
|
||
await set_verify({"inquiry": {"enabled": False}})
|
||
models = Models(monkeypatch, shop)
|
||
paused = await pause(shop)
|
||
[approval] = await pending(paused.id)
|
||
r = await ask(client, paused.id, report="write", unit=None, question=QUESTION, approval_id=approval.id)
|
||
assert r.status_code == 409 and r.json()["code"] == "inquiry_disabled"
|
||
assert "「审批质询 · 启用」" in r.json()["detail"]
|
||
assert models.tasks == [] and models.specs == []
|
||
assert await events(paused.id, "approval.inquiry") == []
|
||
|
||
|
||
async def test_the_run_must_be_waiting_on_that_approval(monkeypatch, shop, client):
|
||
models = Models(monkeypatch, shop)
|
||
paused = await pause(shop)
|
||
[approval] = await pending(paused.id)
|
||
|
||
r = await ask(client, paused.id, report="write", unit=None, question=QUESTION)
|
||
assert r.status_code == 422 and r.json() == {"detail": r.json()["detail"], "field": "approval_id"}
|
||
r = await ask(client, paused.id, report="write", unit=None, question=QUESTION, approval_id="no-such-card")
|
||
assert r.status_code == 409 and r.json()["code"] == "inquiry_not_open"
|
||
|
||
# 还在执行的运行
|
||
async with SessionLocal() as session:
|
||
running = Run(workflow_name="还在执行", status="running", graph={}, input={})
|
||
session.add(running)
|
||
await session.commit()
|
||
running_id = running.id
|
||
r = await ask(client, running_id, report="write", unit=None, question=QUESTION)
|
||
assert r.status_code == 409 and r.json()["code"] == "inquiry_not_open"
|
||
|
||
r = await ask(client, "no-such-run", report="write", unit=None, question=QUESTION)
|
||
assert r.status_code == 404
|
||
|
||
# 封存核对不通过的探索运行：报告和证据已不可信，不再质询
|
||
done = await finish(shop)
|
||
async with SessionLocal() as session:
|
||
first = (await session.execute(select(RunEvent).where(RunEvent.run_id == done.id)
|
||
.order_by(RunEvent.seq).limit(1))).scalar_one()
|
||
first.data = {**(first.data or {}), "tampered": True}
|
||
await session.commit()
|
||
r = await ask(client, done.id, report="write", unit=None, question=QUESTION)
|
||
assert r.status_code == 409 and r.json()["code"] == "inquiry_not_open"
|
||
assert models.tasks == [], "开放条件不满足时不问模型"
|
||
|
||
|
||
@pytest.mark.parametrize("body, field_name", [
|
||
({"unit": None, "question": QUESTION}, "report"),
|
||
({"report": 3, "question": QUESTION}, "report"),
|
||
({"report": "write", "unit": None}, "question"),
|
||
({"report": "write", "unit": None, "question": " "}, "question"),
|
||
({"report": "write", "unit": None, "question": "问" * 1001}, "question"),
|
||
({"report": "write", "unit": None, "question": QUESTION, "quote": "引" * 501}, "quote"),
|
||
({"report": "write", "unit": None, "question": QUESTION, "quote": 5}, "quote"),
|
||
({"report": "write", "unit": 12, "question": QUESTION}, "unit"),
|
||
({"report": "write", "unit": 0, "question": QUESTION}, "unit"),
|
||
({"report": "write", "unit": None, "question": QUESTION, "approval_id": 7}, "approval_id"),
|
||
({"report": "write", "unit": None, "question": QUESTION, "approval_id": False}, "approval_id"),
|
||
])
|
||
async def test_bad_requests_name_the_field(monkeypatch, shop, client, body, field_name):
|
||
Models(monkeypatch, shop)
|
||
paused = await pause(shop)
|
||
[approval] = await pending(paused.id)
|
||
r = await ask(client, paused.id, **{"approval_id": approval.id, **body})
|
||
assert r.status_code == 422, r.text
|
||
assert r.json().keys() == {"detail", "field"} and r.json()["field"] == field_name
|
||
assert r.json()["detail"] and "[" not in r.json()["detail"]
|
||
|
||
|
||
async def test_the_report_and_the_sentence_must_exist(monkeypatch, shop, client):
|
||
models = Models(monkeypatch, shop)
|
||
paused = await pause(shop)
|
||
[approval] = await pending(paused.id)
|
||
r = await ask(client, paused.id, report="no-such-report", unit=None, question=QUESTION, approval_id=approval.id)
|
||
assert r.status_code == 422 and r.json()["field"] == "report"
|
||
r = await ask(client, paused.id, report="write", unit="u999", question=QUESTION, approval_id=approval.id)
|
||
assert r.status_code == 422 and r.json()["field"] == "unit"
|
||
r = await client.post(f"/api/runs/{paused.id}/inquiries", content=b"[1, 2]",
|
||
headers={"content-type": "application/json"})
|
||
assert r.status_code == 422 and r.json().keys() == {"detail", "field"}
|
||
r = await client.post(f"/api/runs/{paused.id}/inquiries", content=b"{not json",
|
||
headers={"content-type": "application/json"})
|
||
assert r.status_code == 422 and r.json().keys() == {"detail", "field"}
|
||
assert models.tasks == []
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 回答：出处、裸数字、遮罩
|
||
# --------------------------------------------------------------------------
|
||
|
||
TOP_BUYER = "SELECT buyer_phone, amount AS 金额 FROM sales ORDER BY amount DESC LIMIT 1"
|
||
BUYER_ORDERS = "SELECT COUNT(*) AS 笔数 FROM sales WHERE buyer_phone = '13900000001'"
|
||
|
||
|
||
async def test_bare_numbers_are_flagged_and_masked_values_never_show(monkeypatch, masked_shop, client):
|
||
plays = {"买得最多": Play([TOP_BUYER, BUYER_ORDERS],
|
||
"金额最高的一笔来自 [[v:Q1.r0.buyer_phone]]（手机号 13900000001），金额 [[v:Q1.r0.金额]]；"
|
||
"这位客户共 [[v:Q2.r0.笔数]] 笔订单，另有 3 笔小额订单。")}
|
||
models = Models(monkeypatch, masked_shop, plays=plays)
|
||
paused = await pause(masked_shop)
|
||
[approval] = await pending(paused.id)
|
||
r = await ask(client, paused.id, report="write", unit="", question="买得最多的是哪位客户？",
|
||
approval_id=approval.id)
|
||
assert r.status_code == 200, r.text
|
||
assert r.json()["inquiry"]["unit"] is None, "不指明句子时写空串和写 null 一样"
|
||
[task] = models.tasks
|
||
assert f"设了遮罩的列（回答里不要写出这些列的值）：「{masked_shop}」的 buyer_phone" in task
|
||
assert "1390000000" not in r.text, "回答、查询清单里都不能出现遮罩列的值"
|
||
item = r.json()["inquiry"]
|
||
answer = item["answer"]
|
||
assert answer["ok"] is True
|
||
assert answer["text"] == "金额最高的一笔来自 已遮罩（手机号 已遮罩），金额 1,200；这位客户共 1 笔订单，另有 3 笔小额订单。"
|
||
doc = artifact_store.load(answer["doc_artifact"])
|
||
assert "1390000000" not in json.dumps(doc, ensure_ascii=False)
|
||
# 裸数字被标出来：界面在它下面画「无出处」
|
||
assert [s["text"] for s in iter_segments(doc) if s.get("issue") == "uncited_number"] == ["3"]
|
||
# 遮罩列的格子：出处照记，值是「已遮罩」
|
||
masked = [s["cite"] for s in iter_segments(doc) if (s.get("cite") or {}).get("locator", {}).get("column")
|
||
== "buyer_phone"]
|
||
assert [(c["alias"], c["status"], c["value"]) for c in masked] == [("Q1", "resolved", "已遮罩")]
|
||
first, second = item["queries"]
|
||
assert first["sql"] == TOP_BUYER
|
||
assert second["sql"] == "SELECT COUNT(*) AS 笔数 FROM sales WHERE buyer_phone = '已遮罩'"
|
||
[event] = await events(paused.id, "approval.inquiry")
|
||
assert "1390000000" not in json.dumps(event.data, ensure_ascii=False)
|
||
|
||
# 质询工件：遮罩照证据面板，查询里写着的遮罩列的值同样换掉
|
||
got = await client.get(f"/api/runs/{paused.id}/inquiries/artifacts/{first['artifact']}")
|
||
assert got.status_code == 200, got.text
|
||
assert got.json()["content"]["rows"] == [["已遮罩", 1200]] and got.json()["masked"] == ["buyer_phone"]
|
||
got = await client.get(f"/api/runs/{paused.id}/inquiries/artifacts/{second['artifact']}")
|
||
assert got.status_code == 200 and "1390000000" not in got.text
|
||
assert got.json()["content"]["rows"] == [[1]]
|
||
|
||
|
||
async def test_the_answer_may_cite_the_reports_own_metrics(monkeypatch, client):
|
||
"""报告里已有的出处（口径卡指标）回答里可以直接引用；这次运行没用到查询工具，agent 只能依据证据作答。"""
|
||
metrics = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}]
|
||
graph = chain(
|
||
node("start", "input"),
|
||
node("calc", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"),
|
||
node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=metrics),
|
||
node("write", "report", instructions="写周报"),
|
||
node("gate", "human", mode="approve", title="确认"),
|
||
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]),
|
||
)
|
||
plays = {"销售额": Play([], "按口径卡，本周销售额为 [[m:gmv]]，这是本周的口径值。[[see:m:gmv]]")}
|
||
models = Models(monkeypatch, None, writer="本周销售额 [[m:gmv]]。[[see:m:gmv]]", plays=plays)
|
||
paused = await pause("", graph=graph)
|
||
[approval] = await pending(paused.id)
|
||
doc = await report_doc(paused.id)
|
||
r = await ask(client, paused.id, report="write", unit=unit_of(doc, "本周销售额 45,678.5元。"),
|
||
question="销售额是怎么算的？", approval_id=approval.id)
|
||
assert r.status_code == 200, r.text
|
||
answer = r.json()["inquiry"]["answer"]
|
||
assert answer["text"] == "按口径卡，本周销售额为 45,678.5元，这是本周的口径值。"
|
||
assert artifact_store.load(answer["doc_artifact"])["violations"] == []
|
||
[task] = models.tasks
|
||
assert "[[m:gmv]]" in task and "销售额 = 45,678.5元" in task
|
||
assert r.json()["inquiry"]["queries"] == []
|
||
|
||
|
||
async def test_a_cell_in_a_table_comes_with_the_whole_table(monkeypatch, shop, client):
|
||
"""问的是表格里的一格：一格的值离了行列说不清是什么，任务里另带整张表（表头在前）和这一格的出处摘录。"""
|
||
models = Models(monkeypatch, shop, writer="各区域销售额如下。\n\n[[table:Q1]]")
|
||
paused = await pause(shop)
|
||
[approval] = await pending(paused.id)
|
||
doc = await report_doc(paused.id)
|
||
cell = unit_of(doc, "1,500")
|
||
r = await ask(client, paused.id, report="write", unit=cell, question=QUESTION, approval_id=approval.id)
|
||
assert r.status_code == 200, r.text
|
||
[task] = models.tasks
|
||
assert "「1,500」" in task and "区域 | 销售额\n华东 | 1,500\n华南 | 900\n华北 | 400" in task
|
||
# 摘录照裁判的办法：被引用的列排在前面，只给被引用的那一行
|
||
assert "【原报告 Q1】" in task and "列：销售额 | 区域\nr0：1,500 | 华东\n" in task
|
||
|
||
|
||
def test_masked_values_are_matched_as_whole_values():
|
||
"""遮罩列的值按整值认：更长的数里的一截、千分位写法里的一截不换；一个字的值不换（换了会把不相干的词弄坏）；
|
||
数按原样和千分位两种写法认。"""
|
||
secrets = inquiry_mod._Secrets()
|
||
snapshot = {"columns": ["phone", "name", "gender", "code", "salary", "region"],
|
||
"rows": [["13900000001", "张伟", "男", "390", 12000, "华东"]]}
|
||
secrets.add(snapshot, {"phone", "name", "gender", "code", "salary"})
|
||
scrub = secrets.scrubber()
|
||
assert scrub("客户 13900000001、139000000012、1,390 元、390 笔、张伟买了男装，月薪 12,000，另 12000.5") == (
|
||
"客户 已遮罩、139000000012、1,390 元、已遮罩 笔、已遮罩买了男装，月薪 已遮罩，另 12000.5")
|
||
assert scrub("华东") == "华东", "没遮的列不换"
|
||
assert inquiry_mod._Secrets().scrubber()("13900000001") == "13900000001"
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 触顶与失败
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_hitting_the_query_limit_is_recorded(monkeypatch, shop, client):
|
||
await set_verify({"inquiry": {"max_queries": 1}})
|
||
plays = {"排除测试账号": Play([NO_TEST, QUERY], NO_TEST_ANSWER)}
|
||
models = Models(monkeypatch, shop, plays=plays)
|
||
paused = await pause(shop)
|
||
[approval] = await pending(paused.id)
|
||
r = await ask(client, paused.id, report="write", unit=None, question=QUESTION, approval_id=approval.id)
|
||
assert r.status_code == 200, r.text
|
||
item = r.json()["inquiry"]
|
||
assert (item["answer"]["ok"], item["answer"]["limit"]) == (True, "max_queries"), "触顶照样按查到的结果收尾"
|
||
# 没指明句子：任务里给报告正文，不给哪一句的摘录
|
||
[task] = models.tasks
|
||
assert "提问人没有指明是哪一句" in task and f"报告正文：\n{EAST}{SOUTH}" in task and "原报告" not in task
|
||
assert [q["sql"] for q in item["queries"]] == [NO_TEST]
|
||
[event] = await events(paused.id, "approval.inquiry")
|
||
assert (event.data["ok"], event.data["limit"]) == (True, "max_queries")
|
||
|
||
|
||
async def test_an_unavailable_model_is_recorded_as_a_failure(monkeypatch, shop, client):
|
||
Models(monkeypatch, shop, broken=ProviderNotConfigured("没有配置模型接入"))
|
||
# 写作者照常用演示模型：只有质询 agent 的模型用不了
|
||
paused = await pause(shop)
|
||
[approval] = await pending(paused.id)
|
||
r = await ask(client, paused.id, report="write", unit=None, question=QUESTION, approval_id=approval.id)
|
||
assert r.status_code == 200, r.text
|
||
body = r.json()
|
||
assert body["recorded"] is True
|
||
answer = body["inquiry"]["answer"]
|
||
assert answer["ok"] is False and answer["text"] == "" and answer["doc_artifact"] is None
|
||
assert answer["error"] and "模型" in answer["error"]
|
||
assert body["inquiry"]["queries"] == [] and body["inquiry"]["usage"]["calls"] == 0
|
||
[event] = await events(paused.id, "approval.inquiry")
|
||
assert event.data["ok"] is False and event.data["error"] == answer["error"]
|
||
|
||
|
||
async def test_an_answer_that_cannot_be_put_together_is_recorded_as_a_failure(monkeypatch, shop, client):
|
||
"""回答原文没能组装成文档：原文里的引用没换成值、遮罩也没做，不交出去；查过的、花掉的照记。"""
|
||
Models(monkeypatch, shop)
|
||
|
||
def broken(*args, **kwargs):
|
||
raise RuntimeError("文档结构异常")
|
||
|
||
monkeypatch.setattr(inquiry_mod, "compose_doc", broken)
|
||
paused = await pause(shop)
|
||
[approval] = await pending(paused.id)
|
||
r = await ask(client, paused.id, report="write", unit=None, question=QUESTION, approval_id=approval.id)
|
||
assert r.status_code == 200, r.text
|
||
item = r.json()["inquiry"]
|
||
assert (item["answer"]["ok"], item["answer"]["text"], item["answer"]["doc_artifact"]) == (False, "", None)
|
||
assert item["answer"]["error"].startswith("整理回答时出错")
|
||
assert [q["sql"] for q in item["queries"]] == [NO_TEST] and item["usage"]["calls"] >= 2
|
||
assert r.json()["recorded"] is True
|
||
[event] = await events(paused.id, "approval.inquiry")
|
||
assert event.data["ok"] is False and event.data["usage"] == item["usage"]
|
||
|
||
|
||
async def test_the_daily_cap_is_recorded_as_a_limit(monkeypatch, shop, client):
|
||
async with SessionLocal() as session:
|
||
session.add(Setting(key=spend.SPEND_KEY, value={"date": spend._today(), "usd": 5.0, "calls": 3}))
|
||
await session.commit()
|
||
models = Models(monkeypatch, shop)
|
||
paused = await pause(shop)
|
||
[approval] = await pending(paused.id)
|
||
r = await ask(client, paused.id, report="write", unit=None, question=QUESTION, approval_id=approval.id)
|
||
assert r.status_code == 200, r.text
|
||
answer = r.json()["inquiry"]["answer"]
|
||
assert (answer["ok"], answer["limit"]) == (False, "daily_max_usd") and answer["error"]
|
||
assert models.finals == []
|
||
[event] = await events(paused.id, "approval.inquiry")
|
||
assert event.data["limit"] == "daily_max_usd"
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 读取
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_listing_filters_by_report_and_sentence_and_artifacts_are_whitelisted(monkeypatch, shop, client):
|
||
plays = {"排除测试账号": Play([NO_TEST], NO_TEST_ANSWER), "华南为什么": Play([], "查不到判断所需的数据。")}
|
||
Models(monkeypatch, shop, plays=plays)
|
||
paused = await pause(shop)
|
||
[approval] = await pending(paused.id)
|
||
doc = await report_doc(paused.id)
|
||
east, south = unit_of(doc, EAST), unit_of(doc, SOUTH)
|
||
# 署名照请求头记（前端按 encodeURIComponent 编码后发）
|
||
first = (await client.post(f"/api/runs/{paused.id}/inquiries", headers={"X-Actor": "%E5%BC%A0%E5%B7%A5"},
|
||
json={"report": "write", "unit": east, "question": QUESTION,
|
||
"approval_id": approval.id})).json()["inquiry"]
|
||
assert first["actor"] == "张工"
|
||
second = (await ask(client, paused.id, report="write", unit=south, question="华南为什么低？",
|
||
approval_id=approval.id)).json()["inquiry"]
|
||
assert first["seq"] < second["seq"]
|
||
|
||
listed = await client.get(f"/api/runs/{paused.id}/inquiries")
|
||
assert listed.json() == {"items": [first, second]}
|
||
assert (await client.get(f"/api/runs/{paused.id}/inquiries", params={"report": "write", "unit": south})
|
||
).json() == {"items": [second]}
|
||
assert (await client.get(f"/api/runs/{paused.id}/inquiries", params={"report": "other"})).json() == {"items": []}
|
||
assert (await client.get("/api/runs/no-such-run/inquiries")).status_code == 404
|
||
|
||
[query] = first["queries"]
|
||
got = await client.get(f"/api/runs/{paused.id}/inquiries/artifacts/{query['artifact']}")
|
||
assert got.status_code == 200, got.text
|
||
assert got.json()["id"] == query["artifact"] and got.json()["masked"] == []
|
||
assert got.json()["content"]["rows"] == [["华东", 1200], ["华南", 900], ["华北", 400]]
|
||
# 回答文档也是质询引用过的工件
|
||
got = await client.get(f"/api/runs/{paused.id}/inquiries/artifacts/{first['answer']['doc_artifact']}")
|
||
assert got.status_code == 200 and got.json()["content"]["markdown"] == NO_TEST_TEXT
|
||
# 报告自己的查询快照不是质询引用过的
|
||
[fetched] = [e for e in await events(paused.id, "tool.end", "fetch")]
|
||
got = await client.get(f"/api/runs/{paused.id}/inquiries/artifacts/{fetched.data['query_artifact']}")
|
||
assert got.status_code == 404
|
||
assert (await client.get(f"/api/runs/{paused.id}/inquiries/artifacts/nothing")).status_code == 404
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 重放
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_resuming_after_an_inquiry_does_not_repeat_the_nodes(monkeypatch, shop, client):
|
||
"""质询不发 tool.start（_stale_replay 只看它），批准后从断点继续：取数、报告都不再执行一遍。"""
|
||
models = Models(monkeypatch, shop)
|
||
paused = await pause(shop)
|
||
[approval] = await pending(paused.id)
|
||
r = await ask(client, paused.id, report="write", unit=None, question=QUESTION, approval_id=approval.id)
|
||
assert r.json()["recorded"] is True
|
||
assert [e for e in await events(paused.id, "tool.start") if e.node_id is None] == []
|
||
await client.post(f"/api/approvals/{approval.id}/decide", json={"approved": True})
|
||
final = await wait(paused.id)
|
||
assert final.status == "succeeded", final.error
|
||
for node_id in ("fetch", "write"):
|
||
assert len(await events(paused.id, "node.started", node_id)) == 1, f"{node_id} 又执行了一遍"
|
||
assert len(await events(paused.id, "tool.start", "fetch")) == 1
|
||
assert models.writes == 1
|