mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-16 06:01:57 +00:00
ce4be9cc8f
- 自查按值认(engine/evidence.py 的 masked_values,report.py 的 _MaskedCells):写作目录里确定取自遮罩列的值出现在正文里——照抄成文字(姓名这类文字列数字核对管不到)、写成数字(换了写法的长号码组装时被拆成几个数字片段,按整句认)、经指标带出来([[m:phone_raw]] 的值就是号码)——记一条 masked_cell(by_value),不带值,交回写作者时附上值换成「已遮罩」的那一句。只收确定取自遮罩列的列、不收分档标签;运行输入里写着的值是发起运行的人自己填的,不算。只读数据源现在设了遮罩的快照,认引用时读过的不再读,没设遮罩的运行一份都不读 - 时间线:日志 report_masked_cell 多带 written(其中按值认出的几处),标题分「引用了…格子」「写进了…值」「引用、写进了…」;缺口、日志按两种分别说;违规清单、CSV 的叫法改成「写进了遮罩列的值」 - 只数个数的列(COUNT(*) FILTER (WHERE phone IS NULL)、SUM(phone IS NULL)、AVG(phone IS NOT NULL)、SUM(amount) FILTER (WHERE phone IS NOT NULL)):认遮罩列的那一套把 FILTER 条件、汇总里的真假条件当成值的来历,数据质量报告引用「手机号缺失的人数」在正式运行里判失败。自查认之前先改写(_counted:FILTER 同 WHERE 去掉,汇总里的真假条件改成 CASE WHEN … THEN 1 ELSE 0 END),不判违规;证据面板、推演照可验证报告的认法多遮(只影响展示),根上的改法在 masking 里 - 证据面板:遮没遮核对不了的输入(快照、取值链上的产出读不出来),值和代入式里写「无法核对」,不写「已遮罩」,数据源可能根本没设遮罩 - 整句面板「挂的依据」不再直接读文档目录里的「名称 = 渲染值」:按证据图的标签写(服务端已把用到遮罩列的指标值换成「已遮罩」),证据图没回来时指标只写名字 - on_demand:当场裁判过、文档里却压根没有这句的判定时,不说「原因见判定」 - 测试:合并查询自查的用例补上拼接、upper、WITH;推演对整行、下标不是字面量的一格说「用到了遮罩列」(顺着取值链认出来的,不是按值猜的);计数指标 cnt 的原值照给;SQL 认不准、只数个数的反例
223 lines
12 KiB
Python
223 lines
12 KiB
Python
"""片段接口的 on_demand(这句能不能请模型判断)在正式运行里照实说。
|
||
|
||
验收发现:正式运行的片段接口一律答「正式运行的结论句已在报告撰写节点内完成裁判」,可那次报告的结论句策略是不裁判
|
||
(report.checked 记 claims: off,文档里没有裁判摘要),节点里根本没判过;标题这种不是结论句的片段也答同一句。现在:
|
||
|
||
- 不是结论句的先说(not_a_claim):标题、连接性文字不论哪类运行都无需裁判
|
||
- 正式运行、节点里确实判过(文档里有当场裁判的摘要):照旧说已在节点内完成裁判
|
||
- 正式运行、没开结论句裁判(不裁判、只要求附依据,或者升级前的运行):照实说这份报告没有开启结论句裁判
|
||
- 正式运行、开了裁判节点里却没判(报告没通过核对,节点紧接着判失败,不再花钱裁判):照实说没有在节点内完成裁判
|
||
- 按需裁判接口(POST …/evidence/judge)对正式运行回 409 时说同一句话
|
||
|
||
写作、裁判都用 MockChatModel 照剧本,不调真接口。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import re
|
||
|
||
import pytest
|
||
from httpx import ASGITransport, AsyncClient
|
||
from langchain_core.messages import AIMessage
|
||
|
||
from app.core import artifact_store
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import Run
|
||
from app.engine import judge as judge_mod
|
||
from app.engine.evidence import compose_doc, iter_units
|
||
from app.engine.judge import JUDGE_ROLE
|
||
from app.engine.runner import run_manager
|
||
from app.main import app
|
||
from app.providers.mock_model import MockChatModel
|
||
|
||
METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}]
|
||
#: 一个标题、一句挂了依据的结论、一句没挂依据的归因
|
||
TEXT = "## 本周\n\n本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。"
|
||
#: 标题那一句(正文里去掉了「## 」),和没挂依据的归因
|
||
HEADING, CAUSAL = "本周", "增长主要来自新客首单。"
|
||
JUDGED_WORDS = "已在报告撰写节点内完成裁判"
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
async def engine_up():
|
||
await run_manager.setup()
|
||
yield
|
||
await run_manager.shutdown()
|
||
|
||
|
||
@pytest.fixture
|
||
async def client():
|
||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||
yield c
|
||
|
||
|
||
def node(nid, ntype, **config):
|
||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
||
|
||
|
||
def weekly(**report) -> dict:
|
||
nodes = [
|
||
node("start", "input"),
|
||
node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"),
|
||
node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=METRICS),
|
||
node("write", "report", instructions="写周报", **report),
|
||
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]),
|
||
]
|
||
ids = [n["id"] for n in nodes]
|
||
return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]}
|
||
|
||
|
||
def models(monkeypatch) -> list[str]:
|
||
"""写作者回 TEXT;裁判(claims: judge 时节点里当场判)一律判有依据。返回交给裁判的请求原文。"""
|
||
judged: list[str] = []
|
||
|
||
def decide(chat, messages):
|
||
if JUDGE_ROLE in str(messages[0].content):
|
||
request = str(messages[-1].content)
|
||
judged.append(request)
|
||
items = [{"unit": uid, "verdict": "supported", "rationale": "有依据", "used": []}
|
||
for uid, _ in re.findall(r"^\[(u\d+)\]([^\n]*)$", request, re.M)]
|
||
return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False))
|
||
return AIMessage(content=TEXT)
|
||
|
||
monkeypatch.setattr(MockChatModel, "_decide", decide)
|
||
|
||
async def judge_model_of(session, spec):
|
||
return MockChatModel(model_name=spec.model or "judge-x"), spec.model or "judge-x"
|
||
|
||
monkeypatch.setattr(judge_mod, "get_chat_model", judge_model_of)
|
||
return judged
|
||
|
||
|
||
async def finish(graph, *, run_class="formal") -> Run:
|
||
run = await run_manager.start(graph=graph, input_payload={}, run_class=run_class)
|
||
for _ in range(400):
|
||
await asyncio.sleep(0.05)
|
||
async with SessionLocal() as session:
|
||
row = await session.get(Run, run.id)
|
||
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
|
||
assert row.status == "succeeded", row.error
|
||
return row
|
||
raise AssertionError(f"run {run.id} 没跑完:{row.status}")
|
||
|
||
|
||
def unit_of(doc: dict, text: str) -> str:
|
||
md = doc["markdown"]
|
||
return next(u["id"] for _, u in iter_units(doc) if md[u["span"][0]:u["span"][1]].strip() == text)
|
||
|
||
|
||
async def on_demand(client, row: Run, text: str) -> dict:
|
||
doc = artifact_store.load(row.output["_evidence"]["doc_artifact"])
|
||
uid = unit_of(doc, text)
|
||
seg = next(s["id"] for _, u in iter_units(doc) if u["id"] == uid for s in u["segments"] if s["kind"] != "structural")
|
||
res = await client.get(f"/api/runs/{row.id}/evidence/segments/{seg}?report=write")
|
||
assert res.status_code == 200, res.text
|
||
return res.json()["unit"]["on_demand"]
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 片段接口
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.parametrize("claims", [None, "off", "require_citation"])
|
||
async def test_a_formal_report_without_claim_judging_says_it_was_not_judged(client, monkeypatch, claims):
|
||
"""验收原样:正式运行、结论句策略不裁判(没写也一样,或者只要求附依据)。不能说「已在节点内完成裁判」。"""
|
||
judged = models(monkeypatch)
|
||
row = await finish(weekly(**({"claims": claims} if claims else {})))
|
||
got = await on_demand(client, row, CAUSAL)
|
||
assert got["available"] is False and got["reason"] == "formal", got
|
||
assert JUDGED_WORDS not in got["message"] and "没有开启结论句裁判" in got["message"], got["message"]
|
||
assert judged == [], "节点里确实没请裁判"
|
||
|
||
|
||
async def test_a_heading_in_a_formal_run_is_not_a_claim(client, monkeypatch):
|
||
"""标题不是结论句:不论哪类运行都答 not_a_claim,不答「已在节点内完成裁判」。"""
|
||
models(monkeypatch)
|
||
row = await finish(weekly())
|
||
got = await on_demand(client, row, HEADING)
|
||
assert got["available"] is False and got["reason"] == "not_a_claim" and JUDGED_WORDS not in got["message"], got
|
||
|
||
|
||
async def test_a_formal_report_judged_in_the_node_says_so(client, monkeypatch):
|
||
"""开了裁判、节点里当场判过:照旧说已完成裁判;标题照样是 not_a_claim。"""
|
||
judged = models(monkeypatch)
|
||
row = await finish(weekly(claims="judge"))
|
||
assert judged, "正式运行在节点里当场裁判"
|
||
got = await on_demand(client, row, CAUSAL)
|
||
assert got["available"] is False and got["reason"] == "formal" and JUDGED_WORDS in got["message"], got
|
||
assert (await on_demand(client, row, HEADING))["reason"] == "not_a_claim"
|
||
|
||
|
||
async def test_the_judge_endpoint_says_the_same_for_a_formal_report_without_judging(client, monkeypatch):
|
||
"""按需裁判接口对正式运行照旧 409,说的话和片段接口一致:没开裁判的照实说没开。"""
|
||
models(monkeypatch)
|
||
row = await finish(weekly())
|
||
doc = artifact_store.load(row.output["_evidence"]["doc_artifact"])
|
||
res = await client.post(f"/api/runs/{row.id}/evidence/judge", json={"units": [unit_of(doc, CAUSAL)]})
|
||
assert res.status_code == 409 and res.json()["code"] == "evidence_judge_formal", res.text
|
||
detail = res.json()["detail"]
|
||
assert JUDGED_WORDS not in detail and "没有开启结论句裁判" in detail, detail
|
||
assert detail == (await on_demand(client, row, CAUSAL))["message"]
|
||
# 点名的不是报告撰写节点:认不出是哪份报告,不替它说判没判过,只说正式运行不能按需裁判
|
||
res = await client.post(f"/api/runs/{row.id}/evidence/judge", json={"units": ["u1"], "report": "caliber"})
|
||
assert res.status_code == 409 and res.json()["code"] == "evidence_judge_formal", res.text
|
||
assert res.json()["detail"] == "正式运行的结论句只在报告撰写节点内裁判,不能按需裁判"
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 手搭:开了裁判、节点里却没判(报告没通过核对,节点紧接着判失败)
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def test_a_formal_report_that_opted_in_but_was_never_judged_says_so():
|
||
"""claims: judge 的正式运行,报告没通过核对、节点判失败之前不再花钱裁判:文档里没有裁判摘要。不能说已完成裁判。"""
|
||
from app.api.evidence import _on_demand, _Sealed
|
||
|
||
doc = compose_doc("增长主要来自新客首单。", {}, node_id="write", run_id="r-unjudged")
|
||
[unit] = [u for _, u in iter_units(doc)]
|
||
sealed = _Sealed(run_id="r-unjudged", graph={}, events=[], run_class="formal", status="failed", ledger_on=True,
|
||
seal={"sealed": True, "ok": True, "manifest_seq": 3, "legacy": False})
|
||
got = _on_demand(sealed, doc, unit["id"], None, claims="judge")
|
||
assert got["available"] is False and got["reason"] == "formal", got
|
||
assert JUDGED_WORDS not in got["message"] and "没有在报告撰写节点内完成裁判" in got["message"], got["message"]
|
||
# 同一份文档带上当场裁判的摘要、这句判过了:照旧说已完成裁判
|
||
judged = {**doc, "judge": {"mode": "inline", "counts": {}}}
|
||
assert JUDGED_WORDS in _on_demand(sealed, judged, unit["id"], {"status": "supported"}, claims="judge")["message"]
|
||
# 不传策略(老调用方):看文档里有没有当场裁判的摘要
|
||
assert "没有开启结论句裁判" in _on_demand(sealed, doc, unit["id"], None)["message"]
|
||
|
||
|
||
@pytest.mark.parametrize("verdict", [
|
||
# 触顶:这句没判成,判定里记着原因
|
||
{"status": "unjudged", "reason": "max_claims", "rationale": "已达上限(每份报告最多裁判 1 句),这句未裁判"},
|
||
# 裁判出错、格式不对、被截断都一样是未裁判
|
||
{"status": "unjudged", "reason": "error", "rationale": "裁判失败:连接超时"},
|
||
# 文档里压根没有这句的判定
|
||
None,
|
||
])
|
||
def test_an_inline_judged_report_that_did_not_judge_this_sentence_says_so(verdict):
|
||
"""正式运行在节点里当场裁判过(文档里有裁判摘要),可这一句没判成:触顶、出错,或者没有判定。不能说已完成裁判,
|
||
照实说这句没判成、原因在判定里;判过的句子照旧说已完成裁判。按需裁判接口一次点了几句、有没判成的,也照实说。"""
|
||
from app.api.evidence import FORMAL_INLINE_MISSED_SOME, _formal_message, _on_demand, _Sealed
|
||
|
||
doc = compose_doc("增长主要来自新客首单。复购率也在上升。", {}, node_id="write", run_id="r-capped")
|
||
doc["judge"] = {"mode": "inline", "complete": False, "counts": {"unjudged": 1}, "limits_hit": ["max_claims"]}
|
||
first, second = [u["id"] for _, u in iter_units(doc)]
|
||
sealed = _Sealed(run_id="r-capped", graph={}, events=[], run_class="formal", status="succeeded", ledger_on=True,
|
||
seal={"sealed": True, "ok": True, "manifest_seq": 3, "legacy": False})
|
||
got = _on_demand(sealed, doc, first, verdict, claims="judge")
|
||
assert got["available"] is False and got["reason"] == "formal", got
|
||
assert JUDGED_WORDS not in got["message"] and "不能按需裁判" in got["message"], got["message"]
|
||
if verdict is None:
|
||
# 没有判定可看:不说「原因见判定」
|
||
assert "这句没有判定" in got["message"] and "原因见判定" not in got["message"], got["message"]
|
||
else:
|
||
assert "这句没有判成" in got["message"] and "原因见判定" in got["message"], got["message"]
|
||
# 判过的那一句照旧说已完成裁判
|
||
assert JUDGED_WORDS in _on_demand(sealed, doc, second, {"status": "supported"}, claims="judge")["message"]
|
||
# 一次问几句(按需裁判接口):有一句没判成就照实说
|
||
assert _formal_message(doc, "judge", [verdict, {"status": "supported"}]) == FORMAL_INLINE_MISSED_SOME
|
||
assert _formal_message(doc, "judge", [{"status": "supported"}, {"status": "partial"}]).startswith("正式运行的结论句已在")
|