mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:11:59 +00:00
ThearchyHelios
03c18c4d78
一、文案（前端约 1600 处，后端传到界面的约 880 处） - 术语统一：工作流层面用「运行」（继续运行、重新运行、试运行），节点和工具层面用「执行」，不再用「跑」； 总时长 / 执行时长 / 等待审批；服务端；助手；Agent；token；工作流；固定版本；模型接入；系统提示；批准 / 驳回；受管级别 - 口语改书面：取不到、没取回来、读不懂、连不上、多半、照样、兜底、揉成这类说法改成「无法 / 未能 / 失败 / 请」；去掉拟人写法 - 不再露出实现细节：界面上不写 checkpoint、同一拍、哈希（凭证处除外），不写改版历史（「以前…」「老数据」）； 报错、校验、门禁、自动修复里裸写的配置键名和枚举值换成界面上的中文标签，由新的 backend/app/engine/labels.py 统一提供； 发布弹窗里结论句的选项不再是 off / require_citation - 同一件事只留一种说法：生成阶段文案、出具档位说明、画布锁定提示各合成一份 - 去掉界面上的工业场景示例（MES、ERP、「张工」「王工」）和真实产品名，换成通用示例 二、说错的提示（都有测试证明） - 数据库测连接失败时的专用提示以前从不出现（按报错文字全等比较，永远比不上）：改为按错误类型判断，不再拼出重复的前缀 - 「设置 → 供应商」改指「设置 → 模型接入」；「在设置里建数据源」改指「数据」页 - 缺少 API Key、模型接入已停用的报错不再夹英文，写明去哪里填 - 两处原因说错（「这种引用在后续版本支持」「这一版还不支持」）改成真实原因 - 节点说明里不准确的说法改正：代码节点在 microVM 或系统沙箱中执行；报告撰写、口径卡的说明不再说过头 - 导出的证据 CSV 里两列同名，第三列改为「片段编号」 三、前后端对齐 - 前端按后端原文识别日志、报错的地方，新旧两种写法都认得（历史运行里存的是旧文字），逐条补了新旧原文的核对 - 后端交回模型的原指令放进单独的 for_model 字段，前端在解析响应时去掉，不会显示到界面上 四、检查 - 检查脚本里按原文匹配的断言改成新文案；改完会变成永远成立的反向断言逐条修正，并做了变异验证 - 新增 check-copy（静态扫描前端界面字符串和后端上界面的文案）：再出现「跑」「后端」「多半」「令牌」、 界面上的 Copilot、裸写的配置键名、半角标点等就报错；确需保留的写进 scripts/copy-allow.json 并注明理由
269 lines
12 KiB
Python
269 lines
12 KiB
Python
"""引文：[[q:K1|原话]] 必须在那次检索命中的片段里逐字出现。
|
||
|
||
知识库的结论以前只能写成 [[see:K1]]：读者知道依据是哪次检索，看不到是哪一句。引文把
|
||
原话写进报告，系统拿检索快照（取回时复验哈希）逐字核对：
|
||
|
||
- 空白归一化后逐字出现才算有出处；改一个字就是没有出处（state none，issue unresolved_ref）
|
||
- 正文里显示引文本身；目录里根本没有这次检索时显示占位，和别的解析不了的引用一样
|
||
- eid 带上命中片段的定位：第几条命中、原文里的起止
|
||
- 出口复核重新取快照、重新比对，不信文档里记的结论
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import random
|
||
|
||
import pytest
|
||
from langchain_core.messages import AIMessage
|
||
from sqlalchemy import select
|
||
|
||
from app.core import artifact_store
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import Run, RunEvent
|
||
from app.engine.evidence import (
|
||
LEGACY_LATER_REASON,
|
||
PHASE_TWO_REASON,
|
||
StreamRenderer,
|
||
build_catalog,
|
||
catalog_prompt,
|
||
compose_doc,
|
||
iter_segments,
|
||
make_eid,
|
||
render_markers,
|
||
verify_doc,
|
||
)
|
||
from app.engine.runner import run_manager
|
||
|
||
K_ART = "c" * 64
|
||
HITS = [
|
||
{"chunk_id": "ch-1", "document_id": "doc-1", "title": "售后月报", "ordinal": 3,
|
||
"content": "九月退款主要集中在\n东区，原因是物流延误。退款率 2.35%，高于上月。"},
|
||
{"chunk_id": "ch-2", "document_id": "doc-2", "title": "Ops notes", "ordinal": 0,
|
||
"content": "Net revenue\nrose after the promotion ended. 物流延误已经解决。"},
|
||
]
|
||
SNAPSHOT = {"query": "退款", "collection": "kb", "hits": HITS}
|
||
|
||
|
||
def loader_of(content):
|
||
def load(artifact):
|
||
return content if artifact == K_ART else None
|
||
return load
|
||
|
||
|
||
LOAD = loader_of(SNAPSHOT)
|
||
|
||
|
||
@pytest.fixture
|
||
def catalog():
|
||
return build_catalog(nodes={}, ledger=[{"kind": "retrieval", "node_id": "kb", "exec": 1, "artifact": K_ART,
|
||
"source": "kb", "rows": len(HITS)}])
|
||
|
||
|
||
def quotes(doc):
|
||
return [s for s in iter_segments(doc) if s["kind"] == "quote"]
|
||
|
||
|
||
def test_verbatim_quote_resolves_with_its_location(catalog):
|
||
doc = compose_doc("售后说「[[q:K1|退款主要集中在东区]]」。[[see:K1]]", catalog, loader=LOAD)
|
||
[seg] = quotes(doc)
|
||
assert doc["markdown"] == "售后说「退款主要集中在东区」。"
|
||
assert seg["text"] == "退款主要集中在东区" and seg["state"] == "deterministic"
|
||
cite = seg["cite"]
|
||
assert cite["kind"] == "quote" and cite["role"] == "quote" and cite["alias"] == "K1"
|
||
loc = cite["locator"]
|
||
assert loc["hit"] == 0 and HITS[0]["content"][loc["start"]:loc["end"]] == "退款主要集中在\n东区"
|
||
assert cite["eid"] == make_eid("quote", K_ART, loc)
|
||
assert cite["source"] == {"artifact": K_ART, "document": "doc-1", "chunk": "ch-1", "title": "售后月报",
|
||
"ordinal": 3}
|
||
assert doc["violations"] == [] and doc["stats"]["quotes"] == 1
|
||
assert doc["blocks"][0]["units"][0]["cites"] == ["K1"]
|
||
|
||
|
||
def test_one_changed_character_is_not_a_quote(catalog):
|
||
doc = compose_doc("售后说「[[q:K1|退款主要集中在西区]]」。", catalog, loader=LOAD)
|
||
[seg] = quotes(doc)
|
||
# 正文照样显示引文本身，只是标成没有出处
|
||
assert seg["text"] == "退款主要集中在西区" and seg["state"] == "none" and seg["issue"] == "unresolved_ref"
|
||
assert "找不到这段引文的原文" in seg["cite"]["reason"]
|
||
assert [v["code"] for v in doc["violations"]] == ["unresolved_ref"]
|
||
assert doc["stats"]["quotes"] == 0
|
||
|
||
|
||
@pytest.mark.parametrize("quote", [
|
||
"Net revenue rose after the promotion ended.",
|
||
"Net revenue rose",
|
||
"退款率 2.35%，高于上月",
|
||
"退款主要集中在 东区",
|
||
])
|
||
def test_whitespace_is_normalised(catalog, quote):
|
||
doc = compose_doc(f"原话：[[q:K1|{quote}]]", catalog, loader=LOAD)
|
||
[seg] = quotes(doc)
|
||
assert seg["state"] == "deterministic", seg["cite"].get("reason")
|
||
loc = seg["cite"]["locator"]
|
||
assert " ".join(HITS[loc["hit"]]["content"][loc["start"]:loc["end"]].split()).replace(" ", "") \
|
||
== quote.replace(" ", "")
|
||
|
||
|
||
@pytest.mark.parametrize("quote", ["Netrevenue rose", "net revenue rose", "退款主要集中在东区。九月"])
|
||
def test_normalising_is_not_fuzzy(catalog, quote):
|
||
doc = compose_doc(f"原话：[[q:K1|{quote}]]", catalog, loader=LOAD)
|
||
assert quotes(doc)[0]["state"] == "none"
|
||
|
||
|
||
def test_numbers_inside_a_verified_quote_are_not_bare(catalog):
|
||
doc = compose_doc("[[q:K1|退款率 2.35%，高于上月]]", catalog, loader=LOAD)
|
||
assert doc["violations"] == [] and doc["stats"]["uncited_numbers"] == 0
|
||
|
||
|
||
@pytest.mark.parametrize("text, words", [
|
||
("[[q:K9|退款主要集中在东区]]", "检索结果 K9 不存在"),
|
||
("[[q:K1|]]", "引文为空"),
|
||
("[[q:K1|东区]]", "太短"),
|
||
])
|
||
def test_quotes_that_cannot_be_checked(catalog, text, words):
|
||
doc = compose_doc(text, catalog, loader=LOAD)
|
||
[seg] = quotes(doc)
|
||
assert seg["state"] == "none" and words in seg["cite"]["reason"], seg["cite"]["reason"]
|
||
|
||
|
||
def test_missing_retrieval_shows_a_placeholder(catalog):
|
||
doc = compose_doc("[[q:K9|退款主要集中在东区]]。", catalog, loader=LOAD)
|
||
assert doc["markdown"] == "⟦?q:K9|退款主要集中在东区⟧。"
|
||
|
||
|
||
def test_tampered_snapshot_is_not_evidence(catalog):
|
||
def tampered(artifact):
|
||
raise ValueError("hash mismatch")
|
||
|
||
doc = compose_doc("[[q:K1|退款主要集中在东区]]", catalog, loader=tampered)
|
||
[seg] = quotes(doc)
|
||
assert seg["state"] == "none" and "哈希" in seg["cite"]["reason"]
|
||
|
||
|
||
def test_exit_check_rereads_the_snapshot(catalog):
|
||
doc = compose_doc("[[q:K1|物流延误已经解决]]。", catalog, loader=LOAD)
|
||
assert doc["violations"] == [] and quotes(doc)[0]["cite"]["locator"]["hit"] == 1
|
||
changed = {**SNAPSHOT, "hits": [HITS[0], {**HITS[1], "content": "物流延误仍在持续。"}]}
|
||
checked = verify_doc(doc, catalog, loader=loader_of(changed))
|
||
assert [v["code"] for v in checked["violations"]] == ["unresolved_ref"]
|
||
assert verify_doc(doc, catalog, loader=LOAD)["ok"] is True
|
||
|
||
|
||
def phase_two(doc):
|
||
"""把文档改回二期组装的样子：那时 q / t / c 一律判为解析不了，正文是占位，原因是 LEGACY_LATER_REASON。
|
||
占位的字和现在「目录里没有 K9」时一样，只有记下的原因不同。"""
|
||
for seg in iter_segments(doc):
|
||
if seg.get("issue") == "unresolved_ref" and seg["ref"][:2] in ("q:", "t:", "c:"):
|
||
seg["cite"]["reason"] = LEGACY_LATER_REASON
|
||
return doc
|
||
|
||
|
||
def test_quotes_from_before_the_upgrade_are_rechecked_the_old_way(catalog):
|
||
"""二期的报告文档（升级前组装，停在报告和出口之间）升级后被复核：原话今天查得到也不能算
|
||
「渲染对不上」——那会被出口当成完整性问题记缺口。按当时的规矩：占位，解析不了。"""
|
||
doc = phase_two(compose_doc("售后说[[q:K1|退款主要集中在东区]]。", {}, loader=LOAD))
|
||
[seg] = quotes(doc)
|
||
assert seg["text"] == "⟦?q:K1|退款主要集中在东区⟧"
|
||
checked = verify_doc(doc, catalog, loader=LOAD)
|
||
assert [v["code"] for v in checked["violations"]] == ["unresolved_ref"]
|
||
assert PHASE_TWO_REASON in checked["violations"][0]["message"]
|
||
# 只认占位：正文被改成了引文本身、又自称是二期的，照样按现在的规矩查
|
||
seg["text"] = "退款主要集中在东区"
|
||
doc["markdown"] = "售后说退款主要集中在东区。"
|
||
tail = [x for x in iter_segments(doc)][-1]
|
||
seg["span"] = [3, 3 + len(seg["text"])]
|
||
tail["span"] = [seg["span"][1], seg["span"][1] + len(tail["text"])]
|
||
[unit] = [u for b in doc["blocks"] for u in b["units"]]
|
||
unit["span"] = [0, tail["span"][1]]
|
||
assert [v["code"] for v in verify_doc(doc, catalog, loader=LOAD)["violations"]] == ["state_mismatch"]
|
||
|
||
|
||
def test_entity_markers_from_before_the_upgrade_are_rechecked_the_old_way():
|
||
ledger = [{"kind": "query", "node_id": "f", "exec": 1, "artifact": "a" * 64, "tool": "db_query__w", "source": "w",
|
||
"columns": ["gmv"], "rows": 1, "truncated": False, "schema_artifact": "5" * 64, "tables": ["orders"]},
|
||
{"kind": "schema", "node_id": "f", "exec": 1, "artifact": "5" * 64, "source": "w",
|
||
"tables": {"orders": ["id", "gmv"]}}]
|
||
doc = phase_two(compose_doc("看 [[t:orders]] 和 [[c:orders.gmv]]。", {}))
|
||
assert [s["text"] for s in iter_segments(doc) if s.get("ref")] == ["⟦?t:orders⟧", "⟦?c:orders.gmv⟧"]
|
||
checked = verify_doc(doc, build_catalog(nodes={}, ledger=ledger))
|
||
assert [v["code"] for v in checked["violations"]] == ["unresolved_ref", "unresolved_ref"]
|
||
|
||
|
||
def test_streaming_equals_full_render(catalog):
|
||
text = "开头 [[q:K1|退款主要集中在东区]]，又说 [[q:K1|退款主要集中在西区]]；[[q:K9|无此检索]] 完。[[see:K1]]"
|
||
whole = render_markers(text, catalog, loader=LOAD)
|
||
assert whole == "开头 退款主要集中在东区，又说 退款主要集中在西区；⟦?q:K9|无此检索⟧ 完。"
|
||
rng = random.Random(11)
|
||
for _ in range(200):
|
||
renderer = StreamRenderer(catalog, loader=LOAD)
|
||
out, i = [], 0
|
||
while i < len(text):
|
||
step = rng.randint(1, 7)
|
||
out.append(renderer.feed(text[i:i + step]))
|
||
i += step
|
||
out.append(renderer.flush())
|
||
assert "".join(out) == whole
|
||
|
||
|
||
def test_catalog_prompt_shows_hits_to_quote_from(catalog):
|
||
prompt = catalog_prompt(catalog, loader=LOAD)
|
||
assert "[[q:K1|" in prompt and "逐字" in prompt and "[[see:K1]]" in prompt
|
||
assert "售后月报" in prompt and "退款主要集中在" in prompt and "Ops notes" in prompt
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 跑一次：检索 → 报告撰写，引错了原话会被要求重写
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.fixture
|
||
async def engine_up():
|
||
await run_manager.setup()
|
||
yield
|
||
await run_manager.shutdown()
|
||
|
||
|
||
async def test_report_node_checks_quotes_against_the_retrieval(engine_up, monkeypatch):
|
||
from app.memory import kb
|
||
from app.providers import mock_model
|
||
|
||
async def fake_search(session, **kw):
|
||
return [dict(HITS[0], score=0.9)]
|
||
|
||
monkeypatch.setattr(kb, "search", fake_search)
|
||
replies = ["售后说「[[q:K1|退款主要集中在西区]]」。[[see:K1]]", "售后说「[[q:K1|退款主要集中在东区]]」。[[see:K1]]"]
|
||
seen: list = []
|
||
|
||
def decide(self, messages):
|
||
seen.append(list(messages))
|
||
return AIMessage(content=replies[min(len(seen), len(replies)) - 1])
|
||
|
||
monkeypatch.setattr(mock_model.MockChatModel, "_decide", decide)
|
||
nodes = [{"id": nid, "type": t, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": cfg}} for nid, t, cfg in [
|
||
("start", "input", {}), ("docs", "retrieve", {"query": "退款", "collection": "kb"}),
|
||
("write", "report", {"instructions": "引用售后的原话"}),
|
||
("out", "output", {"fields": [{"name": "r", "value": "{{ nodes.write.text }}"}]})]]
|
||
graph = {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
||
run = await run_manager.start(graph=graph, input_payload={})
|
||
for _ in range(400):
|
||
await asyncio.sleep(0.05)
|
||
async with SessionLocal() as session:
|
||
row = await session.get(Run, run.id)
|
||
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
|
||
break
|
||
assert row.status == "succeeded", row.error
|
||
assert row.output["r"] == "售后说「退款主要集中在东区」。"
|
||
|
||
first = "\n".join(str(m.content) for m in seen[0])
|
||
assert "[[q:K1|" in first and "退款主要集中在" in first and "售后月报" in first
|
||
assert len(seen) == 2 and "找不到这句原话" in str(seen[1][-1].content)
|
||
async with SessionLocal() as session:
|
||
[checked] = (await session.execute(select(RunEvent).where(
|
||
RunEvent.run_id == run.id, RunEvent.type == "report.checked"))).scalars().all()
|
||
assert checked.data["ok"] is True and checked.data["repairs"] == 1 and checked.data["stats"]["quotes"] == 1
|
||
doc = artifact_store.load(checked.data["doc_artifact"])
|
||
[seg] = quotes(doc)
|
||
retrieval = doc["catalog"]["K1"]["artifact"]
|
||
assert seg["cite"]["eid"] == make_eid("quote", retrieval, seg["cite"]["locator"])
|
||
assert seg["cite"]["source"]["chunk"] == "ch-1"
|