From a43c530fb852d1b071ced01e9e1db1c3b39a07e2 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 4 Oct 2026 11:02:06 +0800 Subject: [PATCH 01/15] =?UTF-8?q?=E7=BB=93=E6=9E=84=E5=8C=96=E6=8A=BD?= =?UTF-8?q?=E5=8F=96=E8=A2=AB=E6=88=AA=E6=96=AD=E6=97=B6=E5=8A=A0=E5=A4=A7?= =?UTF-8?q?=E9=A2=9D=E5=BA=A6=E9=87=8D=E8=AF=95=EF=BC=8C=E4=BB=8D=E6=88=AA?= =?UTF-8?q?=E6=96=AD=E5=B0=B1=E7=85=A7=E5=AE=9E=E8=AF=B4=E6=98=AF=E9=A2=9D?= =?UTF-8?q?=E5=BA=A6=E5=9C=A8=E6=8E=A8=E7=90=86=E9=98=B6=E6=AE=B5=E7=94=A8?= =?UTF-8?q?=E5=AE=8C=E4=BA=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 一次真实运行里 4 个开了按出处核对字段的 Agent 抽取全部失败：抽取关掉思考、额度 4096，可经 OpenAI 兼容接口接入的推理模型关不掉思考，额度在推理阶段就用完了（openai 抛 LengthFinishReasonError）；日志却叫人「稍后重试、去设置测试该模型」，两样都不解决问题。 - 额度默认至少 16384，节点配了更大的用节点的 - 认出输出被截断：openai 的 length limit 类错误、finish_reason=length、Anthropic 的 stop_reason=max_tokens；被截断时额度翻倍（最多 32768）重试一次，每次调用各记一行，被截断那次的用量照样记账 - 服务方嫌额度太大（400，报错里有 max_tokens 一类字样）时退回 8192、4096，不能因为额度提高反而抽不成 - 两次都截断：新日志 code agent_field_truncated（带 fields、max_tokens），原话说清额度在推理阶段用完、结构化结论没能写出来，下一步是换非推理模型或调大最大输出 token；字段照旧记空值、不以 0 代替。非截断类的失败照旧是 agent_field_unverified - 时间线给 agent_field_truncated 专门的标题、副标题和下一步，入口落到该 Agent 的最大输出 token；check-decode 补断言 --- backend/app/engine/nodes/llm.py | 149 +++++++++++++--- backend/tests/test_extract_truncation.py | 208 +++++++++++++++++++++++ frontend/src/run/decode.ts | 16 ++ scripts/check-decode.mjs | 41 +++++ 4 files changed, 392 insertions(+), 22 deletions(-) create mode 100644 backend/tests/test_extract_truncation.py diff --git a/backend/app/engine/nodes/llm.py b/backend/app/engine/nodes/llm.py index 5d0c51e4..6e2bd28e 100644 --- a/backend/app/engine/nodes/llm.py +++ b/backend/app/engine/nodes/llm.py @@ -541,25 +541,66 @@ async def run_agent(state: GraphState, ctx: NodeContext) -> dict[str, Any]: async def extract_step(payload: list[BaseMessage], schema: dict[str, Any]) -> dict[str, Any]: # cite_fields 的结构化抽取：进 checkpoint，节点重放时不再调用、不再计费。失败也在这里收住、 # 作为结果返回——agent 的结论还在，字段记空值、发警告，而不是把整个节点赔进去。 - # thinking 关掉：结构化输出靠强制工具调用，和思考模式不兼容，抽取也用不着思考 - spec = model_spec.model_copy(update={ - "thinking": "off", "max_tokens": max(int(ctx.cfg("max_tokens") or 0), 4096)}) - started = time.perf_counter() - ctx.emit(EventType.LLM_START, model=model_id, structured=True, purpose="cite_fields", - message_count=len(payload)) - try: - async with SessionLocal() as session: - extractor, _ = await get_chat_model(session, spec) - got = await extractor.with_structured_output(schema, include_raw=True).ainvoke(payload) - except Exception as e: # noqa: BLE001 + # thinking 关掉：结构化输出靠强制工具调用，和思考模式不兼容，抽取也用不着思考。但经 OpenAI 兼容 + # 接口接入的推理模型关不掉思考，推理照样算在额度里：额度见 EXTRACT_MAX_TOKENS，被截断时加大重试一次 + budget = max(int(ctx.cfg("max_tokens") or 0), EXTRACT_MAX_TOKENS) + spent = {"input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0, "calls": 0} + rejected: int | None = None # 服务方嫌太大、拒收的额度：之后重试不再超过它 + retried = False + + def count(usage: dict[str, Any]) -> None: + for key in ("input_tokens", "output_tokens", "calls"): + spent[key] += int(usage.get(key) or 0) + spent["cost_usd"] = round(spent["cost_usd"] + float(usage.get("cost_usd") or 0), 6) + + while True: + spec = model_spec.model_copy(update={"thinking": "off", "max_tokens": budget}) + started = time.perf_counter() + ctx.emit(EventType.LLM_START, model=model_id, structured=True, purpose="cite_fields", + message_count=len(payload), max_tokens=budget) + failure: Exception | None = None + raw: BaseMessage | None = None + parsed: Any = None + try: + async with SessionLocal() as session: + extractor, _ = await get_chat_model(session, spec) + got = await extractor.with_structured_output(schema, include_raw=True).ainvoke(payload) + raw, parsed = _split_structured(got) + usage = _usage_of(raw if raw is not None else AIMessage(content=""), model_id) + except Exception as e: # noqa: BLE001 + failure, usage = e, _error_usage(e, model_id) + count(usage) + cut = truncated_error(failure) if failure is not None else (parsed is None and truncated_reply(raw)) + elapsed = int((time.perf_counter() - started) * 1000) + if failure is not None and not cut and _budget_rejected(failure) \ + and (lower := next((b for b in _EXTRACT_FALLBACKS if b < budget), None)) is not None: + # 服务方的输出上限比默认额度小（DeepSeek chat 系只收 8K）：退一档再来，别因为额度提高反而抽不成 + ctx.emit(EventType.LLM_END, agent=ctx.node.title, model=model_id, purpose="cite_fields", + duration_ms=elapsed, **usage, + error=f"模型接入不接受 {budget} token 的输出额度，已改用 {lower} token 重试") + rejected, budget = budget, lower + continue + bigger = min(budget * 2, EXTRACT_RETRY_CAP) + if cut and not retried and bigger > budget and (rejected is None or bigger < rejected): + ctx.emit(EventType.LLM_END, agent=ctx.node.title, model=model_id, purpose="cite_fields", + duration_ms=elapsed, **usage, + error=f"输出被截断：{budget} token 的输出额度用完了，结构化结论没有写完，" + f"已把额度加大到 {bigger} token 重试一次") + retried, budget = True, bigger + continue + if cut: + ctx.emit(EventType.LLM_END, agent=ctx.node.title, model=model_id, purpose="cite_fields", + duration_ms=elapsed, **usage, + error=f"输出被截断：{budget} token 的输出额度用完了，结构化结论没有写完") + return {"error": TRUNCATED_EXTRACT.format(budget=budget), "truncated": True, + "max_tokens": budget, **spent} + if failure is not None: + ctx.emit(EventType.LLM_END, agent=ctx.node.title, model=model_id, purpose="cite_fields", + duration_ms=elapsed, **usage, error=describe_exception(failure)) + return {"error": f"结构化抽取失败：{explain_model_error(failure, model_id)}", **spent} ctx.emit(EventType.LLM_END, agent=ctx.node.title, model=model_id, purpose="cite_fields", - duration_ms=int((time.perf_counter() - started) * 1000), error=describe_exception(e)) - return {"error": f"结构化抽取失败：{explain_model_error(e, model_id)}"} - raw, parsed = _split_structured(got) - usage = _usage_of(raw if raw is not None else AIMessage(content=""), model_id) - ctx.emit(EventType.LLM_END, agent=ctx.node.title, model=model_id, purpose="cite_fields", - duration_ms=int((time.perf_counter() - started) * 1000), **usage) - return {"parsed": parsed, **usage} + duration_ms=elapsed, **usage) + return {"parsed": parsed, **spent} final_text = "" #: 模型把工具调用写成了文字：纠正过一次了没有 @@ -778,8 +819,10 @@ async def run_agent(state: GraphState, ctx: NodeContext) -> dict[str, Any]: total_usage[key] += int(extracted.get(key) or 0) total_usage["cost_usd"] = round(total_usage["cost_usd"] + float(extracted.get("cost_usd") or 0), 6) cited = verify_cited_fields(extracted, fields, queries) - for code, message, names in _field_warnings(*cited, extracted.get("error")): - await once(ctx, EventType.LOG, level="warn", code=code, message=message, fields=names) + truncated = extracted.get("truncated") is True + for code, message, names in _field_warnings(*cited, extracted.get("error"), truncated=truncated): + extra = {"max_tokens": extracted.get("max_tokens")} if code == "agent_field_truncated" else {} + await once(ctx, EventType.LOG, level="warn", code=code, message=message, fields=names, **extra) total_usage["total_tokens"] = total_usage["input_tokens"] + total_usage["output_tokens"] result = { @@ -941,6 +984,60 @@ def cited_schema(schema: dict[str, Any]) -> dict[str, Any]: return {"title": "cited_fields", **rewrite(schema)} +#: cite_fields 结构化抽取的输出额度。thinking: off 只对 Anthropic 有用：经 OpenAI 兼容接口接入的推理模型 +#: （DeepSeek 系等）关不掉思考，推理也算在这个额度里。以前是 4096：2026-10 一次运行 4 个 Agent 的抽取全部在 +#: 推理阶段就用完了额度（completion_tokens=4096），一个字段都没写出来。节点配了更大的「最大输出 token」就用节点的 +EXTRACT_MAX_TOKENS = 16384 +#: 被截断时额度翻倍重试一次，最多到这里 +EXTRACT_RETRY_CAP = 32768 +#: 服务方嫌额度太大、拒收时依次退到这几档（DeepSeek chat 系只收 8K）。拒收发生在请求刚发出时，不产生用量 +_EXTRACT_FALLBACKS = (8192, 4096) +#: 服务方嫌额度太大时报错里会出现的字样（和 api/copilot.py 改图时退回小额度是同一个判断） +_BUDGET_WORDS = ("max_tokens", "max_completion_tokens", "max_output_tokens", "maximum context length") +#: 截断类报错的原话：openai 的 LengthFinishReasonError（结构化输出解析时 finish_reason=length） +_LENGTH_WORDS = ("length limit was reached", "finish_reason=length", "finish_reason='length'") +#: 两次都被截断时的原因（给人看）。重试、去设置页测试模型都不解决额度问题，不能照通用的模型报错那样说 +TRUNCATED_EXTRACT = ("结构化抽取没有完成：模型的输出额度在推理阶段就用完了（最后一次的额度为 {budget} token），" + "结构化结论没能写出来") +#: agent_field_truncated 的下一步（接在「N 个字段都记为空值」后面） +TRUNCATED_NEXT = ("可以为该 Agent 换用非推理模型（抽取沿用 Agent 的模型），或者调大该 Agent 的" + "「最大输出 token」后重新运行") + + +def truncated_error(error: BaseException | None) -> bool: + """这个异常是不是在说「输出被截断」：openai 的 LengthFinishReasonError，或者 langchain、网关转述的同一件事。 + + 服务方嫌额度太大的 400（报错里也有 max_tokens）不算：那是请求被拒，模型一个字都没写。 + """ + if error is None: + return False + if type(error).__name__ == "LengthFinishReasonError": + return True + text = str(error).lower() + return any(word in text for word in _LENGTH_WORDS) + + +def truncated_reply(raw: Any) -> bool: + """这条回复是不是因为额度用完停下的：OpenAI 系是 finish_reason=length，Anthropic 是 stop_reason=max_tokens。""" + meta = getattr(raw, "response_metadata", None) or {} + reason = meta.get("finish_reason") or meta.get("stop_reason") + return reason in ("length", "max_tokens") + + +def _budget_rejected(error: BaseException) -> bool: + status = getattr(error, "status_code", None) or getattr(getattr(error, "response", None), "status_code", None) + return status in (400, 422) and any(word in str(error).lower() for word in _BUDGET_WORDS) + + +def _error_usage(error: BaseException, model_id: str) -> dict[str, Any]: + """失败的那次调用花掉的用量。截断时 openai 的异常带着 completion.usage：额度是真花出去了，要记账。""" + usage = getattr(getattr(error, "completion", None), "usage", None) + inp = int(getattr(usage, "prompt_tokens", 0) or 0) + out = int(getattr(usage, "completion_tokens", 0) or 0) + return {"input_tokens": inp, "output_tokens": out, "total_tokens": inp + out, + "cost_usd": round(catalog.estimate_cost(model_id, inp, out), 6), "calls": int(bool(inp or out))} + + EXTRACT_SYSTEM = ( "你是数据抽取员：把智能体查到的数据按给定结构填好，每个字段都写出处。\n" "- value 是字段的值；from 写它在哪次查询的哪一格：call 是【证据 Qn】里的编号，row 是行号（从 0 数），" @@ -1184,9 +1281,12 @@ def _shown(value: Any) -> str: def _field_warnings(data: dict[str, Any], evidence: dict[str, Any], - failed: str | None) -> list[tuple[str, str, list[str]]]: + failed: str | None, *, truncated: bool = False) -> list[tuple[str, str, list[str]]]: """核对结果里要让人看见的：(code, message, 字段)。mismatch 和 unresolved 各一条；missing 是模型 - 照实说没查到，值已经是空的，不另发警告。""" + 照实说没查到，值已经是空的，不另发警告。 + + 抽取两次都被截断（truncated）时 code 是 agent_field_truncated：原因是输出额度在推理阶段就用完了， + 下一步是换非推理模型或调大最大输出 token，不是「稍后重试」——和别的抽取失败分开说。""" out: list[tuple[str, str, list[str]]] = [] mismatched = [(p, e) for p, e in evidence.items() if e.get("status") == "mismatch"] if mismatched: @@ -1196,6 +1296,11 @@ def _field_warnings(data: dict[str, Any], evidence: dict[str, Any], f"有 {len(mismatched)} 个字段与查询快照不一致，已按快照取值：{detail}", [p for p, _ in mismatched])) unresolved = [(p, e) for p, e in evidence.items() if e.get("status") == "unresolved"] if unresolved: + if failed and truncated: + out.append(("agent_field_truncated", + f"{failed}。{len(unresolved)} 个字段都记为空值（未以 0 代替）。{TRUNCATED_NEXT}", + [p for p, _ in unresolved])) + return out if failed: message = f"{failed}。{len(unresolved)} 个字段都记为空值（未以 0 代替）" else: diff --git a/backend/tests/test_extract_truncation.py b/backend/tests/test_extract_truncation.py new file mode 100644 index 00000000..89ee80ee --- /dev/null +++ b/backend/tests/test_extract_truncation.py @@ -0,0 +1,208 @@ +"""cite_fields 的结构化抽取：输出被截断时加大额度重试，仍截断就照实说是额度在推理阶段用完了。 + +2026-10 一次运行里 4 个开了 cite_fields 的 Agent 抽取全部失败：抽取用 thinking: off、额度 4096，而经 OpenAI 兼容接口 +接入的推理模型关不掉思考，4096 在推理阶段就用完了，openai 抛 LengthFinishReasonError（「Could not parse response +content as the length limit was reached」）。日志却写「请稍后重试；如持续失败，请到设置测试该模型」——重试、测试都没用。 + +- 额度默认至少 16384，节点配了更大的用节点的 +- 认出「输出被截断」：openai 的 length limit 类错误、finish_reason=length、Anthropic 的 stop_reason=max_tokens； + 被截断时额度翻倍（最多 32768）重试一次 +- 仍然截断：日志 code agent_field_truncated，说清是额度在推理阶段用完了，给出能做的下一步；字段记空值，不以 0 代替 +- 服务方嫌额度太大（400，报错里有 max_tokens 一类字样）：退回小一档的额度重来，不能因为额度提高反而抽不成 +- 非截断类的失败照旧（agent_field_unverified） + +夹具全部合成：门店、会员这类虚构的表。 +""" +from __future__ import annotations + +import json + +import pytest +from langchain_core.messages import AIMessage +from openai import LengthFinishReasonError +from openai.types import CompletionUsage +from openai.types.chat import ChatCompletion + +from app.engine.nodes import llm as llm_mod +from app.engine.runner import run_manager +from tests.test_agent_cited_fields import ( # noqa: F401 - 夹具按名字引入 + EXTRACTED, + FINAL, + SCHEMA, + SQLS, + TOOL, + Script, + engine_up, + events, + graph, + logs, + output_of, + shop, + wait, +) + + +def length_error(prompt: int = 12000, completion: int = 4096) -> LengthFinishReasonError: + """openai 的结构化输出在 finish_reason=length 时抛的那个错（和真实运行里的原话一样）。""" + usage = CompletionUsage(completion_tokens=completion, prompt_tokens=prompt, total_tokens=prompt + completion) + return LengthFinishReasonError(completion=ChatCompletion.model_construct( + id="x", choices=[], created=0, model="m", object="chat.completion", usage=usage)) + + +class BudgetRejected(Exception): + """服务方嫌输出额度太大时的 400（DeepSeek chat 系只收 8K）。""" + + status_code = 400 + + +class Extraction: + """记下每次抽取调用的额度（thinking 关掉的那次 get_chat_model），并按额度决定这次怎么回。 + + outcome(budget) 返回 None 表示照常回 EXTRACTED，抛异常就是这次调用失败。 + """ + + def __init__(self, monkeypatch, outcome): + from app.providers import mock_model + + self.budgets: list[int | None] = [] + real = llm_mod.get_chat_model + + async def spy(session, spec): + if spec.thinking == "off": + self.budgets.append(spec.max_tokens) + return await real(session, spec) + + monkeypatch.setattr(llm_mod, "get_chat_model", spy) + base = Script(monkeypatch) + loop = mock_model.MockChatModel._decide + + def decide(model, messages): + if model.response_format: + outcome(self.budgets[-1]) + return AIMessage(content=json.dumps(EXTRACTED, ensure_ascii=False)) + return loop(model, messages) + + monkeypatch.setattr(mock_model.MockChatModel, "_decide", decide) + self.script = base + + +# -------------------------------------------------------------------------- +# 认出「输出被截断」 +# -------------------------------------------------------------------------- + + +def test_length_errors_are_recognised(): + assert llm_mod.truncated_error(length_error()) + assert llm_mod.truncated_error(RuntimeError("Could not parse response content as the length limit was reached")) + assert not llm_mod.truncated_error(RuntimeError("网关超时")) + assert not llm_mod.truncated_error(BudgetRejected("max_tokens is too large: 16384")) + + +def test_cut_off_replies_are_recognised_for_both_families(): + assert llm_mod.truncated_reply(AIMessage(content="", response_metadata={"finish_reason": "length"})) + assert llm_mod.truncated_reply(AIMessage(content="", response_metadata={"stop_reason": "max_tokens"})) + assert not llm_mod.truncated_reply(AIMessage(content="", response_metadata={"finish_reason": "stop"})) + assert not llm_mod.truncated_reply(None) + + +# -------------------------------------------------------------------------- +# 额度与重试 +# -------------------------------------------------------------------------- + + +async def test_the_default_budget_leaves_room_for_reasoning(monkeypatch): + spy = Extraction(monkeypatch, lambda budget: None) + row = await wait((await run_manager.start(graph=graph(card=False), input_payload={})).id) + assert row.status == "succeeded", row.error + assert spy.budgets == [16384], "推理模型的思考也算在额度里：默认至少 16384" + + +async def test_a_larger_node_budget_is_used(monkeypatch): + spy = Extraction(monkeypatch, lambda budget: None) + row = await wait((await run_manager.start(graph=graph(card=False, max_tokens=24000), input_payload={})).id) + assert row.status == "succeeded", row.error + assert spy.budgets == [24000] + + +async def test_a_cut_off_extraction_is_retried_with_a_larger_budget(monkeypatch): + def outcome(budget): + if budget < 32768: + raise length_error(completion=budget) + + spy = Extraction(monkeypatch, outcome) + row = await wait((await run_manager.start(graph=graph(card=False), input_payload={})).id) + assert row.status == "succeeded", row.error + assert spy.budgets == [16384, 32768], "截断后翻倍重试一次" + bot = await output_of(row.id, "bot") + assert bot["data"]["gmv"] == 600.5 and bot["data_evidence"]["gmv"]["status"] == "verified" + evts = await events(row.id) + assert not logs(evts, "agent_field_truncated") + # EXTRACTED 里 region 指向本节点没有的 Q9：只有它核对不了，别的字段都照常核对 + assert [w["fields"] for w in logs(evts, "agent_field_unverified")] == [["region"]] + # 两次调用各记一行：第一次写明为什么重来 + ends = [e.data for e in evts if e.type == "llm.end" and e.data.get("purpose") == "cite_fields"] + assert len(ends) == 2 and "32768" in ends[0]["error"] and not ends[1].get("error") + # 被截断那次的用量照样记账（花出去的 token 是真的） + assert ends[0]["output_tokens"] == 16384 + + +async def test_cut_off_twice_is_reported_as_truncation_with_fields_empty(monkeypatch): + def outcome(budget): + raise length_error(completion=budget) + + spy = Extraction(monkeypatch, outcome) + row = await wait((await run_manager.start(graph=graph(card=False), input_payload={})).id) + assert row.status == "succeeded", row.error + assert spy.budgets == [16384, 32768], "只重试一次" + bot = await output_of(row.id, "bot") + assert bot["data"] == {k: None for k in SCHEMA["properties"]}, "记空值，不以 0 代替" + assert all(e["status"] == "unresolved" for e in bot["data_evidence"].values()) + evts = await events(row.id) + assert not logs(evts, "agent_field_unverified") + [warn] = logs(evts, "agent_field_truncated") + assert warn["level"] == "warn" and warn["fields"] == list(SCHEMA["properties"]) + message = warn["message"] + assert "推理阶段就用完了" in message and "结构化结论没能写出来" in message + assert "32768" in message and "非推理模型" in message and "最大输出 token" in message + assert "请稍后重试" not in message and "测试该模型" not in message, "重试、测试模型都不解决额度问题" + assert "未以 0 代替" in message + assert warn.get("max_tokens") == 32768 + + +async def test_a_node_budget_at_the_cap_is_not_retried(monkeypatch): + def outcome(budget): + raise length_error(completion=budget) + + spy = Extraction(monkeypatch, outcome) + row = await wait((await run_manager.start(graph=graph(card=False, max_tokens=40000), input_payload={})).id) + assert row.status == "succeeded", row.error + assert spy.budgets == [40000], "额度已经超过重试上限，再调一次也一样" + [warn] = logs(await events(row.id), "agent_field_truncated") + assert "40000" in warn["message"] + + +async def test_a_budget_the_service_rejects_falls_back(monkeypatch): + def outcome(budget): + if budget > 8192: + raise BudgetRejected(f"Invalid max_tokens value, the valid range of max_tokens is [1, 8192]: {budget}") + + spy = Extraction(monkeypatch, outcome) + row = await wait((await run_manager.start(graph=graph(card=False), input_payload={})).id) + assert row.status == "succeeded", row.error + assert spy.budgets == [16384, 8192] + bot = await output_of(row.id, "bot") + assert bot["data_evidence"]["gmv"]["status"] == "verified" + + +async def test_other_failures_are_reported_as_before(monkeypatch): + def outcome(budget): + raise RuntimeError("网关超时") + + spy = Extraction(monkeypatch, outcome) + row = await wait((await run_manager.start(graph=graph(card=False), input_payload={})).id) + assert row.status == "succeeded", row.error + assert spy.budgets == [16384], "不是截断就不重试" + evts = await events(row.id) + assert not logs(evts, "agent_field_truncated") + [warn] = logs(evts, "agent_field_unverified") + assert "网关超时" in warn["message"] and "未以 0 代替" in warn["message"] diff --git a/frontend/src/run/decode.ts b/frontend/src/run/decode.ts index cd668956..f82b4844 100644 --- a/frontend/src/run/decode.ts +++ b/frontend/src/run/decode.ts @@ -784,6 +784,22 @@ function explainLog(code: string | undefined, message: string, d: Record 0 ? d.max_tokens : undefined + return { + title: `模型的输出额度在推理阶段用完，${n ? `${n} 个` : ''}字段记为空值（未以 0 代替）`, + sub: `按出处抽取字段时输出被截断${budget ? `（已加大额度重试，最后一次为 ${formatNumber(budget)} token）` : ''}，` + + '结构化结论没能写出来', + next: '这些字段以空值传给下游，口径卡按缺少输入处理，报告不得据此断言没有数据。请为该 Agent 换用非推理模型' + + '（抽取沿用 Agent 的模型），或调大它的「最大输出 token」后重新运行', + fix: 'canvas', + fixField: 'max_tokens', + } + } case 'metric_incomplete': { // 结构化字段（metrics.py）：metric_name、reason（「基于被截断的查询结果计算（只取回了前 1000 行），结果不完整」）。 // 老运行的日志没有这两个字段：只给不点名的标题，原话留在展开区，不从中文里解析 diff --git a/scripts/check-decode.mjs b/scripts/check-decode.mjs index c3ef2f87..5045adb1 100644 --- a/scripts/check-decode.mjs +++ b/scripts/check-decode.mjs @@ -1481,6 +1481,47 @@ await section('agent 字段按出处核对（可点击证据第二期）：抽 check('「抽取结果格式有误（不是对象）」：同样说「失败，N 个字段记为空值」，原因放副标题，不说「出处不存在」', notObject?.title === '按出处抽取字段失败，2 个字段记为空值（未以 0 代替）' && notObject?.sub === '抽取结果格式有误（不是对象）' && !notObject?.next?.includes('出处在查询结果中不存在'), `${notObject?.title} / ${notObject?.sub} / ${notObject?.next}`) + // 两次抽取都被截断（2026-10 一次运行：推理模型关不掉思考，4096 的额度在推理阶段就用完了）：单独的 code， + // 说清是额度问题、给出能做的下一步，不说「稍后重试」「去设置测试该模型」（llm.py TRUNCATED_EXTRACT、TRUNCATED_NEXT） + const TRUNCATED = '结构化抽取没有完成：模型的输出额度在推理阶段就用完了（最后一次的额度为 32768 token），结构化结论没能写出来。' + + '5 个字段都记为空值（未以 0 代替）。可以为该 Agent 换用非推理模型（抽取沿用 Agent 的模型），或者调大该 Agent 的「最大输出 token」后重新运行' + check('后端发截断警告的说法还是夹具里这一种', llmPy.includes('"agent_field_truncated"') + && llmPy.includes('结构化抽取没有完成：模型的输出额度在推理阶段就用完了（最后一次的额度为 {budget} token），"') + && llmPy.includes('"结构化结论没能写出来"') && llmPy.includes('可以为该 Agent 换用非推理模型（抽取沿用 Agent 的模型），或者调大该 Agent 的"') + && llmPy.includes('已把额度加大到 {bigger} token 重试一次'), '改了 llm.py 的说法就同步改 decode.ts 的 explainLog 和这里的夹具') + const FIELDS5 = ['members', 'stores', 'coupons', 'redeemed', 'rate'] + const cut = flatten(mod.decodeRun([ + E(1, 'node.started', 'fetch', { node_type: 'agent', label: '会员专项' }), + E(2, 'llm.start', 'fetch', { model: 'm1', structured: true, purpose: 'cite_fields', message_count: 5, max_tokens: 16384 }), + E(3, 'llm.end', 'fetch', { agent: '会员专项', model: 'm1', purpose: 'cite_fields', duration_ms: 40000, output_tokens: 16384, + error: '输出被截断：16384 token 的输出额度用完了，结构化结论没有写完，已把额度加大到 32768 token 重试一次' }), + E(4, 'llm.start', 'fetch', { model: 'm1', structured: true, purpose: 'cite_fields', message_count: 5, max_tokens: 32768 }), + E(5, 'llm.end', 'fetch', { agent: '会员专项', model: 'm1', purpose: 'cite_fields', duration_ms: 80000, output_tokens: 32768, + error: '输出被截断：32768 token 的输出额度用完了，结构化结论没有写完' }), + E(6, 'log', 'fetch', { level: 'warn', code: 'agent_field_truncated', message: TRUNCATED, fields: FIELDS5, max_tokens: 32768 }), + E(7, 'node.finished', 'fetch', { duration_ms: 125000 }), + ])) + const tr = cut.find((s) => s.code === 'agent_field_truncated') + check('agent_field_truncated：标题说额度在推理阶段用完、几个字段记为空值', tr?.title === '模型的输出额度在推理阶段用完，5 个字段记为空值（未以 0 代替）', tr?.title) + check('……副标题说抽取时输出被截断、重试到的额度（读 max_tokens 字段）', + tr?.sub === '按出处抽取字段时输出被截断（已加大额度重试，最后一次为 32,768 token），结构化结论没能写出来', tr?.sub) + check('……下一步：换非推理模型或调大最大输出 token；不说稍后重试，不叫人去测试模型', + !!tr?.next?.includes('非推理模型') && !!tr?.next?.includes('「最大输出 token」') && !tr?.next?.includes('稍后重试') + && !tr?.next?.includes('测试') && !!tr?.next?.includes('不得据此断言没有数据'), tr?.next) + check('……入口打开该 Agent 的设置、落到最大输出 token；是提醒（warn），原话留在展开区', + tr?.fix === 'canvas' && tr?.fixField === 'max_tokens' && tr?.level === 'warn' && tr?.detail === TRUNCATED, + `${tr?.fix} ${tr?.fixField} ${tr?.level}`) + const tries = cut.filter((s) => s.kind === 'llm') + check('……两次抽取各成一行，第一次的展开区写明已加大额度重试', tries.length === 2 + && !!tries[0].detail?.includes('已把额度加大到 32768 token 重试一次') && !!tries[1].detail?.includes('结构化结论没有写完'), + tries.map((s) => s.detail).join(' | ')) + check('……节点本身照样成功', cut.find((s) => s.kind === 'node')?.status === 'done') + const bareCut = flatten(mod.decodeRun([E(1, 'node.started', 'x', { node_type: 'agent' }), + E(2, 'log', 'x', { level: 'warn', code: 'agent_field_truncated', message: '结构化抽取没有完成。3 个字段都记为空值（未以 0 代替）' })])) + .find((s) => s.code === 'agent_field_truncated') + check('……缺 fields、max_tokens 字段时字段数从原话里认，不出 undefined', + bareCut?.title === '模型的输出额度在推理阶段用完，3 个字段记为空值（未以 0 代替）' + && bareCut?.sub === '按出处抽取字段时输出被截断，结构化结论没能写出来', `${bareCut?.title} / ${bareCut?.sub}`) const node = mod.decodeRun([E(1, 'node.started', 'fetch', { node_type: 'agent' }), E(2, 'node.finished', 'fetch', { duration_ms: 1 })]) check('对照：普通 agent 节点不多出任何行', flatten(node).filter((s) => s.kind !== 'node').length === 0) const odd = flatten(mod.decodeRun([E(1, 'node.started', 'x', { node_type: 'agent' }), From b45a4966c59cf3c6dd4fe06e3483a34a0923163f Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 4 Oct 2026 11:05:33 +0800 Subject: [PATCH 02/15] =?UTF-8?q?OpenAI=20=E5=85=BC=E5=AE=B9=E6=8E=A5?= =?UTF-8?q?=E5=85=A5=E5=BC=80=E5=90=AF=E6=B5=81=E5=BC=8F=E7=94=A8=E9=87=8F?= =?UTF-8?q?=E5=9B=9E=E4=BC=A0=EF=BC=9AChatOpenAI=20=E5=8F=AA=E5=9C=A8?= =?UTF-8?q?=E6=B2=A1=E5=A1=AB=20base=5Furl=20=E6=97=B6=E9=BB=98=E8=AE=A4?= =?UTF-8?q?=E8=AF=B7=E6=B1=82=E6=B5=81=E5=B0=BE=E7=9A=84=E7=94=A8=E9=87=8F?= =?UTF-8?q?=EF=BC=8C=E5=85=BC=E5=AE=B9=E6=8E=A5=E5=8F=A3=E9=83=BD=E5=A1=AB?= =?UTF-8?q?=E4=BA=86=20base=5Furl=EF=BC=8C=E4=B8=80=E6=AC=A1=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=20=E4=B8=8A=E7=99=BE=E6=AC=A1=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E8=AE=B0=E6=88=90=200=20token=E3=80=810=20?= =?UTF-8?q?=E7=BE=8E=E5=85=83=E3=80=82=E6=9E=84=E9=80=A0=E6=97=B6=E4=BC=A0?= =?UTF-8?q?=20stream=5Fusage=3DTrue=EF=BC=9B=E6=9C=8D=E5=8A=A1=E6=96=B9?= =?UTF-8?q?=E4=B8=8D=E8=AE=A4=20stream=5Foptions=EF=BC=88=E7=82=B9?= =?UTF-8?q?=E5=90=8D=E5=AE=83=EF=BC=8C=E6=88=96=E5=8F=AA=E5=9B=9E=20400=20?= =?UTF-8?q?/=20422=EF=BC=89=E6=97=B6=E5=90=8C=E4=B8=80=E6=AC=A1=E8=B0=83?= =?UTF-8?q?=E7=94=A8=E5=8E=BB=E6=8E=89=E5=AE=83=E9=87=8D=E5=8F=91=E3=80=81?= =?UTF-8?q?=E7=85=A7=E5=B8=B8=E6=B5=81=E5=BC=8F=E8=BE=93=E5=87=BA=EF=BC=8C?= =?UTF-8?q?=E5=B9=B6=E8=AE=B0=E4=B8=8B=E8=BF=99=E4=B8=AA=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E4=B9=8B=E5=90=8E=E4=B8=8D=E5=86=8D=E5=B8=A6=E5=AE=83=EF=BC=9B?= =?UTF-8?q?=E9=87=8D=E5=8F=91=E4=B9=9F=E5=A4=B1=E8=B4=A5=E5=B0=B1=E6=8A=A5?= =?UTF-8?q?=E9=87=8D=E5=8F=91=E7=9A=84=E9=94=99=E3=80=81=E4=B8=8D=E8=AE=B0?= =?UTF-8?q?=E3=80=82=E6=B5=8B=E8=AF=95=E7=94=A8=E5=81=87=E7=9A=84=20SSE=20?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E6=A0=B8=E5=AF=B9=E8=AF=B7=E6=B1=82=E4=BD=93?= =?UTF-8?q?=E3=80=81=E5=90=88=E5=B9=B6=E5=90=8E=E7=9A=84=E7=94=A8=E9=87=8F?= =?UTF-8?q?=E3=80=81=E9=80=80=E5=9B=9E=E5=92=8C=E8=BF=90=E8=A1=8C=E7=94=A8?= =?UTF-8?q?=E9=87=8F=E9=87=8C=E7=9A=84=20token=20=E4=B8=8E=E9=87=91?= =?UTF-8?q?=E9=A2=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/providers/factory.py | 64 +++++++ backend/tests/test_openai_stream_usage.py | 213 ++++++++++++++++++++++ 2 files changed, 277 insertions(+) create mode 100644 backend/tests/test_openai_stream_usage.py diff --git a/backend/app/providers/factory.py b/backend/app/providers/factory.py index 6a7c3870..0f7e3cee 100644 --- a/backend/app/providers/factory.py +++ b/backend/app/providers/factory.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import os from typing import Any @@ -14,6 +15,8 @@ from app.db.models import Provider from app.providers import catalog from app.providers.mock_model import MockChatModel +logger = logging.getLogger(__name__) + class ModelSpec(BaseModel): """节点上对模型的请求。省略的字段会回退到 provider 默认值。""" @@ -221,6 +224,10 @@ def build_chat_model(provider: Provider, spec: ModelSpec) -> BaseChatModel: kwargs["base_url"] = base_url if extra.get("headers"): kwargs["default_headers"] = extra["headers"] + # 流式调用要服务方在流尾补一块用量（stream_options.include_usage）。ChatOpenAI 只在没填 base_url 时默认开， + # 兼容接口都填了 base_url，于是一次运行上百次调用记成 0 token、0 美元——llm.end 读的 usage_metadata + # 就来自这一块。不认 stream_options 的服务由 ReasoningAwareChatOpenAI 去掉它重发，不让调用失败 + kwargs["stream_usage"] = True return _ReasoningAwareChatOpenAI(**kwargs) @@ -260,9 +267,66 @@ def _reasoning_aware_openai() -> type: gen.message.additional_kwargs["reasoning_content"] = text return gen + async def _astream(self, messages: Any, stop: Any = None, run_manager: Any = None, **kwargs: Any): + """流式调用；服务方不认 stream_options 时去掉它重发一次，之后这个接口不再带它。 + + 只在还没收到任何一块时重发：拒绝都发生在请求刚发出去的时候，重发不会重复输出。""" + usage = not self._use_responses_api({**kwargs, **self.model_kwargs}) \ + and self._should_stream_usage(kwargs.get("stream_usage"), **kwargs) + key = _endpoint_key(self) + first: Exception | None = None + if usage and key not in _NO_STREAM_USAGE: + started = False + try: + async for chunk in super()._astream(messages, stop, run_manager, **dict(kwargs)): + started = True + yield chunk + return + except Exception as e: # noqa: BLE001 - 认得出的拒绝才重发，其余原样抛出 + if started or not _rejects_stream_options(e): + raise + first = e + if usage: + kwargs = {k: v for k, v in kwargs.items() if k != "stream_options"} | {"stream_usage": False} + try: + async for chunk in super()._astream(messages, stop, run_manager, **kwargs): + if first is not None and key not in _NO_STREAM_USAGE: + # 去掉它就成了：记下这个接口不认，之后不再每次先失败一次 + _NO_STREAM_USAGE.add(key) + logger.warning("模型接入 %s 不支持流式用量回传（%s），已改为不请求用量", key, str(first)[:200]) + yield chunk + except Exception as e: + if first is None: + raise + raise e from first + return ReasoningAwareChatOpenAI +#: 认定不支持 stream_options 的接口（base_url）。进程内有效：重启后再试一次，服务方可能已经升级 +_NO_STREAM_USAGE: set[str] = set() + + +def _endpoint_key(model: Any) -> str: + return str(getattr(model, "openai_api_base", None) or "https://api.openai.com/v1") + + +def _rejects_stream_options(exc: Exception) -> bool: + """这个报错是不是服务方不认 stream_options。 + + 点名了 stream_options / include_usage 的一定是；只回 400 / 422、说不清哪个参数的（有的网关只回一句 + Bad Request）也去掉它试一次——重发也失败就说明不是它的事，原样报重发的错。上下文超长同样是 400， + 重发也没用，不算。""" + from langchain_core.exceptions import ContextOverflowError + + if isinstance(exc, ContextOverflowError): + return False + text = str(exc).lower() + if "stream_options" in text or "include_usage" in text: + return True + return getattr(exc, "status_code", None) in (400, 422) + + class _LazyReasoningChatOpenAI: """延迟导入 langchain_openai：它加载得慢，而 mock/anthropic 路径用不上。""" diff --git a/backend/tests/test_openai_stream_usage.py b/backend/tests/test_openai_stream_usage.py new file mode 100644 index 00000000..99c539f7 --- /dev/null +++ b/backend/tests/test_openai_stream_usage.py @@ -0,0 +1,213 @@ +"""OpenAI 兼容接入的流式用量回传（stream_options.include_usage）。 + +真实运行里：模型走 LiteLLM 的 OpenAI 兼容接口，上百次模型调用，运行用量却是 0 token、0 美元。ChatOpenAI +只在没填 base_url 时默认开流式用量回传；填了 base_url（所有兼容接口都填）就不开，流里没有 usage 那一块， +llm.end 记下的全是 0，运行的用量是 llm.end 逐次加总的，于是也是 0。 + +这组守的是： +- 兼容接入构造出来的模型开着流式用量回传，请求里带 stream_options.include_usage +- 流里最后那块 usage 合进响应，llm.end 和运行的用量里 token、金额都对 +- 不认 stream_options 的服务：同一次调用去掉它重发一次，照常流式输出，不让调用失败；同一个接口之后不再带它 +- 和用量回传无关的报错不重发、原样抛出 + +假的是 HTTP：openai SDK 发出去的请求在它的 HTTP 库的传输层被接住，按请求体回 SSE。模型名、内容都是虚构的。 +""" +from __future__ import annotations + +import asyncio +import json +import uuid + +import pytest +from langchain_core.messages import HumanMessage +from sqlalchemy import delete, select + +from app.db.base import SessionLocal +from app.db.models import Provider, Run, RunEvent +from app.engine.runner import run_manager +from app.providers import catalog, factory +from app.providers.factory import ModelSpec, build_chat_model + +try: # openai SDK 3 起换成了 httpx2（和 httpx 同一套接口）：假服务要装在 SDK 真正用的那个库上 + from langchain_openai._compat import httpx +except ImportError: # pragma: no cover - 老版本的 langchain_openai + import httpx + +PRICED = "gpt-5.2-mini" +USAGE = {"prompt_tokens": 1200, "completion_tokens": 300, "total_tokens": 1500} + + +def _chunk(delta: dict | None = None, *, finish: str | None = None, usage: dict | None = None) -> str: + body = {"id": "c-1", "object": "chat.completion.chunk", "created": 1, "model": PRICED, + "choices": [] if delta is None else [{"index": 0, "delta": delta, "finish_reason": finish}]} + if usage is not None: + body["usage"] = usage + return f"data: {json.dumps(body, ensure_ascii=False)}\n\n" + + +class FakeServer: + """OpenAI 兼容服务：带了 stream_options.include_usage 才在流尾补 usage 那一块。 + + rejects：不认 stream_options 的服务，带了就回 status（400 / 422）和 message； + fail：每个请求都回这个错误（和用量回传无关的报错）。""" + + def __init__(self, *, rejects: tuple[int, str] | None = None, fail: tuple[int, str] | None = None) -> None: + self.rejects, self.fail = rejects, fail + self.bodies: list[dict] = [] + + async def handle(self, request: httpx.Request) -> httpx.Response: + body = json.loads(request.content or b"{}") + self.bodies.append(body) + wants = bool((body.get("stream_options") or {}).get("include_usage")) + if self.fail or (self.rejects and "stream_options" in body): + status, message = self.fail or self.rejects + return httpx.Response(status, json={"error": {"message": message, "type": "invalid_request_error"}}, + request=request) + if not body.get("stream"): + reply = {"id": "c-1", "object": "chat.completion", "created": 1, "model": PRICED, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "上周到店 120 人。"}, + "finish_reason": "stop"}], "usage": USAGE} + return httpx.Response(200, json=reply, request=request) + sse = (_chunk({"role": "assistant", "content": "上周"}) + _chunk({"content": "到店 120 人。"}) + + _chunk({}, finish="stop") + (_chunk(usage=USAGE) if wants else "") + "data: [DONE]\n\n") + return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=sse.encode(), + request=request) + + +@pytest.fixture +def server(monkeypatch): + def install(**kwargs) -> FakeServer: + fake = FakeServer(**kwargs) + monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", fake.handle) + return fake + # 记住「这个接口不认 stream_options」的表是进程级的：每个用例从空表开始 + monkeypatch.setattr(factory, "_NO_STREAM_USAGE", set(), raising=False) + return install + + +def _provider(base_url: str | None = None, *, kind: str = "openai_compatible") -> Provider: + return Provider(name=f"compat-{uuid.uuid4().hex[:8]}", kind=kind, + base_url=base_url or f"http://llm-{uuid.uuid4().hex[:6]}.example.test/v1", + api_key=None, models=[{"id": PRICED}], default_model=PRICED, enabled=True, extra={}) + + +async def _stream(model) -> tuple[str, dict | None, int]: + """流式调用一次：(正文, 合并后的 usage_metadata, 块数)。""" + merged, count = None, 0 + async for chunk in model.astream([HumanMessage(content="上周到店多少人")]): + merged = chunk if merged is None else merged + chunk + count += 1 + return str(merged.content), merged.usage_metadata, count + + +# ---------------------------------------------------------------- 构造 + + +@pytest.mark.parametrize("kind", ["openai_compatible", "openai"]) +def test_compatible_providers_turn_on_streaming_usage(kind): + model = build_chat_model(_provider(kind=kind), ModelSpec()) + assert model.stream_usage is True + + +# ---------------------------------------------------------------- 流式调用 + + +async def test_usage_chunk_is_requested_and_merged(server): + fake = server() + model = build_chat_model(_provider(), ModelSpec()) + text, usage, _ = await _stream(model) + assert text == "上周到店 120 人。" + assert fake.bodies[0]["stream_options"] == {"include_usage": True} + assert usage and usage["input_tokens"] == 1200 and usage["output_tokens"] == 300 + + +@pytest.mark.parametrize("status,message", [ + (400, "Unrecognized request argument supplied: stream_options"), + (422, "body -> stream_options: extra fields not permitted"), + (400, "Bad Request"), +], ids=["named-400", "named-422", "generic-400"]) +async def test_service_rejecting_stream_options_falls_back_and_is_remembered(server, status, message): + fake = server(rejects=(status, message)) + provider = _provider() + model = build_chat_model(provider, ModelSpec()) + text, usage, count = await _stream(model) + # 照常流式输出：去掉 stream_options 重发了一次，正文分块到达 + assert text == "上周到店 120 人。" and count >= 2 + assert ["stream_options" in b for b in fake.bodies] == [True, False] + assert usage is None or not usage.get("input_tokens"), "这个服务给不了流式用量，不编" + + # 同一个接口之后不再带 stream_options（新构造的模型也一样），不再每次先失败一次 + again = build_chat_model(provider, ModelSpec()) + await _stream(again) + assert ["stream_options" in b for b in fake.bodies] == [True, False, False] + + +async def test_other_errors_are_not_retried(server): + fake = server(fail=(401, "invalid api key")) + model = build_chat_model(_provider(), ModelSpec()) + with pytest.raises(Exception, match="invalid api key"): + await _stream(model) + assert len(fake.bodies) == 1 + + +async def test_a_failing_retry_is_not_remembered(server): + """去掉 stream_options 重发也失败：说明不是它的事，报重发的那个错，也不记成这个接口不认它。""" + fake = server(fail=(400, "max_tokens is too large")) + provider = _provider() + model = build_chat_model(provider, ModelSpec()) + with pytest.raises(Exception, match="max_tokens is too large"): + await _stream(model) + assert ["stream_options" in b for b in fake.bodies] == [True, False] + assert provider.base_url not in factory._NO_STREAM_USAGE + + +# ---------------------------------------------------------------- 运行的用量 + + +@pytest.fixture +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +async def test_run_usage_counts_streamed_tokens(server, engine_up): + """整条链：兼容接入 → llm 节点流式调用 → llm.end → 运行的用量。修之前这里是 0 token、0 美元。""" + server() + provider = _provider() + async with SessionLocal() as session: + session.add(provider) + await session.commit() + await session.refresh(provider) + session.expunge(provider) + try: + graph = { + "nodes": [ + {"id": "in", "type": "input", "data": {"config": {}}}, + {"id": "ask", "type": "llm", "data": {"label": "问答", "config": { + "prompt": "上周到店多少人", "provider": provider.name, "model": PRICED}}}, + {"id": "out", "type": "output", "data": {"config": { + "fields": [{"name": "回答", "value": "{{ nodes.ask.text }}"}]}}}, + ], + "edges": [{"source": "in", "target": "ask"}, {"source": "ask", "target": "out"}], + } + started = await run_manager.start(graph=graph, input_payload={}) + for _ in range(200): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, started.id) + if row.status in ("succeeded", "failed"): + break + assert row.status == "succeeded", row.error + async with SessionLocal() as session: + ends = list((await session.execute(select(RunEvent).where( + RunEvent.run_id == started.id, RunEvent.type == "llm.end"))).scalars()) + assert [(e.data["input_tokens"], e.data["output_tokens"]) for e in ends] == [(1200, 300)] + assert row.usage["input_tokens"] == 1200 and row.usage["output_tokens"] == 300 + assert row.usage["total_tokens"] == 1500 + expected = round(catalog.estimate_cost(PRICED, 1200, 300), 6) + assert expected > 0 and row.usage["cost_usd"] == pytest.approx(expected) + finally: + async with SessionLocal() as session: + await session.execute(delete(Provider).where(Provider.id == provider.id)) + await session.commit() From fa5cb1d47bb05c1b8842ca1f879021000333e39f Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 4 Oct 2026 11:07:40 +0800 Subject: [PATCH 03/15] =?UTF-8?q?=E5=8A=A9=E6=89=8B=E5=86=99=E4=BA=BA?= =?UTF-8?q?=E5=B7=A5=E5=AE=A1=E6=89=B9=E7=9A=84=E8=AF=B4=E6=98=8E=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E8=AE=B8=E8=AF=BA=E3=80=8C=E6=AD=A3=E5=BC=8F=E5=87=BA?= =?UTF-8?q?=E5=85=B7=E3=80=8D=EF=BC=9A=E7=9C=9F=E5=AE=9E=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E9=87=8C=E4=B8=80=E6=AC=A1=E6=8E=A2=E7=B4=A2=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E7=9A=84=E5=AE=A1=E6=89=B9=E5=86=99=E7=9D=80=E3=80=8C=E6=89=B9?= =?UTF-8?q?=E5=87=86=E5=90=8E=E6=89=8D=E4=BC=9A=E6=AD=A3=E5=BC=8F=E5=87=BA?= =?UTF-8?q?=E5=85=B7=E3=80=8D=EF=BC=8C=E6=88=90=E6=9E=9C=E8=8A=82=E7=82=B9?= =?UTF-8?q?=E4=B9=9F=E6=B2=A1=E6=9C=89=E5=87=BA=E5=85=B7=E5=A5=91=E7=BA=A6?= =?UTF-8?q?=E3=80=82=E8=8A=82=E7=82=B9=E6=89=8B=E5=86=8C=E7=9A=84=20human?= =?UTF-8?q?=20=E4=B8=80=E6=9D=A1=E5=86=99=E6=98=8E=E5=8F=AA=E6=9C=89?= =?UTF-8?q?=E6=88=90=E6=9E=9C=E8=8A=82=E7=82=B9=E9=85=8D=E4=BA=86=E5=87=BA?= =?UTF-8?q?=E5=85=B7=E5=A5=91=E7=BA=A6=E3=80=81=E4=BB=8E=E5=B7=B2=E5=8F=91?= =?UTF-8?q?=E5=B8=83=E7=89=88=E6=9C=AC=E5=8F=91=E8=B5=B7=E7=9A=84=E6=AD=A3?= =?UTF-8?q?=E5=BC=8F=E8=BF=90=E8=A1=8C=E6=89=8D=E4=BC=9A=E5=87=BA=E5=85=B7?= =?UTF-8?q?=EF=BC=8C=E5=AE=A1=E6=89=B9=E8=AF=B4=E6=98=8E=E5=86=99=E3=80=8C?= =?UTF-8?q?=E6=89=B9=E5=87=86=E5=90=8E=E8=BE=93=E5=87=BA=E3=80=8D=EF=BC=8C?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E7=B1=BB=E5=88=AB=E7=94=B1=E5=AE=A1=E6=89=B9?= =?UTF-8?q?=E5=8D=A1=E6=A0=87=E6=98=8E=EF=BC=9B=E8=87=AA=E6=9F=A5=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=20human=5Fformal=5Fissuance=EF=BC=9A=E8=BF=99?= =?UTF-8?q?=E4=B8=80=E8=BD=AE=E5=86=99=E7=9A=84=E5=AE=A1=E6=89=B9=E6=A0=87?= =?UTF-8?q?=E9=A2=98=E3=80=81=E8=AF=B4=E6=98=8E=E9=87=8C=E6=9C=89=E3=80=8C?= =?UTF-8?q?=E6=AD=A3=E5=BC=8F=E5=87=BA=E5=85=B7=E3=80=8D=E5=B0=B1=E4=BA=A4?= =?UTF-8?q?=E5=9B=9E=E6=A8=A1=E5=9E=8B=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/copilot.py | 35 +++++ .../tests/test_copilot_approval_wording.py | 142 ++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 backend/tests/test_copilot_approval_wording.py diff --git a/backend/app/api/copilot.py b/backend/app/api/copilot.py index 54732cfc..be9168a5 100644 --- a/backend/app/api/copilot.py +++ b/backend/app/api/copilot.py @@ -193,6 +193,10 @@ NODE_REFERENCE = """\ 写进去再也读不回来——真发生过：一轮写进 scope"user"，下一轮没写 scope 落回 default，于是「我叫张三」存下来了，问「我是谁」却答不上来 - human：人工审批。config: {mode: approve|input|edit, title, message} + title、message 说批了会怎样时写「批准后输出」「批准后继续」，**不要写「正式出具」「批准后才会出具」**：出具只发生在 + 成果节点配了出具契约、并且从已发布版本发起的正式运行里；画布上试运行、问数据都是探索运行，批准后只是输出。 + 这次是探索运行还是正式运行，审批卡会自己标明。审的是报告时把 human 接在 report 后面（message 里可以引用 + {{ nodes.报告节点id.text }}），审批卡会附上这份报告的核对概况 - validate：JSON Schema 校验，可自动让模型修复。config: {schema, source, max_retries} - transform：数据整形。config: {mode: expression|template|json, expression/template, assign_to} - subgraph：嵌套另一个工作流。config: {workflow_id, input} @@ -1323,6 +1327,7 @@ def authored_issues(spec: GraphSpec, sources: list[Any], baseline: list[dict[str changed = _changed(spec, baseline) out: list[dict[str, Any]] = [] out += _unsourced_reports(spec, changed) + out += _issuance_promises(spec, changed) for issue in text_parse_issues(spec): if issue.node_id in changed: # 模型的文字放在模板引号里的那种，改法是 | json，不是换结构化输出 @@ -1356,6 +1361,36 @@ def authored_issues(spec: GraphSpec, sources: list[Any], baseline: list[dict[str return out +#: 人工审批的标题、说明里不该出现的说法：出具要成果节点配了出具契约、从已发布版本发起的正式运行才谈得上 +_ISSUANCE_PROMISE = "正式出具" + + +def _issuance_promises(spec: GraphSpec, changed: set[str]) -> list[dict[str, Any]]: + """这一轮写的人工审批节点在标题、说明里许诺「正式出具」（human_formal_issuance）。 + + 真实运行里一次探索运行的审批写着「批准后才会正式出具」，审批人以为自己在签发正式的东西。画布上试运行、问数据 + 都是探索运行，成果节点也常常没有出具契约；同一张图发布后发起的正式运行才谈得上出具。审批说明写「批准后输出」， + 这次是哪一类运行由审批卡标明。""" + out: list[dict[str, Any]] = [] + for node in spec.nodes: + if node.id not in changed or node.type != NodeType.HUMAN: + continue + field = next((key for key in ("title", "message") + if _ISSUANCE_PROMISE in str(node.config.get(key) or "")), None) + if field is None: + continue + out.append({ + "level": "error", "node_id": node.id, "edge_id": None, "field": field, "code": "human_formal_issuance", + "message": f"人工审批「{node.title}」的{'标题' if field == 'title' else '说明'}写了「{_ISSUANCE_PROMISE}」。" + "只有成果节点配了出具契约、从已发布版本发起的正式运行才会出具，画布上试运行和问数据都是探索运行，" + "批准后只是输出。请改成「批准后输出」这类说法", + "for_model": f"人工审批的 {field} 里不要写「{_ISSUANCE_PROMISE}」：只有成果节点配了出具契约、从已发布版本发起的" + "正式运行才会出具，这张图在画布上试运行、问数据时都是探索运行，批准后只是输出。改成「批准后输出」" + "「批准后继续」这类说法，运行类别由审批卡标明", + }) + return out + + def _catalog_issues(node_id: str, sql: str, checker: Any) -> list[dict[str, Any]]: """一个调用工具节点的 SQL 对照数据目录查出来的问题，形状和别的自查问题一样（node_id、field 都在）， 另带 table、column、relation_id、sql_excerpt 给界面定位。""" diff --git a/backend/tests/test_copilot_approval_wording.py b/backend/tests/test_copilot_approval_wording.py new file mode 100644 index 00000000..36e36f66 --- /dev/null +++ b/backend/tests/test_copilot_approval_wording.py @@ -0,0 +1,142 @@ +"""助手写人工审批的说明时不说「正式出具」。 + +真实运行里：助手搭了「几个 Agent 查库 → 报告 → 人工审批 → 成果」，审批节点的标题和说明写着「请审核后正式出具」 +「批准后才会正式出具」。可那是一次探索运行，成果节点也没有配出具契约——根本不存在「正式出具」。审批人照着这句 +话批准，以为自己在签发一份正式的东西。 + +出具只发生在成果节点配了出具契约、从已发布版本发起的正式运行里；画布上试运行、问数据都是探索运行。审批说明 +写「批准后输出」这类说法，这次是哪一类运行由审批卡自己标明。 + +这组守的是： +- 节点手册里 human 那一条写明这条规矩和替代说法 +- 助手写出来的审批标题、说明里有「正式出具」时，自查交回去改（给人看的那句不带键名，给模型的那句说怎么改） +- 这一轮没碰过的审批节点不替人改 +""" +from __future__ import annotations + +import json + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.api.copilot import NODE_REFERENCE, authored_issues +from app.engine.schema import GraphSpec +from app.main import app + + +def _human_rule() -> str: + start = NODE_REFERENCE.index("- human：人工审批") + return NODE_REFERENCE[start:NODE_REFERENCE.index("\n- validate：", start)] + + +def test_node_reference_tells_the_assistant_not_to_promise_formal_issuance(): + rule = _human_rule() + assert "不要写「正式出具」" in rule + assert "批准后输出" in rule + # 说清什么时候才谈得上出具：配了出具契约、从已发布版本发起的正式运行 + assert "出具契约" in rule and "已发布版本" in rule and "正式运行" in rule and "探索运行" in rule + + +def _node(nid, ntype, **config): + return {"id": nid, "type": ntype, "data": {"label": nid, "config": config}} + + +def _spec(*nodes): + return GraphSpec.model_validate({"nodes": list(nodes), "edges": [ + {"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}) + + +def _graph(**human): + return _spec(_node("start", "input"), + _node("write", "report", instructions="写周报"), + _node("gate", "human", mode="approve", **human), + _node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}])) + + +@pytest.mark.parametrize("human, field", [ + ({"title": "请审核后正式出具", "message": "请审核本周报告"}, "title"), + ({"title": "审批", "message": "请审核本周报告，批准后才会正式出具。\n\n{{ nodes.write.text }}"}, "message"), +]) +def test_promising_formal_issuance_is_sent_back(human, field): + issues = [i for i in authored_issues(_graph(**human), []) if i["code"] == "human_formal_issuance"] + assert len(issues) == 1, issues + [issue] = issues + assert issue["level"] == "error" and issue["node_id"] == "gate" and issue["field"] == field + assert "正式出具" in issue["message"] and "批准后输出" in issue["message"] + assert "出具契约" in issue["for_model"] and "批准后输出" in issue["for_model"] + + +def test_accurate_wording_passes(): + spec = _graph(title="请审核本周报告", message="批准后输出这份报告。\n\n{{ nodes.write.text }}") + assert not [i for i in authored_issues(spec, []) if i["code"] == "human_formal_issuance"] + + +def test_approvals_this_round_did_not_touch_are_left_alone(): + spec = _graph(title="请审核后正式出具") + gate = next(n for n in spec.model_dump()["nodes"] if n["id"] == "gate") + assert not [i for i in authored_issues(spec, [], baseline=[gate]) if i["code"] == "human_formal_issuance"] + + +class Scripted: + """第一轮吐 first；收到自查请求（human 里有「上线前自查」）时吐 fix。""" + + def __init__(self, first: list[dict], fix: list[dict]) -> None: + self.first = [json.dumps(o, ensure_ascii=False) for o in first] + self.fix = [json.dumps(o, ensure_ascii=False) for o in fix] + self.calls: list[str] = [] + + async def astream(self, messages): + from langchain_core.messages import AIMessageChunk + + human = next(text for role, text in messages if role == "human") + self.calls.append(human) + for line in (self.fix if "上线前自查" in human else self.first): + yield AIMessageChunk(content=line + "\n") + + +def _add(nid, ntype, label, **config): + return {"op": "add_node", "node": {"id": nid, "type": ntype, "label": label, "config": config}} + + +def _link(a, b, handle=None): + return {"op": "add_edge", "edge": {"source": a, "target": b, **({"sourceHandle": handle} if handle else {})}} + + +async def test_the_self_check_rewrites_the_approval_before_delivery(monkeypatch): + import app.api.copilot as copilot + + first = [ + _add("start", "input", "输入", fields=[{"name": "week"}]), + _add("fetch", "transform", "取数", mode="expression", expression="{'gmv': 1}", assign_to="kpi"), + _add("card", "metrics", "口径卡", metrics=[{"id": "gmv", "name": "销售额", "expression": "vars.kpi.gmv"}]), + _add("write", "report", "写周报", instructions="写周报"), + _add("gate", "human", "审批", mode="approve", title="请审核后正式出具", + message="批准后才会正式出具。\n\n{{ nodes.write.text }}"), + _add("out", "output", "成果", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]), + _link("start", "fetch"), _link("fetch", "card"), _link("card", "write"), _link("write", "gate"), + _link("gate", "out", "approved"), + {"op": "done", "explanation": "周报", "run": False}, + ] + fix = [{"op": "update_node", "id": "gate", "config": { + "mode": "approve", "title": "请审核本周报告", "message": "批准后输出这份报告。\n\n{{ nodes.write.text }}"}}, + {"op": "done"}] + model = Scripted(first, fix) + + async def _model(*_a, **_k): + return model, "mock-fast" + + monkeypatch.setattr(copilot, "get_chat_model", _model) + events: list[dict] = [] + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + async with client.stream("POST", "/api/copilot/generate-stream", json={"instruction": "写周报并审批"}) as r: + async for line in r.aiter_lines(): + if line.startswith("data: "): + events.append(json.loads(line[6:])) + checks = [e for e in events if e.get("op") == "check"] + assert checks[0]["status"] == "repairing" + assert any(i.get("code") == "human_formal_issuance" for i in checks[0]["issues"]) + assert "出具契约" in model.calls[-1], "交回模型的修正请求里写明为什么不能这么说" + assert checks[-1]["status"] == "passed" + final = next(e for e in events if e.get("op") == "final") + gate = next(n for n in final["graph"]["nodes"] if n["id"] == "gate") + assert "正式出具" not in json.dumps(gate, ensure_ascii=False) From a46bad05925e7a8ea2ae77847785f894c7e1a30a Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 4 Oct 2026 11:09:18 +0800 Subject: [PATCH 04/15] =?UTF-8?q?=E5=8A=A9=E6=89=8B=E6=90=AD=E5=A4=9A?= =?UTF-8?q?=E6=9D=A5=E6=BA=90=E6=B1=87=E6=80=BB=E7=9A=84=E5=88=86=E6=9E=90?= =?UTF-8?q?=E6=8A=A5=E5=91=8A=E6=97=B6=E9=BB=98=E8=AE=A4=E5=BC=80=E7=BB=93?= =?UTF-8?q?=E8=AE=BA=E5=8F=A5=E8=A3=81=E5=88=A4=EF=BC=9A=E8=8A=82=E7=82=B9?= =?UTF-8?q?=E6=89=8B=E5=86=8C=E7=9A=84=20report=20=E4=B8=80=E6=9D=A1?= =?UTF-8?q?=E5=86=99=E6=98=8E=E5=87=A0=E4=B8=AA=20Agent=20=E5=88=86?= =?UTF-8?q?=E5=A4=B4=E6=9F=A5=E5=BA=93=E3=80=81=E6=88=96=E8=80=85=E5=87=A0?= =?UTF-8?q?=E6=AC=A1=E6=9F=A5=E8=AF=A2=E5=90=88=E5=9C=A8=E4=B8=80=E8=B5=B7?= =?UTF-8?q?=E5=86=99=E6=88=90=E7=9A=84=E5=88=86=E6=9E=90=E6=8A=A5=E5=91=8A?= =?UTF-8?q?=E9=BB=98=E8=AE=A4=E5=86=99=20claims:=20judge=EF=BC=8C=E4=B8=8D?= =?UTF-8?q?=E5=86=99=20judge=20=E7=9A=84=E9=A2=84=E7=AE=97=EF=BC=88?= =?UTF-8?q?=E5=8F=96=E8=AE=BE=E7=BD=AE=E9=87=8C=E7=9A=84=E9=BB=98=E8=AE=A4?= =?UTF-8?q?=EF=BC=89=EF=BC=8C=E6=AD=A3=E5=BC=8F=E8=BF=90=E8=A1=8C=E5=9C=A8?= =?UTF-8?q?=E8=8A=82=E7=82=B9=E9=87=8C=E8=87=AA=E5=8A=A8=E5=88=A4=E3=80=81?= =?UTF-8?q?=E6=8E=A2=E7=B4=A2=E8=BF=90=E8=A1=8C=E7=94=B1=E8=AF=BB=E7=9A=84?= =?UTF-8?q?=E4=BA=BA=E6=8C=89=E9=9C=80=E5=88=A4=EF=BC=8C=E7=94=A8=E6=88=B7?= =?UTF-8?q?=E6=98=8E=E7=A1=AE=E4=B8=8D=E8=A6=81=E7=9A=84=E4=B8=8D=E5=8A=A0?= =?UTF-8?q?=E3=80=82=E5=92=8C=E5=8F=97=E7=AE=A1=E7=BA=A7=E5=88=AB=E3=80=8C?= =?UTF-8?q?judge=20=E5=BF=85=E9=A1=BB=E5=86=99=E9=A2=84=E7=AE=97=E3=80=81?= =?UTF-8?q?=E6=8B=BF=E4=B8=8D=E5=87=86=E5=B0=B1=E9=97=AE=E3=80=8D=E7=9A=84?= =?UTF-8?q?=E5=8F=96=E8=88=8D=EF=BC=9A=E9=82=A3=E6=9D=A1=E5=8F=AA=E7=AE=A1?= =?UTF-8?q?=E8=A6=81=E6=8C=89=E5=8F=97=E7=AE=A1=E7=BA=A7=E5=88=AB=E5=8F=91?= =?UTF-8?q?=E5=B8=83=E7=9A=84=E6=A8=A1=E6=9D=BF=EF=BC=8C=E6=99=AE=E9=80=9A?= =?UTF-8?q?=E8=8D=89=E7=A8=BF=E9=87=8C=E9=BB=98=E8=AE=A4=E5=BC=80=E7=9A=84?= =?UTF-8?q?=20judge=20=E4=B8=8D=E4=B8=BA=E9=A2=84=E7=AE=97=E9=97=AE?= =?UTF-8?q?=E4=BA=BA=EF=BC=9B=E5=8F=91=E5=B8=83=E5=89=8D=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E7=BC=BA=E9=A2=84=E7=AE=97=E8=A6=81=E9=97=AE=E7=9A=84=E8=A7=84?= =?UTF-8?q?=E7=9F=A9=E4=B8=8D=E5=8F=98=E3=80=82=E9=97=AE=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E5=8E=9F=E6=9C=AC=E5=AE=9A=E4=BA=86=E4=B8=8D?= =?UTF-8?q?=E6=9B=BF=E4=BA=BA=E6=89=93=E5=BC=80=E7=BB=93=E8=AE=BA=E5=8F=A5?= =?UTF-8?q?=E7=AD=96=E7=95=A5=EF=BC=8C=E4=BF=9D=E7=95=99=EF=BC=8C=E5=B9=B6?= =?UTF-8?q?=E5=86=99=E6=98=8E=E8=BF=99=E6=9D=A1=E9=BB=98=E8=AE=A4=E5=9C=A8?= =?UTF-8?q?=E9=97=AE=E6=95=B0=E9=87=8C=E4=B8=8D=E9=80=82=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/copilot.py | 9 +++- backend/tests/test_copilot_claims_default.py | 53 ++++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 backend/tests/test_copilot_claims_default.py diff --git a/backend/app/api/copilot.py b/backend/app/api/copilot.py index be9168a5..2c49e468 100644 --- a/backend/app/api/copilot.py +++ b/backend/app/api/copilot.py @@ -236,6 +236,10 @@ NODE_REFERENCE = """\ judge.provider 指定裁判模型（建议和写作模型不同，同一个模型等于自己审自己；不写就用设置里的「证据裁判模型」）； judge.max_cost_usd、judge.max_claims、judge.timeout_s 是每份报告的裁判金额、句数、时长上限，写 null 表示不限， 不写取设置里的默认；judge.rewrite_once 默认 false，打开后把证据相矛盾、证据不足的句子交回写作者只改一次。 + **多个来源汇总成的分析报告**（几个 agent 分头查库，或者几次查询的结果合在一起写成的分析、结论）默认写 claims: "judge"， + 不写 judge 的预算（judge 整个不写，金额、句数、时长上限都取设置里的默认）：正式运行在节点里自动判，探索运行由读的人 + 点开哪句判哪句，不在节点里花钱。用户明确说不要核对结论句的不加；只回答一个问题、一次查询的不必加。要按受管级别 + 发布的模板不适用这条默认，按下面受管级别的规则写预算 entities 默认 link （核对表名、字段名），报告里根本不提表名时才写 off - output 的出具契约：config.contract = {report_from: 报告节点id, metrics_from:[口径卡id], @@ -245,7 +249,7 @@ NODE_REFERENCE = """\ （问数据的典型图），契约不写 metrics_from，也不写 required / expected 要按受管级别发布的模板（发布前检查不过就发不出去）：报告节点写明 numbers: "strict"、on_violation: "fail"、 claims: "require_citation"（或者更严的 claims: "judge"，这时 judge 里必须写 judge.max_cost_usd——写一个金额，或者 - 写 null 表示不限，预算要用户定，拿不准就问）；契约用 report_from（受管级别不收 narrative），成果字段只取报告节点的正文；给口径卡 + 写 null 表示不限，预算要用户定，拿不准就问；只有受管级别要这样，普通草稿里默认开的 judge 不写预算、不为预算问人）；契约用 report_from（受管级别不收 narrative），成果字段只取报告节点的正文；给口径卡 供数的 agent 配 output_schema 和 cite_fields: true，code 节点喂口径卡的必须是真取数（evidence_role: "source"）， 否则把计算写进口径卡；llm、agent、协作团队写的文字不能绕过报告节点直接进出口，llm 也不能喂口径卡 @@ -355,7 +359,8 @@ def _user_message(payload: GenerateIn, *, patch: bool) -> str: "- report 提到表名、字段名时写 [[t:表名]] / [[c:表名.字段名]]（英文名也可以放进反引号，中文名不行：" "反引号里的中文名不核对），只写 agent 真查过的名字，" "编造的会被标出来；要引用知识库原话就在 report 前接 retrieve，用 [[q:K1|原话]] 逐字引用。问数是探索运行，" - "report 的 claims 不用写：结论句支不支持，由读答案的人点开时按需请模型判断\n" + "report 的 claims 不用写：结论句支不支持，由读答案的人点开时按需请模型判断。节点说明里「多个来源汇总成的" + "分析报告默认开结论句裁判」那条在问数里不适用，几个 agent 分头查的也一样\n" "- 用户要的是周报、要三档出具时，按搭图规则配口径卡和 report_from 契约\n" "- 不要用 transform 解析 agent / llm 的文字：要把 agent 查到的数交给下游，就给它配 output_schema + " "cite_fields\n" diff --git a/backend/tests/test_copilot_claims_default.py b/backend/tests/test_copilot_claims_default.py new file mode 100644 index 00000000..979d4c2b --- /dev/null +++ b/backend/tests/test_copilot_claims_default.py @@ -0,0 +1,53 @@ +"""助手搭多来源汇总的分析报告时，默认给报告撰写节点开结论句裁判（claims: judge）。 + +真实运行里：四个 Agent 分头查库、汇总成一份分析报告，报告节点没写 claims。有一句结论和证据相矛盾，审批人批准时 +完全不知道，事后在证据面板点开那句才判出来。分析报告的结论句恰恰最需要核对：几个来源拼在一起，最容易写出 +证据撑不住的归因和判断。 + +取舍：默认写 claims: "judge"、不写 judge 的预算（取设置里的默认）——正式运行在节点里自动判，探索运行不在节点里 +花钱、由读的人按需判。和「受管级别的 judge 必须写预算、拿不准就问」不冲突：那条只管要按受管级别发布的模板， +普通草稿不问预算；发布前修复里缺预算要问的规矩不变。用户明确说不要核对结论句的不加。问数据那条路径原本定了 +不替人打开结论句策略（一律探索运行，读的人按需判），那条保留，并写明节点说明里的默认在问数里不适用。 + +这组守的是节点手册和问数据那条路径里的说法，以及它们和受管级别规则、发布前修复规则不打架。 +""" +from __future__ import annotations + +from app.api.copilot import NODE_REFERENCE, GenerateIn, _publish_fix_request, _user_message + + +def _report_rule() -> str: + start = NODE_REFERENCE.index("- report：报告撰写") + return NODE_REFERENCE[start:NODE_REFERENCE.index("\n- output 的出具契约", start)] + + +def test_multi_source_analysis_reports_default_to_the_judge(): + rule = _report_rule() + assert '默认写 claims: "judge"' in rule + # 什么算多来源：几个 Agent 分头查、或者几次查询的结果合在一起写 + assert "多个来源汇总成的分析报告" in rule and "几个 agent" in rule and "几次查询" in rule + # 不写预算，交给设置里的默认值 + assert "不写 judge 的预算" in rule and "设置里的默认" in rule + # 正式运行在节点里自动判，探索运行由读的人按需判 + assert "正式运行在节点里自动判" in rule and "探索运行由读的人" in rule + # 用户明确不要的不加 + assert "用户明确说不要" in rule + + +def test_the_managed_level_budget_rule_still_applies_and_says_so(): + # 受管级别仍然要写预算、拿不准就问；默认开裁判的那一条说明它让位给受管级别的规则 + assert "judge 里必须写 judge.max_cost_usd" in NODE_REFERENCE + assert "拿不准就问" in NODE_REFERENCE + rule = _report_rule() + assert "受管级别" in rule + # 发布前修复（受管级别）缺预算时照旧问人，不替人定 + request = _publish_fix_request({"nodes": [], "edges": []}, [], "governed") + assert "judge 缺预算（judge.max_cost_usd）时输出 question 问写多少" in request + + +def test_answer_path_keeps_its_own_rule_and_says_which_one_wins(): + """问数据那条路径原本就定了不替人打开结论句策略（test_copilot_entity_guide 守着）：问数一律是探索运行，读答案的人 + 点开哪句判哪句。节点说明里新加的默认和它说法相反，所以这里写明那条默认在问数里不适用，免得模型两头为难。""" + text = _user_message(GenerateIn(instruction="各渠道上月表现怎么样", intent="answer"), patch=False) + assert "report 的 claims 不用写" in text + assert "分析报告默认开结论句裁判」那条在问数里不适用" in text From f4a9955cc3b673a5b690b4578bf7c13c2626c333 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 4 Oct 2026 11:12:00 +0800 Subject: [PATCH 05/15] =?UTF-8?q?=E5=8A=A9=E6=89=8B=E5=86=99=E6=9F=A5?= =?UTF-8?q?=E5=BA=93=20SQL=20=E6=97=B6=E7=BB=99=E7=BB=93=E6=9E=9C=E5=88=97?= =?UTF-8?q?=E8=B5=B7=E4=B8=AD=E6=96=87=E5=88=AB=E5=90=8D=EF=BC=9A=E7=9C=9F?= =?UTF-8?q?=E5=AE=9E=E8=BF=90=E8=A1=8C=E9=87=8C=E6=8A=A5=E5=91=8A=E8=A1=A8?= =?UTF-8?q?=E6=A0=BC=E7=9A=84=E8=A1=A8=E5=A4=B4=E5=85=A8=E6=98=AF=E8=8B=B1?= =?UTF-8?q?=E6=96=87=E7=BC=A9=E5=86=99=E7=9A=84=E5=8E=9F=E5=A7=8B=E5=88=97?= =?UTF-8?q?=E5=90=8D=EF=BC=88[[table:]]=20=E5=8E=9F=E6=A0=B7=E7=94=A8?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E7=BB=93=E6=9E=9C=E7=9A=84=E5=88=97=E5=90=8D?= =?UTF-8?q?=EF=BC=89=E3=80=82=E8=8A=82=E7=82=B9=E6=89=8B=E5=86=8C=E9=87=8C?= =?UTF-8?q?=20Agent=20=E6=9F=A5=E5=BA=93=E7=9A=84=E8=AF=B4=E6=98=8E?= =?UTF-8?q?=E5=92=8C=E5=86=99=20SQL=20=E7=9A=84=E7=A1=AC=E6=80=A7=E8=A6=81?= =?UTF-8?q?=E6=B1=82=E9=83=BD=E5=8A=A0=E4=B8=8A=E4=B8=80=E6=9D=A1=EF=BC=9A?= =?UTF-8?q?=E7=BB=93=E6=9E=9C=E5=88=97=E7=94=A8=E4=B8=AD=E6=96=87=E5=88=AB?= =?UTF-8?q?=E5=90=8D=EF=BC=8C=E6=8C=89=E6=96=B9=E8=A8=80=E5=8A=A0=E6=A0=87?= =?UTF-8?q?=E8=AF=86=E7=AC=A6=E5=BC=95=E5=8F=B7=EF=BC=88MySQL=20/=20MariaD?= =?UTF-8?q?B=20=E7=94=A8=E5=8F=8D=E5=BC=95=E5=8F=B7=EF=BC=8CPostgreSQL?= =?UTF-8?q?=E3=80=81Oracle=E3=80=81SQLite=20=E7=94=A8=E5=8F=8C=E5=BC=95?= =?UTF-8?q?=E5=8F=B7=EF=BC=8C=E4=B8=8D=E7=94=A8=E5=8D=95=E5=BC=95=E5=8F=B7?= =?UTF-8?q?=EF=BC=89=EF=BC=8C=E5=8F=AA=E7=BB=99=E6=9C=80=E5=A4=96=E5=B1=82?= =?UTF-8?q?=20SELECT=20=E8=B5=B7=E5=88=AB=E5=90=8D=EF=BC=9B=E8=87=AA?= =?UTF-8?q?=E6=9F=A5=E7=9A=84=E5=88=97=E5=90=8D=E6=A0=B8=E5=AF=B9=E6=8A=8A?= =?UTF-8?q?=E5=B8=A6=E5=BC=95=E5=8F=B7=E7=9A=84=E4=B8=AD=E6=96=87=E6=A0=87?= =?UTF-8?q?=E8=AF=86=E7=AC=A6=E6=8A=B9=E6=8E=89=E5=86=8D=E5=88=A4=EF=BC=8C?= =?UTF-8?q?=E4=B8=8D=E5=86=8D=E5=9B=A0=E4=B8=BA=E6=9C=89=E5=B8=A6=E5=BC=95?= =?UTF-8?q?=E5=8F=B7=E7=9A=84=E4=B8=AD=E6=96=87=E5=88=AB=E5=90=8D=E5=B0=B1?= =?UTF-8?q?=E6=95=B4=E6=9D=A1=E6=94=BE=E5=BC=83=E6=A0=B8=E5=AF=B9=E3=80=81?= =?UTF-8?q?=E6=8A=8A=E7=8C=9C=E9=94=99=E7=9A=84=E8=8B=B1=E6=96=87=E5=88=97?= =?UTF-8?q?=E5=90=8D=E4=B8=80=E8=B5=B7=E6=94=BE=E8=BF=87=E5=8E=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/copilot.py | 23 ++++++++- backend/app/api/copilot_context.py | 4 ++ backend/tests/test_copilot_sql_alias.py | 67 +++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 backend/tests/test_copilot_sql_alias.py diff --git a/backend/app/api/copilot.py b/backend/app/api/copilot.py index 2c49e468..fe0334c9 100644 --- a/backend/app/api/copilot.py +++ b/backend/app/api/copilot.py @@ -152,6 +152,11 @@ NODE_REFERENCE = """\ 查不到的记空值、不兜底成 0。assign_to 拿到的就是核对过的对象，口径卡直接写 vars..字段。 只写 output_schema 不开 cite_fields 不生效。**不要用 transform 解析 agent / llm 的文字**：模型写的 JSON 常夹着没转义的引号，解析一失败整个节点就失败，也核对不了出处 + **agent 查库时**，prompt 里写明查询结果列用中文别名（AS 月份、AS 曝光量）：报告用 [[table:]] 生成的表格表头、 + [[v:]] 引用、口径卡的 cell() 都用这个列名，写成英文缩写的原始列名读的人看不懂。中文别名按数据源的方言 + 加标识符引号：MySQL / MariaDB 用反引号（AS `月份`），PostgreSQL、Oracle、SQLite 用双引号（AS "月份"），不要用 + 单引号（那是字符串）。只给最外层 SELECT 的结果列起别名，WHERE、JOIN、GROUP BY 里照旧写原列名；Oracle 旧版本 + 的标识符最长 30 字节，别名不超过 10 个字 **默认不要写 max_steps、budget_tokens、budget_usd**。平台用的是护栏而不是固定步数： 步数只是很高的兜底（跟随设置，默认 100），重复调用、连续几步没有新信息、预算用完、 上下文快满时，它会按查到的部分收尾。写死一个小数字（见过 8）会让它查完表结构就没额度回答了。 @@ -1463,6 +1468,22 @@ _DATEPART_ARG = re.compile( _WINDOW_NAME = re.compile(r"(?:\bwindow|,)\s*([A-Za-z_][\w$#]*)\s+as\s*\(", re.I) +def _blank_unicode_identifiers(sql: str) -> str: + """带引号、名字里有非 ASCII 字符的标识符（中文别名 AS "月份"、AS `曝光量`）连引号一起换成空格。 + + 这项核对只认 ASCII 的列名，中文名本来就不判；不抹掉的话，SQL 里一出现带引号的标识符就整条放弃核对，照「结果列 + 起中文别名」写的 SQL 会把猜错的英文列名一起放过去。英文的带引号标识符（"StatDate"）大小写、空格都可能，照旧不判。""" + from app.data.guard import _GENERIC, _tokens + + out = [] + for token in _tokens(sql, _GENERIC): + if token.kind == "ident" and not token.text.isascii(): + out.append(" " * len(token.text)) + else: + out.append(token.text) + return "".join(out) + + def sql_unknown_columns(sql: str, source: Any) -> tuple[list[str], list[str]] | None: """SQL 里用到、数据源结构里查不到的列：(查不到的列, 这几张表的全部列)。判断不了、或者都查得到时返回 None。""" from app.data import introspect as _introspect @@ -1471,7 +1492,7 @@ def sql_unknown_columns(sql: str, source: Any) -> tuple[list[str], list[str]] | cache = getattr(source, "schema_cache", None) or {} if not cache.get("tables") or cache.get("failed") or first_verb(sql) != "select": return None - text = re.sub(r"\{\{.*?\}\}", " 0 ", strip_comments(sql), flags=re.S) + text = _blank_unicode_identifiers(re.sub(r"\{\{.*?\}\}", " 0 ", strip_comments(sql), flags=re.S)) if '"' in text.replace("''", "") and _blank_quoted(text) != _blank_quoted(text, identifiers=False): return None # 引号括起来的标识符：大小写、空格都可能，不猜 text = _blank_quoted(text) diff --git a/backend/app/api/copilot_context.py b/backend/app/api/copilot_context.py index 6c527ebc..5b065047 100644 --- a/backend/app/api/copilot_context.py +++ b/backend/app/api/copilot_context.py @@ -99,6 +99,10 @@ SQL_RULES_PROMPT = ( "去对结构，对不上会打回来让你改\n" "- 比率、增幅、占比这类派生计算进口径卡（metrics 节点）；查询结果里原样的数，报告撰写节点（report）" "可以直接用 [[v:Q1.r0.列名]] 引用。别让 llm 节点直接对数字做算术\n" + # 真实运行里报告表格的表头全是英文缩写的原始列名：[[table:]] 原样用查询结果的列名 + "- 结果列用中文别名（AS 月份、AS 曝光量），报告里的表格表头、引用直接可读。按上面标的方言加标识符引号：" + "MySQL / MariaDB 用反引号（AS `月份`），PostgreSQL、Oracle、SQLite 用双引号（AS \"月份\"），不要用单引号" + "（那是字符串）；只给最外层 SELECT 的结果列起别名，WHERE、JOIN、GROUP BY 里照旧写原列名\n" ) PICK_SYSTEM = ( diff --git a/backend/tests/test_copilot_sql_alias.py b/backend/tests/test_copilot_sql_alias.py new file mode 100644 index 00000000..59a79893 --- /dev/null +++ b/backend/tests/test_copilot_sql_alias.py @@ -0,0 +1,67 @@ +"""助手写查库 SQL 时给结果列起中文别名，报告里的表格表头直接可读。 + +真实运行里：报告节点用 [[table:…]] 把查询结果原样生成表格，表头就是 Agent 写的 SQL 里的列名——一串英文缩写的 +原始列名，读报告的人得猜。结果列起中文别名（AS 月份、AS 曝光量）以后，表头、单元格引用、口径卡的 +cell() 都直接用中文名。 + +中文别名要按方言加标识符引号：MySQL / MariaDB 用反引号，PostgreSQL、Oracle、SQLite 用双引号；单引号是字符串。 + +这组守的是： +- 节点手册里 Agent 查库的说明、写 SQL 的硬性要求（搭查库节点时的规则）都有这一条，按方言给出示例 +- 加了引号的中文别名不让自查放弃列名核对：以前 SQL 里只要有带引号的标识符就整条不判，照这条规矩写的 SQL + 会把猜错的英文列名一起放过去 +""" +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from app.api.copilot import NODE_REFERENCE, sql_unknown_columns +from app.api.copilot_context import SQL_RULES_PROMPT + + +def _agent_rule() -> str: + start = NODE_REFERENCE.index("- agent：带工具循环的 agent") + return NODE_REFERENCE[start:NODE_REFERENCE.index("\n- supervisor：", start)] + + +@pytest.mark.parametrize("text", [_agent_rule(), SQL_RULES_PROMPT], ids=["agent", "sql-rules"]) +def test_guidance_asks_for_chinese_aliases_with_dialect_quotes(text): + assert "中文别名" in text + assert "AS `月份`" in text and "MySQL" in text # MySQL / MariaDB：反引号 + assert 'AS "月份"' in text and "PostgreSQL" in text and "Oracle" in text and "SQLite" in text + assert "单引号" in text # 单引号是字符串，不是别名 + + +def _source(**tables): + return SimpleNamespace(name="shop", schema_cache={"tables": { + name: {"qualified": name, "columns": [{"name": c} for c in cols]} for name, cols in tables.items()}}) + + +SHOP = _source(daily_stats=["id", "stat_date", "impressions", "clicks", "region"]) + + +@pytest.mark.parametrize("sql", [ + 'SELECT stat_date AS "日期", SUM(impressions) AS "曝光量" FROM daily_stats GROUP BY stat_date ORDER BY "日期"', + "SELECT stat_date AS `日期`, SUM(impressions) AS `曝光量` FROM daily_stats GROUP BY stat_date ORDER BY `曝光量` DESC", + "SELECT stat_date AS 日期, SUM(clicks) AS 点击量 FROM daily_stats GROUP BY stat_date", + 'SELECT region AS "区域", COUNT(*) AS "天数" FROM daily_stats WHERE region = \'华东\' GROUP BY region', +]) +def test_quoted_chinese_aliases_are_fine(sql): + assert sql_unknown_columns(sql, SHOP) is None, sql_unknown_columns(sql, SHOP) + + +@pytest.mark.parametrize("sql, unknown", [ + ('SELECT stat_date AS "日期", SUM(impresions) AS "曝光量" FROM daily_stats GROUP BY stat_date', ["impresions"]), + ("SELECT stat_month AS `月份`, SUM(impressions) AS `曝光量` FROM daily_stats GROUP BY stat_month", + ["stat_month"]), +]) +def test_guessed_columns_next_to_quoted_chinese_aliases_are_still_caught(sql, unknown): + found = sql_unknown_columns(sql, SHOP) + assert found is not None and found[0] == unknown, found + + +def test_quoted_ascii_identifiers_are_still_not_guessed(): + # 带引号的英文标识符：大小写、空格都可能，照旧不判 + assert sql_unknown_columns('SELECT "StatDate", impresions FROM daily_stats', SHOP) is None From bf98ce16b8d0b8e486426f14a27cc474960e8905 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 4 Oct 2026 11:14:41 +0800 Subject: [PATCH 06/15] =?UTF-8?q?=E8=BF=90=E8=A1=8C=E5=A4=8D=E6=A0=B8?= =?UTF-8?q?=E6=8A=8A=20agent=5Ffield=5Ftruncated=20=E7=AE=97=E4=BD=9C?= =?UTF-8?q?=E7=BC=BA=E5=8F=A3=EF=BC=88degraded=EF=BC=89=EF=BC=8C=E5=92=8C?= =?UTF-8?q?=20agent=5Ffield=5Funverified=20=E5=90=8C=E4=B8=80=E6=A1=A3?= =?UTF-8?q?=EF=BC=9A=E5=AD=97=E6=AE=B5=E9=83=BD=E8=AE=B0=E7=A9=BA=E5=80=BC?= =?UTF-8?q?=E3=80=81=E7=AD=94=E6=A1=88=E8=83=BD=E7=94=A8=EF=BC=8C=E7=BC=BA?= =?UTF-8?q?=E5=8F=A3=E6=98=AF=E8=BE=93=E5=87=BA=E9=A2=9D=E5=BA=A6=E5=9C=A8?= =?UTF-8?q?=E6=8E=A8=E7=90=86=E9=98=B6=E6=AE=B5=E7=94=A8=E5=AE=8C=E4=BA=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/engine/review.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/app/engine/review.py b/backend/app/engine/review.py index 1f671611..594ff9c2 100644 --- a/backend/app/engine/review.py +++ b/backend/app/engine/review.py @@ -95,6 +95,8 @@ WARN_CODES: dict[str, tuple[str, str]] = { # （记成空值，不兜底成 0）。答案能用，缺口要说清是哪一种 "agent_field_mismatch": ("agent_field_mismatch", DEGRADED), "agent_field_unverified": ("agent_field_unverified", DEGRADED), + # 结构化抽取两次都被截断（输出额度在推理阶段就用完了）：字段都记空值，答案能用，缺口要说清是额度问题 + "agent_field_truncated": ("agent_field_truncated", DEGRADED), # 结论句裁判（claims: judge）：触顶、没跑成的句子记为未裁判，出具不能判完整——是缺口，要说明 "judge_limit": ("judge_limit", DEGRADED), "judge_failed": ("judge_failed", DEGRADED), From 4e876ff9d816f2f4f9a9569036c82eb3e3e921f0 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 4 Oct 2026 11:17:13 +0800 Subject: [PATCH 07/15] =?UTF-8?q?=E5=86=99=E4=BD=9C=E8=80=85=E7=9A=84?= =?UTF-8?q?=E5=87=BA=E5=A4=84=E6=B8=85=E5=8D=95=E6=8C=89=E5=86=99=E4=BD=9C?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E7=9A=84=E7=AA=97=E5=8F=A3=E5=AE=9A=E7=AF=87?= =?UTF-8?q?=E5=B9=85=EF=BC=8C=E6=94=BE=E4=B8=8D=E4=B8=8B=E6=97=B6=E5=88=86?= =?UTF-8?q?=E4=BC=98=E5=85=88=E7=BA=A7=E3=80=81=E6=8C=89=E6=9D=A5=E6=BA=90?= =?UTF-8?q?=E5=9D=87=E8=A1=A1=EF=BC=8C=E7=9C=81=E7=95=A5=E4=BA=86=E4=BB=80?= =?UTF-8?q?=E4=B9=88=E7=85=A7=E5=AE=9E=E8=AF=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 一次真实运行里证据台账有上百次查询（4 个 Agent 查询次数悬殊），清单写死 12000 字，十万字以上的目录被截到一万两千字以内，只有前二十来次查询进了写作者的视野；查询按编号排，前面的 Agent 占满了预算，有的 Agent 一次都没露面。 - 预算：写作模型上下文窗口的 40%（字数按 1 字约 1 token 保守算），窗口先看模型接入里给这个模型登记的 context，再看模型目录；拿不到按 128K（和 Agent 护栏同一个保守值），即 51200 字；下限 12000、上限 160000 - 第一优先：各 Agent 经按出处核对交来的结构化字段（换成报告目录里的全局编号，值按单元格引用的渲染规则写，给出可照抄的引用标记）、口径卡指标和运行输入，一律列出；结构化结论为空的来源写明不代表没有数据 - 查询按产出它的节点分组、轮流展开：字段引用到的查询先展开，再给每次查询列前 3 行，有余量再补到 20 行；表和字段、知识库检索最多占预算的四分之一 - 没展开的查询列出编号和一行摘要（表、行数）；约定只能在句末写 [[see:Qn]] 作依据，不猜行列、不凭摘要写数，数字照样逐字核对 - 清单开头明说目录有省略、各来源各省略了几次；报告节点发 report_catalog_trimmed 警告（带 queries、omitted、nodes），时间线给它专门的标题和下一步，运行复核算作缺口 - 放得下时清单和以前一字不差（多出的只有结构化字段那一节） --- backend/app/engine/evidence.py | 333 +++++++++++++++++-- backend/app/engine/nodes/report.py | 74 ++++- backend/app/engine/review.py | 2 + backend/tests/fixtures/evidence/store_run.py | 106 ++++++ backend/tests/test_report_catalog_budget.py | 209 ++++++++++++ frontend/src/run/decode.ts | 16 + scripts/check-copy.mjs | 5 +- scripts/check-decode.mjs | 20 ++ 8 files changed, 734 insertions(+), 31 deletions(-) create mode 100644 backend/tests/fixtures/evidence/store_run.py create mode 100644 backend/tests/test_report_catalog_budget.py diff --git a/backend/app/engine/evidence.py b/backend/app/engine/evidence.py index 62ebaf79..02a62c8b 100644 --- a/backend/app/engine/evidence.py +++ b/backend/app/engine/evidence.py @@ -47,7 +47,9 @@ import json import math import re import unicodedata +from collections import Counter from collections.abc import Callable +from dataclasses import dataclass, field from decimal import ROUND_HALF_UP, Context, Decimal from typing import Any, Literal, TypedDict @@ -57,6 +59,7 @@ from app.data.names import FILLERS, name_key from app.data.provenance_types import DOC_PROVENANCE from app.data.tabular import UNSHAPED_NOTE from app.engine.expressions import CellError, cell_value, column_kind, locate_cell +from app.engine.guards import FALLBACK_CONTEXT from app.engine.issuance import _tolerance, extract_numbers, number_allowance from app.engine.labels import option_label @@ -3094,10 +3097,12 @@ def _sql_check_note(artifact: Any, snaps: _Snapshots) -> str | None: "引用时须如实说明，不要当作确定的结论）") -def _query_prompt(entry: dict[str, Any], snaps: _Snapshots, cells_allowed: bool) -> list[str]: +def _query_prompt(entry: dict[str, Any], snaps: _Snapshots, cells_allowed: bool, + shown_rows: int = _PROMPT_ROWS) -> list[str]: """一次查询在写作目录里的样子：编号、来源、列，以及能引用的行（行号 + 渲染后的值）。 - 值按 [[v:]] 的渲染规则写出来：写作者在这里看到的，就是引用之后报告里显示的字。 + 值按 [[v:]] 的渲染规则写出来：写作者在这里看到的，就是引用之后报告里显示的字。shown_rows 是列出几行： + 目录放不下时先给每次查询列前几行（_PROMPT_COMPACT_ROWS），再补到 _PROMPT_ROWS。 """ alias = entry["alias"] cols = entry.get("columns") or [] @@ -3115,7 +3120,7 @@ def _query_prompt(entry: dict[str, Any], snaps: _Snapshots, cells_allowed: bool) columns = [str(c) for c in snapshot.get("columns") or []][:_PROMPT_COLS] rows = snapshot["rows"] example = None - for r, record in enumerate(rows[:_PROMPT_ROWS]): + for r, record in enumerate(rows[:shown_rows]): cells = [] for c in columns: try: @@ -3130,22 +3135,78 @@ def _query_prompt(entry: dict[str, Any], snaps: _Snapshots, cells_allowed: bool) lines.append(f" r{r}：" + "，".join(cells)) if not rows: lines.append(" （0 行，没有可以引用的格）") - elif len(rows) > _PROMPT_ROWS: - lines.append(f" …共 {len(rows)} 行，这里只列出前 {_PROMPT_ROWS} 行；后面的行也能按行号引用") + elif len(rows) > shown_rows: + lines.append(f" …共 {len(rows)} 行，这里只列出前 {shown_rows} 行；后面的行也能按行号引用") if rows and columns: lines.append(f" 例：[[v:{alias}.r0.{example or columns[0]}]]") return lines -def catalog_prompt(catalog: dict[str, Any], *, budget: int = 12000, cells_allowed: bool = True, - loader: Loader | None = None) -> str: - """证据目录的文字版，放进写作者的 prompt。超过预算就截断，并照实说截断了。 +#: 写作目录的字数预算（catalog_budget）按写作模型的上下文窗口定，不再写死 12000： +#: - 占窗口的 40%。其余留给写作规则和指令（约 4 千字）、重写轮次（每轮把上一稿和违规清单追加进对话，最多 3 轮）、 +#: 推理模型的思考和报告正文（输出额度）。字数按 1 字 ≈ 1 token 算：中文一个字约 0.6–1 个 token，目录里的数字、 +#: 列名、标点每个字更少，按 1:1 算偏保守，实际占用低于 40% +#: - 拿不到窗口时按 128K（和 Agent 护栏的 FALLBACK_CONTEXT 同一个保守值），即 51200 字。原来的 12000 字只放得下 +#: 20 多次查询：2026-10 一次上百次查询的运行，十万字以上的目录被截到一万两千字以内，写作者只看到前二十来次 +#: - 下限 12000：窗口很小（32K 以下）的本地模型维持原来的篇幅；上限 160000：百万窗口的模型每写一稿、每重写一次 +#: 都要重发整份目录，花费随之上涨，而 160000 字已经放得下那次运行的查询全部展开 +CATALOG_WINDOW_SHARE = 0.4 +CATALOG_MIN_CHARS = 12_000 +CATALOG_MAX_CHARS = 160_000 +#: 目录放不下时，先给每次查询列几行 +_PROMPT_COMPACT_ROWS = 3 +#: 放不下时表和字段、知识库检索两节合起来最多占预算的几分之一（剩下的给查询） +_PROMPT_TAIL_SHARE = 4 +_PROMPT_CUT = "\n…（目录太长，后面的省略了）" - 查询结果会列出能引用的行（要读快照）；cells_allowed=False 时只列编号，说明只能当依据。 - 沙箱代码节点不列：它的产出不能引用。 + +def catalog_budget(window: int | None) -> int: + """写作目录的字数预算：写作模型上下文窗口（token）的 40%，夹在上下限之间；拿不到窗口按 FALLBACK_CONTEXT。""" + size = window if isinstance(window, int) and not isinstance(window, bool) and window > 0 else FALLBACK_CONTEXT + return max(CATALOG_MIN_CHARS, min(CATALOG_MAX_CHARS, int(size * CATALOG_WINDOW_SHARE))) + + +DEFAULT_CATALOG_BUDGET = catalog_budget(None) + +#: 目录有省略时，开头跟在省略情况后面的约定（给模型看）。和数字核对一致：没展开的查询只能当依据，数照样逐字核对 +OMITTED_RULES = ( + "没展开的查询照样在证据台账里，只是这里没有列出它的数据：\n" + "- 不要据此断言「没有查到」「不存在」「缺失」「没有数据」，只能写「本报告未覆盖」，并说明依据不足；\n" + "- 要用没展开的查询，只能在句末写 [[see:Qn]] 作依据（Qn 换成它的编号）；不要写 [[v:]]、[[table:]] 去猜它的行和列，" + "也不要凭摘要写任何数字——数字照样逐字核对，没有出处的数会被打回重写。结构化结论里写好的引用标记可以照抄。" +) +#: 结构化结论为空的来源后面跟的那句（给模型看） +EMPTY_SOURCE_PROMPT = ("结构化结论为空（字段都记为空值）。这不代表没有数据：不得据此断言没有查到、不存在或缺失，" + "只能写本报告未覆盖") + + +@dataclass +class WriterCatalog: + """给写作者的出处清单（writer_catalog）：文字，以及省略了什么。 + + - trimmed：放不下、有省略；nodes 是各来源的查询数 [{node_id, title, total, full, compact, omitted}]， + full 列出前 20 行、compact 只列前几行、omitted 只列编号和摘要；omitted 是只列摘要的查询编号 + - empty_sources：结构化结论为空的来源（Agent 的名字） + 清单有省略、或有来源的结构化结论为空时，报告节点加写作规则（ABSENCE_RULE）；有省略时组装文档另查缺失断言 """ - snaps = _Snapshots(loader) - lines = ["可引用的证据（写数字只能用下面的引用标记，系统会换成真实数值）："] + + text: str + budget: int + queries: int = 0 + trimmed: bool = False + nodes: list[dict[str, Any]] = field(default_factory=list) + omitted: list[str] = field(default_factory=list) + empty_sources: list[str] = field(default_factory=list) + + @property + def guarded(self) -> bool: + """写作者看不全证据：缺失断言没有依据。""" + return self.trimmed or bool(self.empty_sources) + + +def _metric_prompt(catalog: dict[str, Any]) -> list[str]: + """口径卡指标和运行输入在写作目录里的样子。""" + lines: list[str] = [] groups: dict[tuple[str, str, str], list[dict[str, Any]]] = {} for entry in catalog.values(): if entry.get("kind") == "metric": @@ -3172,23 +3233,249 @@ def catalog_prompt(catalog: dict[str, Any], *, budget: int = 12000, cells_allowe if inputs: lines.append("\n运行输入：") lines.extend(f"- [[{e['alias']}]] = {e.get('rendered')}" for e in inputs) + return lines + + +def _dig_path(data: Any, path: str) -> Any: + for key in path.split("."): + data = data.get(key) if isinstance(data, dict) else None + return data + + +def _field_prompt(path: str, value: Any, ev: dict[str, Any], aliases: dict[str, str], snaps: _Snapshots, + cells_allowed: bool) -> str: + """一个结构化字段在写作目录里的那一行：值（按 [[v:]] 的渲染规则）和引用它的标记。""" + status = ev.get("status") + if status == "missing": + return f"- {path}：抽取时没有找到这个值（记为空，不代表库里没有）" + if status not in ("verified", "mismatch"): + return f"- {path}：出处核对不了（记为空）" + locator = ev.get("locator") if isinstance(ev.get("locator"), dict) else {} + alias = aliases.get(str(ev.get("artifact") or "")) + if alias is None: + return f"- {path}：所依据的查询不在这份目录里，不能引用" + if not cells_allowed: + return f"- {path}：出自 {alias}，只能在句末写 [[see:{alias}]] 作依据" + if isinstance(locator.get("rows"), list) and len(locator["rows"]) == 2: + first, last = locator["rows"] + named = locator.get("columns") if isinstance(locator.get("columns"), dict) else {"": locator.get("column")} + cols = ",".join(dict.fromkeys(str(c) for c in named.values() if c)) + many = f"（共 {last - first + 1} 行，整表最多 {TABLE_MAX_ROWS} 行，要分成几张表）" \ + if last - first + 1 > TABLE_MAX_ROWS else "" + return (f"- {path}：数组，出自 {alias} 第 {first}–{last} 行，可以单独一行写 " + f"[[table:{alias} cols={cols} rows={first}-{last}]]{many}") + row, column = locator.get("row"), locator.get("column") + shown = MISSING + snapshot, why = snaps.get(alias, ev.get("artifact")) + try: + if why: + raise CellError(why) + raw, column = locate_cell(snapshot, row, column) + shown = render_cell(raw, kind=column_kind(snapshot, column)) + except (CellError, RenderError): + try: + shown = render_cell(value) + except RenderError: + pass + return f"- {path} = {shown}：[[v:{alias}.r{row}.{column}]]" + + +def _fields_prompt(sources: list[dict[str, Any]], catalog: dict[str, Any], snaps: _Snapshots, + cells_allowed: bool) -> tuple[list[str], list[str], list[str]]: + """各 Agent 经按出处核对交来的结构化字段：(写作目录里的几行, 结构化结论为空的来源, 字段引用到的查询编号)。 + + 字段的出处记的是 Agent 自己的编号（它的第几次查询），这里换成报告目录里的全局编号：按快照工件认，同一个节点的 + 优先（工件内容寻址，两次一模一样的查询共用一份快照，指哪一次都是同一格）。 + """ + mine: dict[tuple[str, str], str] = {} + anyone: dict[str, str] = {} + for e in catalog.values(): + if e.get("kind") == "query": + mine.setdefault((str(e.get("node_id") or ""), str(e.get("artifact") or "")), e["alias"]) + anyone.setdefault(str(e.get("artifact") or ""), e["alias"]) + lines: list[str] = [] + empty: list[str] = [] + cited: list[str] = [] + for source in sources: + evidence = source.get("evidence") + if not isinstance(evidence, dict) or not evidence: + continue + node_id = str(source.get("node_id") or "") + title = str(source.get("title") or node_id) + data = source.get("data") if isinstance(source.get("data"), dict) else {} + if all(_dig_path(data, path) is None for path in evidence): + empty.append(title) + lines.append(f"「{title}」：{EMPTY_SOURCE_PROMPT}") + continue + aliases = {artifact: mine.get((node_id, artifact)) or alias for artifact, alias in anyone.items()} + lines.append(f"「{title}」：") + for path, ev in evidence.items(): + ev = ev if isinstance(ev, dict) else {} + lines.append(_field_prompt(path, _dig_path(data, path), ev, aliases, snaps, cells_allowed)) + if ev.get("status") in ("verified", "mismatch") and (alias := aliases.get(str(ev.get("artifact") or ""))): + cited.append(alias) + if not lines: + return [], [], [] + return (["\n各 Agent 交来的结构化结论（系统已逐格核对到查询快照；要写这些数，用后面的引用标记）：", *lines], + empty, list(dict.fromkeys(cited))) + + +def _query_brief(entry: dict[str, Any], tables: list[str]) -> str: + """没展开的查询在目录里的一行摘要：编号、查的表（没有表结构时写工具名）、行数。""" + rows = entry.get("rows") + what = ("、".join(tables[:3]) + (f" 等 {len(tables)} 张表" if len(tables) > 3 else "")) if tables \ + else str(entry.get("tool") or entry.get("source") or "查询") + size = f"{rows} 行" if isinstance(rows, int) else "行数不明" + return f" - {entry['alias']}：{what}，{size}" + ("（结果已截断）" if entry.get("truncated") else "") + + +def _cut_lines(lines: list[str], limit: int, note: str) -> list[str]: + """从头取到放得下 limit 个字为止，后面换成一句 note。""" + out: list[str] = [] + used = len(note) + 1 + for line in lines: + if used + len(line) + 1 > limit: + return [*out, note] + out.append(line) + used += len(line) + 1 + return out + + +def writer_catalog(catalog: dict[str, Any], *, budget: int = DEFAULT_CATALOG_BUDGET, cells_allowed: bool = True, + loader: Loader | None = None, fields: list[dict[str, Any]] | None = None, + titles: dict[str, str] | None = None) -> WriterCatalog: + """证据目录的文字版，放进写作者的 prompt，连同省略了什么（WriterCatalog）。 + + 查询结果会列出能引用的行（要读快照）；cells_allowed=False 时只列编号，说明只能当依据。 + 沙箱代码节点不列：它的产出不能引用。fields 是上游 Agent 经按出处核对交来的结构化字段 + [{node_id, title, data, evidence}]，titles 是节点在画布上的名字（分组标题用）。 + + 放得下就整份给（没有结构化字段时和以前一字不差）。放不下时按优先级、按来源均衡（_trimmed_catalog）： + 口径卡指标、运行输入、结构化字段一律列出；查询按产出它的节点分组、轮流展开；没展开的列编号和摘要； + 开头照实说目录有省略、各来源省略了几次。 + """ + snaps = _Snapshots(loader) + head = ["可引用的证据（写数字只能用下面的引用标记，系统会换成真实数值）："] + field_lines, empty, cited = _fields_prompt(fields or [], catalog, snaps, cells_allowed) + first = [*_metric_prompt(catalog), *field_lines] queries = [e for e in catalog.values() if e.get("kind") == "query"] - if queries: - lines.append(f"\n{CELL_RULES}" if cells_allowed else - "\n查询结果（这次出具不允许直接引用单元格，只能写在 [[see:Q1]] 这样的依据里）：") - for e in queries: - lines.extend(_query_prompt(e, snaps, cells_allowed)) + rules = ([f"\n{CELL_RULES}" if cells_allowed else + "\n查询结果（这次出具不允许直接引用单元格，只能写在 [[see:Q1]] 这样的依据里）："] if queries else []) + tail: list[str] = [] if (index := snaps.entities(catalog)).active: - lines.extend(_entity_prompt(catalog, partial=index.partial, snaps=snaps)) + tail.extend(_entity_prompt(catalog, partial=index.partial, snaps=snaps)) retrievals = [e for e in catalog.values() if e.get("kind") == "retrieval"] if retrievals: - lines.append(f"\n{QUOTE_RULES}") + tail.append(f"\n{QUOTE_RULES}") for e in retrievals: - lines.extend(_retrieval_prompt(e, snaps)) - text = "\n".join(lines) + tail.extend(_retrieval_prompt(e, snaps)) + full = {e["alias"]: _query_prompt(e, snaps, cells_allowed) for e in queries} + text = "\n".join([*head, *first, *rules, *(line for e in queries for line in full[e["alias"]]), *tail]) + if len(text) <= budget: + return WriterCatalog(text=text, budget=budget, queries=len(queries), empty_sources=empty) + return _trimmed_catalog(catalog, head, first, rules, queries, full, tail, budget=budget, snaps=snaps, + cells_allowed=cells_allowed, titles=titles or {}, empty=empty, pinned=cited) + + +def _trimmed_catalog(catalog: dict[str, Any], head: list[str], first: list[str], rules: list[str], + queries: list[dict[str, Any]], full: dict[str, list[str]], tail: list[str], *, budget: int, + snaps: _Snapshots, cells_allowed: bool, titles: dict[str, str], empty: list[str], + pinned: list[str]) -> WriterCatalog: + """放不下时的目录。以前从头截断：查询按编号排，前面的节点占满预算，后面的节点一次都不露面。 + + 1. 口径卡指标、运行输入、结构化字段一律列出（第一优先） + 2. 表和字段、知识库检索两节合起来最多占预算的 1/_PROMPT_TAIL_SHARE + 3. 每次查询先只占一行摘要；结构化字段引用到的查询先展开（pinned），再按产出它的节点分组、轮流展开： + 第一轮每次列前 _PROMPT_COMPACT_ROWS 行，第二轮补到 _PROMPT_ROWS 行，放不下为止 + 4. 开头写明目录有省略、各来源省略了几次，以及没展开的查询怎么用（OMITTED_RULES） + 最后整份量一次：超了就把最后展开的退回去；指标和字段本身就超（极少见）才从尾部截断。 + """ + if tail and len("\n".join(tail)) > budget // _PROMPT_TAIL_SHARE: + tail = _cut_lines(tail, budget // _PROMPT_TAIL_SHARE, "…（表和字段、知识库检索太长，后面的省略了）") + tables: dict[str, list[str]] = {} + for e in catalog.values(): + if e.get("kind") == "table": + for alias in e.get("queries") or []: + tables.setdefault(alias, []).append(str(e.get("name"))) + groups: dict[str, list[dict[str, Any]]] = {} + for e in queries: + groups.setdefault(str(e.get("node_id") or ""), []).append(e) + compact = {e["alias"]: _query_prompt(e, snaps, cells_allowed, _PROMPT_COMPACT_ROWS) for e in queries} + brief = {e["alias"]: [_query_brief(e, tables.get(e["alias"], []))] for e in queries} + shapes = {"brief": brief, "compact": compact, "full": full} + tier = {e["alias"]: "brief" for e in queries} + + def rows() -> list[dict[str, Any]]: + out = [] + for nid, entries in groups.items(): + got = Counter(tier[e["alias"]] for e in entries) + out.append({"node_id": nid, "title": titles.get(nid) or nid or "其他", "total": len(entries), + "full": got["full"], "compact": got["compact"], "omitted": got["brief"]}) + return out + + def assemble() -> str: + stats = rows() + per = "、".join(f"{r['title']} {r['total']} 次（展开 {r['full'] + r['compact']} 次，省略 {r['omitted']} 次）" + for r in stats) + n = {k: sum(r[k] for r in stats) for k in ("full", "compact", "omitted")} + notice = (f"注意：证据太多，这份目录有省略（篇幅上限约 {budget} 字）。共 {len(queries)} 次查询：" + f"{n['full']} 次列出前 {_PROMPT_ROWS} 行，{n['compact']} 次只列前 {_PROMPT_COMPACT_ROWS} 行，" + f"{n['omitted']} 次只列编号和摘要。各来源：{per}。") + body = [*head, notice, OMITTED_RULES, *first, *rules] + for r, entries in zip(stats, groups.values()): + body.append(f"\n来自「{r['title']}」的查询（共 {r['total']} 次，展开 {r['full'] + r['compact']} 次）：") + for e in entries: + if tier[e["alias"]] != "brief": + body.extend(shapes[tier[e["alias"]]][e["alias"]]) + if r["omitted"]: + body.append(f" 没展开（{r['omitted']} 次，只列编号和摘要）：") + body.extend(brief[e["alias"]][0] for e in entries if tier[e["alias"]] == "brief") + return "\n".join([*body, *tail]) + + def size(lines: list[str]) -> int: + return sum(len(line) + 1 for line in lines) + + # 省略情况那句里的数字会变长几位：留一点余量，最后再整份量一次 + remaining = budget - len(assemble()) - 8 * (len(groups) + 3) + upgraded: list[tuple[str, str]] = [] + for alias in pinned: + if tier.get(alias) == "brief" and (delta := size(compact[alias]) - size(brief[alias])) <= remaining: + tier[alias] = "compact" + remaining -= delta + upgraded.append((alias, "brief")) + for target, source in (("compact", "brief"), ("full", "compact")): + moved = True + while moved: + moved = False + for entries in groups.values(): + for e in entries: + alias = e["alias"] + if tier[alias] != source: + continue + delta = size(shapes[target][alias]) - size(shapes[source][alias]) + if delta <= remaining: + tier[alias] = target + remaining -= delta + upgraded.append((alias, source)) + moved = True + break + text = assemble() + while len(text) > budget and upgraded: + alias, source = upgraded.pop() + tier[alias] = source + text = assemble() if len(text) > budget: - text = text[:budget].rsplit("\n", 1)[0] + "\n…（目录太长，后面的省略了）" - return text + text = text[:budget - len(_PROMPT_CUT)].rsplit("\n", 1)[0] + _PROMPT_CUT + return WriterCatalog(text=text, budget=budget, queries=len(queries), trimmed=True, nodes=rows(), + omitted=[e["alias"] for e in queries if tier[e["alias"]] == "brief"], empty_sources=empty) + + +def catalog_prompt(catalog: dict[str, Any], *, budget: int = DEFAULT_CATALOG_BUDGET, cells_allowed: bool = True, + loader: Loader | None = None, fields: list[dict[str, Any]] | None = None, + titles: dict[str, str] | None = None) -> str: + """证据目录的文字版（writer_catalog 的文字）。超过预算时分优先级、按来源均衡，并照实说省略了什么。""" + return writer_catalog(catalog, budget=budget, cells_allowed=cells_allowed, loader=loader, fields=fields, + titles=titles).text def unshaped_table(entry: dict[str, Any], fetch: Callable[[Any], Any]) -> bool: diff --git a/backend/app/engine/nodes/report.py b/backend/app/engine/nodes/report.py index 1fcd50c4..a7d7cabc 100644 --- a/backend/app/engine/nodes/report.py +++ b/backend/app/engine/nodes/report.py @@ -50,11 +50,13 @@ from app.engine.evidence import ( MARKER_RULES, SEE_AND_CODE_RULES, StreamRenderer, + WriterCatalog, build_catalog, - catalog_prompt, + catalog_budget, compose_doc, describe_violations, ledger_enabled, + writer_catalog, ) from app.engine.nodes.llm import _invoke_streaming, _model_spec, _report_call from app.engine.schema import ( @@ -70,7 +72,8 @@ from app.engine.schema import ( ) from app.engine.state import GraphState, message_text, thinking_text from app.engine.toolcalls import leaked_markup, markup_warning -from app.providers.factory import ProviderNotConfigured, get_chat_model +from app.providers import catalog as model_catalog +from app.providers.factory import ModelSpec, ProviderNotConfigured, get_chat_model, resolve_provider ON_VIOLATION = ("fail", "flag") NUMBERS = ("strict", "off") @@ -140,6 +143,57 @@ def report_catalog(state: GraphState, spec: GraphSpec, node: GraphNode) -> dict[ return catalog +def report_fields(state: GraphState, spec: GraphSpec, node: GraphNode) -> list[dict[str, Any]]: + """上游 Agent 经按出处核对交来的结构化字段：[{node_id, title, data, evidence}]，按画布上的顺序。 + + 范围和 report_catalog 收查询的范围一致：祖先节点，写了 evidence_from 就只收那几个。只进写作目录（第一优先， + 放不下也列出），不进目录本身：出口复核按目录重建，字段的引用落到的是查询单元格，和以前一样核对。 + """ + sources = _setting(spec, node, "evidence_from", "ancestors") + if isinstance(sources, str) and sources != "ancestors": + sources = [sources] + keep = {str(s) for s in sources} if isinstance(sources, list) else None + allowed = _ancestors(spec, node.id) + outputs = state.get("nodes") or {} + found: list[dict[str, Any]] = [] + for other in spec.nodes: + result = outputs.get(other.id) + if other.id not in allowed or (keep is not None and other.id not in keep) or not isinstance(result, dict): + continue + if isinstance(result.get("data_evidence"), dict) and result["data_evidence"]: + found.append({"node_id": other.id, "title": other.title or other.id, "data": result.get("data"), + "evidence": result["data_evidence"]}) + return found + + +async def model_window(session: Any, spec: ModelSpec, model_id: str) -> int | None: + """写作模型的上下文窗口（token）：模型接入里给这个模型登记的 context 优先，其次模型目录 + （providers/catalog.py）；都没有返回 None，由 catalog_budget 取保守值。""" + try: + provider = await resolve_provider(session, spec.provider, spec.model or model_id) + except ProviderNotConfigured: + provider = None + for entry in (provider.models or []) if provider is not None else []: + if isinstance(entry, dict) and entry.get("id") == model_id: + raw = entry.get("context") + try: + value = int(raw) if not isinstance(raw, bool) and raw not in (None, "") else 0 + except (TypeError, ValueError): + value = 0 + if value > 0: + return value + return model_catalog.find_context(model_id) + + +def _trimmed_message(writer: WriterCatalog) -> str: + """写作目录有省略时的那条警告（给人看）：各来源省略了几次，写作模型看不到哪些明细。""" + per = "、".join(f"{r['title']}省略 {r['omitted']} 次" for r in writer.nodes if r["omitted"]) + shown = writer.queries - len(writer.omitted) + return (f"证据目录超出篇幅（约 {writer.budget} 字，按写作模型的上下文窗口确定）：{writer.queries} 次查询中 " + f"{shown} 次列出了数据，{len(writer.omitted)} 次只列编号和摘要（{per}）。写作模型看不到这些查询的明细，" + "报告不得据此断言没有数据") + + def report_entities(spec: GraphSpec, node: GraphNode) -> str: """报告节点的实体层：link（默认）/ off。出口复核调 verify_doc 时传 entities=(这个值 == "link")。 @@ -351,8 +405,7 @@ def _repair_request(violations: list[dict[str, Any]]) -> str: ) -def _messages(ctx: NodeContext, state: GraphState, catalog: dict[str, Any], cells: bool = True, - claims: str = "off") -> list[BaseMessage]: +def _messages(ctx: NodeContext, state: GraphState, writer: WriterCatalog, claims: str = "off") -> list[BaseMessage]: system = ctx.render_str(ctx.cfg("system", ""), state) instructions = ctx.render_str(ctx.cfg("instructions", ""), state).strip() \ or "根据下面的证据写一份简洁的报告，先总后分。" @@ -363,7 +416,7 @@ def _messages(ctx: NodeContext, state: GraphState, catalog: dict[str, Any], cell return [ SystemMessage(content="\n\n".join(p for p in (system, ROLE) if p)), HumanMessage(content=( - f"{instructions}\n\n{catalog_prompt(catalog, cells_allowed=cells)}\n\n{rules}\n\n" + f"{instructions}\n\n{writer.text}\n\n{rules}\n\n" "只输出报告正文（Markdown），不要写撰写说明。" )), ] @@ -392,6 +445,8 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]: model, model_id = await get_chat_model(session, _model_spec(ctx)) except ProviderNotConfigured as e: raise NodeError(ctx.node.id, not_configured(e)) from e + # 写作目录的篇幅按这个模型的上下文窗口定（evidence.catalog_budget） + window = await model_window(session, _model_spec(ctx), model_id) spec = ctx.run.spec catalog = report_catalog(state, spec, ctx.node) @@ -408,7 +463,14 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]: if warning := _no_evidence(catalog, cells): ctx.emit(EventType.LOG, level="warn", code="report_no_evidence", message=warning) - messages = _messages(ctx, state, catalog, cells, claims) + writer = writer_catalog(catalog, budget=catalog_budget(window), cells_allowed=cells, + fields=report_fields(state, spec, ctx.node), + titles={n.id: n.title or n.id for n in spec.nodes}) + if writer.trimmed: + ctx.emit(EventType.LOG, level="warn", code="report_catalog_trimmed", message=_trimmed_message(writer), + budget=writer.budget, window=window, queries=writer.queries, omitted=len(writer.omitted), + nodes=[{k: r[k] for k in ("node_id", "title", "total", "omitted")} for r in writer.nodes]) + messages = _messages(ctx, state, writer, claims) started = time.perf_counter() spent: list[dict[str, Any]] = [] nudged = False diff --git a/backend/app/engine/review.py b/backend/app/engine/review.py index 594ff9c2..98e0d36d 100644 --- a/backend/app/engine/review.py +++ b/backend/app/engine/review.py @@ -89,6 +89,8 @@ WARN_CODES: dict[str, tuple[str, str]] = { "report_repair": ("report_repair", DEGRADED), # 报告撰写节点的上游没有可引用的口径卡指标：报告里的数都没有出处 "report_no_evidence": ("report_no_evidence", DEGRADED), + # 证据目录超出篇幅，有的查询只列了编号和摘要：写作模型没看全证据，报告的覆盖面有缺口 + "report_catalog_trimmed": ("report_catalog_trimmed", DEGRADED), # 口径卡的指标集、报告文档没能落进工件库：答案照常，只是点不开证据 "evidence_store_failed": ("evidence_store_failed", DEGRADED), # agent 的结构化字段：模型报的值和查询快照对不上（已按快照取值），或者出处核对不了 diff --git a/backend/tests/fixtures/evidence/store_run.py b/backend/tests/fixtures/evidence/store_run.py new file mode 100644 index 00000000..138eb477 --- /dev/null +++ b/backend/tests/fixtures/evidence/store_run.py @@ -0,0 +1,106 @@ +"""合成的「4 个专项 Agent 并行查库 → 报告」场景：查询多、分布不均，每个 Agent 都交了按出处核对的字段。 + +形状仿一次真实运行（4 个 Agent 并行、查询次数悬殊，合计上百次），次数和内容全部虚构：门店、会员、卡券、活动。 +不连库、不调模型：台账条目、查询快照、表结构快照、Agent 的 data / data_evidence 都在内存里拼好，快照经 loader 取。 + +台账按节点跑完的先后排（state["evidence"] 是各节点结束时追加的），所以全局编号 Q1… 前 26 个都是会员专项的。 +""" +from __future__ import annotations + +from typing import Any + +from app.engine.evidence import cell_eid, make_eid + +#: (节点 id, 画布上的名字, 查询次数, 查的表) +AGENTS: list[tuple[str, str, int, str]] = [ + ("agent_member", "会员专项", 26, "member_daily"), + ("agent_store", "门店专项", 17, "store_daily"), + ("agent_coupon", "卡券核销", 79, "coupon_redemption"), + ("agent_campaign", "活动内容", 42, "campaign_view"), +] +COLUMNS = ["store_name", "biz_time", "cnt", "amount"] +TYPES = {"store_name": "text", "biz_time": "datetime", "cnt": "number", "amount": "number"} +ROWS = 30 +#: 结构化抽取失败（两次都被截断）的那个 Agent：字段全是空值 +EMPTY_AGENT = "agent_coupon" +FIELDS = ["total_cnt", "top_store", "latest_time", "top_stores", "churn_rate"] + + +def artifact_of(node_id: str, i: int) -> str: + return f"{node_id}-q{i:03d}".ljust(64, "0") + + +def snapshot(node_id: str, i: int, rows: int = ROWS) -> dict[str, Any]: + """第 i 次查询的快照：每行一个门店，时间是 ISO 写法（快照里存的就是这样）。""" + seed = sum(map(ord, node_id)) + i * 7 + return { + "columns": list(COLUMNS), + "column_types": dict(TYPES), + "rows": [[f"门店{(seed + r) % 90 + 1:02d}", f"2026-06-{r % 28 + 1:02d}T08:{(seed + r) % 60:02d}:39", + (seed * 13 + r * 17) % 900 + 10, f"{(seed * 31 + r * 7) % 9000 + 100}.50"] for r in range(rows)], + "truncated": False, + } + + +def schema_snapshot(table: str) -> dict[str, Any]: + return {"source": "shop", "tables": {table: {"columns": [{"name": c, "type": TYPES[c]} for c in COLUMNS]}}} + + +def build() -> tuple[list[dict[str, Any]], dict[str, Any], dict[str, Any], dict[str, str]]: + """(台账, 快照 {工件 id: 内容}, state["nodes"], {节点 id: 名字})。""" + ledger: list[dict[str, Any]] = [] + store: dict[str, Any] = {} + nodes: dict[str, Any] = {} + titles: dict[str, str] = {} + for node_id, title, count, table in AGENTS: + titles[node_id] = title + schema_id = f"schema-{table}".ljust(64, "0") + store[schema_id] = schema_snapshot(table) + ledger.append({"kind": "schema", "node_id": node_id, "exec": 1, "artifact": schema_id, "source": "shop", + "tables": {table: list(COLUMNS)}}) + for i in range(1, count + 1): + aid = artifact_of(node_id, i) + store[aid] = snapshot(node_id, i) + ledger.append({"kind": "query", "node_id": node_id, "exec": 1, "artifact": aid, + "via": f"tool-{aid}"[:64], "call_id": f"call_{i}", "tool": "db_query__shop", "source": "shop", + "columns": list(COLUMNS), "rows": ROWS, "truncated": False, + "schema_artifact": schema_id, "tables": [table]}) + nodes[node_id] = agent_output(node_id, store) + return ledger, store, nodes, titles + + +def agent_output(node_id: str, store: dict[str, Any]) -> dict[str, Any]: + """Agent 交出的结构化字段：本节点的 Q3 那一格、Q5 的前 5 行；EMPTY_AGENT 抽取失败，字段全空。""" + if node_id == EMPTY_AGENT: + reason = "结构化抽取没有完成：模型的输出额度在推理阶段就用完了（最后一次的额度为 32768 token），结构化结论没能写出来" + return {"text": "查到了大量核销和兑换记录。", "data": {f: None for f in FIELDS}, + "data_evidence": {f: {"status": "unresolved", "reason": reason} for f in FIELDS}} + q3, q5 = artifact_of(node_id, 3), artifact_of(node_id, 5) + first = store[q3]["rows"][0] + + def cell(column: str, index: int) -> dict[str, Any]: + return {"ref": f"Q3.r0.{column}", "call": "Q3", "artifact": q3, "via": f"tool-{q3}"[:64], + "locator": {"row": 0, "column": column}, "eid": cell_eid(q3, 0, column), "status": "verified"} + + rows_locator = {"rows": [0, 4], "column": "store_name"} + return { + "text": "结论见字段。", + "data": {"total_cnt": first[2], "top_store": first[0], "latest_time": first[1], + "top_stores": [r[0] for r in store[q5]["rows"][:5]], "churn_rate": None}, + "data_evidence": { + "total_cnt": cell("cnt", 2), "top_store": cell("store_name", 0), "latest_time": cell("biz_time", 1), + "top_stores": {"ref": "Q5.r0-4.store_name", "call": "Q5", "artifact": q5, "via": f"tool-{q5}"[:64], + "locator": rows_locator, "eid": make_eid("rows", q5, rows_locator), "status": "verified"}, + "churn_rate": {"ref": None, "status": "missing"}, + }, + } + + +def global_alias(node_id: str, local: int) -> str: + """本节点第 local 次查询在报告目录里的全局编号（台账按节点先后排）。""" + offset = 0 + for nid, _, count, _ in AGENTS: + if nid == node_id: + return f"Q{offset + local}" + offset += count + raise KeyError(node_id) diff --git a/backend/tests/test_report_catalog_budget.py b/backend/tests/test_report_catalog_budget.py new file mode 100644 index 00000000..3ea3da91 --- /dev/null +++ b/backend/tests/test_report_catalog_budget.py @@ -0,0 +1,209 @@ +"""写作者的出处清单：预算按写作模型的上下文窗口定；超出预算时分优先级、按来源均衡，省略了什么照实说。 + +2026-10 一次运行：证据台账里上百次查询（4 个 Agent 查询次数悬殊），出处清单写死 12000 字，十万字以上的目录 +被截到一万两千字以内，只有前二十来次查询进了写作者的视野，后面只有一句「目录太长，后面的省略了」。查询按编号排，前面的 +Agent 占满了预算，有的 Agent 一次都没露面；写作者于是写出「本轮没有查到任何核销结果」，可那个 Agent 查到了上百条。 + +- 预算：写作模型上下文窗口的 40%（字数按 1 字 ≈ 1 token 保守算），拿不到窗口按 128K；下限 12000、上限 160000 +- 第一优先：各 Agent 经按出处核对交来的结构化字段、口径卡指标，一律列出 +- 查询按产出它的节点分组、轮流取：先给每次查询前几行，再补到 20 行；每个节点都有代表 +- 没展开的查询列出编号和一行摘要（表、行数）；约定只能写 [[see:Qn]] 作依据，数字照样逐字核对 +- 清单开头明说「目录有省略」，各节点各省略了多少条 + +夹具全部合成（tests/fixtures/evidence/store_run.py）。 +""" +from __future__ import annotations + +import re + +import pytest + +import app.db.models # noqa: F401 - 建表（conftest 的 create_all）前要先登记模型 +from app.engine.evidence import ( + CATALOG_MAX_CHARS, + CATALOG_MIN_CHARS, + DEFAULT_CATALOG_BUDGET, + build_catalog, + catalog_budget, + catalog_prompt, + writer_catalog, +) +from tests.fixtures.evidence import store_run + +LEDGER, STORE, NODES, TITLES = store_run.build() + + +def load(artifact): + return STORE.get(artifact) + + +@pytest.fixture +def catalog(): + return build_catalog(nodes=NODES, ledger=LEDGER) + + +def sources(): + return [{"node_id": nid, "title": TITLES[nid], "data": NODES[nid]["data"], + "evidence": NODES[nid]["data_evidence"]} for nid, *_ in store_run.AGENTS] + + +def written(catalog, budget=DEFAULT_CATALOG_BUDGET, **kw): + return writer_catalog(catalog, budget=budget, loader=load, fields=sources(), titles=TITLES, **kw) + + +def group_of(text: str, title: str) -> str: + """清单里「来自「<名字>」的查询」那一段（到下一个来源或下一节为止）。""" + start = text.index(f"来自「{title}」的查询") + rest = text[start + 1:] + ends = [m.start() for m in re.finditer(r"\n(?:来自「|\n)", rest)] + return text[start:start + 1 + (ends[0] if ends else len(rest))] + + +# -------------------------------------------------------------------------- +# 预算 +# -------------------------------------------------------------------------- + + +def test_budget_follows_the_model_window(): + assert DEFAULT_CATALOG_BUDGET == catalog_budget(None) == 51_200, "拿不到窗口按 128K 的 40%" + assert catalog_budget(128_000) == 51_200 + assert catalog_budget(200_000) == 80_000 + assert catalog_budget(32_000) == 12_800 + assert catalog_budget(8_000) == CATALOG_MIN_CHARS == 12_000, "很小的窗口维持原来的篇幅" + assert catalog_budget(1_000_000) == CATALOG_MAX_CHARS == 160_000, "百万窗口也有上限：每写一稿都要重发整份目录" + assert catalog_budget(0) == catalog_budget(-5) == DEFAULT_CATALOG_BUDGET + + +def test_the_default_is_far_above_the_old_cut(): + assert DEFAULT_CATALOG_BUDGET >= 4 * 12_000 + + +# -------------------------------------------------------------------------- +# 放得下：照旧 +# -------------------------------------------------------------------------- + + +def test_a_catalog_that_fits_is_not_trimmed(): + small = build_catalog(nodes={}, ledger=[e for e in LEDGER if e.get("node_id") == "agent_store"]) + got = writer_catalog(small, loader=load) + assert not got.trimmed and not got.omitted + assert "目录有省略" not in got.text + assert got.text == catalog_prompt(small, loader=load), "没有结构化字段时和 catalog_prompt 一字不差" + assert "Q17：" in got.text and " r19：" in got.text + + +# -------------------------------------------------------------------------- +# 放不下：分优先级、按来源均衡（那次运行同样形状的场景） +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("budget", [DEFAULT_CATALOG_BUDGET, 12_000]) +def test_every_agent_stays_in_view_within_the_budget(catalog, budget): + got = written(catalog, budget) + text = got.text + assert got.trimmed and len(text) <= budget, len(text) + for nid, title, count, table in store_run.AGENTS: + group = group_of(text, title) + assert f"共 {count} 次" in group + assert re.search(r"\n {2}r0：", group), f"「{title}」一次查询都没展开" + # 每次查询都在清单里：展开的有行，没展开的有编号和摘要 + total = sum(c for *_, c, _ in store_run.AGENTS) + for n in range(1, total + 1): + assert re.search(rf"(?:^|\n)\s*- Q{n}：", text), f"Q{n} 不见了" + + +def test_omitted_queries_are_listed_with_a_summary(catalog): + got = written(catalog) + assert got.omitted, "这个场景放不下全部展开" + for alias in got.omitted: + node = catalog[alias]["node_id"] + table = next(t for nid, _, _, t in store_run.AGENTS if nid == node) + assert re.search(rf"\n {{2}}- {alias}：{table}，30 行", got.text), alias + # 没展开的查询没有行 + first = got.omitted[0] + block = got.text[got.text.index(f"- {first}："):].split("\n", 2) + assert not block[1].lstrip().startswith("r0：") + + +def test_the_head_says_what_was_left_out_per_source(catalog): + got = written(catalog) + head = got.text[:800] + assert "目录有省略" in head + for row in got.nodes: + assert f"{row['title']} {row['total']} 次" in head and f"省略 {row['omitted']} 次" in head, row + assert sum(r["omitted"] for r in got.nodes) == len(got.omitted) + assert [r["node_id"] for r in got.nodes] == [nid for nid, *_ in store_run.AGENTS] + + +def test_how_to_use_an_omitted_query_is_spelled_out(catalog): + text = written(catalog).text + assert "[[see:Qn]]" in text and "本报告未覆盖" in text + assert "不要凭摘要写" in text and "数字照样逐字核对" in text + for phrase in ("没有查到", "不存在", "缺失", "没有数据"): + assert phrase in text + + +def test_structured_fields_come_first_and_point_at_global_cells(catalog): + got = written(catalog, 12_000) + text = got.text + fields_at = text.index("各 Agent 交来的结构化结论") + assert fields_at < text.index("来自「"), "结构化结论排在查询前面" + member_q3 = store_run.global_alias("agent_member", 3) + store_q3 = store_run.global_alias("agent_store", 3) + campaign_q5 = store_run.global_alias("agent_campaign", 5) + assert f"[[v:{member_q3}.r0.cnt]]" in text and f"[[v:{store_q3}.r0.store_name]]" in text + assert f"[[table:{campaign_q5} cols=store_name rows=0-4]]" in text + assert "churn_rate：抽取时没有找到这个值" in text + # 抽取失败的那个 Agent：结构化结论为空，不能据此断言没有数据 + block = text[text.index("「卡券核销」：", fields_at):] + assert block.startswith("「卡券核销」：结构化结论为空") and "本报告未覆盖" in block.split("\n", 1)[0] + assert got.empty_sources == ["卡券核销"] + + +def test_queries_the_fields_cite_are_expanded_first(catalog): + """结构化字段引用到的那几次查询先展开：写作者照抄字段的引用标记时，看得到那几行。""" + got = written(catalog, 12_000) + for nid in ("agent_member", "agent_store", "agent_campaign"): + for local in (3, 5): + alias = store_run.global_alias(nid, local) + assert alias not in got.omitted, alias + assert re.search(rf"\n- {alias}：.*\n {{2}}r0：", got.text), alias + + +def test_a_field_value_reads_like_the_cell_it_cites(catalog): + """字段后面写的值和引用之后报告里显示的一样（[[v:]] 的渲染规则）。""" + text = written(catalog).text + q3 = STORE[store_run.artifact_of("agent_member", 3)] + assert f"total_cnt = {q3['rows'][0][2]:,}：[[v:Q3.r0.cnt]]" in text + + +# -------------------------------------------------------------------------- +# 报告节点：窗口从哪来 +# -------------------------------------------------------------------------- + + +async def test_the_window_comes_from_the_provider_entry_then_the_catalog(): + """模型接入里给这个模型登记的 context 优先，其次模型目录；都没有返回 None（预算按保守值）。""" + from sqlalchemy import delete + + from app.db.base import SessionLocal + from app.db.models import Provider + from app.engine.nodes.report import model_window + from app.providers.factory import ModelSpec + + async with SessionLocal() as session: + row = Provider(name="fa-window-test", kind="mock", enabled=True, default_model="mock-wide", + models=[{"id": "mock-wide", "context": 64_000}, {"id": "mock-text", "context": "96000"}, + {"id": "mock-bare"}]) + session.add(row) + await session.commit() + try: + assert await model_window(session, ModelSpec(model="mock-wide"), "mock-wide") == 64_000 + assert await model_window(session, ModelSpec(model="mock-text"), "mock-text") == 96_000 + # 接入里没登记窗口：按模型目录（deepseek-chat 登记了 128K）；目录里也没有就是 None + assert await model_window(session, ModelSpec(model="deepseek-chat"), "deepseek-chat") == 128_000 + assert await model_window(session, ModelSpec(model="mock-bare"), "mock-bare") is None + assert catalog_budget(await model_window(session, ModelSpec(model="mock-wide"), "mock-wide")) == 25_600 + finally: + await session.execute(delete(Provider).where(Provider.id == row.id)) + await session.commit() diff --git a/frontend/src/run/decode.ts b/frontend/src/run/decode.ts index f82b4844..f9bab20c 100644 --- a/frontend/src/run/decode.ts +++ b/frontend/src/run/decode.ts @@ -756,6 +756,22 @@ function explainLog(code: string | undefined, message: string, d: Record typeof r?.omitted === 'number' && r.omitted > 0 && str(r?.title)) + .map((r) => `${r.title} 省略 ${formatNumber(r.omitted)}/${formatNumber(r.total)} 次`) + return { + title: omitted != null + ? `证据目录超出篇幅，${total != null ? `${formatNumber(total)} 次查询中 ` : ''}${formatNumber(omitted)} 次只列了编号和摘要` + : '证据目录超出篇幅，部分查询只列了编号和摘要', + sub: per.length ? per.join('、') : undefined, + next: '写作模型按来源均衡地看到了一部分查询的明细，被省略的查询仍在证据台账里，报告不得据此断言没有数据。' + + '需要完整明细时，请换用上下文窗口更大的写作模型，或让 Agent 在 SQL 中先汇总再交出结果', + } + } case 'agent_field_mismatch': { // 「有 1 个字段和查询快照对不上，已按快照取值：order_cnt 模型报 1240，快照是 1234」（llm.py _field_warnings） const n = message.match(/有\s*(\d+)\s*个字段/)?.[1] diff --git a/scripts/check-copy.mjs b/scripts/check-copy.mjs index 2376d6f1..e023b249 100644 --- a/scripts/check-copy.mjs +++ b/scripts/check-copy.mjs @@ -230,12 +230,13 @@ const MODEL_ONLY = { 'NODE_REFERENCE', 'ROLE', 'MARKUP_NUDGE', 'JUDGE_RULES', 'JUDGE_ROLE', 'JUDGE_RULE', 'REVIEW_SYSTEM', 'MARKER_RULES', 'GATE_SYSTEM', 'TOOL_MARKUP_NUDGE', '_UNKNOWN_ENTITY_FOR_MODEL', 'NODE_FIELD_FIX', '_FIX_TAIL', '_STREAM_PROTOCOL', '_RULES_HEAD', '_NO_LOWERING', '_ASK_HUMAN', '_UPGRADE_TASK', 'CELL_RULES', 'ENTITY_RULES', - 'EXTRACT_SYSTEM', 'GRAPH_SCHEMA', 'REVIEW_SCHEMA', '_CN_N', + 'EXTRACT_SYSTEM', 'GRAPH_SCHEMA', 'REVIEW_SCHEMA', '_CN_N', 'OMITTED_RULES', 'ABSENCE_RULE', ]), // 拼提示词的函数：生成 / 修正 / 升级请求、抽取消息、交给抽取模型的输出结构； // 助手的数据源上下文（api/copilot_context.py 的各段渲染、挑表请求，copilot.py 里挑表用的需求） funcs: new Set(['_user_message', '_repair_request', '_publish_fix_request', '_upgrade_assist_request', - '_extract_messages', '_from_rows', 'catalog_prompt', + '_extract_messages', '_from_rows', 'catalog_prompt', 'writer_catalog', '_trimmed_catalog', '_metric_prompt', + '_fields_prompt', '_field_prompt', '_query_brief', '_pick_messages', '_pick_need', '_publish_fix_need', '_upgrade_need', '_detail_block', '_selected_block', '_join_lines', '_route_line', '_source_head']), // 第二个参数是交回模型改写的指令（第一个是给人看的那句）：engine/evidence.py 的 _Reason(text, fix)、fail(text, fix) diff --git a/scripts/check-decode.mjs b/scripts/check-decode.mjs index 5045adb1..f11e795a 100644 --- a/scripts/check-decode.mjs +++ b/scripts/check-decode.mjs @@ -1300,6 +1300,26 @@ await section('报告核对 report.checked、口径卡的台账（可点击证 `${flagged?.status} ${flagged?.level}`) const bare = flatten(mod.decodeRun([E(1, 'report.checked', null, {})])).find((s) => s.code === 'report_checked') check('载荷缺字段也不崩，只说核对了报告', bare?.title === '核对报告' && !bare.artifact, bare?.title) + + // 写作目录超出篇幅（2026-10 一次运行：上百次查询只有前二十来次进了写作者的视野，界面上一个字都没说）： + // 现在按来源均衡地展开，没展开的列编号和摘要，并发一条警告。界面读结构化字段（queries、omitted、nodes） + check('后端发目录有省略的警告，带着结构化字段', reportPy.includes('code="report_catalog_trimmed"') + && reportPy.includes('queries=writer.queries, omitted=len(writer.omitted)') && reportPy.includes('"node_id", "title", "total", "omitted"'), + '改了 report.py 的字段就同步改 decode.ts 的 explainLog 和这里的夹具') + const TRIMMED = '证据目录超出篇幅（约 51200 字，按写作模型的上下文窗口确定）：164 次查询中 90 次列出了数据，74 次只列编号和摘要' + + '（卡券核销省略 52 次、活动内容省略 22 次）。写作模型看不到这些查询的明细，报告不得据此断言没有数据' + const trimmed = flatten(mod.decodeRun([E(1, 'node.started', 'w', { node_type: 'report', label: '写报告' }), + E(2, 'log', 'w', { level: 'warn', code: 'report_catalog_trimmed', message: TRIMMED, budget: 51200, window: null, queries: 164, + omitted: 74, nodes: [{ node_id: 'a', title: '会员专项', total: 26, omitted: 0 }, { node_id: 'b', title: '卡券核销', total: 79, omitted: 52 }, + { node_id: 'c', title: '活动内容', total: 42, omitted: 22 }] })])).find((s) => s.code === 'report_catalog_trimmed') + check('report_catalog_trimmed：标题说几次查询只列了编号和摘要', trimmed?.title === '证据目录超出篇幅，164 次查询中 74 次只列了编号和摘要', trimmed?.title) + check('……副标题按来源列省略了几次（没省略的来源不列）', trimmed?.sub === '卡券核销 省略 52/79 次、活动内容 省略 22/42 次', trimmed?.sub) + check('……下一步说被省略的仍在台账里、不得据此断言没有数据，以及怎么看全', !!trimmed?.next?.includes('不得据此断言没有数据') + && !!trimmed?.next?.includes('上下文窗口更大') && trimmed?.level === 'warn' && trimmed?.detail === TRIMMED, trimmed?.next) + const trimmedBare = flatten(mod.decodeRun([E(1, 'log', 'w', { level: 'warn', code: 'report_catalog_trimmed', message: '目录有省略' })])) + .find((s) => s.code === 'report_catalog_trimmed') + check('……缺字段时标题照样说人话，不出 undefined', trimmedBare?.title === '证据目录超出篇幅，部分查询只列了编号和摘要' + && trimmedBare?.sub === undefined, `${trimmedBare?.title} / ${trimmedBare?.sub}`) }) await section('工具门控 tool.gated（工具信任三档）', async () => { From 799c80b97e4ffd6e32f7c035ba6f61a73b1148d1 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 4 Oct 2026 11:28:23 +0800 Subject: [PATCH 08/15] =?UTF-8?q?=E6=B8=85=E5=8D=95=E6=9C=89=E7=9C=81?= =?UTF-8?q?=E7=95=A5=E6=97=B6=E4=B8=8D=E8=AE=B8=E6=96=AD=E8=A8=80=E3=80=8C?= =?UTF-8?q?=E6=B2=A1=E6=9C=89=E6=95=B0=E6=8D=AE=E3=80=8D=EF=BC=9A=E5=86=99?= =?UTF-8?q?=E4=BD=9C=E8=A7=84=E5=88=99=E5=8A=A0=E4=B8=80=E6=9D=A1=EF=BC=8C?= =?UTF-8?q?=E6=A0=B8=E5=AF=B9=E8=AE=A4=E5=87=BA=E7=BC=BA=E5=A4=B1=E6=96=AD?= =?UTF-8?q?=E8=A8=80=E4=BA=A4=E5=9B=9E=E9=87=8D=E5=86=99=E4=B8=80=E6=AC=A1?= =?UTF-8?q?=EF=BC=8C=E4=BB=8D=E6=9C=89=E7=9A=84=E5=8F=AA=E6=A0=87=E6=B3=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 一次真实运行里写作者只看到上百次查询里的前二十来次，却写出「某类数据缺失、本轮没有查到任何结果」这样的断言，可那个 Agent 实际查到了上百条记录。 - 写作规则：清单有省略、或有来源的结构化结论为空时，不得断言没有查到、不存在、缺失、没有数据，只能写「本报告未覆盖」并说明依据不足 - 组装文档：清单有省略时按一组明确的说法认出缺失断言（没有查到、未查询到、查不到、没有任何记录、数据为空、数据缺失、不存在相关记录、暂无数据等；「不存在明显差异」「不代表没有数据」这类不算，代码块不查），记新违规 absence_claim，带 span、所在片段和原句 - 和句中的 [[see:]] 一样归入 REWRITE_ONLY：交回重写一次，重写后仍有的只标注、不判失败；只剩这几类时不再按最多重写次数反复重写。仍有时报告节点发 report_absence_claim（带 count、texts），运行复核算作缺口 - 证据面板的违规清单标「缺失断言」并写明为什么依据不足，时间线给 report_absence_claim 专门的一行，文案在 terms.ts；check-decode、check-evidence 补断言 - 没展开的查询的摘要在没有表结构快照时从查询快照里的 SQL 认表名 - 回归测试：合成的门店、会员、卡券、活动库真跑「4 个 Agent 并行查库 → 报告」（查询次数虚构、分布悬殊），覆盖抽取截断重试、两次截断记空值、清单均衡和缺失断言的重写 --- backend/app/engine/evidence.py | 74 +++- backend/app/engine/nodes/report.py | 22 +- backend/app/engine/review.py | 2 + backend/tests/test_report_absence_claim.py | 82 +++++ .../tests/test_report_evidence_regression.py | 316 ++++++++++++++++++ frontend/src/lib/terms.ts | 13 + frontend/src/run/EvidencePanel.tsx | 12 +- frontend/src/run/decode.ts | 12 +- scripts/check-decode.mjs | 20 ++ scripts/check-evidence.mjs | 29 +- 10 files changed, 568 insertions(+), 14 deletions(-) create mode 100644 backend/tests/test_report_absence_claim.py create mode 100644 backend/tests/test_report_evidence_regression.py diff --git a/backend/app/engine/evidence.py b/backend/app/engine/evidence.py index 02a62c8b..ff830cfb 100644 --- a/backend/app/engine/evidence.py +++ b/backend/app/engine/evidence.py @@ -2474,11 +2474,13 @@ def compose_doc( cells_allowed: bool = True, loader: Loader | None = None, entities: bool = True, + absence: bool = False, ) -> dict[str, Any]: """模型写的原文（带标记）→ 报告文档：渲染后的 markdown、块、句、片段、统计、违规。 entities=False（报告节点 entities: off）：表名、字段名整层不管——不链接、不核对反引号， [[t:]] / [[c:]] 解析不了，原因是 ENTITIES_OFF_REASON。 + absence=True（写作目录有省略）：另查缺失断言（absence_claims），和句中的 [[see:]] 一样只在组装时查。 stats / violations 由 verify_doc 算出，和出口契约复核用的是同一个函数。 整表标记先展开成每格一个 [[v:]] 的 Markdown 表，之后和模型自己写的表格走同一条路； @@ -2634,8 +2636,10 @@ def compose_doc( "markdown": markdown, "source": source, "catalog": kept, "blocks": out_blocks, } checked = verify_doc(doc, catalog, allow_numbers=allow_numbers, cells_allowed=cells_allowed, loader=snaps) - # 句中的 [[see:]] 只在组装时查（交回写作者重写）：出口复核看的是数字和出处，句子通不通不影响出具 - doc["stats"], doc["violations"] = checked["stats"], checked["violations"] + inline_see(source) + # 句中的 [[see:]]、目录有省略时的缺失断言只在组装时查（交回写作者重写）：出口复核看的是数字和出处， + # 句子通不通、结论有没有越过证据不影响出具 + doc["stats"], doc["violations"] = checked["stats"], \ + checked["violations"] + inline_see(source) + (absence_claims(doc) if absence else []) return doc @@ -3035,6 +3039,58 @@ def inline_see(source: str) -> list[dict[str, Any]]: return [_violation("inline_see", INLINE_SEE_MESSAGE, text=m.group(0)) for m in _INLINE_SEE.finditer(source or "")] +#: 写作目录有省略、或者有来源的结构化结论为空时，拼在写作规则后面（给模型看） +ABSENCE_RULE = ( + "这次你看不到全部证据（证据目录有省略，或者有来源的结构化结论为空）：不得断言「没有查到」「未查到」「不存在」" + "「缺失」「没有数据」「无数据」这类结论。某个方向的证据没有展开或为空时，只能写「本报告未覆盖……」，并说明依据不足" + "（比如「目录有省略，相关查询未展开」），句末可以挂 [[see:Qn]] 指向相关查询。" +) + +#: 缺失断言：断言「没有数据」的几种说法。只认拿得准的，不泛化：「不存在明显差异」「缺少口径说明」这类说的是看到的 +#: 数据里的情况，不算；「不存在」「为空」「缺失」只认跟在数据、记录、结果后面（或者前面）的 +_ABSENCE = re.compile( + r"(?:没有|没|未|未能|没能)(?:查询|查)到|查不到|查无" + r"|(?:没有|暂无|并无)(?:任何)?(?:相关的?)?(?:数据|记录|结果)" + r"|无(?:任何)?(?:相关的?)?(?:数据|记录)(?![库源集表权字项列])" + r"|不存在(?:任何)?(?:相关的?)?(?:数据|记录|结果)|(?:数据|记录|结果)(?:均|都|全部)?(?:不存在|为空|缺失|空缺)" +) +#: 前面是这些词的不算：「不代表没有数据」「并非查不到」说的恰恰是不能下这个结论 +_ABSENCE_NEGATED = re.compile(r"(?:不代表|并不代表|不等于|不意味着|并非|不是|不能说明|不能断言|不得断言|不应断言)[^。！？\n]{0,6}$") +ABSENCE_MESSAGE = _Reason( + "证据目录有省略时写了缺失断言：写作模型没有看到全部证据，「没有查到」「不存在」「缺失」这类结论依据不足", + "证据目录有省略（没展开的查询只列了编号和摘要），这一句却断言了没有数据。看不到的证据不能当成不存在：" + "改成「本报告未覆盖……」，并说明依据不足（目录有省略、相关查询未展开），不要写「没有查到」「不存在」「缺失」「没有数据」", +) + + +def absence_claims(doc: dict[str, Any]) -> list[dict[str, Any]]: + """正文里的缺失断言（_ABSENCE），每处一条违规 absence_claim。只在写作目录有省略时由报告节点要求查（compose_doc 的 + absence）：目录全给了的时候，「本周没有退款记录」可能就是看了数据说的。 + + 在渲染后的正文里找（标记已经换成了字），代码块不查。带着 span、所在片段和句子：证据面板能定位、能显示原句。 + """ + markdown = str(doc.get("markdown") or "") + found: list[dict[str, Any]] = [] + for block in doc.get("blocks") or []: + if block.get("type") == "code": + continue + for unit in block.get("units") or []: + span = unit.get("span") + if not (isinstance(span, list) and len(span) == 2): + continue + sentence = markdown[span[0]:span[1]] + for m in _ABSENCE.finditer(sentence): + if _ABSENCE_NEGATED.search(sentence[:m.start()]): + continue + at = span[0] + m.start() + seg = next((g for g in unit.get("segments") or [] if g.get("kind") != "structural" + and isinstance(g.get("span"), list) and g["span"][0] <= at < g["span"][1]), None) + found.append(_violation("absence_claim", ABSENCE_MESSAGE, text=m.group(0), + span=[at, span[0] + m.end()], context=sentence.strip()[:120], + segment=seg.get("id") if seg else None, unit=unit.get("id"))) + return found + + #: 引用查询结果的写法。只跟着目录里的查询出现（catalog_prompt）：写作规则本身保持一期那份，升级前 #: 发起的运行跑到报告节点时提示一字不差；只有口径卡的目录也不该教写作者去引用不存在的 Q1 CELL_RULES = ( @@ -3320,8 +3376,16 @@ def _fields_prompt(sources: list[dict[str, Any]], catalog: dict[str, Any], snaps empty, list(dict.fromkeys(cited))) -def _query_brief(entry: dict[str, Any], tables: list[str]) -> str: - """没展开的查询在目录里的一行摘要：编号、查的表（没有表结构时写工具名）、行数。""" +def _query_brief(entry: dict[str, Any], tables: list[str], snaps: _Snapshots) -> str: + """没展开的查询在目录里的一行摘要：编号、查的表、行数。 + + 表先看目录里的表条目（带表结构快照的查询才有），没有就从查询快照里存的 SQL 认（sql_tables）， + 再没有写工具名。 + """ + if not tables: + snapshot, why = snaps.get(entry["alias"], entry.get("artifact")) + if not why and isinstance(snapshot.get("sql"), str): + tables = sql_tables(snapshot["sql"]) rows = entry.get("rows") what = ("、".join(tables[:3]) + (f" 等 {len(tables)} 张表" if len(tables) > 3 else "")) if tables \ else str(entry.get("tool") or entry.get("source") or "查询") @@ -3401,7 +3465,7 @@ def _trimmed_catalog(catalog: dict[str, Any], head: list[str], first: list[str], for e in queries: groups.setdefault(str(e.get("node_id") or ""), []).append(e) compact = {e["alias"]: _query_prompt(e, snaps, cells_allowed, _PROMPT_COMPACT_ROWS) for e in queries} - brief = {e["alias"]: [_query_brief(e, tables.get(e["alias"], []))] for e in queries} + brief = {e["alias"]: [_query_brief(e, tables.get(e["alias"], []), snaps)] for e in queries} shapes = {"brief": brief, "compact": compact, "full": full} tier = {e["alias"]: "brief" for e in queries} diff --git a/backend/app/engine/nodes/report.py b/backend/app/engine/nodes/report.py index a7d7cabc..c9bedf60 100644 --- a/backend/app/engine/nodes/report.py +++ b/backend/app/engine/nodes/report.py @@ -44,6 +44,7 @@ from app.engine import judge as judging from app.engine.context import NodeContext, NodeError from app.engine.labels import field_label, option_label from app.engine.evidence import ( + ABSENCE_RULE, CELLS_REASON, CLAIMS_RULE, ENTITY_KINDS, @@ -84,8 +85,10 @@ CLAIMS = CLAIMS_VALUES #: 只标注、不让写作者重写也不判失败的违规：可疑实体（受管的正式出具按缺口降档）、表结构快照不全时 #: 核对不了的名字（出口只标注） SOFT = frozenset({"unknown_entity", "unverified_entity"}) -#: 交回写作者重写、重写不好也不判失败的违规：句中的 [[see:]]（渲染后留下断句，数字和出处都没问题） -REWRITE_ONLY = frozenset({"inline_see"}) +#: 交回写作者重写一次、重写不好也不判失败的违规：句中的 [[see:]]（渲染后留下断句，数字和出处都没问题）； +#: 写作目录有省略时的缺失断言（「没有查到」「数据缺失」：写作者没看全证据，结论依据不足，但数字和出处都核对过）。 +#: 只剩这几类时不按 max_repairs 反复重写：重写一次没改掉，再写也多半一样，留在核对结果里标注 +REWRITE_ONLY = frozenset({"inline_see", "absence_claim"}) #: 重写一次就是多一次整篇的模型调用，写不对的模型多给几次也大多写不对 MAX_REPAIRS = 3 @@ -410,6 +413,9 @@ def _messages(ctx: NodeContext, state: GraphState, writer: WriterCatalog, claims instructions = ctx.render_str(ctx.cfg("instructions", ""), state).strip() \ or "根据下面的证据写一份简洁的报告，先总后分。" rules = f"{MARKER_RULES}\n{SEE_AND_CODE_RULES}" + if writer.guarded: + # 写作者看不全证据（目录有省略、有来源的结构化结论为空）：「没有查到」这类结论没有依据 + rules = f"{rules}\n{ABSENCE_RULE}" if claims in ("require_citation", CLAIMS_JUDGE): # 裁判只看写作者挂的依据：要判，就更得每句都挂上 rules = f"{rules}\n{CLAIMS_RULE}" + (f"\n{judging.JUDGE_RULE}" if claims == CLAIMS_JUDGE else "") @@ -500,12 +506,13 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]: def compose(raw: str) -> dict[str, Any]: return compose_doc(raw, catalog, node_id=ctx.node.id, run_id=ctx.run.run_id, allow_numbers=allow, - cells_allowed=cells, entities=entities) + cells_allowed=cells, entities=entities, absence=writer.trimmed) response, raw = await draft(messages) doc = compose(raw) repairs = 0 - while (blocking := _blocking(doc["violations"], numbers)) and repairs < max_repairs: + while (blocking := _blocking(doc["violations"], numbers)) and repairs < max_repairs \ + and not (repairs and not _fatal(blocking)): repairs += 1 ctx.emit(EventType.LOG, level="warn", code="report_repair", message=f"报告有 {len(blocking)} 处未通过核对（{_named(blocking)}），" @@ -528,6 +535,13 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]: draft=draft, compose=compose, numbers=numbers, blocking=blocking) spent.extend(usages) + if absent := [v for v in doc["violations"] if v.get("code") == "absence_claim"]: + # 重写之后还在（或者没让重写）：报告照常产出，这几处标注出来，时间线上单独说清为什么依据不足 + texts = list(dict.fromkeys(str(v.get("text") or "") for v in absent)) + tried = f"已要求模型重写 {repairs} 次仍未改掉" if repairs else "未要求重写" + ctx.emit(EventType.LOG, level="warn", code="report_absence_claim", count=len(absent), texts=texts, + message=f"报告在证据目录有省略时写了 {len(absent)} 处缺失断言（{_named(absent)}），{tried}：" + "被省略的查询仍在证据台账里，这类结论依据不足，应写作本报告未覆盖。报告照常产出，这几处已标注") doc_artifact = await _store(doc, ctx) ctx.emit(EventType.REPORT_CHECKED, doc_artifact=doc_artifact, ok=not doc["violations"], stats=doc["stats"], violations=doc["violations"][:20], repairs=repairs, diff --git a/backend/app/engine/review.py b/backend/app/engine/review.py index 98e0d36d..65c0b752 100644 --- a/backend/app/engine/review.py +++ b/backend/app/engine/review.py @@ -91,6 +91,8 @@ WARN_CODES: dict[str, tuple[str, str]] = { "report_no_evidence": ("report_no_evidence", DEGRADED), # 证据目录超出篇幅，有的查询只列了编号和摘要：写作模型没看全证据，报告的覆盖面有缺口 "report_catalog_trimmed": ("report_catalog_trimmed", DEGRADED), + # 目录有省略时写了「没有查到」这类缺失断言、重写后仍在：报告照常产出并标注，这几句结论依据不足 + "report_absence_claim": ("report_absence_claim", DEGRADED), # 口径卡的指标集、报告文档没能落进工件库：答案照常，只是点不开证据 "evidence_store_failed": ("evidence_store_failed", DEGRADED), # agent 的结构化字段：模型报的值和查询快照对不上（已按快照取值），或者出处核对不了 diff --git a/backend/tests/test_report_absence_claim.py b/backend/tests/test_report_absence_claim.py new file mode 100644 index 00000000..cdd3c7f1 --- /dev/null +++ b/backend/tests/test_report_absence_claim.py @@ -0,0 +1,82 @@ +"""缺失断言（absence_claim）：写作目录有省略时，「没有查到」「不存在」「缺失」「没有数据」这类结论依据不足。 + +2026-10 一次运行，写作者只看到上百次查询里的前二十来次，写出「某类数据缺失：本轮没有查到任何结果」这样的断言， +可那个 Agent 实际查到了上百条记录。 + +- 写作规则：清单有省略、或有来源的结构化结论为空时，不得断言没有数据，只能写「本报告未覆盖」并说明依据不足 +- 组装文档：清单有省略时按一组明确的说法认出缺失断言，记 absence_claim，交回重写一次；重写后仍有的只标注、不判失败 + （report.REWRITE_ONLY，和句中的 [[see:]] 同一种处理） +- 只认拿得准的说法：「不存在明显差异」「本报告未覆盖」「不代表没有数据」都不算 +""" +from __future__ import annotations + +import pytest + +from app.engine.evidence import ABSENCE_RULE, build_catalog, compose_doc +from app.engine.nodes.report import REWRITE_ONLY + + +@pytest.fixture +def catalog(): + card = {"kind": "metric_set", "caliber": "周报口径", "caliber_version": "v1", + "metrics": [{"id": "gmv", "name": "销售额", "value": 45678.5, "unit": "元", "format": "thousands"}]} + return build_catalog(nodes={"card": card}, ledger=[{"kind": "metric_set", "node_id": "card", "artifact": "c" * 64}]) + + +def claims(doc): + return [v for v in doc["violations"] if v["code"] == "absence_claim"] + + +def test_the_rule_tells_the_writer_what_to_write_instead(): + for phrase in ("没有查到", "不存在", "缺失", "没有数据", "本报告未覆盖", "依据不足"): + assert phrase in ABSENCE_RULE + assert "absence_claim" in REWRITE_ONLY and "inline_see" in REWRITE_ONLY + + +@pytest.mark.parametrize("text, found", [ + ("卡券数据缺失：本轮没有查到核销、兑换、退券的任何结果。", ["数据缺失", "没有查到"]), + ("卡券核销没有任何记录。", ["没有任何记录"]), + ("本周门店客流数据为空。", ["数据为空"]), + ("活动内容无数据。", ["无数据"]), + ("未查询到会员流失的情况。", ["未查询到"]), + ("库里不存在相关记录。", ["不存在相关记录"]), + ("兑换记录均不存在。", ["记录均不存在"]), + ("暂无数据，查不到核销明细。", ["暂无数据", "查不到"]), + ("| 方向 | 结论 |\n| --- | --- |\n| 卡券 | 没有查到 |", ["没有查到"]), +]) +def test_absence_claims_are_recognised(catalog, text, found): + doc = compose_doc(text, catalog, absence=True) + got = claims(doc) + assert [v["text"] for v in got] == found, doc["violations"] + for v in got: + assert doc["markdown"][v["span"][0]:v["span"][1]] == v["text"], "span 指着正文里的那几个字" + assert v.get("unit") and v.get("context") + assert "依据不足" in v["message"] and "本报告未覆盖" in v["for_model"] + + +@pytest.mark.parametrize("text", [ + "本报告未覆盖卡券核销的明细：证据目录有省略，相关查询未展开，依据不足。", + "两家门店的客单价不存在明显差异。", + "目录里没展开的查询不代表没有数据。", + "这并非没有查到，而是没有展开。", + "本周无数据库变更，无数据源切换。", + "本周销售额 [[m:gmv]]，缺少口径说明的部分已注明。", + "```sql\nSELECT * FROM t -- 没有查到\n```", +]) +def test_other_wording_is_left_alone(catalog, text): + assert not claims(compose_doc(text, catalog, absence=True)) + + +def test_only_checked_when_the_catalog_was_trimmed(catalog): + text = "卡券核销数据缺失：没有查到任何记录。" + assert not claims(compose_doc(text, catalog)) + assert claims(compose_doc(text, catalog, absence=True)) + + +def test_the_claim_can_be_located_in_the_document(catalog): + doc = compose_doc("## 卡券\n\n本周销售额 [[m:gmv]]。核销数据缺失。", catalog, absence=True) + [v] = claims(doc) + segments = {s["id"]: s for b in doc["blocks"] for u in b["units"] for s in u["segments"]} + seg = segments[v["segment"]] + assert seg["kind"] == "text" and seg["span"][0] <= v["span"][0] < seg["span"][1] + assert v["context"] == "核销数据缺失。" diff --git a/backend/tests/test_report_evidence_regression.py b/backend/tests/test_report_evidence_regression.py new file mode 100644 index 00000000..0d7eb648 --- /dev/null +++ b/backend/tests/test_report_evidence_regression.py @@ -0,0 +1,316 @@ +"""回归：「4 个专项 Agent 并行查库 → 报告」整条跑一遍，复现 2026-10 那次运行的形状。 + +那次运行状态是 succeeded，报告却不可靠：4 个 Agent 的结构化抽取全部被截断（推理模型的额度在推理阶段就用完了）； +写作者的出处清单写死 12000 字，上百次查询只有前二十来次露了面；写作者于是在清单被截断的情况下写了「数据缺失： +本轮没有查到任何核销结果」，可那个 Agent 查到了上百条。 + +这里用合成的门店、会员、卡券、活动库（SQLite）真跑：4 个 Agent 分别查 26、17、79、42 次（虚构的次数，一次并行发完），都配了 +output_schema 和按出处核对字段；写作模型、抽取模型都是脚本。验证： + +- 抽取被截断后加大额度重试成功；两次都截断的记 agent_field_truncated，字段为空值 +- 出处清单在预算内，每个 Agent 都有代表，被省略的查询有编号和摘要，开头说明各来源省略了几次 +- 清单有省略时的缺失断言被认出来（absence_claim）、交回重写；重写后仍有的只标注、不判失败 +""" +from __future__ import annotations + +import asyncio +import json +import re +import sqlite3 + +import pytest +from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage +from openai import LengthFinishReasonError +from openai.types import CompletionUsage +from openai.types.chat import ChatCompletion +from sqlalchemy import select + +from app.core import artifact_store +from app.data.engine import engines +from app.db.base import SessionLocal +from app.db.models import DataSource, Run, RunEvent +from app.engine.evidence import ABSENCE_RULE, DEFAULT_CATALOG_BUDGET, MARKER_RULES +from app.engine.nodes import llm as llm_mod +from app.engine.nodes import report as report_mod +from app.engine.runner import run_manager +from tests.fixtures.sources import drop_source + +SOURCE = "fa_store" +TOOL = f"db_query__{SOURCE}" +#: (节点 id, 画布上的名字, 记号, 查询次数, 表, 列) +AGENTS = [ + ("agent_member", "会员专项", "member", 26, "member_daily", ["store_name", "biz_date", "new_members", "active_members"]), + ("agent_store", "门店专项", "store", 17, "store_daily", ["store_name", "biz_date", "visits", "sales"]), + ("agent_coupon", "卡券核销", "coupon", 79, "coupon_redemption", ["store_name", "redeemed_at", "coupon_type", "amount"]), + ("agent_campaign", "活动内容", "campaign", 42, "campaign_view", ["campaign", "biz_date", "views", "clicks"]), +] +SCHEMA = {"type": "object", "properties": { + "first_store": {"type": "string", "description": "第一次查询第一行的门店或活动"}, + "first_value": {"type": "number", "description": "第一次查询第一行的第三列"}, + "churn_rate": {"type": "number", "description": "流失率（要计算，填 null）"}, +}} +ROWS_PER_QUERY = 5 + +ABSENT = ("## 概览\n\n各专项都已完成查询。[[see:Q1]]\n\n" + "## 卡券核销\n\n卡券数据缺失：本轮没有查到核销、兑换的任何结果。") +COVERED = ("## 概览\n\n各专项都已完成查询。[[see:Q1]]\n\n" + "## 卡券核销\n\n本报告未覆盖卡券核销的明细：证据目录有省略，相关查询未展开，依据不足。[[see:Q1]]") + + +def length_error(completion: int) -> LengthFinishReasonError: + usage = CompletionUsage(completion_tokens=completion, prompt_tokens=14000, total_tokens=14000 + completion) + return LengthFinishReasonError(completion=ChatCompletion.model_construct( + id="x", choices=[], created=0, model="m", object="chat.completion", usage=usage)) + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +@pytest.fixture +async def store_db(tmp_path): + """合成库：每张表 120 行，第 i 次查询取 OFFSET i 的 5 行，次次结果不同。""" + path = tmp_path / "store.db" + db = sqlite3.connect(path) + for _, _, _, _, table, cols in AGENTS: + db.execute(f"CREATE TABLE {table} ({cols[0]} TEXT, {cols[1]} TEXT, {cols[2]} {'TEXT' if table == 'coupon_redemption' else 'INTEGER'}, {cols[3]} REAL)") + rows = [] + for r in range(120): + when = f"2026-06-{r % 28 + 1:02d}" + (f"T08:{r % 60:02d}:39" if table == "coupon_redemption" else "") + third = f"券型{r % 4 + 1}" if table == "coupon_redemption" else (r * 7) % 300 + 5 + rows.append((f"门店{r % 30 + 1:02d}", when, third, round((r * 13) % 900 + 0.5, 1))) + db.executemany(f"INSERT INTO {table} VALUES (?, ?, ?, ?)", rows) + db.commit() + db.close() + async with SessionLocal() as session: + row = DataSource(name=SOURCE, kind="sqlite", database=str(path), readonly=True, enabled=True) + session.add(row) + await session.commit() + source_id = row.id + await engines.invalidate(source_id) + yield path + await drop_source(source_id) + + +def sql_of(table: str, cols: list[str], i: int) -> str: + return f"SELECT {', '.join(cols)} FROM {table} ORDER BY rowid LIMIT {ROWS_PER_QUERY} OFFSET {i}" + + +def node(nid, ntype, label, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label, "config": config}} + + +def graph(counts: dict[str, int] | None = None, **report): + nodes = [node("start", "input", "开始")] + for nid, title, key, count, table, cols in AGENTS: + nodes.append(node(nid, "agent", title, prompt=f"【{key}】按{title}的口径查数，每次查一段", tools=[TOOL], + approval="never", parallel_tools=True, output_schema=SCHEMA, cite_fields=True, + assign_to=f"{key}_findings")) + nodes.append(node("write", "report", "专项报告", instructions="汇总各专项的发现", **report)) + nodes.append(node("out", "output", "成果", fields=[{"name": "报告", "value": "{{ nodes.write.text }}"}])) + edges = [{"source": "start", "target": a[0]} for a in AGENTS] + [{"source": a[0], "target": "write"} for a in AGENTS] + edges.append({"source": "write", "target": "out"}) + return {"nodes": nodes, "edges": edges} + + +class World: + """脚本化的模型：Agent 一步并行发完自己的查询再给结论；抽取按 outcome 决定；写作者依次回 drafts。""" + + def __init__(self, monkeypatch, db_path, *, counts: dict[str, int], drafts: list[str], outcome): + from app.providers import mock_model + + self.counts = counts + self.drafts = drafts + self.writer: list[list] = [] + self.extract_calls: dict[str, list[int | None]] = {a[2]: [] for a in AGENTS} + self.budgets: list[int | None] = [] + db = sqlite3.connect(db_path) + self.first_rows = {key: db.execute(sql_of(table, cols, 1)).fetchone() for _, _, key, _, table, cols in AGENTS} + db.close() + real = llm_mod.get_chat_model + + async def spy(session, spec): + if spec.thinking == "off": + self.budgets.append(spec.max_tokens) + return await real(session, spec) + + monkeypatch.setattr(llm_mod, "get_chat_model", spy) + world = self + + def text(messages) -> str: + return "\n".join(m.content if isinstance(m.content, str) else json.dumps(m.content, ensure_ascii=False) + for m in messages) + + def who(messages) -> str: + found = re.search(r"【(member|store|coupon|campaign)】", text(messages)) + assert found, "认不出是哪个 Agent" + return found.group(1) + + def decide(model, messages): + if model.response_format: + key = who(messages) + world.extract_calls[key].append(world.budgets[-1]) + outcome(key, len(world.extract_calls[key]), world.budgets[-1]) + return AIMessage(content=json.dumps(world.extracted(key), ensure_ascii=False)) + if any(isinstance(m, SystemMessage) and "报告撰写人" in str(m.content) for m in messages): + world.writer.append(list(messages)) + return AIMessage(content=world.drafts[min(len(world.writer), len(world.drafts)) - 1]) + key = who(messages) + if model.tools and not any(isinstance(m, ToolMessage) for m in messages): + _, _, _, _, table, cols = next(a for a in AGENTS if a[2] == key) + return AIMessage(content="分段查询。", tool_calls=[ + {"name": TOOL, "args": {"sql": sql_of(table, cols, i)}, "id": f"{key}_{i}"} + for i in range(1, world.counts[key] + 1)]) + return AIMessage(content=f"【{key}】查完了，结论见各次查询。") + + monkeypatch.setattr(mock_model.MockChatModel, "_decide", decide) + + def extracted(self, key: str) -> dict: + first = self.first_rows[key] + _, _, _, _, _, cols = next(a for a in AGENTS if a[2] == key) + return {"first_store": {"value": first[0], "from": {"call": "Q1", "row": 0, "column": cols[0]}}, + "first_value": {"value": first[3], "from": {"call": "Q1", "row": 0, "column": cols[3]}}, + "churn_rate": {"value": None, "from": None}} + + +async def finish(graph_json) -> Run: + run = await run_manager.start(graph=graph_json, input_payload={}) + for _ in range(1200): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run.id) + if row.status in ("succeeded", "failed") and row.manifest_seq is not None: + return row + raise AssertionError(f"run {run.id} 没跑完：{row.status}") + + +async def events(run_id: str, etype: str | None = None) -> list[RunEvent]: + async with SessionLocal() as session: + q = select(RunEvent).where(RunEvent.run_id == run_id) + if etype: + q = q.where(RunEvent.type == etype) + return list((await session.execute(q.order_by(RunEvent.seq))).scalars()) + + +def logs(evts, code): + return [e.data for e in evts if e.type == "log" and e.data.get("code") == code] + + +async def output_of(run_id: str, node_id: str) -> dict: + [finished] = [e for e in await events(run_id, "node.finished") if e.node_id == node_id] + return artifact_store.load(finished.data["artifact"]) + + +def catalog_text(messages) -> str: + """写作者第一次收到的提示里出处清单那一段（从「可引用的证据」到写作规则之前）。""" + prompt = next(m.content for m in messages if isinstance(m, HumanMessage)) + return prompt[prompt.index("可引用的证据"):prompt.index(MARKER_RULES)].rstrip() + + +# -------------------------------------------------------------------------- +# 和那次运行同样的形状：上百次查询、分布不均 +# -------------------------------------------------------------------------- + + +async def test_the_reported_run_shape_end_to_end(monkeypatch, store_db): + counts = {key: count for _, _, key, count, _, _ in AGENTS} + + def outcome(key, attempt, budget): + if key == "coupon": + raise length_error(budget) # 两次都截断 + if key == "member" and attempt == 1: + raise length_error(budget) # 截断一次，加大额度后成功 + + world = World(monkeypatch, store_db, counts=counts, drafts=[ABSENT, COVERED], outcome=outcome) + row = await finish(graph()) + assert row.status == "succeeded", row.error + evts = await events(row.id) + + # 1. 抽取：会员专项截断后重试成功；卡券核销两次都截断，字段为空值 + assert world.extract_calls["member"] == [16384, 32768] + assert world.extract_calls["coupon"] == [16384, 32768] + assert world.extract_calls["store"] == world.extract_calls["campaign"] == [16384] + member = await output_of(row.id, "agent_member") + assert member["data"]["first_store"] == world.first_rows["member"][0] + assert member["data_evidence"]["first_store"]["status"] == "verified" + coupon = await output_of(row.id, "agent_coupon") + assert coupon["data"] == {"first_store": None, "first_value": None, "churn_rate": None} + [cut] = logs(evts, "agent_field_truncated") + assert cut["fields"] == ["first_store", "first_value", "churn_rate"] and "推理阶段就用完了" in cut["message"] + assert not logs(evts, "agent_field_unverified") + + # 2. 出处清单：预算内，每个 Agent 都有代表，被省略的有编号和摘要，开头说明各来源省略了几次 + first = world.writer[0] + text = catalog_text(first) + assert len(text) <= DEFAULT_CATALOG_BUDGET, len(text) + head = text[:900] + assert "目录有省略" in head + for _, title, _, count, table, _ in AGENTS: + assert f"{title} {count} 次" in head + group = text[text.index(f"来自「{title}」的查询"):] + group = group[:group.index("\n\n", 1)] if "\n\n" in group[1:] else group + assert f"共 {count} 次" in group and re.search(r"\n {2}r0：", group), f"「{title}」没有展开的查询" + briefs = re.findall(r"\n {2}- (Q\d+)：(\w+)，5 行", text) + assert briefs, "没有列出被省略的查询" + assert {t for _, t in briefs} <= {a[4] for a in AGENTS} + assert all(f"Q{n}：" in text for n in range(1, 165)), "每次查询都要在清单里（展开的或摘要）" + # 结构化结论排在查询前面；抽取失败的卡券核销写明结构化结论为空、不代表没有数据 + assert text.index("各 Agent 交来的结构化结论") < text.index("来自「") + assert "「卡券核销」：结构化结论为空" in text + # 写作规则里有缺失断言那一条 + assert ABSENCE_RULE in next(m.content for m in first if isinstance(m, HumanMessage)) + [trimmed] = logs(evts, "report_catalog_trimmed") + assert trimmed["queries"] == 164 and trimmed["budget"] == DEFAULT_CATALOG_BUDGET + assert [n["title"] for n in sorted(trimmed["nodes"], key=lambda n: n["title"])] == \ + sorted(a[1] for a in AGENTS) + assert trimmed["omitted"] == sum(n["omitted"] for n in trimmed["nodes"]) > 0 + + # 3. 缺失断言：认出来、交回重写一次；改成「本报告未覆盖」后通过 + assert len(world.writer) == 2 + request = str(world.writer[1][-1].content) + assert "本报告未覆盖" in request and "没有查到" in request + [repair] = logs(evts, "report_repair") + assert "数据缺失" in repair["message"] or "没有查到" in repair["message"] + [checked] = [e.data for e in evts if e.type == "report.checked"] + assert not [v for v in checked["violations"] if v["code"] == "absence_claim"] + assert not logs(evts, "report_absence_claim") + + +# -------------------------------------------------------------------------- +# 重写后仍然写缺失断言：只标注、不判失败 +# -------------------------------------------------------------------------- + +SMALL = {"member": 6, "store": 4, "coupon": 12, "campaign": 8} + + +async def test_an_absence_claim_that_survives_the_rewrite_is_flagged_not_failed(monkeypatch, store_db): + monkeypatch.setattr(report_mod, "catalog_budget", lambda window: 4000) + world = World(monkeypatch, store_db, counts=SMALL, drafts=[ABSENT], outcome=lambda *a: None) + row = await finish(graph(on_violation="fail", max_repairs=3)) + assert row.status == "succeeded", row.error + assert len(world.writer) == 2, "缺失断言只交回重写一次，不按最多重写次数反复重写" + evts = await events(row.id) + [checked] = [e.data for e in evts if e.type == "report.checked"] + assert checked["failed"] is False + claims = [v for v in checked["violations"] if v["code"] == "absence_claim"] + assert {v["text"] for v in claims} == {"数据缺失", "没有查到"} + assert all(v.get("segment") and v.get("context") for v in claims), "带着片段和原句：证据面板能定位" + [flag] = logs(evts, "report_absence_claim") + assert flag["level"] == "warn" and flag["count"] == 2 and flag["texts"] == ["数据缺失", "没有查到"] + assert "本报告未覆盖" in flag["message"] or "依据不足" in flag["message"] + + +async def test_without_omissions_an_absence_statement_is_left_alone(monkeypatch, store_db): + """目录全给了：「没有查到」可能就是看了数据说的，不查。""" + counts = {"member": 1, "store": 1, "coupon": 1, "campaign": 1} + world = World(monkeypatch, store_db, counts=counts, drafts=[ABSENT], outcome=lambda *a: None) + row = await finish(graph()) + assert row.status == "succeeded", row.error + assert len(world.writer) == 1 + evts = await events(row.id) + assert not logs(evts, "report_catalog_trimmed") and not logs(evts, "report_absence_claim") + prompt = next(m.content for m in world.writer[0] if isinstance(m, HumanMessage)) + assert ABSENCE_RULE not in prompt and "目录有省略" not in prompt diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 37388a16..b47b1eec 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -203,6 +203,19 @@ export function evidenceTally(cited: number, total: number, other = 0): string { return parts.join(' · ') } +/** + * 缺失断言：证据目录有省略时，报告写了「没有查到」「不存在」「缺失」这类结论（违规 absence_claim，重写后仍有时 + * 报告节点另发日志 report_absence_claim）。证据面板的违规清单和运行时间线共用这一套说法 + */ +export const ABSENCE_CLAIM_TEXT = { + label: '缺失断言', + hint: '写作模型没有看到全部证据：被省略的查询仍在证据台账里，看不到不等于不存在，这类结论应写作「本报告未覆盖」。' + + '报告照常产出，这一处只标注、不判失败', + title: (n?: number) => `报告写了${n ? ` ${formatNumber(n)} 处` : ''}缺失断言，已标注（证据目录有省略）`, + next: '被省略的查询仍在证据台账里，可在证据面板中查看。请在报告撰写节点的指令中要求未展开的方向写作「本报告未覆盖」，' + + '或换用上下文窗口更大的写作模型后重新运行', +} + /** 证据面板和证据条的文案 */ export const EVIDENCE_TEXT = { allCited: (n: number) => `${formatNumber(n)} 个数字都有出处`, diff --git a/frontend/src/run/EvidencePanel.tsx b/frontend/src/run/EvidencePanel.tsx index 00a35156..b4b9dd14 100644 --- a/frontend/src/run/EvidencePanel.tsx +++ b/frontend/src/run/EvidencePanel.tsx @@ -18,7 +18,7 @@ import { } from '../lib/evidence' import { humanizeError } from '../lib/errors' import { formatDateTime, formatNumber, NONE, shortId } from '../lib/format' -import { EVIDENCE_TEXT, JUDGE_TEXT, MERGE_TEXT, NODE_TYPE_LABEL, RUN_SQL_CHECK_TEXT, SQL_CHECK_TEXT, nodeTypeLabel } from '../lib/terms' +import { ABSENCE_CLAIM_TEXT, EVIDENCE_TEXT, JUDGE_TEXT, MERGE_TEXT, NODE_TYPE_LABEL, RUN_SQL_CHECK_TEXT, SQL_CHECK_TEXT, nodeTypeLabel } from '../lib/terms' import { useCatalog } from '../store/catalog' import { askKey, segmentKey, useEvidence, useExplore, useVerdicts } from '../store/evidence' import type { @@ -1552,6 +1552,7 @@ const CODE_LABEL: Record = { unresolved_ref: '引用无法解析', unknown_entity: '疑似不存在的名称', unverified_entity: '无法核实的名称', + absence_claim: ABSENCE_CLAIM_TEXT.label, } /** 违规清单：报告撰写节点核对出来的全部问题，画得出线的能定位，画不出的说清在哪 */ @@ -1563,7 +1564,9 @@ function ViolationList({ doc, onLocate }: { doc: EvidenceDocData; onLocate: (seg

{EVIDENCE_TEXT.violationsHint}

    {list.map((v, i) => { - const seg = locatable(doc, v) + // 缺失断言说的是一整句话，正文里普通文字不是可聚焦的片段：不给「定位」，原句写在下面 + const sentence = v.code === 'absence_claim' + const seg = sentence ? null : locatable(doc, v) return (
  • @@ -1576,8 +1579,11 @@ function ViolationList({ doc, onLocate }: { doc: EvidenceDocData; onLocate: (seg )} {v.message &&

    {v.message}

    } + {v.code === 'absence_claim' && ( +

    {ABSENCE_CLAIM_TEXT.hint}

    + )} {v.context &&

    「{v.context.trim()}」

    } - {!seg && ( + {!seg && !sentence && (

    {v.segment ? EVIDENCE_TEXT.structural : EVIDENCE_TEXT.noSegment}

    diff --git a/frontend/src/run/decode.ts b/frontend/src/run/decode.ts index f9bab20c..6ab8bfc7 100644 --- a/frontend/src/run/decode.ts +++ b/frontend/src/run/decode.ts @@ -2,7 +2,7 @@ import type { CatalogDriftTable, RunEvent, SqlCheckItem, TeamMember, TeamRound, import type { NodeState, RunPhase } from './trace' import { formatDuration, formatNumber } from '../lib/format' import { - CATALOG_DRIFT_TEXT, CATALOG_PATCH_TEXT, COPILOT_CONTEXT_TEXT, JUDGE_TEXT, MERGE_TEXT, RUN_SQL_CHECK_TEXT, SCHEMA_PARTIAL_TEXT, SQL_CHECK_TEXT, TYPE_LABEL, + ABSENCE_CLAIM_TEXT, CATALOG_DRIFT_TEXT, CATALOG_PATCH_TEXT, COPILOT_CONTEXT_TEXT, JUDGE_TEXT, MERGE_TEXT, RUN_SQL_CHECK_TEXT, SCHEMA_PARTIAL_TEXT, SQL_CHECK_TEXT, TYPE_LABEL, claimTally, evidenceTally, issuanceLabel, nodeTypeLabel, } from '../lib/terms' @@ -756,6 +756,16 @@ function explainLog(code: string | undefined, message: string, d: Record 0 ? d.count : undefined + const texts = strs(d.texts) + return { + title: ABSENCE_CLAIM_TEXT.title(n), + sub: texts.length ? texts.map((t) => `「${t}」`).join('、') : undefined, + next: ABSENCE_CLAIM_TEXT.next, + } + } case 'report_catalog_trimmed': { // 写作目录超出篇幅（report.py _trimmed_message）。读结构化字段（queries、omitted、nodes），不从原话里认 const total = typeof d.queries === 'number' ? d.queries : undefined diff --git a/scripts/check-decode.mjs b/scripts/check-decode.mjs index f11e795a..ca9ad388 100644 --- a/scripts/check-decode.mjs +++ b/scripts/check-decode.mjs @@ -1320,6 +1320,26 @@ await section('报告核对 report.checked、口径卡的台账（可点击证 .find((s) => s.code === 'report_catalog_trimmed') check('……缺字段时标题照样说人话，不出 undefined', trimmedBare?.title === '证据目录超出篇幅，部分查询只列了编号和摘要' && trimmedBare?.sub === undefined, `${trimmedBare?.title} / ${trimmedBare?.sub}`) + + // 目录有省略时报告写了缺失断言（「没有查到」「数据缺失」），交回重写一次仍在：只标注、不判失败，时间线单独一行说清 + check('后端发缺失断言的警告，带着 count、texts', reportPy.includes('code="report_absence_claim"') + && reportPy.includes('count=len(absent), texts=texts'), '改了 report.py 的字段就同步改 decode.ts 的 explainLog 和这里的夹具') + const ABSENT = '报告在证据目录有省略时写了 2 处缺失断言（「数据缺失」、「没有查到」），已要求模型重写 1 次仍未改掉：' + + '被省略的查询仍在证据台账里，这类结论依据不足，应写作本报告未覆盖。报告照常产出，这几处已标注' + const absentRun = flatten(mod.decodeRun([E(1, 'node.started', 'w', { node_type: 'report', label: '写报告' }), + E(2, 'log', 'w', { level: 'warn', code: 'report_absence_claim', message: ABSENT, count: 2, texts: ['数据缺失', '没有查到'] }), + E(3, 'report.checked', 'w', { doc_artifact: 'abc', repairs: 1, ok: false, on_violation: 'fail', failed: false, + stats: { numbers: 0, numbers_cited: 0, uncited_numbers: 0, unresolved: 0 }, + violations: [{ code: 'absence_claim', text: '数据缺失', message: '证据目录有省略时写了缺失断言：写作模型没有看到全部证据，「没有查到」「不存在」「缺失」这类结论依据不足' }] }), + ])) + const ab = absentRun.find((s) => s.code === 'report_absence_claim') + check('report_absence_claim：标题说几处缺失断言、已标注、为什么（目录有省略）', ab?.title === '报告写了 2 处缺失断言，已标注（证据目录有省略）', ab?.title) + check('……副标题点名那几处说法', ab?.sub === '「数据缺失」、「没有查到」', ab?.sub) + check('……下一步说被省略的仍在台账里、应写「本报告未覆盖」；是提醒（warn），原话留在展开区', + !!ab?.next?.includes('本报告未覆盖') && !!ab?.next?.includes('证据台账') && ab?.level === 'warn' && ab?.detail === ABSENT, ab?.next) + const rcAbsent = absentRun.find((s) => s.code === 'report_checked') + check('……核对那一行照常完成（不判失败），违规清单里写着这条的说明', rcAbsent?.status === 'done' + && !!rcAbsent?.detail?.includes('缺失断言') && !!rcAbsent?.detail?.includes('依据不足'), `${rcAbsent?.status} ${rcAbsent?.detail}`) }) await section('工具门控 tool.gated（工具信任三档）', async () => { diff --git a/scripts/check-evidence.mjs b/scripts/check-evidence.mjs index 5126e1c0..3da07423 100644 --- a/scripts/check-evidence.mjs +++ b/scripts/check-evidence.mjs @@ -1094,7 +1094,34 @@ await section('sqlcheck', 'SQL 检查（数据目录阶段 4B）：查询步骤 await bad.ctx.close() }) -await section('q-narrow', '查询步骤在 360px：栏内展开、底部抽屉都不横向滚动；减少动效', async () => { +await section('absence', '缺失断言：证据目录有省略时写了「没有查到」，违规清单写明是什么、为什么依据不足，带着原句', async () => { + // 违规的形状：backend/app/engine/evidence.py 的 absence_claims（code、message、text、span、segment、unit、context） + const MESSAGE = '证据目录有省略时写了缺失断言：写作模型没有看到全部证据，「没有查到」「不存在」「缺失」这类结论依据不足' + const r = await probeQ((id, body) => body, { + patchQuery: (f) => { + const units = f.doc.blocks.flatMap((b) => b.units ?? []) + const unit = units.find((u) => (u.segments ?? []).some((s) => s.kind === 'text' && s.text.trim().length > 2)) + const seg = unit.segments.find((s) => s.kind === 'text' && s.text.trim().length > 2) + f.doc.violations = [...(f.doc.violations ?? []), { code: 'absence_claim', message: MESSAGE, text: '没有查到', + span: [seg.span[0], seg.span[0] + 1], segment: seg.id, unit: unit.id, context: '核销数据缺失：本轮没有查到任何结果。' }] + return f + }, + }) + await openQ(r.page, '1,288') + await r.page.locator('[data-evidence-panel] [data-ev-open-violations]').click() + await r.page.waitForSelector('[data-ev-violations]') + const item = r.page.locator('[data-ev-violation="absence_claim"]') + const text = await item.innerText().catch(() => '') + check('违规清单里有这一条，标「缺失断言」，不是笼统的「文档核对未通过」', text.includes('缺失断言') && !text.includes('文档核对未通过'), text) + check('……写明为什么依据不足、该怎么写（文案在 terms.ts）', text.includes(MESSAGE) + && (await item.locator('[data-ev-absence-hint]').innerText().catch(() => '')).includes('本报告未覆盖'), text.replace(/\s+/g, ' ')) + check('……带着原句；说的是一整句，不给「定位」，也不说「正文里没有对应文本」', text.includes('本轮没有查到任何结果') + && await item.getByRole('button', { name: '定位' }).count() === 0 && !text.includes('没有对应文本'), text.replace(/\s+/g, ' ')) + check('没有运行时报错', r.errors.length === 0, r.errors.join(' | ')) + await r.ctx.close() +}) + +await section('q-narrow','查询步骤在 360px：栏内展开、底部抽屉都不横向滚动；减少动效', async () => { const overflow = () => page.evaluate((sel) => { const el = document.querySelector(sel) return { sw: el.scrollWidth, cw: el.clientWidth } From 546d0e93260724491e63c82246a89426d66cd89c Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 4 Oct 2026 11:28:39 +0800 Subject: [PATCH 09/15] =?UTF-8?q?=E6=B2=A1=E6=9C=89=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E7=9B=AE=E5=BD=95=E7=9A=84=E5=A4=A7=E5=BA=93=EF=BC=9A=E5=8A=A9?= =?UTF-8?q?=E6=89=8B=E3=80=8C=E5=8F=82=E8=80=83=E4=BA=86=20N=20=E5=BC=A0?= =?UTF-8?q?=E8=A1=A8=E3=80=8D=E9=82=A3=E4=B8=80=E8=A1=8C=E6=8F=90=E7=A4=BA?= =?UTF-8?q?=E5=85=88=E8=B5=B7=E8=8D=89=E7=9B=AE=E5=BD=95=E3=80=82=E7=9C=9F?= =?UTF-8?q?=E5=AE=9E=E8=BF=90=E8=A1=8C=E9=87=8C=E7=9A=84=E5=A4=A7=E5=BA=93?= =?UTF-8?q?=E4=B8=80=E5=BC=A0=E7=9B=AE=E5=BD=95=E9=83=BD=E6=B2=A1=E6=9C=89?= =?UTF-8?q?=EF=BC=8C=E6=8C=91=E8=A1=A8=E5=92=8C=20SQL=20=E6=A3=80=E6=9F=A5?= =?UTF-8?q?=E9=83=BD=E7=94=A8=E4=B8=8D=E4=B8=8A=E7=9B=AE=E5=BD=95=EF=BC=8C?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E4=B9=9F=E4=B8=8D=E7=9F=A5=E9=81=93=E8=AF=A5?= =?UTF-8?q?=E5=85=88=E5=8E=BB=E8=B5=B7=E8=8D=89=E3=80=82=E6=9C=8D=E5=8A=A1?= =?UTF-8?q?=E7=AB=AF=20context=20=E6=93=8D=E4=BD=9C=E7=9A=84=E6=AF=8F?= =?UTF-8?q?=E4=B8=AA=E6=95=B0=E6=8D=AE=E6=BA=90=E5=A4=9A=E5=B8=A6=20source?= =?UTF-8?q?=5Fid=20=E5=92=8C=20catalog=5Ftables=EF=BC=88=E5=8E=BB=E6=8E=89?= =?UTF-8?q?=E9=A9=B3=E5=9B=9E=E9=A1=B9=E5=90=8E=E8=BF=98=E6=9C=89=E5=86=85?= =?UTF-8?q?=E5=AE=B9=E7=9A=84=E8=A1=A8=E6=95=B0=EF=BC=89=EF=BC=9B=E5=89=8D?= =?UTF-8?q?=E7=AB=AF=20copilotContext=20=E5=9C=A8=E8=A6=81=E6=8C=91?= =?UTF-8?q?=E8=A1=A8=E7=9A=84=E5=A4=A7=E5=BA=93=20catalog=5Ftables=20?= =?UTF-8?q?=E4=B8=BA=200=20=E6=97=B6=E7=BB=99=E5=87=BA=E4=B8=8B=E4=B8=80?= =?UTF-8?q?=E6=AD=A5=E3=80=8C=E8=BF=99=E4=B8=AA=E6=95=B0=E6=8D=AE=E6=BA=90?= =?UTF-8?q?=E8=BF=98=E6=B2=A1=E6=9C=89=E6=95=B0=E6=8D=AE=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=EF=BC=8C=E8=B5=B7=E8=8D=89=E5=90=8E=E6=8C=91=E8=A1=A8=E5=92=8C?= =?UTF-8?q?=20SQL=20=E6=A3=80=E6=9F=A5=E4=BC=9A=E6=9B=B4=E5=87=86=E3=80=8D?= =?UTF-8?q?=EF=BC=8C=E5=85=A5=E5=8F=A3=E7=9B=B4=E8=BE=BE=20/data/catalog/:?= =?UTF-8?q?sourceId=EF=BC=88=E5=87=A0=E4=B8=AA=E5=BA=93=E6=97=B6=E7=82=B9?= =?UTF-8?q?=E5=90=8D=EF=BC=8C=E8=80=81=E6=9C=8D=E5=8A=A1=E7=AB=AF=E4=B8=8D?= =?UTF-8?q?=E7=BB=99=E7=9B=AE=E5=BD=95=E8=A1=A8=E6=95=B0=E6=97=B6=E4=B8=8D?= =?UTF-8?q?=E6=8F=90=E7=A4=BA=EF=BC=89=E3=80=82check-decode=E3=80=81check-?= =?UTF-8?q?stream=E3=80=81check-chat=20=E8=A1=A5=E6=96=AD=E8=A8=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/copilot_context.py | 19 ++++++++---- backend/tests/test_copilot_table_pick.py | 37 ++++++++++++++++++++---- frontend/src/lib/terms.ts | 3 ++ frontend/src/run/AssistantStream.tsx | 12 ++++++-- frontend/src/run/__tests__/synthetic.ts | 10 +++++-- frontend/src/run/decode.ts | 32 ++++++++++++++++++-- scripts/check-chat.mjs | 11 ++++++- scripts/check-decode.mjs | 22 ++++++++++++++ scripts/check-stream.mjs | 12 ++++++++ 9 files changed, 138 insertions(+), 20 deletions(-) diff --git a/backend/app/api/copilot_context.py b/backend/app/api/copilot_context.py index 5b065047..3e963b5d 100644 --- a/backend/app/api/copilot_context.py +++ b/backend/app/api/copilot_context.py @@ -214,6 +214,11 @@ class SourcePlan: """给模型看的目录：去掉驳回项。""" return {t: catalog.visible_notes(n) for t, n in self.raw_notes.items()} + @property + def catalog_tables(self) -> int: + """有数据目录的表数：去掉驳回项之后还有内容的。整份被驳回的不算——挑表、SQL 检查都用不上它。""" + return sum(1 for n in self.raw_notes.values() if catalog.visible_notes(n)) + def _modes(sources: list[Any]) -> list[Literal["all", "pick"]]: """每个源给详细摘要还是先挑表。 @@ -565,16 +570,20 @@ class DatasourceContext: def op(self) -> dict[str, Any] | None: """告诉前端这一轮参考了哪些表：{"op": "context", "sources": [...], "elapsed_ms"?}。没有可查的表时为 None。 - 每个源一项：source 源名，tables 给了全字段的表（全名），selected_by，total 这个源一共几张表（表结构缓存里 - 记的真实总数）；探查截断了的再带 explored（探查到几张），界面写「共 N 张，只探查了 200 张」；挑表失败的 - 带上 reason。只列了名字的表不进 tables：界面上说的「参考了 N 张表」是模型看得到字段的那些。 + 每个源一项：source 源名，source_id 源的 id（界面据此给去数据目录页的入口），tables 给了全字段的表（全名）， + selected_by，total 这个源一共几张表（表结构缓存里记的真实总数），catalog_tables 有数据目录的表数（没有目录的 + 大库，界面提示先去起草目录：挑表和 SQL 检查都靠它）；探查截断了的再带 explored（探查到几张），界面写「共 N 张， + 只探查了 200 张」；挑表失败的带上 reason。只列了名字的表不进 tables：界面上说的「参考了 N 张表」是模型看得到 + 字段的那些。 """ rows = [] for s in self.sources: if not s.explored: continue - row: dict[str, Any] = {"source": s.source.name, "tables": [s.qualified(t) for t in s.tables], - "selected_by": s.selected_by, "total": s.total} + row: dict[str, Any] = {"source": s.source.name, "source_id": getattr(s.source, "id", None), + "tables": [s.qualified(t) for t in s.tables], + "selected_by": s.selected_by, "total": s.total, + "catalog_tables": s.plan.catalog_tables} if s.plan.unexplored: row["explored"] = s.explored if s.selected_by == "fallback" and self.reason: diff --git a/backend/tests/test_copilot_table_pick.py b/backend/tests/test_copilot_table_pick.py index 7e249404..53361e70 100644 --- a/backend/tests/test_copilot_table_pick.py +++ b/backend/tests/test_copilot_table_pick.py @@ -322,8 +322,9 @@ async def test_small_source_keeps_detail_and_carries_the_catalog(small_source): assert f"中文名：订单{INFERRED_MARK}" in text and "单位：元" in text assert f"orders.customer_id = customers.id（多对一）{INFERRED_MARK}" in text op = ctx.op() - assert op["sources"] == [{"source": small_source.name, "tables": ["customers", "orders"], "selected_by": "all", - "total": 2}] + assert op["sources"] == [{"source": small_source.name, "source_id": small_source.id, + "tables": ["customers", "orders"], "selected_by": "all", "total": 2, + "catalog_tables": 1}] # ---------------------------------------------------------------- 回退：绝不阻断生成 @@ -346,8 +347,9 @@ async def test_pick_failures_fall_back_to_names_only(client, monkeypatch, scenic "instruction": NEED, "intent": "answer", "datasource_ids": [scenic_source.id]}) op = _context_op(events) - assert op["sources"] == [{"source": scenic_source.name, "tables": [], "selected_by": "fallback", - "total": scenic.TABLE_COUNT + len(scenic.VIEWS), "reason": op["sources"][0]["reason"]}] + assert op["sources"] == [{"source": scenic_source.name, "source_id": scenic_source.id, "tables": [], + "selected_by": "fallback", "total": scenic.TABLE_COUNT + len(scenic.VIEWS), + "catalog_tables": 0, "reason": op["sources"][0]["reason"]}] assert op["sources"][0]["reason"] system = model.systems[0] # 和以前一样只列表名、提醒先查字段 @@ -371,8 +373,9 @@ async def test_stream_reports_the_context_and_prompt_carries_selected_tables(cli "instruction": NEED, "intent": "answer", "datasource_ids": [scenic_source.id]}) op = _context_op(events) - assert op["sources"] == [{"source": scenic_source.name, "tables": ["visits", "channels", "channel_visits"], - "selected_by": "model", "total": scenic.TABLE_COUNT + len(scenic.VIEWS)}] + assert op["sources"] == [{"source": scenic_source.name, "source_id": scenic_source.id, + "tables": ["visits", "channels", "channel_visits"], "selected_by": "model", + "total": scenic.TABLE_COUNT + len(scenic.VIEWS), "catalog_tables": 0}] assert isinstance(op["elapsed_ms"], int) and op["elapsed_ms"] >= 0 # context 在模型首帧之后、第一条图操作之前 kinds = [e["op"] for e in events] @@ -423,6 +426,28 @@ async def test_self_check_repair_reuses_the_same_context(client, monkeypatch, sc assert len(picker.calls) == 1 +async def test_context_op_says_how_much_of_each_source_has_a_catalog(scenic_source): + """context 操作里每个源带上有数据目录的表数（catalog_tables）和源的 id：没有目录的大库，界面提示先去起草目录。 + + 真实运行里数据源一张目录都没有，挑表和 SQL 检查都用不上目录，用户也不知道该先去起草。只算有可见项的表： + 整份被驳回的不算有目录。""" + from app.api import copilot_context + + async def op() -> dict: + plan = await copilot_context.plan_context([scenic_source]) + return (await plan.resolve(Picker(PICKED), need=NEED)).op()["sources"][0] + + first = await op() + assert first["catalog_tables"] == 0 and first["source_id"] == scenic_source.id + async with SessionLocal() as session: + await catalog.write_entry(session, scenic_source.id, "visits", { + "label": make_item("入园记录", "llm")}, if_version=0, actor="王敏") + rejected = make_item("门店", "llm") + rejected["status"] = "rejected" + await catalog.write_entry(session, scenic_source.id, "stores", {"label": rejected}, if_version=0, actor="王敏") + assert (await op())["catalog_tables"] == 1 + + async def test_small_sources_do_not_call_the_picker(client, monkeypatch, small_source): picker = Picker(error=AssertionError("小库不该挑表")) _use_picker(monkeypatch, picker) diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 37388a16..6b017e2f 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -2657,4 +2657,7 @@ export const COPILOT_CONTEXT_TEXT = { withFields: (n: number) => `参考了 ${formatNumber(n)} 张表的字段`, namesOnly: (n: number) => `另有 ${formatNumber(n)} 张表只提供了表名`, notPicked: '未能按需求挑选', + /** 没有数据目录的大库：只有一个数据源时说「这个数据源」，几个时点名 */ + noCatalog: (who: string | null) => `${who ?? '这个数据源'}还没有数据目录，起草后挑表和 SQL 检查会更准`, + toCatalog: '去数据目录起草', } diff --git a/frontend/src/run/AssistantStream.tsx b/frontend/src/run/AssistantStream.tsx index 9ecf00fb..ecdc7eb0 100644 --- a/frontend/src/run/AssistantStream.tsx +++ b/frontend/src/run/AssistantStream.tsx @@ -15,7 +15,9 @@ import { humanizeError } from '../lib/errors' import { explainRunError } from '../lib/explain' import { formatClock, formatDuration, formatNumber, NONE, shortId } from '../lib/format' import { statusLabel } from '../lib/status' -import { EVIDENCE_TEXT, JUDGE_TEXT, MERGE_TEXT, RUN_SQL_CHECK_TEXT, evidenceTally, issuanceLabel, nodeTypeLabel } from '../lib/terms' +import { + COPILOT_CONTEXT_TEXT, EVIDENCE_TEXT, JUDGE_TEXT, MERGE_TEXT, RUN_SQL_CHECK_TEXT, evidenceTally, issuanceLabel, nodeTypeLabel, +} from '../lib/terms' import { SqlCheckList } from './SqlChecks' import { ISSUANCE_HINT, childrenByExec, compactSteps, parseQueryResult, progressOf, spread, teamVerdictOf, @@ -901,6 +903,12 @@ function FixAction({ fix, nodeId, label, to, first, field }: { if (fix === 'settings') { return 去模型接入 } + if (fix === 'catalog' && to) { + // 助手参考的大库还没有数据目录：直达那个数据源的目录页去起草 + return ( + {COPILOT_CONTEXT_TEXT.toCatalog} + ) + } if (fix === 'tools') { // 数据源工具（db_query__ 这类）不在工具库里接入，在数据页 const data = !!to?.startsWith('/data') @@ -1170,7 +1178,7 @@ function StepLine({ step, depth, turnMs }: { step: Step; depth: number; turnMs?: {/* 要改的是另一个节点（指标的问题出在来源查询的 SQL 上）：入口落到那个节点的那一栏 */} + label={step.fixNode ? step.fixLabel : undefined} to={step.fixTo} /> )} diff --git a/frontend/src/run/__tests__/synthetic.ts b/frontend/src/run/__tests__/synthetic.ts index 3177ed1f..46fea4ed 100644 --- a/frontend/src/run/__tests__/synthetic.ts +++ b/frontend/src/run/__tests__/synthetic.ts @@ -503,8 +503,10 @@ export const COPILOT_CONTEXT = [ { op: 'model', model: 'claude-sonnet-4' }, { op: 'heartbeat', phase: 'planning', elapsed_ms: 3000 }, { op: 'context', elapsed_ms: 3420, sources: [ - { source: 'scenic', tables: ['visits', 'channels', 'channel_visits'], selected_by: 'model', total: 51 }, - { source: 'shop', tables: ['orders', 'customers'], selected_by: 'all', total: 2 }, + { source: 'scenic', source_id: 'src-scenic', tables: ['visits', 'channels', 'channel_visits'], selected_by: 'model', total: 51, + catalog_tables: 12 }, + // 小库全部表都带字段：没有数据目录也不提示 + { source: 'shop', source_id: 'src-shop', tables: ['orders', 'customers'], selected_by: 'all', total: 2, catalog_tables: 0 }, ] }, { op: 'heartbeat', phase: 'planning', elapsed_ms: 6000 }, { op: 'thinking', delta: '入园记录不带渠道，要经 channel_visits 连到渠道表，再按渠道汇总上月的入园人次。' }, @@ -539,7 +541,9 @@ export const COPILOT_CONTEXT_FALLBACK = [ { op: 'model', model: 'claude-sonnet-4' }, { op: 'heartbeat', phase: 'planning', elapsed_ms: 18000 }, { op: 'context', elapsed_ms: 20004, sources: [ - { source: 'scenic', tables: [], selected_by: 'fallback', total: 51, reason: '挑选数据表超过 20 秒未完成' }, + // 大库一张目录都没有：过程里提示先去起草目录 + { source: 'scenic', source_id: 'src-scenic', tables: [], selected_by: 'fallback', total: 51, catalog_tables: 0, + reason: '挑选数据表超过 20 秒未完成' }, ] }, { op: 'plan', summary: '先查表结构，再取数' }, { op: 'add_node', node: { id: 's', type: 'tool', label: '查看表结构' } }, diff --git a/frontend/src/run/decode.ts b/frontend/src/run/decode.ts index cd668956..e352daf3 100644 --- a/frontend/src/run/decode.ts +++ b/frontend/src/run/decode.ts @@ -48,7 +48,8 @@ export const teamVerdictOf = (team: TeamRun | undefined | null): TeamVerdict | u (team as TeamRunEx | undefined | null)?.verdict /** 下一步该去哪儿改。界面据此给一个直达的入口，而不只是一句话 */ -export type FixKind = 'canvas' | 'settings' | 'tools' +/** catalog：去数据目录页（fixTo 指到哪个数据源），助手参考的大库还没有数据目录时给 */ +export type FixKind = 'canvas' | 'settings' | 'tools' | 'catalog' /** * 事件 → 人能看懂的步骤。 @@ -147,6 +148,8 @@ export interface Step { next?: string /** 下一步该去的地方 */ fix?: FixKind + /** fix 要去的页面地址（catalog：/data/catalog/:sourceId） */ + fixTo?: string /** 工具的时限（秒，tool.start.timeout_s）。进行中的行超过它就要说破 */ limitS?: number /** 出具那一行的档位（formal / degraded / withheld） */ @@ -2649,6 +2652,10 @@ export interface CopilotContextSource { explored?: number /** 没能挑出来的原因 */ reason?: string + /** 数据源 id：去数据目录页的入口用（老服务端不给） */ + sourceId?: string + /** 有数据目录的表数（老服务端不给：不知道就不提示） */ + catalogTables?: number } /** 挑表失败时的说法：模型这一轮只看到表名，字段要它自己去查 */ @@ -2668,10 +2675,24 @@ function contextSources(op: CopilotOp): CopilotContextSource[] { ...(total != null ? { total } : {}), ...(total != null && explored != null && explored < total ? { explored } : {}), ...(typeof x.reason === 'string' && x.reason ? { reason: x.reason } : {}), + ...(typeof x.source_id === 'string' && x.source_id ? { sourceId: x.source_id } : {}), + ...(num(x.catalog_tables) != null ? { catalogTables: num(x.catalog_tables) } : {}), } }) } +/** + * 没有数据目录的大库（要按需求挑表的那种）：挑表只能照表名猜，SQL 检查也没有目录可对照。真实运行里数据源一张目录 + * 都没有，用户也不知道该先去起草。只认服务端明说了 0 张的；小库全部表都带字段，不提示 + */ +function noCatalog(groups: CopilotContextSource[]): { next: string; fixTo?: string } | null { + const bare = groups.filter((g) => g.catalogTables === 0 && g.selectedBy !== 'all') + if (!bare.length) return null + const who = groups.length === 1 ? null : `「${bare.map((g) => g.source).join('」「')}」` + const to = bare.find((g) => g.sourceId)?.sourceId + return { next: COPILOT_CONTEXT_TEXT.noCatalog(who), ...(to ? { fixTo: `/data/catalog/${encodeURIComponent(to)}` } : {}) } +} + /** * 一个库在展开区里的那一行：怎么来的 + 表名。探查结构截断了的库写明「共 205 张，只探查了 200 张」： * 模型只看得到探查到的那些，用户说的表不在里面时，先得知道是没探查到，不是库里没有 @@ -2702,7 +2723,8 @@ function contextLine(g: CopilotContextSource): string { * 先得知道模型当时看到的是哪几张。标题只说几张（「参考了 8 张表」），展开按数据源分组列表名；挑表失败时 * 标题直说只给了表名，原因放在展开区。认不出的形状返回 null：不出这一行，也不报错。 */ -export function copilotContext(op: CopilotOp): { title: string; sub?: string; detail: string; ms?: number } | null { +export function copilotContext(op: CopilotOp): + { title: string; sub?: string; detail: string; ms?: number; next?: string; fixTo?: string } | null { // 展开区里按需求挑过表的库排在前面：请求里点名的那个库往往就是它，排在一串小库后面得在内层滚动框里翻才看得到 const order: Record = { model: 0, all: 1, fallback: 2 } const groups = contextSources(op).map((g, i) => ({ g, i })) @@ -2721,11 +2743,13 @@ export function copilotContext(op: CopilotOp): { title: string; sub?: string; de const reasons = [...new Set(failed.map((g) => g.reason).filter(Boolean))] // 挑过表才有 elapsed_ms：挑表是生成之前多出来的一次模型调用，没挑成（比如超时）也要让人看到等了多久 const ms = num(op.elapsed_ms) + const hint = noCatalog(groups) return { title: count ? (namesOnly ? COPILOT_CONTEXT_TEXT.withFields(count) : `参考了 ${count} 张表`) : CONTEXT_FALLBACK_TEXT, ...(sub ? { sub } : {}), detail: [...groups.map(contextLine), ...reasons.map((r) => `原因：${r}`)].join('\n'), ...(ms != null && ms >= 10 ? { ms } : {}), + ...(hint ?? {}), } } @@ -2848,7 +2872,9 @@ export function decodeCopilot(ops: CopilotOp[], opts?: { add({ id: `cx-${i}`, seq: i, kind: 'schema', status: 'done', code: 'copilot_context', title: ctx.title, detail: ctx.detail, ...(ctx.sub ? { sub: ctx.sub } : {}), - ...(ctx.ms != null ? { ms: ctx.ms, meta: formatDuration(ctx.ms) } : {}) }) + ...(ctx.ms != null ? { ms: ctx.ms, meta: formatDuration(ctx.ms) } : {}), + // 没有数据目录的大库：下一步去起草目录，入口直达那个数据源的目录页 + ...(ctx.next ? { next: ctx.next, ...(ctx.fixTo ? { fix: 'catalog' as const, fixTo: ctx.fixTo } : {}) } : {}) }) break } case 'catalog_patch': { diff --git a/scripts/check-chat.mjs b/scripts/check-chat.mjs index 69b7bd90..d1f65737 100644 --- a/scripts/check-chat.mjs +++ b/scripts/check-chat.mjs @@ -487,8 +487,10 @@ async function fakeApi(route) { const ops = [ { op: 'heartbeat', phase: 'planning', elapsed_ms: 800 }, // 服务端按需求挑了表（copilot_context）：过程里多一行「参考了 N 张表」 + // 这个库还没有数据目录（catalog_tables: 0）：那一行下面提示先去起草目录 { op: 'context', elapsed_ms: 1200, sources: [ - { source: 'shop', tables: ['orders', 'order_items'], selected_by: 'model', total: 48 }] }, + { source: 'shop', source_id: 'src-shop', tables: ['orders', 'order_items'], selected_by: 'model', total: 48, + catalog_tables: 0 }] }, { op: 'plan', summary: '查一下再回答' }, ...GRAPH.nodes.map((n) => ({ op: 'add_node', node: n })), { op: 'done', explanation: '一张三步的图' }, @@ -1069,6 +1071,13 @@ for (const theme of THEMES) { await ctxRow.locator('button').first().click() check('再点开看到按数据源分组的表名', await ctxRow.getByText('「shop」按需求从 48 张表中挑出 2 张：orders、order_items') .waitFor({ timeout: 3000 }).then(() => true, () => false)) + // 真实运行里数据源一张目录都没有，挑表和 SQL 检查都用不上目录：这一行下面提示起草，并给去目录页的入口 + const noCatalog = ctxRow.locator('[data-step-next]') + check('没有数据目录的大库：提示先起草目录', await noCatalog.count() === 1 + && (await noCatalog.innerText()).includes('这个数据源还没有数据目录，起草后挑表和 SQL 检查会更准'), + await noCatalog.count() ? await noCatalog.innerText() : '没有这一句') + check('……入口直达这个数据源的目录页', await ctxRow.locator('a[data-fix="catalog"]').getAttribute('href').catch(() => null) + === '/data/catalog/src-shop') check('没有运行时报错', errors.length === 0, errors[0] ?? '') await ctx.close() }) diff --git a/scripts/check-decode.mjs b/scripts/check-decode.mjs index c3ef2f87..b021e395 100644 --- a/scripts/check-decode.mjs +++ b/scripts/check-decode.mjs @@ -682,6 +682,28 @@ await section('Copilot：这一轮参考了哪些表（context 操作）', async check('挑表失败：原因和库的规模放在展开区', !!fb?.detail?.includes('原因：挑选数据表超过 20 秒未完成') && fb.detail.includes('「scenic」共 51 张表'), fb?.detail) check('挑表失败也说等了多久', fb?.ms === 20004, String(fb?.ms)) + // 真实运行里数据源一张目录都没有，挑表、SQL 检查都用不上目录，用户也不知道该先去起草 + check('没有数据目录的大库：下一步提示起草目录，入口直达那个源的目录页', + fb?.next === '这个数据源还没有数据目录，起草后挑表和 SQL 检查会更准' && fb?.fix === 'catalog' + && fb?.fixTo === '/data/catalog/src-scenic', JSON.stringify({ next: fb?.next, fix: fb?.fix, fixTo: fb?.fixTo })) + check('有目录的大库、没有目录的小库：不提示', !row?.next && !row?.fix && !row?.fixTo, JSON.stringify({ next: row?.next })) + const bare = mod.copilotContext({ op: 'context', sources: [ + { source: 'a', source_id: 'id-a', tables: ['x'], selected_by: 'model', total: 60, catalog_tables: 0 }, + { source: 'b', source_id: 'id-b', tables: ['y'], selected_by: 'all', total: 3, catalog_tables: 0 }, + { source: 'c', source_id: 'id c', tables: [], selected_by: 'fallback', total: 90, catalog_tables: 0, reason: 'r' }] }) + check('几个数据源：点名没有目录的大库，入口去第一个', bare?.next === '「a」「c」还没有数据目录，起草后挑表和 SQL 检查会更准' + && bare?.fixTo === '/data/catalog/id-a', JSON.stringify({ next: bare?.next, fixTo: bare?.fixTo })) + const partly = mod.copilotContext({ op: 'context', sources: [ + { source: 'a', source_id: 'id-a', tables: ['x'], selected_by: 'model', total: 60, catalog_tables: 3 }] }) + check('起草过一部分目录：不再提示', !partly?.next, partly?.next) + const old = mod.copilotContext({ op: 'context', sources: [{ source: 'a', tables: ['x'], selected_by: 'model', total: 60 }] }) + check('老服务端不给目录表数：不猜，不提示', !old?.next && !old?.fixTo, old?.next) + const noId = mod.copilotContext({ op: 'context', sources: [ + { source: 'a', tables: [], selected_by: 'fallback', total: 60, catalog_tables: 0 }] }) + check('没给数据源 id：照样提示，只是不给入口', !!noId?.next && !noId?.fixTo, JSON.stringify(noId)) + const step = mod.decodeCopilot([{ op: 'context', sources: [ + { source: 'a', tables: [], selected_by: 'fallback', total: 60, catalog_tables: 0 }] }]).find((s) => s.code === 'copilot_context') + check('没有入口时不带 fix：不画一个点了没处去的按钮', !!step?.next && !step?.fix, JSON.stringify(step)) const seeded = mod.copilotContext({ op: 'context', sources: [ { source: 'scenic', tables: ['stores'], selected_by: 'fallback', total: 51, reason: '模型未挑出任何表' }] }) diff --git a/scripts/check-stream.mjs b/scripts/check-stream.mjs index f4660f41..db54b27e 100644 --- a/scripts/check-stream.mjs +++ b/scripts/check-stream.mjs @@ -774,12 +774,24 @@ await section('参考了哪些表：画布右栏和问数据页（context 操作 && expanded.includes('channel_visits') && expanded.includes('「shop」全部 2 张表'), expanded.slice(0, 160)) const overflow = await c.page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth) check('展开后不撑破 380px 窄栏', overflow <= 0, `${overflow}px`) + check('有数据目录的大库：不提示起草目录', await row.locator('[data-step-next]').count() === 0 + && await row.locator('[data-fix="catalog"]').count() === 0) check('没有运行时报错', c.errors.length === 0, c.errors.join(' | ')) await c.page.close() const f = await open('syn=copilot-fallback&dense=1', { w: 380, h: 800, name: 'copilot-fallback' }) const fb = f.page.locator('[data-step-code="copilot_context"]') check('挑表失败：写明只给了表名', await fb.count() === 1 && (await fb.innerText()).includes('未能按需求挑选，已提供全部表名')) + // 真实运行里数据源一张目录都没有：这一行下面常显下一步，入口去那个源的目录页 + const hint = fb.locator('[data-step-next]') + check('没有数据目录的大库：常显「还没有数据目录」，不收进展开区', await hint.count() === 1 + && (await hint.innerText()).includes('这个数据源还没有数据目录，起草后挑表和 SQL 检查会更准'), + await hint.count() ? await hint.innerText() : '没有这一句') + const go = fb.locator('a[data-fix="catalog"]') + check('……附去数据目录页的入口', await go.count() === 1 && await go.getAttribute('href') === '/data/catalog/src-scenic' + && (await go.innerText()).includes('去数据目录起草'), await go.count() ? await go.getAttribute('href') : '没有入口') + const narrow = await f.page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth) + check('……不撑破 380px 窄栏', narrow <= 0, `${narrow}px`) await fb.locator('button').first().click() await f.page.waitForTimeout(150) check('挑表失败：展开看到原因', (await fb.innerText()).includes('原因：挑选数据表超过 20 秒未完成')) From 4e1adec205a01421485749ee2612c83fcc345e40 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 4 Oct 2026 11:29:04 +0800 Subject: [PATCH 10/15] =?UTF-8?q?=E5=AE=A1=E6=89=B9=E5=8D=A1=E6=A0=87?= =?UTF-8?q?=E6=98=8E=E8=BF=90=E8=A1=8C=E7=B1=BB=E5=88=AB=E3=80=81=E9=99=84?= =?UTF-8?q?=E4=B8=8A=E5=AE=A1=E7=9A=84=E6=8A=A5=E5=91=8A=E7=9A=84=E6=A0=B8?= =?UTF-8?q?=E5=AF=B9=E6=A6=82=E5=86=B5=EF=BC=9A=E7=9C=9F=E5=AE=9E=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E9=87=8C=E5=AE=A1=E6=89=B9=E8=AF=B4=E6=98=8E=E5=86=99?= =?UTF-8?q?=E7=9D=80=E3=80=8C=E6=89=B9=E5=87=86=E5=90=8E=E6=89=8D=E4=BC=9A?= =?UTF-8?q?=E6=AD=A3=E5=BC=8F=E5=87=BA=E5=85=B7=E3=80=8D=EF=BC=8C=E9=82=A3?= =?UTF-8?q?=E5=8D=B4=E6=98=AF=E4=B8=80=E6=AC=A1=E6=8E=A2=E7=B4=A2=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=EF=BC=9B=E6=8A=A5=E5=91=8A=E9=87=8C=E4=B8=80=E5=8F=A5?= =?UTF-8?q?=E7=BB=93=E8=AE=BA=E5=92=8C=E8=AF=81=E6=8D=AE=E7=9B=B8=E7=9F=9B?= =?UTF-8?q?=E7=9B=BE=EF=BC=8C=E5=AE=A1=E6=89=B9=E4=BA=BA=E6=89=B9=E5=87=86?= =?UTF-8?q?=E6=97=B6=E5=AE=8C=E5=85=A8=E4=B8=8D=E7=9F=A5=E9=81=93=E3=80=82?= =?UTF-8?q?=E4=BA=BA=E5=B7=A5=E5=AE=A1=E6=89=B9=E7=9A=84=E5=8D=A1=E4=B8=8A?= =?UTF-8?q?=E5=8A=A0=E4=B8=80=E8=A1=8C=E8=BF=90=E8=A1=8C=E7=B1=BB=E5=88=AB?= =?UTF-8?q?=E5=92=8C=E6=89=B9=E4=BA=86=E4=BC=9A=E6=80=8E=E6=A0=B7=EF=BC=88?= =?UTF-8?q?=E6=8E=A2=E7=B4=A2=E8=BF=90=E8=A1=8C=E5=86=99=E3=80=8C=E6=89=B9?= =?UTF-8?q?=E5=87=86=E5=90=8E=E8=BE=93=E5=87=BA=EF=BC=8C=E8=BF=99=E6=98=AF?= =?UTF-8?q?=E6=8E=A2=E7=B4=A2=E8=BF=90=E8=A1=8C=EF=BC=8C=E4=B8=8D=E6=98=AF?= =?UTF-8?q?=E6=AD=A3=E5=BC=8F=E5=87=BA=E5=85=B7=E3=80=8D=EF=BC=8C=E6=AD=A3?= =?UTF-8?q?=E5=BC=8F=E8=BF=90=E8=A1=8C=E5=86=99=E4=BB=8E=E5=B7=B2=E5=8F=91?= =?UTF-8?q?=E5=B8=83=E7=89=88=E6=9C=AC=E5=8F=91=E8=B5=B7=EF=BC=89=EF=BC=9B?= =?UTF-8?q?human=20=E8=8A=82=E7=82=B9=E6=8C=82=E8=B5=B7=E6=97=B6=EF=BC=8C?= =?UTF-8?q?=E4=B8=8A=E6=B8=B8=E7=9B=B4=E6=8E=A5=E8=BF=9E=E7=9D=80=E7=9A=84?= =?UTF-8?q?=E3=80=81=E6=88=96=E6=A0=87=E9=A2=98=E5=92=8C=E8=AF=B4=E6=98=8E?= =?UTF-8?q?=E9=87=8C=E5=BC=95=E7=94=A8=E4=BA=86=E6=AD=A3=E6=96=87=EF=BC=88?= =?UTF-8?q?nodes.=20=E6=88=96=E6=8A=A5=E5=91=8A=E7=9A=84=20assign=5Fto?= =?UTF-8?q?=EF=BC=89=E7=9A=84=E6=8A=A5=E5=91=8A=E6=92=B0=E5=86=99=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E5=90=84=E5=B8=A6=E4=B8=80=E4=BB=BD=E6=A0=B8=E5=AF=B9?= =?UTF-8?q?=E6=A6=82=E5=86=B5=EF=BC=88payload.reports=EF=BC=9A=E5=90=8C=20?= =?UTF-8?q?report.checked=20=E7=9A=84=20stats=E3=80=81=E7=BB=93=E8=AE=BA?= =?UTF-8?q?=E5=8F=A5=E7=AD=96=E7=95=A5=E3=80=81=E8=A3=81=E5=88=A4=E6=91=98?= =?UTF-8?q?=E8=A6=81=E3=80=81=E6=8A=A5=E5=91=8A=E6=96=87=E6=A1=A3=E5=B7=A5?= =?UTF-8?q?=E4=BB=B6=20id=EF=BC=89=EF=BC=8C=E5=8D=A1=E4=B8=8A=E5=86=99?= =?UTF-8?q?=E3=80=8CN=20=E4=B8=AA=E6=95=B0=E5=AD=97=E9=83=BD=E6=9C=89?= =?UTF-8?q?=E5=87=BA=E5=A4=84=E3=80=8D=E5=92=8C=E7=BB=93=E8=AE=BA=E5=8F=A5?= =?UTF-8?q?=E7=9A=84=E6=A0=B8=E5=AF=B9=E7=8A=B6=E6=80=81=EF=BC=88=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E9=87=8C=E5=88=A4=E8=BF=87=E7=9A=84=E5=86=99=E5=88=A4?= =?UTF-8?q?=E4=BA=86=E5=87=A0=E5=8F=A5=E3=80=81=E5=87=A0=E5=8F=A5=E8=AF=81?= =?UTF-8?q?=E6=8D=AE=E7=9B=B8=E7=9F=9B=E7=9B=BE=EF=BC=9B=E6=B2=A1=E5=BC=80?= =?UTF-8?q?=E8=A3=81=E5=88=A4=E6=88=96=E6=8E=A2=E7=B4=A2=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E5=86=99=E3=80=8C=E7=BB=93=E8=AE=BA=E5=8F=A5=E5=B0=9A=E6=9C=AA?= =?UTF-8?q?=E6=A0=B8=E5=AF=B9=E3=80=8D=EF=BC=89=EF=BC=8C=E5=B9=B6=E7=BB=99?= =?UTF-8?q?=E6=9F=A5=E7=9C=8B=E9=80=90=E6=AE=B5=E8=AF=81=E6=8D=AE=E7=9A=84?= =?UTF-8?q?=E5=85=A5=E5=8F=A3=EF=BC=8C=E7=82=B9=E6=95=B0=E5=AD=97=E6=89=93?= =?UTF-8?q?=E5=BC=80=E8=AF=81=E6=8D=AE=E9=9D=A2=E6=9D=BF=E3=80=82check-str?= =?UTF-8?q?eam=20=E8=A1=A5=E6=96=AD=E8=A8=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/engine/nodes/human.py | 48 ++++++ backend/tests/test_approval_report_check.py | 154 ++++++++++++++++++++ frontend/src/dev/preview.tsx | 33 ++++- frontend/src/lib/terms.ts | 27 ++++ frontend/src/run/ApprovalReport.tsx | 128 ++++++++++++++++ frontend/src/run/RunPanel.tsx | 16 +- frontend/src/types.ts | 15 ++ scripts/check-stream.mjs | 57 +++++++- 8 files changed, 472 insertions(+), 6 deletions(-) create mode 100644 backend/tests/test_approval_report_check.py create mode 100644 frontend/src/run/ApprovalReport.tsx diff --git a/backend/app/engine/nodes/human.py b/backend/app/engine/nodes/human.py index 83e83d95..29bd3fc1 100644 --- a/backend/app/engine/nodes/human.py +++ b/backend/app/engine/nodes/human.py @@ -33,6 +33,8 @@ async def run_human(state: GraphState, ctx: NodeContext) -> dict[str, Any]: } if mode == "edit": payload["draft"] = ctx.render(ctx.cfg("draft", ""), state) + if reports := _reviewed_reports(state, ctx): + payload["reports"] = reports response = await ask(ctx, payload, **payload) # 谁批的跟着事件走：时间线要写得出"张工驳回了"，没署名就是 None，不再默认写"你" @@ -71,6 +73,52 @@ async def run_human(state: GraphState, ctx: NodeContext) -> dict[str, Any]: return updates +#: 模板里对某个节点产出的引用：{{ nodes.write.text }}、{{ vars.weekly }} +_NODE_REF = re.compile(r"\bnodes\s*\.\s*([\w-]+)") +_VAR_REF = re.compile(r"\bvars\s*\.\s*([\w-]+)") +#: 审批卡要的裁判摘要：判了几句、各判成什么、有没有判完。其余（上限、费用、模型名）留在证据面板里看 +_JUDGE_KEYS = ("mode", "counts", "complete", "unjudged", "limits_hit") + + +def _reviewed_reports(state: GraphState, ctx: NodeContext) -> list[dict[str, Any]]: + """这次审批审的报告各自的核对概况：上游直接连着的报告撰写节点，加上标题、说明里引用了正文的 + （{{ nodes..text }}，或者报告的 assign_to 变量）。 + + 真实运行里报告有一句结论和证据相矛盾，审批人批准时完全不知道：审批卡只有正文。这里带上报告节点产出里的 + stats（同 report.checked）、结论句策略、裁判摘要和报告文档的工件 id，审批卡据此写核对概况、给打开证据的入口。 + 只认已经跑完、带着核对统计的报告；重放时 state 一样，算出来的也一样。""" + from app.engine.schema import NodeType + + spec = ctx.run.spec + nodes = spec.node_map() + text = " ".join(json.dumps(ctx.config.get(key), ensure_ascii=False, default=str) + for key in ("title", "message", "context", "draft") if ctx.config.get(key)) + # 报告节点的 assign_to 也能把正文交给下游：{{ vars. }} 和 {{ nodes..text }} 一样算引用 + by_var = {str(n.config.get("assign_to")): n.id for n in spec.nodes + if n.type == NodeType.REPORT and n.config.get("assign_to")} + wanted = ([e.source for e in spec.edges if e.target == ctx.node.id] + _NODE_REF.findall(text) + + [by_var[v] for v in _VAR_REF.findall(text) if v in by_var]) + outputs = state.get("nodes") or {} + out: list[dict[str, Any]] = [] + for rid in dict.fromkeys(wanted): + node = nodes.get(rid) + result = outputs.get(rid) + if node is None or node.type != NodeType.REPORT or not isinstance(result, dict) \ + or not isinstance(result.get("stats"), dict): + continue + item: dict[str, Any] = { + "node_id": rid, "label": node.title, + "stats": {k: v for k, v in result["stats"].items() if isinstance(v, int) and not isinstance(v, bool)}, + "claims": result.get("claims") or "off", + } + if isinstance(result.get("doc_artifact"), str) and result["doc_artifact"]: + item["doc_artifact"] = result["doc_artifact"] + if isinstance(result.get("judge"), dict): + item["judge"] = {k: result["judge"][k] for k in _JUDGE_KEYS if k in result["judge"]} + out.append(item) + return out + + # -------------------------------------------------------------------------- # 结构化校验 # -------------------------------------------------------------------------- diff --git a/backend/tests/test_approval_report_check.py b/backend/tests/test_approval_report_check.py new file mode 100644 index 00000000..cd13c261 --- /dev/null +++ b/backend/tests/test_approval_report_check.py @@ -0,0 +1,154 @@ +"""人工审批审的是报告时，审批卡附上这份报告的核对概况（human 节点的请求里带 reports）。 + +真实运行里：报告里有一句结论和证据相矛盾，审批人批准时完全不知道——报告节点只把正文交给了审批卡，几个数字有出处、 +结论句判没判过，一个字都没有。用户事后在证据面板点开那句，裁判才判它相矛盾。 + +现在 human 节点挂起时，上游的报告节点（直接连着的，或者标题、说明里引用了它正文的）各带一份核对概况： +数字核对的统计（report.checked 的 stats）、结论句策略、裁判摘要（几句判了、几句相矛盾），以及报告文档的工件 id， +审批卡据此写「N 个数字都有出处」「结论句已裁判 X 句，证据相矛盾 Y 句」或「结论句尚未核对」，并给出打开证据的入口。 + +表名、指标、数字都是虚构的。 +""" +from __future__ import annotations + +import asyncio +import json +import re + +import pytest +from langchain_core.messages import AIMessage +from sqlalchemy import delete, select + +from app.db.base import SessionLocal +from app.db.models import Approval, Run, RunEvent, Setting +from app.engine import judge as judge_mod +from app.engine.judge import JUDGE_ROLE, SPEND_KEY +from app.engine.runner import run_manager +from app.providers.mock_model import MockChatModel + +PRICED = "claude-sonnet-5" +METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}] +TEXT = "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。" + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + async with SessionLocal() as session: + await session.execute(delete(Setting).where(Setting.key.in_([SPEND_KEY, "judge", "copilot"]))) + await session.commit() + yield + await run_manager.shutdown() + + +@pytest.fixture(autouse=True) +def models(monkeypatch): + """写作者回 TEXT；裁判把「新客」那句判成证据相矛盾，其余有依据。""" + def decide(model, messages): + if JUDGE_ROLE in str(messages[0].content): + request = str(messages[-1].content) + items = [{"unit": uid, "verdict": "contradicted" if "新客" in line else "supported", + "rationale": "虚构的理由", "used": []} + for uid, line in re.findall(r"^\[(u\d+)\]([^\n]*)$", request, re.M)] + return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False)) + return AIMessage(content=TEXT) + + monkeypatch.setattr(MockChatModel, "_decide", decide) + + async def judge_model_of(session, spec): + return MockChatModel(model_name=PRICED), PRICED + + monkeypatch.setattr(judge_mod, "get_chat_model", judge_model_of) + + +def node(nid, ntype, label=None, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}} + + +def graph(*, gate: dict, wire_report_to_gate: bool = True, **report): + nodes = [ + node("start", "input"), + node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"), + node("caliber", "metrics", caliber="周报口径", caliber_version="v1", metrics=METRICS), + node("write", "report", "写周报", instructions="写周报", **report), + node("note", "transform", mode="expression", expression="'ok'", assign_to="note"), + node("gate", "human", "主管审批", mode="approve", **gate), + node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]), + ] + chain = ["start", "fetch", "caliber", "write"] + (["gate"] if wire_report_to_gate else ["note", "gate"]) + edges = [{"source": a, "target": b} for a, b in zip(chain, chain[1:])] + edges.append({"source": "gate", "target": "out", "sourceHandle": "approved"}) + return {"nodes": nodes, "edges": edges} + + +async def paused(g, *, run_class: str) -> tuple[Run, Approval]: + started = await run_manager.start(graph=g, input_payload={}, run_class=run_class) + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, started.id) + if row.status in ("interrupted", "failed", "succeeded"): + approval = (await session.execute(select(Approval).where(Approval.run_id == row.id))).scalar_one_or_none() + if row.status != "interrupted" or approval is not None: + break + assert row.status == "interrupted", row.error + assert approval is not None + return row, approval + + +async def test_formal_run_carries_the_judged_claims_to_the_approval(): + row, approval = await paused(graph(gate={"title": "审核周报", "message": "{{ nodes.write.text }}"}, + claims="judge"), run_class="formal") + [report] = approval.payload["reports"] + assert report["node_id"] == "write" and report["label"] == "写周报" + assert isinstance(report["doc_artifact"], str) and report["doc_artifact"] + assert report["stats"]["numbers"] == report["stats"]["numbers_cited"] == 1 + assert report["claims"] == "judge" + judge = report["judge"] + assert judge["mode"] == "inline" + assert judge["counts"]["contradicted"] == 1 and judge["counts"]["supported"] == 1 + assert judge["counts"]["unjudged"] == 0 + # 和 report.checked 是同一份：审批卡写的和时间线那一行对得上 + async with SessionLocal() as session: + [checked] = list((await session.execute(select(RunEvent).where( + RunEvent.run_id == row.id, RunEvent.type == "report.checked"))).scalars()) + assert checked.data["doc_artifact"] == report["doc_artifact"] + assert {k: checked.data["stats"][k] for k in report["stats"]} == report["stats"] + assert checked.data["judge"]["counts"] == judge["counts"] + # human.requested 事件里也带着，回放时界面一样读得到 + async with SessionLocal() as session: + [requested] = list((await session.execute(select(RunEvent).where( + RunEvent.run_id == row.id, RunEvent.type == "human.requested"))).scalars()) + assert requested.data["reports"] == approval.payload["reports"] + + +async def test_exploratory_run_says_claims_are_judged_on_demand(): + _, approval = await paused(graph(gate={"title": "审核周报"}, claims="judge"), run_class="exploratory") + [report] = approval.payload["reports"] + assert report["claims"] == "judge" and report["judge"]["mode"] == "on_demand" + assert report["judge"]["counts"]["contradicted"] == 0, "探索运行不在节点里判" + + +async def test_report_without_claims_policy_still_reports_numbers(): + _, approval = await paused(graph(gate={"title": "审核周报"}), run_class="exploratory") + [report] = approval.payload["reports"] + assert report["claims"] == "off" and "judge" not in report + assert report["stats"]["numbers"] == 1 + + +async def test_a_report_quoted_in_the_message_counts_even_when_not_wired_directly(): + _, approval = await paused(graph(gate={"title": "审核", "message": "请看：{{ nodes.write.text }}"}, + wire_report_to_gate=False), run_class="exploratory") + assert [r["node_id"] for r in approval.payload["reports"]] == ["write"] + + +async def test_a_report_quoted_through_its_variable_counts_too(): + _, approval = await paused(graph(gate={"title": "审核", "message": "{{ vars.weekly }}"}, wire_report_to_gate=False, + assign_to="weekly"), run_class="exploratory") + assert [r["node_id"] for r in approval.payload["reports"]] == ["write"] + + +async def test_approvals_unrelated_to_a_report_carry_nothing(): + _, approval = await paused(graph(gate={"title": "确认", "message": "继续吗"}, wire_report_to_gate=False), + run_class="exploratory") + assert "reports" not in approval.payload diff --git a/frontend/src/dev/preview.tsx b/frontend/src/dev/preview.tsx index 15d215d9..89dc111c 100644 --- a/frontend/src/dev/preview.tsx +++ b/frontend/src/dev/preview.tsx @@ -488,11 +488,36 @@ function Expanding({ dense, n }: { dense: boolean; n: number }) { const APPROVAL = { id: 'ap-1', run_id: 'syn-mixed', node_id: 'review', mode: 'approve' as const, title: '出勤结论可以发出吗？', status: 'pending', response: {}, - payload: { message: '早班出勤率 **94.23%**，晚班 **73.08%**。' }, + payload: { kind: 'human_node', message: '早班出勤率 **94.23%**，晚班 **73.08%**。' }, created_at: new Date(Date.now() - 8 * 86_400_000 - 3_600_000).toISOString().replace('Z', ''), workflow_name: '出勤日报', node_label: '班长复核', run_class: 'formal' as const, } +/** + * ?approval=report：探索运行里审一份报告。核对概况取 evidence-doc 夹具（7/12 数字有出处、1 处引用解析不了）， + * 结论句没在节点里判；?approval=report-formal：正式运行，节点里判过，1 句证据相矛盾 + */ +const APPROVAL_REPORT = { + id: 'ap-2', run_id: evidenceFixture.run_id, node_id: 'review', mode: 'approve' as const, + title: '本周周报可以发出吗？', status: 'pending', response: {}, + payload: { + kind: 'human_node', message: '请审核本周周报。', + reports: [{ node_id: 'write', label: '写周报', stats: evidenceFixture.doc.stats, claims: 'judge', + doc_artifact: evidenceFixture.doc_artifact, + judge: { mode: 'on_demand', counts: { unjudged: 4 }, complete: false } }], + }, + created_at: new Date(Date.now() - 120_000).toISOString().replace('Z', ''), + workflow_name: '周报', node_label: '主管审批', run_class: 'exploratory' as const, +} +const APPROVAL_REPORT_FORMAL = { + ...APPROVAL_REPORT, id: 'ap-3', run_class: 'formal' as const, + payload: { + ...APPROVAL_REPORT.payload, + reports: [{ ...APPROVAL_REPORT.payload.reports[0], stats: { ...evidenceFixture.doc.stats, numbers_cited: 12, uncited_numbers: 0, unresolved: 0 }, + judge: { mode: 'inline', counts: { supported: 5, contradicted: 1, unjudged: 0 }, complete: true } }], + }, +} + function Preview() { // ?case=db 只看一个用例，?theme=light 直接出浅色——都是为了截图可复现， // 不用靠点按钮 @@ -525,11 +550,13 @@ function Preview() { if (md) return
    if (params.get('grow') === '1') return if (params.get('expand')) return - if (params.get('approval') === '1') { + const approvalCase = params.get('approval') + if (approvalCase === '1' || approvalCase === 'report' || approvalCase === 'report-formal') { + const approval = approvalCase === 'report' ? APPROVAL_REPORT : approvalCase === 'report-formal' ? APPROVAL_REPORT_FORMAL : APPROVAL return (
    - undefined} /> + undefined} />
    ) diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 6b017e2f..bbcfea34 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -87,6 +87,33 @@ export function runClassLabel(runClass: 'formal' | 'exploratory' | null | undefi return '探索运行' } +/** + * 审批卡上人工审批的运行类别和审的报告的核对概况。出具只发生在成果节点配了出具契约、从已发布版本发起的正式运行里： + * 探索运行批准后只是输出，卡上要说破，不能让审批人以为自己在签发正式的东西 + */ +export const APPROVAL_TEXT = { + /** 人工审批批了会怎样，按运行类别说 */ + outcome: (runClass: 'formal' | 'exploratory' | null | undefined, approve: boolean) => (runClass === 'formal' + ? `从已发布版本发起，${approve ? '批准' : '提交'}后继续运行` + : `${approve ? '批准后输出' : '提交后继续运行'}，这是探索运行，不是正式出具`), + reportHead: (label: string) => `报告「${label}」的核对`, + noNumbers: '报告中没有需要核对的数字', + /** 节点里当场裁判过（正式运行开了结论句裁判）：判了几句、几句证据相矛盾、几句没判成 */ + claimsJudged: (judged: number, contradicted: number, unjudged: number) => + `结论句已裁判 ${formatNumber(judged)} 句，证据相矛盾 ${formatNumber(contradicted)} 句` + + (unjudged ? `，未裁判 ${formatNumber(unjudged)} 句` : ''), + claimsNone: '报告中没有需要裁判的结论句', + claimsPending: '结论句尚未核对', + /** 探索运行停在审批上时还没封存，按需裁判要等运行结束 */ + claimsLater: '探索运行不在节点内裁判，运行结束后可在证据中逐句请模型判断', + /** 正式运行、报告没开结论句裁判 */ + claimsOff: '这份报告没有开启结论句裁判', + openEvidence: '查看逐段证据', + closeEvidence: '收起证据', + docLoading: '正在加载报告文档…', + docMissing: '无法获取报告文档', +} + /** * 没挂在工作流上的运行：画布或问数据当场搭的，从没存成工作流。后端给它的名字老数据是 * 「临时图」、新的是「未保存的工作流」，也可能是空的；界面上一律叫「未保存的工作流」。 diff --git a/frontend/src/run/ApprovalReport.tsx b/frontend/src/run/ApprovalReport.tsx new file mode 100644 index 00000000..40fb718f --- /dev/null +++ b/frontend/src/run/ApprovalReport.tsx @@ -0,0 +1,128 @@ +import { useEffect, useId, useState } from 'react' +import { ChevronRight, ShieldCheck } from 'lucide-react' +import { EVIDENCE_STATE, claimCountsOf, claimProblems, statsTally } from '../lib/evidence' +import { formatNumber } from '../lib/format' +import { APPROVAL_TEXT, EVIDENCE_TEXT, JUDGE_TEXT, evidenceTally, runClassLabel } from '../lib/terms' +import { useEvidence } from '../store/evidence' +import type { ApprovalReport } from '../types' +import { EvidenceDoc } from './EvidenceDoc' + +type RunClass = 'formal' | 'exploratory' | null | undefined + +/** + * 人工审批这次是探索运行还是正式运行，批了会怎样。 + * + * 真实运行里审批说明写着「批准后才会正式出具」，那却是一次探索运行：出具只发生在成果节点配了出具契约、从已发布 + * 版本发起的正式运行里。审批卡自己标明运行类别，不靠模型写的说明 + */ +export function ApprovalRunClass({ runClass, approve }: { runClass: RunClass; approve: boolean }) { + const formal = runClass === 'formal' + return ( +
    + {runClassLabel(runClass)} + {APPROVAL_TEXT.outcome(runClass, approve)} +
    + ) +} + +/** 结论句那一句：节点里判过的说判了几句、几句相矛盾；没判过的说尚未核对，并说什么时候能核对 */ +function claimsLine(report: ApprovalReport, runClass: RunClass): { text: string; sub?: string; warn: boolean } { + const judge = report.judge + if (judge?.mode === 'inline' && judge.counts && typeof judge.counts === 'object') { + const c = claimCountsOf(judge.counts) + if (!c.total) return { text: APPROVAL_TEXT.claimsNone, warn: false } + const contradicted = c.contradicted + c.unsupported + return { + text: APPROVAL_TEXT.claimsJudged(c.total - c.unjudged, contradicted, c.unjudged), + warn: claimProblems(c) > 0 || c.unjudged > 0, + } + } + return { + text: APPROVAL_TEXT.claimsPending, + sub: runClass === 'formal' ? (report.claims === 'judge' ? undefined : APPROVAL_TEXT.claimsOff) : APPROVAL_TEXT.claimsLater, + warn: true, + } +} + +/** + * 审批审的报告的核对概况：几个数字有出处（同证据条的说法），结论句判没判过、几句证据相矛盾，再给一个查看逐段证据 + * 的入口——点开是和成果里同一份可逐段点开的报告文档，点数字、句末徽标打开证据面板。 + * + * 真实运行里报告有一句结论和证据相矛盾，审批人批准时完全不知道：卡上只有正文 + */ +export function ApprovalReports({ reports, runId, runClass }: { + reports: ApprovalReport[] + runId: string + runClass: RunClass +}) { + if (!reports.length) return null + return ( + <> + {reports.map((r) => )} + + ) +} + +function ReportCheck({ report, runId, runClass }: { report: ApprovalReport; runId: string; runClass: RunClass }) { + const [open, setOpen] = useState(false) + const docId = useId() + const counts = statsTally(report.stats) + const suspect = typeof report.stats?.unknown_entities === 'number' ? report.stats.unknown_entities : 0 + const numbers = counts ? evidenceTally(counts.cited, counts.total, counts.other) : '' + const claims = claimsLine(report, runClass) + const gaps = !!counts && (counts.none > 0 || counts.other > 0) + return ( +
    +
    + {APPROVAL_TEXT.reportHead(report.label || report.node_id)} +
    +
    + + {numbers || APPROVAL_TEXT.noNumbers} + + {suspect > 0 && ( + + · {EVIDENCE_STATE.suspect.glyph} {EVIDENCE_TEXT.suspectTag} {formatNumber(suspect)} + + )} + {/* 判定是模型给的：全都有依据也不用确定性的绿，有问题、没核对的用提醒色 */} + + · {claims.text} + +
    + {claims.sub &&
    {claims.sub}
    } + {report.doc_artifact && ( + <> + + {open && ( +
    + +
    + )} + + )} +
    + ) +} + +/** 报告文档：和成果里同一份可逐段点开的文档，自带证据条（定位下一处、违规清单），点开的证据面板在卡内展开 */ +function ReportDoc({ artifact, runId, runClass }: { artifact: string; runId: string; runClass: RunClass }) { + const slot = useEvidence((s) => s.docs[artifact]) + const loadDoc = useEvidence((s) => s.loadDoc) + useEffect(() => { void loadDoc(artifact) }, [artifact, loadDoc]) + if (slot?.status === 'error') return
    {APPROVAL_TEXT.docMissing}
    + const doc = slot?.status === 'ok' ? slot.data : undefined + if (!doc) return
    {APPROVAL_TEXT.docLoading}
    + return ( +
    + +
    + ) +} diff --git a/frontend/src/run/RunPanel.tsx b/frontend/src/run/RunPanel.tsx index 325d4886..fe83abac 100644 --- a/frontend/src/run/RunPanel.tsx +++ b/frontend/src/run/RunPanel.tsx @@ -11,7 +11,8 @@ import { formatDateTime, formatSpan, formatTime, parseServerTime } from '../lib/ import { runClassLabel, runName, TOOL_TRUST_TEXT } from '../lib/terms' import { useRunClock } from './useRunClock' import { Markdown } from './Markdown' -import type { Approval } from '../types' +import { ApprovalReports, ApprovalRunClass } from './ApprovalReport' +import type { Approval, ApprovalReport } from '../types' // 发起运行的表单搬去了 run/RunControl.tsx 的工具栏控件里。它原先常驻在 // 助手栏顶部，和 Copilot 输入框两个"主要动作"互相压着——而它们是两种不同的 @@ -99,6 +100,12 @@ export function ApprovalCard({ approval, onResolved, showWorkflow = true }: { } const where = approval.node_label || approval.node_id + // 人工审批节点挂起的审批（工具、代码的审批另有 kind）：卡上标明运行类别、批了会怎样，审的是报告时附上核对概况 + const humanNode = payload.kind === 'human_node' + const reports: ApprovalReport[] = Array.isArray(payload.reports) + ? payload.reports.filter((r: unknown): r is ApprovalReport => + !!r && typeof r === 'object' && typeof (r as ApprovalReport).node_id === 'string') + : [] return (
    @@ -109,7 +116,7 @@ export function ApprovalCard({ approval, onResolved, showWorkflow = true }: { 人工审批 {showWorkflow && approval.workflow_name && <>·{runName(approval)}} {where && <>·节点「{where}」} - {approval.run_class && <>·{runClassLabel(approval.run_class)}} + {approval.run_class && !humanNode && <>·{runClassLabel(approval.run_class)}} {created && ( <> · @@ -126,6 +133,9 @@ export function ApprovalCard({ approval, onResolved, showWorkflow = true }: { )}
    + {/* 老后端的审批没有运行类别：不猜，不画这一行 */} + {humanNode && approval.run_class && } + {payload.message && ( // 待审的往往是一整段模型写的文案，同样是 Markdown。要人家判断"能不能 // 发"，却让他对着一堆星号看，是在给审批这件事添难度 @@ -134,6 +144,8 @@ export function ApprovalCard({ approval, onResolved, showWorkflow = true }: { )} + + {payload.tool && (
    diff --git a/frontend/src/types.ts b/frontend/src/types.ts index dc8a1c4c..3d176275 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -173,6 +173,21 @@ export interface Approval { resolved_at?: string | null } +/** + * 人工审批审的报告的核对概况（human 节点请求里的 payload.reports，服务端 nodes/human._reviewed_reports）： + * 上游直接连着的、或者标题和说明里引用了正文的报告撰写节点，各一项。stats 同 report.checked，judge 是裁判摘要 + * 的几项（mode：inline 节点里当场判的 / on_demand 探索运行按需判）。老后端、审的不是报告时没有 + */ +export interface ApprovalReport { + node_id: string + label?: string + stats?: EvidenceStats + /** 结论句策略：off / require_citation / judge */ + claims?: string + doc_artifact?: string + judge?: { mode?: string; counts?: Record; complete?: boolean; unjudged?: unknown; limits_hit?: unknown } +} + /** * 后端记着的最近一次测连接（模型接入、数据源、MCP 的列表接口都平铺这四项）。 * 没测过、或者连接配置改过之后全是 null。用 lib/health 的 healthFromServer 转成 diff --git a/scripts/check-stream.mjs b/scripts/check-stream.mjs index db54b27e..c95b0319 100644 --- a/scripts/check-stream.mjs +++ b/scripts/check-stream.mjs @@ -923,10 +923,65 @@ await section('审批卡的上下文', async () => { check('没署名时提醒并给去处', body.includes('未署名') && await page.locator('a[href="/settings/prefs"]').count() === 1) await page.close() const s = await open('approval=1&actor=张工', { w: 820, h: 500, name: 'approval-signed' }) - check('署了名就写明以谁的名义', (await s.page.locator('body').innerText()).includes('将以「张工」签批')) + const signed = await s.page.locator('body').innerText() + check('署了名就写明以谁的名义', signed.includes('将以「张工」签批')) + // 人工审批自己标明运行类别：正式运行从已发布版本发起 + check('人工审批：正式运行标明类别和批了会怎样', await s.page.locator('[data-approval-run-class="formal"]').count() === 1 + && signed.includes('从已发布版本发起，批准后继续运行'), signed.replace(/\n/g, ' ').slice(0, 200)) await s.page.close() }) +await section('审批卡：审的报告的核对概况', async () => { + // 真实运行里审批说明写着「批准后才会正式出具」，那却是一次探索运行；报告里一句结论和证据相矛盾，审批人批准时完全 + // 不知道。审批卡自己标明运行类别，附上报告的核对概况和查看逐段证据的入口。文档、片段、证据图都用夹具伪造，非 GET 拦掉 + const { readFileSync } = await import('node:fs') + const fx = JSON.parse(readFileSync(new URL('../frontend/src/run/__tests__/evidence-doc.json', import.meta.url), 'utf8')) + const page = await browser.newPage({ viewport: { width: 420, height: 900 } }) + const errors = [] + page.on('pageerror', (e) => errors.push(e.message)) + await page.route((u) => new URL(u).pathname.startsWith('/api/'), (r) => { + const url = new URL(r.request().url()) + if (r.request().method() !== 'GET') return r.abort() + const seg = url.pathname.match(/\/evidence\/segments\/([^/]+)$/) + if (seg) return r.fulfill({ json: fx.segments[seg[1]] ?? {} }) + if (url.pathname === `/api/runs/${fx.run_id}/evidence`) return r.fulfill({ json: fx.graph }) + if (url.pathname === `/api/artifacts/${fx.doc_artifact}`) return r.fulfill({ json: { id: fx.doc_artifact, content: fx.doc } }) + return r.continue() + }) + await page.goto(`${WEB}/preview.html?approval=report&dense=1&actor=张工`, { waitUntil: 'networkidle' }) + await page.waitForSelector('[data-approval]', { timeout: 6000 }) + const card = page.locator('[data-approval]') + const text = await card.innerText() + check('探索运行：标明类别，写明批准后输出、不是正式出具', await card.locator('[data-approval-run-class="exploratory"]').count() === 1 + && text.includes('批准后输出，这是探索运行，不是正式出具'), text.replace(/\n/g, ' ').slice(0, 160)) + const report = card.locator('[data-approval-report="write"]') + check('报告的核对概况：标题写哪份报告', (await report.innerText()).includes('报告「写周报」的核对')) + check('数字和证据条同一种说法', (await report.locator('[data-approval-report-numbers]').innerText()) + .includes('7/12 数字有出处 · 无证据 5 · 另有 1 处引用无法解析'), await report.locator('[data-approval-report-numbers]').innerText()) + check('探索运行：结论句尚未核对，并说运行结束后才能逐句判断', (await report.locator('[data-approval-report-claims]').innerText()) + .includes('结论句尚未核对') && (await report.innerText()).includes('运行结束后可在证据中逐句请模型判断')) + await report.locator('[data-approval-evidence]').click() + await page.waitForSelector('[data-approval-evidence-doc] [data-evidence-doc]', { timeout: 6000 }) + check('查看逐段证据：卡里展开可点的报告文档，自带证据条', await report.locator('[data-evidence-doc] [data-seg]').count() > 0 + && await report.locator('[data-evidence-tally]').count() === 1) + await report.locator('[data-seg="s6"]').click() + await page.waitForSelector('[data-approval] [data-evidence-panel]', { timeout: 6000 }) + check('点数字打开证据面板，在卡内展开', await page.locator('[data-approval] [data-evidence-panel]').getAttribute('data-evidence-panel') === 'inline') + const overflow = await page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth) + check('展开后不撑破窄栏', overflow <= 0, `${overflow}px`) + await report.locator('[data-approval-evidence]').click() + check('再点一下收起', await report.locator('[data-approval-evidence-doc]').count() === 0) + check('没有运行时报错', errors.length === 0, errors.join(' | ')) + await page.close() + + const f = await open('approval=report-formal&dense=1&actor=张工', { w: 420, h: 700, name: 'approval-report-formal' }) + const formal = await f.page.locator('[data-approval-report="write"]').innerText() + check('正式运行：节点里判过的写判了几句、几句证据相矛盾', formal.includes('结论句已裁判 6 句，证据相矛盾 1 句'), formal.replace(/\n/g, ' ')) + check('正式运行：数字都有出处', formal.includes('12 个数字都有出处')) + check('正式运行：卡上标明正式运行', await f.page.locator('[data-approval-run-class="formal"]').count() === 1) + await f.page.close() +}) + await section('画布右栏：运行视图', async () => { // 真页面、真 store：往 window.__studio 灌事件（同 check-canvas-fx）。工作流、会话、审批、 // 运行全用 page.route 伪造，写操作一律回 409——检查脚本不写库 From 470f8b6641d91a637e4c1274413b3ec84b84e4f3 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 4 Oct 2026 11:33:09 +0800 Subject: [PATCH 11/15] =?UTF-8?q?=E6=8A=A5=E5=91=8A=E9=87=8C=E5=8D=95?= =?UTF-8?q?=E5=85=83=E6=A0=BC=E7=9A=84=20ISO=20=E6=97=A5=E6=9C=9F=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E6=98=BE=E7=A4=BA=E6=88=90=E3=80=8C2026-06-30=2008:21?= =?UTF-8?q?:39=E3=80=8D=EF=BC=8C=E5=8F=AA=E6=94=B9=E6=98=BE=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 报告表格里的时间以前照快照原样显示成 2026-06-30T08:21:39，读起来费劲。[[v:]] 和 [[table:]] 渲染出来的 ISO 日期时间改成用空格隔开（小数秒、时区照留），写作目录和流式输出同一套写法。快照里的值、引用记下的 value、eid、证据下钻都不变。 复核按文档生成时的规则：老文档存的是带 T 的字，拿新规则重新渲染会对不上、被当成文档被改过。所以显示规则也记版本：新文档里真有日期时间被改写时在顶层记 cell_display: 2（参与内容哈希），没有被改写的文档和以前一字不差；没有这个字段的按 1 版（原样）复核，记了认不出的值报 bad_schema。出口复核、期 4 的金样（test_provenance_unchanged）都照旧通过；裁判摘录是给模型看的，不改 --- backend/app/engine/evidence.py | 60 +++++++-- backend/tests/test_report_datetime_display.py | 118 ++++++++++++++++++ 2 files changed, 165 insertions(+), 13 deletions(-) create mode 100644 backend/tests/test_report_datetime_display.py diff --git a/backend/app/engine/evidence.py b/backend/app/engine/evidence.py index ff830cfb..e7fa5237 100644 --- a/backend/app/engine/evidence.py +++ b/backend/app/engine/evidence.py @@ -72,6 +72,15 @@ DOC_SCHEMA = "agentlab.report/1" #: 和记下的状态对不上，会被当成文档被改过（state_mismatch）。 ENTITY_SYNTAX = 2 ENTITY_SYNTAXES = frozenset({1, 2}) +#: 单元格显示规则的版本（文档顶层 cell_display）： +#: - 1：照快照里的原样显示，ISO 日期时间带 T（2026-06-30T08:21:39）。没有这个字段的老文档按它复核 +#: - 2：ISO 日期时间显示成「2026-06-30 08:21:39」（小数秒、时区照留）。只改显示：快照里的值、引用记下的 value、 +#: eid、证据下钻都不变 +#: 新组装的文档里真有日期时间被改写时才记 cell_display: 2（参与内容哈希），没有的文档和以前一字不差。复核必须按 +#: 文档生成时的那一版：老文档存的是带 T 的字，拿新规则重新渲染对不上，会被当成文档被改过（render_mismatch） +CELL_DISPLAY = 2 +CELL_DISPLAYS = frozenset({1, 2}) +_ISO_DATETIME = re.compile(r"^(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?)$") # -------------------------------------------------------------------------- @@ -1273,7 +1282,8 @@ class _Snapshots: compose / verify / 流式渲染里：出口复核是另一次调用，照样重新取、重新验。 """ - def __init__(self, loader: Loader | None = None, *, entities: bool = True, syntax: int | None = None) -> None: + def __init__(self, loader: Loader | None = None, *, entities: bool = True, syntax: int | None = None, + display: int | None = None) -> None: self.loader = loader or load_artifact self.memo: dict[str, tuple[Any, str | None]] = {} self._index: tuple[dict[str, Any], _EntityIndex] | None = None @@ -1281,6 +1291,17 @@ class _Snapshots: self.entities_on = entities #: 这一次解析按哪一版实体语法：组装、流式渲染用当前版，复核按文档记下的版本（verify_doc） self.syntax = ENTITY_SYNTAX if syntax is None else syntax + #: 单元格按哪一版显示规则渲染（CELL_DISPLAY）：组装、流式渲染用当前版，复核按文档记下的版本 + self.display = CELL_DISPLAY if display is None else display + #: 这一次解析里有没有哪一格因为显示规则改了写法：组装时据此决定文档记不记 cell_display + self.reformatted = False + + def shown(self, raw: Any) -> Any: + """按显示规则要显示的原值：2 版起 ISO 日期时间的 T 换成空格。没改就是原值本身。""" + if self.display >= 2 and isinstance(raw, str) and (m := _ISO_DATETIME.match(raw.strip())): + self.reformatted = True + return f"{m.group(1)} {m.group(2)}" + return raw def entities(self, catalog: dict[str, Any]) -> _EntityIndex: if self._index is None or self._index[0] is not catalog: @@ -1289,11 +1310,12 @@ class _Snapshots: self._index = (catalog, _EntityIndex(shown, syntax=self.syntax)) return self._index[1] - def with_syntax(self, syntax: int) -> "_Snapshots": - """换一版实体语法的同一次解析。取过的快照共用：快照内容和语法版本无关，不必再取、再验一遍。""" - if syntax == self.syntax: + def with_syntax(self, syntax: int, display: int | None = None) -> "_Snapshots": + """换一版实体语法（和单元格显示规则）的同一次解析。取过的快照共用：快照内容和版本无关，不必再取、再验一遍。""" + display = self.display if display is None else display + if syntax == self.syntax and display == self.display: return self - twin = _Snapshots(self.loader, entities=self.entities_on, syntax=syntax) + twin = _Snapshots(self.loader, entities=self.entities_on, syntax=syntax, display=display) twin.memo = self.memo return twin @@ -1339,11 +1361,11 @@ class _Snapshots: def _snapshots(loader: Loader | _Snapshots | None, *, entities: bool = True, - syntax: int | None = None) -> _Snapshots: - """loader 已经是 _Snapshots 时沿用它（entities 不看）；给了 syntax 就换成那一版。""" + syntax: int | None = None, display: int | None = None) -> _Snapshots: + """loader 已经是 _Snapshots 时沿用它（entities 不看）；给了 syntax、display 就换成那一版。""" if isinstance(loader, _Snapshots): - return loader if syntax is None else loader.with_syntax(syntax) - return _Snapshots(loader, entities=entities, syntax=syntax) + return loader if syntax is None and display is None else loader.with_syntax(syntax or loader.syntax, display) + return _Snapshots(loader, entities=entities, syntax=syntax, display=display) _CELL_REF = re.compile(r"^(?P[QK]\d+)\.r(?P\d+)\.(?P.+)$") @@ -1393,7 +1415,7 @@ def _resolve_cell(marker: dict[str, Any], catalog: dict[str, Any], snaps: _Snaps try: raw, column = locate_cell(snapshot, parsed["row"], parsed["column"]) kind = column_kind(snapshot, column) - rendered = render_cell(raw, conv, kind=kind) + rendered = render_cell(snaps.shown(raw), conv, kind=kind) except CellError as e: return _unresolved(marker, alias, "cell", f"{alias} {e}") except RenderError as e: @@ -2630,8 +2652,10 @@ def compose_doc( # entity_syntax：这份文档按哪一版实体语法组装，复核照它走（verify_doc）。在文档顶层，参与内容哈希 # provenance：期 4 的文档标记（P4-SPEC 1.4）。带它的文档才有查询步骤的提示字段、推断来源、裁判摘录的新行； # 没有它的（期 4 之前组装的）一律按原样展示。写在顶层、参与内容哈希，事后补不上：补了哈希就变，封存链就断 + # cell_display：有单元格因为显示规则改了写法（ISO 日期时间）才记，复核照它渲染；没有的文档和以前一字不差 doc: dict[str, Any] = { "schema": DOC_SCHEMA, "entity_syntax": snaps.syntax, "provenance": DOC_PROVENANCE, + **({"cell_display": snaps.display} if snaps.reformatted and snaps.display != 1 else {}), "run_id": run_id, "node_id": node_id, "markdown": markdown, "source": source, "catalog": kept, "blocks": out_blocks, } @@ -2785,7 +2809,8 @@ def verify_doc( 有知识库检索时才有（它们才可能不是 0）：升级前的运行统计的键和以前一模一样。 """ entity_syntax = doc_entity_syntax(doc) - snaps = _snapshots(loader, entities=entities, syntax=entity_syntax or ENTITY_SYNTAX) + display = doc_cell_display(doc) + snaps = _snapshots(loader, entities=entities, syntax=entity_syntax or ENTITY_SYNTAX, display=display or CELL_DISPLAY) index = snaps.entities(catalog) violations: list[dict[str, Any]] = [] uncited: list[dict[str, Any]] = [] @@ -2803,6 +2828,8 @@ def verify_doc( if entity_syntax is None: violations.append(_violation("bad_schema", f"报告文档记录的表名、字段名规则版本「{doc.get('entity_syntax')}」无法识别")) + if display is None: + violations.append(_violation("bad_schema", f"报告文档记录的单元格显示规则版本「{doc.get('cell_display')}」无法识别")) markdown = str(doc.get("markdown") or "") masks: list[tuple[int, int]] = [] # 引用渲染出来的字 @@ -2949,6 +2976,13 @@ def verify_doc( return {"ok": not violations, "violations": violations, "stats": stats, "uncited": uncited} +def doc_cell_display(doc: Any) -> int | None: + """文档按哪一版单元格显示规则组装：没有 cell_display 字段的是 1 版（老文档、没有日期时间被改写的文档）， + 记了认不出的值返回 None。""" + raw = doc.get("cell_display", 1) if isinstance(doc, dict) else 1 + return raw if type(raw) is int and raw in CELL_DISPLAYS else None + + def doc_entity_syntax(doc: Any) -> int | None: """文档按哪一版实体语法组装：没有 entity_syntax 字段的是老文档（1 版），记了认不出的值返回 None。""" raw = doc.get("entity_syntax", 1) if isinstance(doc, dict) else 1 @@ -3182,7 +3216,7 @@ def _query_prompt(entry: dict[str, Any], snaps: _Snapshots, cells_allowed: bool, try: raw, _ = locate_cell(snapshot, r, c) kind = column_kind(snapshot, c) - shown, numeric = render_cell(raw, kind=kind), is_number(cell_value(raw, kind)) + shown, numeric = render_cell(snaps.shown(raw), kind=kind), is_number(cell_value(raw, kind)) except (CellError, RenderError): shown, numeric = MISSING, False cells.append(f"{c}={shown}") @@ -3327,7 +3361,7 @@ def _field_prompt(path: str, value: Any, ev: dict[str, Any], aliases: dict[str, if why: raise CellError(why) raw, column = locate_cell(snapshot, row, column) - shown = render_cell(raw, kind=column_kind(snapshot, column)) + shown = render_cell(snaps.shown(raw), kind=column_kind(snapshot, column)) except (CellError, RenderError): try: shown = render_cell(value) diff --git a/backend/tests/test_report_datetime_display.py b/backend/tests/test_report_datetime_display.py new file mode 100644 index 00000000..946e6eda --- /dev/null +++ b/backend/tests/test_report_datetime_display.py @@ -0,0 +1,118 @@ +"""报告里单元格的 ISO 日期时间显示成「2026-06-30 08:21:39」，不再是「2026-06-30T08:21:39」。 + +只改显示：快照里的值、引用记下的 value、eid、证据下钻都不变。复核要按文档生成时的规则：老文档里存的是带 T 的字， +拿新规则重新渲染会对不上，被当成文档被改过（render_mismatch）——所以新文档在顶层记 cell_display: 2（只在真有日期 +时间被改写时记，没有的文档和以前一字不差），没有这个字段的老文档按 1 版（原样）复核。期 4 的金样 +（test_provenance_unchanged）照旧逐字不变。 + +夹具合成：门店、卡券。 +""" +from __future__ import annotations + +import pytest + +from app.core.artifact_store import canonical_json +from app.engine import evidence as ev +from app.engine.evidence import ( + CELL_DISPLAY, + StreamRenderer, + build_catalog, + cell_eid, + compose_doc, + doc_cell_display, + iter_segments, + verify_doc, + writer_catalog, +) + +ART = "d" * 64 +SNAP = {"columns": ["store_name", "redeemed_at", "biz_date", "amount", "note"], + "column_types": {"store_name": "text", "redeemed_at": "datetime", "biz_date": "date", "amount": "number", + "note": "text"}, + "rows": [["门店01", "2026-06-30T08:21:39", "2026-06-30", 120, "T 开头的备注"], + ["门店02", "2026-06-29T23:05:00.250000+08:00", "2026-06-29", 80, "2026-06-29T10:00:00Z"]], + "truncated": False} +STORE = {ART: SNAP} + + +def load(artifact): + return STORE.get(artifact) + + +@pytest.fixture +def catalog(): + ledger = [{"kind": "query", "node_id": "fetch", "exec": 1, "artifact": ART, "tool": "db_query__shop", + "columns": SNAP["columns"], "rows": 2, "truncated": False}] + return build_catalog(nodes={}, ledger=ledger) + + +def cells(doc): + return [s for s in iter_segments(doc) if s.get("ref", "").startswith("v:")] + + +def test_iso_datetimes_read_with_a_space(catalog): + doc = compose_doc("最近一笔核销在 [[v:Q1.r0.redeemed_at]]，门店 [[v:Q1.r0.store_name]]。[[see:Q1]]", catalog, loader=load) + [when, store] = cells(doc) + assert when["text"] == "2026-06-30 08:21:39" and "2026-06-30 08:21:39" in doc["markdown"] + # 只改显示：引用记下的值、证据标识照旧 + assert when["cite"]["value"] == "2026-06-30T08:21:39" + assert when["cite"]["eid"] == cell_eid(ART, 0, "redeemed_at") + assert store["text"] == "门店01" + assert doc["cell_display"] == CELL_DISPLAY == 2 and doc_cell_display(doc) == 2 + assert '"cell_display":2' in canonical_json(doc), "在文档顶层，参与内容哈希" + assert verify_doc(doc, catalog, loader=load)["ok"] + + +def test_fractions_and_offsets_are_kept_and_dates_are_untouched(catalog): + doc = compose_doc("时间 [[v:Q1.r1.redeemed_at]]，日期 [[v:Q1.r1.biz_date]]，备注 [[v:Q1.r1.note]]。", catalog, + loader=load) + assert [s["text"] for s in cells(doc)] == ["2026-06-29 23:05:00.250000+08:00", "2026-06-29", "2026-06-29 10:00:00Z"] + assert verify_doc(doc, catalog, loader=load)["ok"] + + +def test_tables_read_the_same_way(catalog): + doc = compose_doc("[[table:Q1 cols=store_name,redeemed_at]]", catalog, loader=load) + assert "| 门店01 | 2026-06-30 08:21:39 |" in doc["markdown"] + assert "T08:21:39" not in doc["markdown"] + assert verify_doc(doc, catalog, loader=load)["ok"] + + +def test_documents_without_datetimes_are_unchanged(catalog): + doc = compose_doc("门店 [[v:Q1.r0.store_name]]，金额 [[v:Q1.r0.amount]]。", catalog, loader=load) + assert "cell_display" not in doc, "没有被改写的日期时间：文档和以前一字不差" + assert doc_cell_display(doc) == 1 + + +def test_old_documents_still_verify_as_they_were(catalog, monkeypatch): + """改动前组装的文档：存的是带 T 的字、没有 cell_display。按 1 版复核，不报 render_mismatch。""" + monkeypatch.setattr(ev, "CELL_DISPLAY", 1) + old = compose_doc("最近一笔核销在 [[v:Q1.r0.redeemed_at]]。", catalog, loader=load) + monkeypatch.undo() + assert "cell_display" not in old and "2026-06-30T08:21:39" in old["markdown"] + checked = verify_doc(old, catalog, loader=load) + assert checked["ok"], checked["violations"] + + +def test_a_document_edited_back_is_caught(catalog): + doc = compose_doc("最近一笔核销在 [[v:Q1.r0.redeemed_at]]。", catalog, loader=load) + [seg] = cells(doc) + start = seg["span"][0] + doc["markdown"] = doc["markdown"].replace("2026-06-30 08:21:39", "2026-06-30T08:21:39") + seg["text"] = "2026-06-30T08:21:39" + seg["span"] = [start, start + len(seg["text"])] + codes = {v["code"] for v in verify_doc(doc, catalog, loader=load)["violations"]} + assert "render_mismatch" in codes + + +def test_an_unknown_display_version_is_reported(catalog): + doc = compose_doc("最近一笔核销在 [[v:Q1.r0.redeemed_at]]。", catalog, loader=load) + doc["cell_display"] = 9 + assert "bad_schema" in {v["code"] for v in verify_doc(doc, catalog, loader=load)["violations"]} + + +def test_the_writer_and_the_stream_see_what_the_report_will_show(catalog): + text = writer_catalog(catalog, loader=load).text + assert "redeemed_at=2026-06-30 08:21:39" in text and "T08:21:39" not in text + stream = StreamRenderer(catalog, loader=load) + shown = stream.feed("最近一笔在 [[v:Q1.r0.redeemed_at]]。") + stream.flush() + assert "2026-06-30 08:21:39" in shown From fa842a7bec1e9eb451feab409118788284d68c51 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 4 Oct 2026 12:03:14 +0800 Subject: [PATCH 12/15] =?UTF-8?q?=E4=BA=A4=E7=BB=99=E4=BA=BA=E5=B7=A5?= =?UTF-8?q?=E5=AE=A1=E6=89=B9=E7=9A=84=E6=8A=A5=E5=91=8A=EF=BC=8C=E6=8E=A2?= =?UTF-8?q?=E7=B4=A2=E8=BF=90=E8=A1=8C=E4=B9=9F=E5=9C=A8=E8=8A=82=E7=82=B9?= =?UTF-8?q?=E9=87=8C=E8=A3=81=E5=88=A4=E7=BB=93=E8=AE=BA=E5=8F=A5=EF=BC=9A?= =?UTF-8?q?=E5=AE=A1=E6=89=B9=E4=BA=BA=E6=89=B9=E5=87=86=E5=89=8D=E5=B0=B1?= =?UTF-8?q?=E8=83=BD=E7=9C=8B=E5=88=B0=E5=88=A4=E4=BA=86=E5=87=A0=E5=8F=A5?= =?UTF-8?q?=E3=80=81=E5=87=A0=E5=8F=A5=E8=AF=81=E6=8D=AE=E7=9B=B8=E7=9F=9B?= =?UTF-8?q?=E7=9B=BE=E3=80=82=E6=8C=89=E9=9C=80=E8=A3=81=E5=88=A4=E8=A6=81?= =?UTF-8?q?=E7=AD=89=E8=BF=90=E8=A1=8C=E5=B0=81=E5=AD=98=E4=B9=8B=E5=90=8E?= =?UTF-8?q?=E6=89=8D=E8=83=BD=E5=88=A4=EF=BC=8C=E5=B0=81=E5=AD=98=E5=89=8D?= =?UTF-8?q?=E6=94=BE=E5=BC=80=E4=BC=9A=E5=8A=A8=E8=AF=81=E6=8D=AE=E6=B8=85?= =?UTF-8?q?=E5=8D=95=E7=9A=84=E5=AE=8C=E6=95=B4=E6=80=A7=E7=BA=A6=E6=9D=9F?= =?UTF-8?q?=EF=BC=8C=E6=89=80=E4=BB=A5=E8=BF=99=E7=A7=8D=E6=8A=A5=E5=91=8A?= =?UTF-8?q?=E6=94=B9=E5=9C=A8=E8=8A=82=E7=82=B9=E9=87=8C=E5=BD=93=E5=9C=BA?= =?UTF-8?q?=E5=88=A4=EF=BC=8C=E5=92=8C=E6=AD=A3=E5=BC=8F=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E5=90=8C=E4=B8=80=E6=9D=A1=E8=B7=AF=EF=BC=9B=E6=B2=A1=E6=9C=89?= =?UTF-8?q?=E5=AE=A1=E6=89=B9=E7=9A=84=E6=8E=A2=E7=B4=A2=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E7=85=A7=E6=97=A7=E6=8C=89=E9=9C=80=E5=88=A4=EF=BC=8C=E4=B8=8D?= =?UTF-8?q?=E5=9C=A8=E8=8A=82=E7=82=B9=E9=87=8C=E8=8A=B1=E9=92=B1=E3=80=82?= =?UTF-8?q?=E5=AE=A1=E6=89=B9=E8=8A=82=E7=82=B9=E5=AE=A1=E5=93=AA=E5=87=A0?= =?UTF-8?q?=E4=BB=BD=E6=8A=A5=E5=91=8A=E7=9A=84=E8=AE=A4=E6=B3=95=E6=8A=BD?= =?UTF-8?q?=E6=88=90=20reviewed=5Freport=5Fids=EF=BC=8C=E5=AE=A1=E6=89=B9?= =?UTF-8?q?=E5=8D=A1=E5=92=8C=E6=8A=A5=E5=91=8A=E8=8A=82=E7=82=B9=E5=85=B1?= =?UTF-8?q?=E7=94=A8=EF=BC=8C=E4=B8=A4=E8=BE=B9=E8=AF=B4=E7=9A=84=E3=80=8C?= =?UTF-8?q?=E8=A6=81=E4=BA=BA=E5=AE=A1=E3=80=8D=E5=AF=B9=E5=BE=97=E4=B8=8A?= =?UTF-8?q?=E3=80=82=E5=AE=A1=E6=89=B9=E5=8D=A1=E4=B8=8A=E6=B2=A1=E5=BC=80?= =?UTF-8?q?=E8=A3=81=E5=88=A4=E7=9A=84=E6=8E=A2=E7=B4=A2=E6=8A=A5=E5=91=8A?= =?UTF-8?q?=E6=94=B9=E5=86=99=E3=80=8C=E8=BF=99=E4=BB=BD=E6=8A=A5=E5=91=8A?= =?UTF-8?q?=E6=B2=A1=E6=9C=89=E5=BC=80=E5=90=AF=E7=BB=93=E8=AE=BA=E5=8F=A5?= =?UTF-8?q?=E8=A3=81=E5=88=A4=E2=80=A6=E2=80=A6=E5=BC=80=E5=90=AF=E5=90=8E?= =?UTF-8?q?=EF=BC=8C=E4=BA=A4=E7=BB=99=E5=AE=A1=E6=89=B9=E7=9A=84=E6=8A=A5?= =?UTF-8?q?=E5=91=8A=E4=BC=9A=E5=9C=A8=E6=89=B9=E5=87=86=E5=89=8D=E5=88=A4?= =?UTF-8?q?=E5=AE=8C=E3=80=8D=EF=BC=9B=E5=BC=80=E4=BA=86=E8=A3=81=E5=88=A4?= =?UTF-8?q?=E5=8D=B4=E6=B2=A1=E5=9C=A8=E8=8A=82=E7=82=B9=E9=87=8C=E5=88=A4?= =?UTF-8?q?=E7=9A=84=E8=80=81=E8=BF=90=E8=A1=8C=E5=8F=A6=E7=BB=99=E4=B8=80?= =?UTF-8?q?=E5=8F=A5=E3=80=82=E5=8A=A9=E6=89=8B=E6=8C=87=E5=BC=95=E3=80=81?= =?UTF-8?q?=E5=87=BA=E5=8F=A3=E5=A5=91=E7=BA=A6=E6=B3=A8=E9=87=8A=E3=80=81?= =?UTF-8?q?=E5=89=8D=E7=AB=AF=E7=B1=BB=E5=9E=8B=E6=B3=A8=E9=87=8A=E5=90=8C?= =?UTF-8?q?=E6=AD=A5=EF=BC=9B=E5=AE=A1=E6=89=B9=E6=9A=82=E5=81=9C=E6=97=B6?= =?UTF-8?q?=E6=8C=89=E9=9C=80=E8=A3=81=E5=88=A4=E4=BB=8D=E6=8B=92=E7=BB=9D?= =?UTF-8?q?=EF=BC=8C=E6=B5=8B=E8=AF=95=E6=94=B9=E6=88=90=E5=8F=AA=E7=9C=8B?= =?UTF-8?q?=E7=82=B9=E5=87=BB=E4=B8=8D=E5=A4=9A=E4=B8=80=E6=AC=A1=E8=B0=83?= =?UTF-8?q?=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/copilot.py | 5 +-- backend/app/engine/nodes/human.py | 39 +++++++++++++++------ backend/app/engine/nodes/io.py | 6 ++-- backend/app/engine/nodes/report.py | 8 +++-- backend/tests/test_approval_report_check.py | 17 +++++++-- backend/tests/test_evidence_judge_api.py | 6 +++- frontend/src/dev/preview.tsx | 9 +++-- frontend/src/lib/terms.ts | 6 ++-- frontend/src/run/ApprovalReport.tsx | 4 ++- frontend/src/types.ts | 9 ++--- scripts/check-stream.mjs | 5 +-- 11 files changed, 79 insertions(+), 35 deletions(-) diff --git a/backend/app/api/copilot.py b/backend/app/api/copilot.py index fe0334c9..6b849d96 100644 --- a/backend/app/api/copilot.py +++ b/backend/app/api/copilot.py @@ -237,13 +237,14 @@ NODE_REFERENCE = """\ 挂引用，没挂的计入缺口、出具降档）/ judge（在 require_citation 之外，再请另一个模型按每句挂的依据判断支不支持： 正式运行在节点里判，证据相矛盾的按 judge.on_unsupported 处置——degrade 降档（默认）或 withhold 不予出具， 证据不足的（摘录里没有判断所需的信息）只降档； - 探索运行不在节点里花钱，读的人点开哪句再按需判哪句，只标注）。judge 的子配置写在节点的 judge 里：judge.model、 + 探索运行不在节点里花钱，读的人点开哪句再按需判哪句，只标注；要交给人工审批的报告例外，探索运行也在节点里判， + 审批人批准前就能看到判了几句、几句证据相矛盾）。judge 的子配置写在节点的 judge 里：judge.model、 judge.provider 指定裁判模型（建议和写作模型不同，同一个模型等于自己审自己；不写就用设置里的「证据裁判模型」）； judge.max_cost_usd、judge.max_claims、judge.timeout_s 是每份报告的裁判金额、句数、时长上限，写 null 表示不限， 不写取设置里的默认；judge.rewrite_once 默认 false，打开后把证据相矛盾、证据不足的句子交回写作者只改一次。 **多个来源汇总成的分析报告**（几个 agent 分头查库，或者几次查询的结果合在一起写成的分析、结论）默认写 claims: "judge"， 不写 judge 的预算（judge 整个不写，金额、句数、时长上限都取设置里的默认）：正式运行在节点里自动判，探索运行由读的人 - 点开哪句判哪句，不在节点里花钱。用户明确说不要核对结论句的不加；只回答一个问题、一次查询的不必加。要按受管级别 + 点开哪句判哪句，不在节点里花钱；报告要交给人工审批的，探索运行也在节点里判。用户明确说不要核对结论句的不加；只回答一个问题、一次查询的不必加。要按受管级别 发布的模板不适用这条默认，按下面受管级别的规则写预算 entities 默认 link （核对表名、字段名），报告里根本不提表名时才写 off diff --git a/backend/app/engine/nodes/human.py b/backend/app/engine/nodes/human.py index 29bd3fc1..f314011b 100644 --- a/backend/app/engine/nodes/human.py +++ b/backend/app/engine/nodes/human.py @@ -80,9 +80,35 @@ _VAR_REF = re.compile(r"\bvars\s*\.\s*([\w-]+)") _JUDGE_KEYS = ("mode", "counts", "complete", "unjudged", "limits_hit") +def reviewed_report_ids(spec: Any, human: Any) -> list[str]: + """一个人工审批节点审的是哪几份报告：上游直接连着的报告撰写节点，加上标题、说明里引用了正文的 + （{{ nodes..text }}，或者报告的 assign_to 变量）。只看图，不看运行状态。 + + 审批卡（_reviewed_reports）和报告节点（reviewed_by_human：要交给人审的报告，探索运行也在节点里判） + 用的是同一套认法，两边说的「这份报告要人审」不会对不上。""" + from app.engine.schema import NodeType + + config = human.config or {} + text = " ".join(json.dumps(config.get(key), ensure_ascii=False, default=str) + for key in ("title", "message", "context", "draft") if config.get(key)) + # 报告节点的 assign_to 也能把正文交给下游：{{ vars. }} 和 {{ nodes..text }} 一样算引用 + by_var = {str(n.config.get("assign_to")): n.id for n in spec.nodes + if n.type == NodeType.REPORT and n.config.get("assign_to")} + nodes = spec.node_map() + wanted = ([e.source for e in spec.edges if e.target == human.id] + _NODE_REF.findall(text) + + [by_var[v] for v in _VAR_REF.findall(text) if v in by_var]) + return [rid for rid in dict.fromkeys(wanted) if (n := nodes.get(rid)) is not None and n.type == NodeType.REPORT] + + +def reviewed_by_human(spec: Any, report_id: str) -> bool: + """这份报告会不会交给人工审批（有审批节点审它，认法同 reviewed_report_ids）。""" + from app.engine.schema import NodeType + + return any(report_id in reviewed_report_ids(spec, n) for n in spec.nodes if n.type == NodeType.HUMAN) + + def _reviewed_reports(state: GraphState, ctx: NodeContext) -> list[dict[str, Any]]: - """这次审批审的报告各自的核对概况：上游直接连着的报告撰写节点，加上标题、说明里引用了正文的 - （{{ nodes..text }}，或者报告的 assign_to 变量）。 + """这次审批审的报告各自的核对概况（审的是哪几份见 reviewed_report_ids）。 真实运行里报告有一句结论和证据相矛盾，审批人批准时完全不知道：审批卡只有正文。这里带上报告节点产出里的 stats（同 report.checked）、结论句策略、裁判摘要和报告文档的工件 id，审批卡据此写核对概况、给打开证据的入口。 @@ -91,16 +117,9 @@ def _reviewed_reports(state: GraphState, ctx: NodeContext) -> list[dict[str, Any spec = ctx.run.spec nodes = spec.node_map() - text = " ".join(json.dumps(ctx.config.get(key), ensure_ascii=False, default=str) - for key in ("title", "message", "context", "draft") if ctx.config.get(key)) - # 报告节点的 assign_to 也能把正文交给下游：{{ vars. }} 和 {{ nodes..text }} 一样算引用 - by_var = {str(n.config.get("assign_to")): n.id for n in spec.nodes - if n.type == NodeType.REPORT and n.config.get("assign_to")} - wanted = ([e.source for e in spec.edges if e.target == ctx.node.id] + _NODE_REF.findall(text) - + [by_var[v] for v in _VAR_REF.findall(text) if v in by_var]) outputs = state.get("nodes") or {} out: list[dict[str, Any]] = [] - for rid in dict.fromkeys(wanted): + for rid in reviewed_report_ids(spec, ctx.node): node = nodes.get(rid) result = outputs.get(rid) if node is None or node.type != NodeType.REPORT or not isinstance(result, dict) \ diff --git a/backend/app/engine/nodes/io.py b/backend/app/engine/nodes/io.py index a7de8542..e289aa84 100644 --- a/backend/app/engine/nodes/io.py +++ b/backend/app/engine/nodes/io.py @@ -467,10 +467,10 @@ def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, An 里 ignore 一律当 degrade：没有哪道门禁查契约里的 claims，不能让它把用户要的底线拉低。 报告节点写的是 judge 时，判定在文档里（按哈希取回的那份：判定是模型给的，出口复算不了，只能信封存的 - 文档）。正式运行在节点里判过：证据相矛盾的（contradicted，旧文档里的 unsupported 同样算）按 on_unsupported + 文档）。节点里判过的（正式运行，以及交给人工审批的探索运行）：证据相矛盾的（contradicted，旧文档里的 unsupported 同样算）按 on_unsupported 判档（报告节点和契约取更严的）；证据不足的（insufficient）记一条缺口——最多降档，不会不予出具，也不会 完整出具；没判完的（裁判没跑成、触顶）把裁判摘要里的缺口照抄进来——不能判完整出具，也不当成「矛盾」。 - 探索运行按需裁判，这里只标注。契约要 judge、报告节点却没开的，照实记缺口：出口替不了报告节点去裁判。 + 其余探索运行按需裁判，这里只标注。契约要 judge、报告节点却没开的，照实记缺口：出口替不了报告节点去裁判。 数的是这里重新核对出来的 uncited，不信文档里记的 cites。升级前写的报告（节点产出里没有 claims 这个键， 包括跨着升级还没跑完的运行）一律不管，契约写了也只记一条缺口：报告生成于升级前，结论句未核对。 @@ -520,7 +520,7 @@ def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, An inline = summary.get("mode") == "inline" flagged: dict[str, list[dict[str, Any]]] = {s: [] for s in ("contradicted", "insufficient", UNSUPPORTED, "partial")} if inline: - # 探索运行按需裁判的判定不在文档里（在封存之后追加的 evidence.judged 事件里），这里只数正式运行的 + # 按需裁判的判定不在文档里（在封存之后追加的 evidence.judged 事件里），这里只数节点里判过的 markdown = str((doc or {}).get("markdown") or "") for _, unit in iter_units(doc or {}): verdict = unit.get("verdict") diff --git a/backend/app/engine/nodes/report.py b/backend/app/engine/nodes/report.py index c9bedf60..920930bc 100644 --- a/backend/app/engine/nodes/report.py +++ b/backend/app/engine/nodes/report.py @@ -523,9 +523,13 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]: judgement: dict[str, Any] | None = None if claims == CLAIMS_JUDGE: + from app.engine.nodes.human import reviewed_by_human + judge_cfg = report_judge(spec, ctx.node) - if await _run_class(ctx.run.run_id) != "formal": - # 探索运行默认按需裁判：点开哪句才判哪句，节点里不花这笔钱 + if await _run_class(ctx.run.run_id) != "formal" and not reviewed_by_human(spec, ctx.node.id): + # 探索运行默认按需裁判：点开哪句才判哪句，节点里不花这笔钱。要交给人工审批的报告例外：审批人据此 + # 拍板，批准前就该知道哪句结论站不住；而按需裁判要等运行封存之后才能判，封存前放开会动证据清单的 + # 完整性约束，所以这种报告探索运行也在节点里当场判（和正式运行同一条路） judgement = judging.on_demand(doc, on_unsupported=judge_cfg["on_unsupported"], rewrite_once=judge_cfg["rewrite_once"], writer_model=model_id) elif not (_fatal(blocking) and on_violation == "fail"): diff --git a/backend/tests/test_approval_report_check.py b/backend/tests/test_approval_report_check.py index cd13c261..f727ff42 100644 --- a/backend/tests/test_approval_report_check.py +++ b/backend/tests/test_approval_report_check.py @@ -122,11 +122,22 @@ async def test_formal_run_carries_the_judged_claims_to_the_approval(): assert requested.data["reports"] == approval.payload["reports"] -async def test_exploratory_run_says_claims_are_judged_on_demand(): +async def test_exploratory_report_under_approval_is_judged_in_the_node(): + """要交给人审批的报告，探索运行也在节点里判：审批人据此拍板，批准前就该知道哪句结论站不住。 + 真实运行里探索运行的报告有一句和证据相矛盾，审批卡只能写「结论句尚未核对」——按需裁判要等封存 + （运行结束）之后才能判，而封存前放开按需裁判会动证据清单的完整性约束，所以改在节点里当场判。 + 没有审批的探索运行照旧按需判，不在节点里花这笔钱（test_contract_judge.test_explore_runs_only_annotate）。""" _, approval = await paused(graph(gate={"title": "审核周报"}, claims="judge"), run_class="exploratory") [report] = approval.payload["reports"] - assert report["claims"] == "judge" and report["judge"]["mode"] == "on_demand" - assert report["judge"]["counts"]["contradicted"] == 0, "探索运行不在节点里判" + assert report["claims"] == "judge" and report["judge"]["mode"] == "inline" + assert report["judge"]["counts"]["contradicted"] == 1 and report["judge"]["counts"]["unjudged"] == 0 + + +async def test_report_quoted_in_the_approval_message_is_judged_in_the_node_too(): + _, approval = await paused(graph(gate={"title": "审核", "message": "请看：{{ nodes.write.text }}"}, + wire_report_to_gate=False, claims="judge"), run_class="exploratory") + [report] = approval.payload["reports"] + assert report["judge"]["mode"] == "inline" and report["judge"]["counts"]["contradicted"] == 1 async def test_report_without_claims_policy_still_reports_numbers(): diff --git a/backend/tests/test_evidence_judge_api.py b/backend/tests/test_evidence_judge_api.py index 1f87e71e..598d5cad 100644 --- a/backend/tests/test_evidence_judge_api.py +++ b/backend/tests/test_evidence_judge_api.py @@ -374,11 +374,15 @@ async def test_formal_runs_are_judged_inside_the_node_not_on_demand(client, monk async def test_a_run_paused_at_an_approval_is_not_judged(client, monkeypatch): + """停在审批上的运行还没封存，按需裁判不接。交给审批的报告探索运行也在节点里判过了 + （test_approval_report_check），这里只看点击：不再多一次裁判调用，也不追加判定事件。""" models = Models(monkeypatch) row = await finish(weekly(claims="judge", gate=True), statuses=("interrupted",)) + judged_in_node = list(models.judges) + assert judged_in_node res = await judge(client, row.id, ["u1"]) assert res.status_code == 409 and res.json()["code"] == "evidence_judge_unsealed" - assert models.judges == [] and await judged_events(row.id) == [] + assert models.judges == judged_in_node and await judged_events(row.id) == [] async def test_runs_from_before_the_upgrade_are_not_judged(client, monkeypatch): diff --git a/frontend/src/dev/preview.tsx b/frontend/src/dev/preview.tsx index 89dc111c..67dd671c 100644 --- a/frontend/src/dev/preview.tsx +++ b/frontend/src/dev/preview.tsx @@ -495,16 +495,15 @@ const APPROVAL = { /** * ?approval=report：探索运行里审一份报告。核对概况取 evidence-doc 夹具（7/12 数字有出处、1 处引用解析不了）， - * 结论句没在节点里判；?approval=report-formal：正式运行，节点里判过，1 句证据相矛盾 + * 报告没开结论句裁判；?approval=report-formal：正式运行，节点里判过，1 句证据相矛盾 */ const APPROVAL_REPORT = { id: 'ap-2', run_id: evidenceFixture.run_id, node_id: 'review', mode: 'approve' as const, title: '本周周报可以发出吗？', status: 'pending', response: {}, payload: { kind: 'human_node', message: '请审核本周周报。', - reports: [{ node_id: 'write', label: '写周报', stats: evidenceFixture.doc.stats, claims: 'judge', - doc_artifact: evidenceFixture.doc_artifact, - judge: { mode: 'on_demand', counts: { unjudged: 4 }, complete: false } }], + reports: [{ node_id: 'write', label: '写周报', stats: evidenceFixture.doc.stats, claims: 'off', + doc_artifact: evidenceFixture.doc_artifact }], }, created_at: new Date(Date.now() - 120_000).toISOString().replace('Z', ''), workflow_name: '周报', node_label: '主管审批', run_class: 'exploratory' as const, @@ -513,7 +512,7 @@ const APPROVAL_REPORT_FORMAL = { ...APPROVAL_REPORT, id: 'ap-3', run_class: 'formal' as const, payload: { ...APPROVAL_REPORT.payload, - reports: [{ ...APPROVAL_REPORT.payload.reports[0], stats: { ...evidenceFixture.doc.stats, numbers_cited: 12, uncited_numbers: 0, unresolved: 0 }, + reports: [{ ...APPROVAL_REPORT.payload.reports[0], claims: 'judge', stats: { ...evidenceFixture.doc.stats, numbers_cited: 12, uncited_numbers: 0, unresolved: 0 }, judge: { mode: 'inline', counts: { supported: 5, contradicted: 1, unjudged: 0 }, complete: true } }], }, } diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index ec222135..cb37afe9 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -104,8 +104,10 @@ export const APPROVAL_TEXT = { + (unjudged ? `，未裁判 ${formatNumber(unjudged)} 句` : ''), claimsNone: '报告中没有需要裁判的结论句', claimsPending: '结论句尚未核对', - /** 探索运行停在审批上时还没封存，按需裁判要等运行结束 */ - claimsLater: '探索运行不在节点内裁判，运行结束后可在证据中逐句请模型判断', + /** 探索运行、报告没开结论句裁判：按需裁判要等运行结束封存之后。开了裁判的报告交给审批时会在节点里当场判，不走这句 */ + claimsLater: '这份报告没有开启结论句裁判，运行结束后可在证据中逐句请模型判断；开启后，交给审批的报告会在批准前判完', + /** 开了结论句裁判、却没在节点里判（这条规则之前的探索运行）：按需裁判要等运行结束封存之后 */ + claimsAfterRun: '这次没有在节点内裁判，运行结束后可在证据中逐句请模型判断', /** 正式运行、报告没开结论句裁判 */ claimsOff: '这份报告没有开启结论句裁判', openEvidence: '查看逐段证据', diff --git a/frontend/src/run/ApprovalReport.tsx b/frontend/src/run/ApprovalReport.tsx index 40fb718f..c1abb21d 100644 --- a/frontend/src/run/ApprovalReport.tsx +++ b/frontend/src/run/ApprovalReport.tsx @@ -40,7 +40,9 @@ function claimsLine(report: ApprovalReport, runClass: RunClass): { text: string; } return { text: APPROVAL_TEXT.claimsPending, - sub: runClass === 'formal' ? (report.claims === 'judge' ? undefined : APPROVAL_TEXT.claimsOff) : APPROVAL_TEXT.claimsLater, + sub: runClass === 'formal' ? (report.claims === 'judge' ? undefined : APPROVAL_TEXT.claimsOff) + // 开了裁判却没在节点里判的，只会是这条规则之前的探索运行（现在交给审批的报告探索运行也当场判） + : report.claims === 'judge' ? APPROVAL_TEXT.claimsAfterRun : APPROVAL_TEXT.claimsLater, warn: true, } } diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 3d176275..0b1c6c84 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -176,7 +176,8 @@ export interface Approval { /** * 人工审批审的报告的核对概况（human 节点请求里的 payload.reports，服务端 nodes/human._reviewed_reports）： * 上游直接连着的、或者标题和说明里引用了正文的报告撰写节点，各一项。stats 同 report.checked，judge 是裁判摘要 - * 的几项（mode：inline 节点里当场判的 / on_demand 探索运行按需判）。老后端、审的不是报告时没有 + * 的几项（mode：inline 节点里当场判的——交给审批的报告探索运行也当场判 / on_demand 按需判，老服务端的探索运行）。 + * 老后端、审的不是报告时没有 */ export interface ApprovalReport { node_id: string @@ -1797,8 +1798,8 @@ export interface EvidenceUnit { /** 列表项的缩进层级 */ depth?: number /** - * 结论句裁判的判定（四期）：只有送了裁判的候选句有。正式运行里节点当场判的进封存（post_seal false）； - * 探索运行里候选句先标 unjudged + reason on_demand，点开再判 + * 结论句裁判的判定（四期）：只有送了裁判的候选句有。正式运行（以及交给人工审批的探索运行）里节点当场判的 + * 进封存（post_seal false）；其余探索运行里候选句先标 unjudged + reason on_demand，点开再判 */ verdict?: EvidenceVerdict } @@ -1835,7 +1836,7 @@ export interface EvidenceVerdict { /** 文档（和 report.checked、节点产出）里的裁判摘要：键的顺序同后端 judge.SUMMARY_KEYS */ export interface EvidenceJudgeSummary { - /** inline：正式运行节点里当场判；on_demand：探索运行按需 */ + /** inline：节点里当场判（正式运行，以及交给人工审批的探索运行）；on_demand：其余探索运行按需 */ mode?: 'inline' | 'on_demand' | string model?: string | null on_unsupported?: 'degrade' | 'withhold' | string diff --git a/scripts/check-stream.mjs b/scripts/check-stream.mjs index c95b0319..651db9af 100644 --- a/scripts/check-stream.mjs +++ b/scripts/check-stream.mjs @@ -958,8 +958,9 @@ await section('审批卡：审的报告的核对概况', async () => { check('报告的核对概况：标题写哪份报告', (await report.innerText()).includes('报告「写周报」的核对')) check('数字和证据条同一种说法', (await report.locator('[data-approval-report-numbers]').innerText()) .includes('7/12 数字有出处 · 无证据 5 · 另有 1 处引用无法解析'), await report.locator('[data-approval-report-numbers]').innerText()) - check('探索运行：结论句尚未核对，并说运行结束后才能逐句判断', (await report.locator('[data-approval-report-claims]').innerText()) - .includes('结论句尚未核对') && (await report.innerText()).includes('运行结束后可在证据中逐句请模型判断')) + check('探索运行、报告没开裁判：结论句尚未核对，说明运行结束后可逐句判断、开启后会在批准前判完', + (await report.locator('[data-approval-report-claims]').innerText()).includes('结论句尚未核对') + && (await report.innerText()).includes('没有开启结论句裁判') && (await report.innerText()).includes('交给审批的报告会在批准前判完')) await report.locator('[data-approval-evidence]').click() await page.waitForSelector('[data-approval-evidence-doc] [data-evidence-doc]', { timeout: 6000 }) check('查看逐段证据：卡里展开可点的报告文档，自带证据条', await report.locator('[data-evidence-doc] [data-seg]').count() > 0 From 720f842b823f24aaca16603eefb940f3fa38f397 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 4 Oct 2026 12:27:38 +0800 Subject: [PATCH 13/15] =?UTF-8?q?=E7=BB=93=E8=AE=BA=E5=8F=A5=E8=A3=81?= =?UTF-8?q?=E5=88=A4=E5=B0=8F=E6=89=B9=E5=88=86=E6=B3=A2=E5=9C=B0=E9=97=AE?= =?UTF-8?q?=EF=BC=9A=E4=B8=80=E6=AC=A1=E8=B0=83=E7=94=A8=E6=9C=80=E5=A4=9A?= =?UTF-8?q?=208=20=E5=8F=A5=EF=BC=88=E5=8E=9F=E6=9D=A5=2040=20=E5=8F=A5?= =?UTF-8?q?=EF=BC=89=EF=BC=8C=E7=AC=AC=E4=B8=80=E6=89=B9=E5=8D=95=E7=8B=AC?= =?UTF-8?q?=E9=97=AE=E3=80=81=E4=B9=8B=E5=90=8E=E6=AF=8F=E6=B3=A2=E6=9C=80?= =?UTF-8?q?=E5=A4=9A=204=20=E6=89=B9=E5=90=8C=E6=97=B6=E9=97=AE=EF=BC=8C?= =?UTF-8?q?=E5=87=A0=E6=89=B9=E5=85=B1=E7=94=A8=E5=89=A9=E4=B8=8B=E7=9A=84?= =?UTF-8?q?=E6=97=B6=E9=95=BF=EF=BC=9B=E8=B6=85=E6=97=B6=E5=8F=AA=E4=B8=A2?= =?UTF-8?q?=E6=B2=A1=E5=9B=9E=E6=9D=A5=E7=9A=84=E9=82=A3=E5=87=A0=E6=89=B9?= =?UTF-8?q?=EF=BC=8C=E5=9B=9E=E6=9D=A5=E7=9A=84=E5=88=A4=E5=AE=9A=E7=85=A7?= =?UTF-8?q?=E6=A0=B7=E7=95=99=E4=B8=8B=E3=80=82=E7=9C=9F=E5=AE=9E=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E9=87=8C=E6=8E=A8=E7=90=86=E6=A8=A1=E5=9E=8B=E6=8B=BF?= =?UTF-8?q?=E7=9D=80=2040=20=E5=8F=A5=E7=9A=84=E6=91=98=E5=BD=95=E5=9C=A8?= =?UTF-8?q?=2030=20=E7=A7=92=E7=9A=84=E6=97=B6=E9=95=BF=E4=B8=8A=E9=99=90?= =?UTF-8?q?=E5=86=85=E6=B2=A1=E5=9B=9E=E6=9D=A5=EF=BC=8C=E4=BA=A4=E7=BB=99?= =?UTF-8?q?=E5=AE=A1=E6=89=B9=E7=9A=84=E6=8A=A5=E5=91=8A=E4=B8=80=E5=8F=A5?= =?UTF-8?q?=E9=83=BD=E6=B2=A1=E5=88=A4=E6=88=90=EF=BC=8C=E5=AE=A1=E6=89=B9?= =?UTF-8?q?=E5=8D=A1=E5=8F=AA=E8=83=BD=E5=86=99=E5=85=A8=E9=83=A8=E6=9C=AA?= =?UTF-8?q?=E8=A3=81=E5=88=A4=EF=BC=9B=E5=90=8C=E4=B8=80=E4=B8=AA=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=205=20=E5=8F=A5=E4=B8=80=E6=89=B9=203=20=E7=A7=92?= =?UTF-8?q?=E5=A4=9A=E5=B0=B1=E5=9B=9E=E6=9D=A5=E4=BA=86=E3=80=82=E9=87=91?= =?UTF-8?q?=E9=A2=9D=E4=B8=8A=E9=99=90=E6=8C=89=E5=B7=B2=E8=8A=B1=E7=9A=84?= =?UTF-8?q?=E5=8A=A0=E8=BF=99=E4=B8=80=E6=B3=A2=E5=B7=B2=E5=AE=9A=E4=B8=8B?= =?UTF-8?q?=E7=9A=84=E4=BC=B0=E7=AE=97=E7=AE=97=EF=BC=9B=E7=AC=AC=E4=B8=80?= =?UTF-8?q?=E6=89=B9=E8=B0=83=E4=B8=8D=E9=80=9A=EF=BC=88=E9=89=B4=E6=9D=83?= =?UTF-8?q?=E3=80=81=E7=BD=91=E5=85=B3=E6=8C=82=E4=BA=86=EF=BC=89=E6=97=B6?= =?UTF-8?q?=E5=8F=AA=E6=92=9E=E8=BF=99=E4=B8=80=E6=AC=A1=EF=BC=8C=E5=90=8E?= =?UTF-8?q?=E9=9D=A2=E7=9A=84=E7=BB=9F=E4=B8=80=E8=AE=B0=E6=9C=AA=E8=A3=81?= =?UTF-8?q?=E5=88=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/engine/judge.py | 166 ++++++++++++++++----------- backend/tests/test_evidence_judge.py | 32 ++++++ 2 files changed, 128 insertions(+), 70 deletions(-) diff --git a/backend/app/engine/judge.py b/backend/app/engine/judge.py index 8e8d67a8..1dc0e67d 100644 --- a/backend/app/engine/judge.py +++ b/backend/app/engine/judge.py @@ -98,8 +98,12 @@ MISSING_MAX = 30 #: 判定的规则版本：新判的判定带着它（rule_version），也进 request_key。报告节点的断点里换了规则的旧判定不复用； #: 按需裁判遇到早于当前规则的判定（没有版本、版本更小、旧取值 unsupported）重判一次 RULES_VERSION = 2 -#: 一次调用最多判几句：再多，4096 个输出令牌装不下每句一条判定加理由 -BATCH_SIZE = 40 +#: 一次调用最多判几句。以前是 40：真实运行里推理模型拿着 40 句的摘录 30 秒没回来，整份报告一句都没判成 +#: （同一个模型 5 句一批 3 秒多就回来了）。批小了，超时最多丢掉没回来的那几批 +BATCH_SIZE = 8 +#: 一波最多几批同时问：几批共用剩下的时长，句数上限内的句子一两波就判完。第一批单独问：接入调不通（鉴权、 +#: 网关挂了）时只撞这一次，不让一整波一起失败 +PARALLEL_BATCHES = 4 MAX_TOKENS = 4096 #: 证据摘录：SQL 前几个字、每次查询最多几行几列、每段知识库片段最多几个字、整份引用的检索给几段。 #: 列数上限只管没被引用的列：句子引用的格子所在的列一律给（遮罩的除外），再用别的列补满到这么多列 @@ -1180,80 +1184,102 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, emit(EventType.LOG, level="warn", code="judge_unpriced", message=note) spent = 0.0 + width = 1 while pending: remaining = None if budget.timeout_s is None else budget.timeout_s - (time.monotonic() - started) if remaining is not None and remaining <= 0: stop(pending, "timeout_s") break - batch = pending[:BATCH_SIZE] - cost = request.cost(model_id, batch) if priced else None - if cost is not None: - caps = [] - if budget.max_cost_usd is not None: - caps.append((budget.max_cost_usd - spent, "max_cost_usd")) - if budget.daily_max_usd is not None: - caps.append((budget.daily_max_usd - (await daily_spend())["usd"], "daily_max_usd")) - if caps: - cap, binding = min(caps, key=lambda x: x[0]) - trimmed = False - while batch and (cost := request.cost(model_id, batch) or 0.0) > cap + 1e-12: - batch, trimmed = batch[:-1], True # 按优先级从后往前丢：先判挂了依据的 - if not batch: - stop(pending, binding) - break - if trimmed: - # 这一批已经是剩下的钱装得下的最多句子，后面的句子更装不下 - stop(pending[len(batch):], binding) - pending = pending[:len(batch)] - pending = pending[len(batch):] - charged = cost or 0.0 - day = await _charge(charged, limit=budget.daily_max_usd if priced else None, priced=priced) - if day is None: # 估算之后、记账之前别的裁判先花掉了今天剩下的钱 - stop([*batch, *pending], "daily_max_usd") - break - estimated += charged - began = time.perf_counter() - emit(EventType.LLM_START, model=model_id, structured=True, purpose="judge", message_count=2, - units=len(batch)) - items: list[Any] | None = None - raws: list[BaseMessage] = [] - problem: str | None = None - calls = 1 - try: - async with asyncio.timeout(remaining): - items, raws, problem, calls = await _ask(model, request.messages(batch)) - except TimeoutError: - problem = "timeout_s" - tokens = [_usage_of(r, model_id) for r in raws] - inp, out = sum(t["input_tokens"] for t in tokens), sum(t["output_tokens"] for t in tokens) - # 回复没带用量（有的网关不给、调用超时）就按估算记：宁可多记，不能让每日上限漏掉 - actual = (pricing.estimate_cost(model_id, inp, out) if inp or out else charged) if priced else 0.0 - await _settle(day, actual - charged) - spent += actual - usage["input_tokens"] += inp - usage["output_tokens"] += out - usage["calls"] += calls - usage["cost_usd"] += actual - end = {"model": model_id, "purpose": "judge", "duration_ms": int((time.perf_counter() - began) * 1000), - "input_tokens": inp, "output_tokens": out, "total_tokens": inp + out, "cost_usd": round(actual, 6), - "calls": calls, "units": len(batch)} - if problem is not None and problem != "format": - reason = "timeout_s" if problem == "timeout_s" else "error" - detail = problem.partition(":")[2] - emit(EventType.LLM_END, **end, error=detail or _why_unjudged(reason, label, click=click)) - if reason == "error": - # 调用本身失败（鉴权、网关挂了）：后面几批多半一样，不再一次次去撞 - details["error"] = detail - stop([*batch, *pending], reason, detail) + # 一波最多 PARALLEL_BATCHES 批：逐批按金额上限定下来（这一波已经定下的按估算先占着），再一起问 + wave: list[tuple[list[Candidate], float, Any]] = [] + reserved = 0.0 + while pending and len(wave) < width: + batch = pending[:BATCH_SIZE] + cost = request.cost(model_id, batch) if priced else None + if cost is not None: + caps = [] + if budget.max_cost_usd is not None: + caps.append((budget.max_cost_usd - spent - reserved, "max_cost_usd")) + if budget.daily_max_usd is not None: + caps.append((budget.daily_max_usd - (await daily_spend())["usd"], "daily_max_usd")) + if caps: + cap, binding = min(caps, key=lambda x: x[0]) + trimmed = False + while batch and (cost := request.cost(model_id, batch) or 0.0) > cap + 1e-12: + batch, trimmed = batch[:-1], True # 按优先级从后往前丢：先判挂了依据的 + if not batch: + stop(pending, binding) + pending = [] + break + if trimmed: + # 这一批已经是剩下的钱装得下的最多句子，后面的句子更装不下 + stop(pending[len(batch):], binding) + pending = pending[:len(batch)] + pending = pending[len(batch):] + charged = cost or 0.0 + day = await _charge(charged, limit=budget.daily_max_usd if priced else None, priced=priced) + if day is None: # 估算之后、记账之前别的裁判先花掉了今天剩下的钱 + stop([*batch, *pending], "daily_max_usd") + pending = [] break - stop(batch, reason, detail) - continue - emit(EventType.LLM_END, **end) - if items is None: - stop(batch, "format") - continue - _take(items, batch, request, verdicts, judge=judge, post_seal=post_seal) - stop(batch, "missing") + estimated += charged + reserved += charged + wave.append((batch, charged, day)) + if not wave: + break + + began = time.perf_counter() + for batch, _, _ in wave: + emit(EventType.LLM_START, model=model_id, structured=True, purpose="judge", message_count=2, + units=len(batch)) + tasks = [asyncio.create_task(_ask(model, request.messages(batch))) for batch, _, _ in wave] + done, late = await asyncio.wait(tasks, timeout=remaining) + for task in late: + task.cancel() + if late: + await asyncio.gather(*late, return_exceptions=True) + failed: str | None = None + for (batch, charged, day), task in zip(wave, tasks): + items: list[Any] | None = None + raws: list[BaseMessage] = [] + problem: str | None = None + calls = 1 + if task in done: + items, raws, problem, calls = task.result() + else: + problem = "timeout_s" + tokens = [_usage_of(r, model_id) for r in raws] + inp, out = sum(t["input_tokens"] for t in tokens), sum(t["output_tokens"] for t in tokens) + # 回复没带用量（有的网关不给、调用超时）就按估算记：宁可多记，不能让每日上限漏掉 + actual = (pricing.estimate_cost(model_id, inp, out) if inp or out else charged) if priced else 0.0 + await _settle(day, actual - charged) + spent += actual + usage["input_tokens"] += inp + usage["output_tokens"] += out + usage["calls"] += calls + usage["cost_usd"] += actual + end = {"model": model_id, "purpose": "judge", "duration_ms": int((time.perf_counter() - began) * 1000), + "input_tokens": inp, "output_tokens": out, "total_tokens": inp + out, + "cost_usd": round(actual, 6), "calls": calls, "units": len(batch)} + if problem is not None and problem != "format": + reason = "timeout_s" if problem == "timeout_s" else "error" + detail = problem.partition(":")[2] + emit(EventType.LLM_END, **end, error=detail or _why_unjudged(reason, label, click=click)) + if reason == "error": + # 调用本身失败（鉴权、网关挂了）：后面几批多半一样，不再一次次去撞 + details["error"] = failed = detail + stop(batch, reason, detail) + continue + emit(EventType.LLM_END, **end) + if items is None: + stop(batch, "format") + continue + _take(items, batch, request, verdicts, judge=judge, post_seal=post_seal) + stop(batch, "missing") + if failed is not None: + stop(pending, "error", failed) + break + width = PARALLEL_BATCHES for c in request.cands: # 兜底：每个候选都有判定 verdicts.setdefault(c.unit, _unjudged("missing", _why_unjudged("missing", label), judge=judge, diff --git a/backend/tests/test_evidence_judge.py b/backend/tests/test_evidence_judge.py index 1c3c421c..75a3b995 100644 --- a/backend/tests/test_evidence_judge.py +++ b/backend/tests/test_evidence_judge.py @@ -17,6 +17,7 @@ """ from __future__ import annotations +import asyncio import json import re @@ -464,6 +465,37 @@ async def test_cost_truncation_follows_priority(monkeypatch, stored): assert {status_of(outcome, doc, t)["status"] for t in (CITED, CITED2)} == {"supported"} +async def test_a_slow_batch_times_out_alone_and_the_others_keep_their_verdicts(monkeypatch, stored): + """真实运行里推理模型拿着 40 句一批，30 秒没回来，整份报告一句都没判成（同一个模型 5 句一批 3 秒多就回来）。 + 现在小批：第一批单独问，之后几批同时问、共用剩下的时长；超时只丢没回来的那一批，回来的判定照样留下。""" + Judge(monkeypatch) + catalog = make_catalog(stored) + raw = "".join(f"第 {i} 个区域的销售额增长来自新客。[[see:m:gmv]]" for i in range(BATCH_SIZE * 3)) + doc = compose_doc(raw, catalog) + real_ask = judge_mod._ask + asked: list[int] = [] + running = peak = 0 + + async def ask(model, messages): + nonlocal running, peak + asked.append(len(re.findall(r"^\[(u\d+)\]", str(messages[-1].content), re.M))) + running += 1 + peak = max(peak, running) + try: + await asyncio.sleep(5 if len(asked) == 3 else 0.05) # 第三批卡住 + return await real_ask(model, messages) + finally: + running -= 1 + + monkeypatch.setattr(judge_mod, "_ask", ask) + outcome = await judge_doc(doc, catalog, spec=SPEC, + budget=Budget(max_claims=None, max_cost_usd=None, timeout_s=1, daily_max_usd=None)) + assert asked == [BATCH_SIZE] * 3 + assert peak == 2, "第一批单独问，后两批同时问" + assert outcome["counts"]["supported"] == BATCH_SIZE * 2 + assert outcome["unjudged"] == {"timeout_s": BATCH_SIZE} and outcome["limits_hit"] == ["timeout_s"] + + async def test_unlimited_never_truncates(monkeypatch, stored): """每一项都写 null：句数再多也全判（按每批上限分批），金额、时长、每日都不拦。""" judge = Judge(monkeypatch) From 46330dfd9673a479cd05e4656c556b4ff410b397 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 4 Oct 2026 12:27:38 +0800 Subject: [PATCH 14/15] =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E4=B8=8D=E5=9C=A8?= =?UTF-8?q?=E4=BB=B7=E6=A0=BC=E7=9B=AE=E5=BD=95=E9=87=8C=E6=97=B6=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E9=87=91=E9=A2=9D=E5=86=99=E3=80=8C=E9=87=91=E9=A2=9D?= =?UTF-8?q?=E6=97=A0=E6=B3=95=E4=BC=B0=E7=AE=97=E3=80=8D=EF=BC=8C=E4=B8=8D?= =?UTF-8?q?=E5=86=99=20$0=EF=BC=9A=E7=9C=9F=E5=AE=9E=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E9=87=8C=20token=20=E8=AE=B0=E5=88=B0=E4=BA=86=E4=B8=8A?= =?UTF-8?q?=E7=99=BE=E4=B8=87=EF=BC=8C=E9=87=91=E9=A2=9D=E5=8D=B4=E6=98=AF?= =?UTF-8?q?=20$0=EF=BC=8C=E7=9C=8B=E7=9D=80=E5=83=8F=E6=B2=A1=E8=8A=B1?= =?UTF-8?q?=E9=92=B1=E3=80=82llm.end=20=E5=8F=91=E5=87=BA=E6=97=B6?= =?UTF-8?q?=EF=BC=8C=E7=94=A8=E4=BA=86=20token=E3=80=81=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E5=8D=B4=E4=B8=8D=E5=9C=A8=E4=BB=B7=E6=A0=BC=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E9=87=8C=E7=9A=84=E6=A0=87=20unpriced=EF=BC=88=E7=9B=AE?= =?UTF-8?q?=E5=BD=95=E9=87=8C=E6=A0=87=200=20=E4=BB=B7=E7=9A=84=E6=9C=AC?= =?UTF-8?q?=E5=9C=B0=E6=A8=A1=E5=9E=8B=E4=B8=8D=E7=AE=97=EF=BC=89=EF=BC=9B?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E7=94=A8=E9=87=8F=E6=8C=89=20llm.end=20?= =?UTF-8?q?=E6=95=B0=E5=87=BA=20unpriced=5Fcalls=EF=BC=8C=E6=88=90?= =?UTF-8?q?=E5=8A=9F=E7=9A=84=E8=BF=90=E8=A1=8C=E4=B9=9F=E8=A1=A5=E4=B8=8A?= =?UTF-8?q?=EF=BC=88=E7=8A=B6=E6=80=81=E9=87=8C=E7=9A=84=E7=94=A8=E9=87=8F?= =?UTF-8?q?=E5=88=86=E4=B8=8D=E5=87=BA=200=20=E6=98=AF=E5=85=8D=E8=B4=B9?= =?UTF-8?q?=E8=BF=98=E6=98=AF=E6=B2=A1=E4=BB=B7=EF=BC=89=E3=80=82=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E8=AF=A6=E6=83=85=E3=80=81=E7=94=BB=E5=B8=83=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E9=9D=A2=E6=9D=BF=E3=80=81=E8=88=AA=E8=BF=B9=E7=9A=84?= =?UTF-8?q?=E9=87=91=E9=A2=9D=EF=BC=9A=E4=B8=80=E5=88=86=E9=83=BD=E6=B2=A1?= =?UTF-8?q?=E4=BC=B0=E5=87=BA=E6=9D=A5=E5=86=99=E3=80=8C=E9=87=91=E9=A2=9D?= =?UTF-8?q?=E6=97=A0=E6=B3=95=E4=BC=B0=E7=AE=97=E3=80=8D=EF=BC=8C=E4=BC=B0?= =?UTF-8?q?=E5=87=BA=E4=B8=80=E9=83=A8=E5=88=86=E5=86=99=E3=80=8C=E2=89=A5?= =?UTF-8?q?=20$x=E3=80=8D=EF=BC=8C=E6=82=AC=E5=81=9C=E8=AF=B4=E5=87=A0?= =?UTF-8?q?=E6=AC=A1=E8=B0=83=E7=94=A8=E4=B8=8D=E5=9C=A8=E4=BB=B7=E6=A0=BC?= =?UTF-8?q?=E7=9B=AE=E5=BD=95=E4=B8=AD=E3=80=82check-trace=20=E8=A1=A5?= =?UTF-8?q?=E6=96=AD=E8=A8=80=EF=BC=9B=E6=B5=81=E5=BC=8F=E7=94=A8=E9=87=8F?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E5=92=8C=E6=B3=A8=E9=87=8A=E9=87=8C=E7=9A=84?= =?UTF-8?q?=E7=9C=9F=E5=AE=9E=E8=B0=83=E7=94=A8=E6=AC=A1=E6=95=B0=E6=8D=A2?= =?UTF-8?q?=E6=88=90=E6=A6=82=E6=8B=AC=E8=AF=B4=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/engine/context.py | 6 +++ backend/app/engine/runner.py | 15 ++++++- backend/app/providers/catalog.py | 16 +++++++ backend/tests/test_openai_stream_usage.py | 51 +++++++++++++++++++---- frontend/src/lib/terms.ts | 20 ++++++++- frontend/src/pages/runs/Telemetry.tsx | 7 +++- frontend/src/pages/runs/TracePane.tsx | 6 ++- frontend/src/run/RunHud.tsx | 13 ++++-- frontend/src/run/trace.ts | 6 ++- frontend/src/types.ts | 2 + scripts/check-trace.mjs | 29 +++++++++++++ 11 files changed, 151 insertions(+), 20 deletions(-) diff --git a/backend/app/engine/context.py b/backend/app/engine/context.py index 65fb8a85..ad197844 100644 --- a/backend/app/engine/context.py +++ b/backend/app/engine/context.py @@ -56,6 +56,12 @@ class NodeContext: writer = get_stream_writer() except Exception: # noqa: BLE001 - 不在图执行上下文里（比如单元测试） return + if str(event_type) == EventType.LLM_END and "unpriced" not in data: + from app.providers.catalog import unpriced_call + + if unpriced_call(data): + # 金额估不出来（模型不在价格目录里）：标出来，界面不把它的 0 当成「没花钱」 + data = {**data, "unpriced": True} writer( { "__agentlab__": True, diff --git a/backend/app/engine/runner.py b/backend/app/engine/runner.py index d47639c8..1be770d4 100644 --- a/backend/app/engine/runner.py +++ b/backend/app/engine/runner.py @@ -837,11 +837,15 @@ class RunManager: run = await session.get(Run, run_id) if run: timing = await _timing(session, run, elapsed_ms, now) + events_usage = await _usage_of_events(session, run_id) if status != "succeeded": # 状态里的用量只算跑完了的节点。失败、取消、停在审批上的那个节点已经 # 花出去的调用（校验修复、agent 的前几步）只在 llm.end 里——以前这部分 # 在运行记录上凭空消失，失败的运行看着像没花钱 - usage = _spent(usage, await _usage_of_events(session, run_id)) + usage = _spent(usage, events_usage) + elif events_usage.get("unpriced_calls"): + # 金额估不出来的调用只有 llm.end 数得出（状态里的用量只有金额，0 分不出是免费还是没价） + usage = {**usage, "unpriced_calls": events_usage["unpriced_calls"]} # duration_ms 是各段执行时长之和（= active_ms）。以前用这一段的耗时 # 覆盖它：87 秒、中间等过一次审批的运行，列表上写着 22ms usage.update(duration_ms=timing["active_ms"], **timing) @@ -1251,13 +1255,20 @@ async def _usage_of_events(session: Any, run_id: str) -> dict[str, Any]: rows = await session.execute( select(RunEvent.data).where(RunEvent.run_id == run_id, RunEvent.type == EventType.LLM_END) ) + from app.providers.catalog import unpriced_call + total: dict[str, Any] = dict.fromkeys(_USAGE_KEYS, 0) + unpriced = 0 for (data,) in rows: for key in _USAGE_KEYS: value = (data or {}).get(key, 1 if key == "calls" else 0) if isinstance(value, (int, float)) and not isinstance(value, bool): total[key] += value + unpriced += unpriced_call(data) total["cost_usd"] = round(total["cost_usd"], 6) + if unpriced: + # 有几次调用的金额估不出来（模型不在价格目录里）：cost_usd 只是估得出的那部分，界面据此不写 $0 + total["unpriced_calls"] = unpriced return total @@ -1269,6 +1280,8 @@ def _spent(state_usage: dict[str, Any], events_usage: dict[str, Any]) -> dict[st mine, theirs = out.get(key) or 0, events_usage.get(key) or 0 if isinstance(mine, (int, float)) and theirs > mine: out[key] = theirs + if events_usage.get("unpriced_calls"): + out["unpriced_calls"] = events_usage["unpriced_calls"] if "input_tokens" in out or "output_tokens" in out: out["total_tokens"] = int(out.get("input_tokens") or 0) + int(out.get("output_tokens") or 0) return out diff --git a/backend/app/providers/catalog.py b/backend/app/providers/catalog.py index 116850ab..a610c82d 100644 --- a/backend/app/providers/catalog.py +++ b/backend/app/providers/catalog.py @@ -229,6 +229,22 @@ def find_context(model: str) -> int | None: return None +def unpriced_call(data: Any) -> bool: + """一次模型调用（llm.end 的数据）的金额估不出来：用了 token，模型却不在价格目录里，estimate_cost 只能给 0。 + + 这时写 $0 像是没花钱（真实运行里上百次调用、上百万 token，运行页写着 $0）。本地模型在目录里标的是 0 价， + 不算估不出来。事件自己标了 unpriced 的以它为准。""" + if not isinstance(data, dict): + return False + if isinstance(data.get("unpriced"), bool): + return data["unpriced"] + tokens = sum(v for v in (data.get("input_tokens"), data.get("output_tokens")) + if isinstance(v, (int, float)) and not isinstance(v, bool)) + model = data.get("model") + return tokens > 0 and isinstance(model, str) and bool(model) and not data.get("cost_usd") \ + and find_pricing(model) is None + + def estimate_cost(model: str, input_tokens: int, output_tokens: int) -> float: """按目录价估算一次调用的花费（美元）。""" pricing = find_pricing(model) diff --git a/backend/tests/test_openai_stream_usage.py b/backend/tests/test_openai_stream_usage.py index 99c539f7..14429cb4 100644 --- a/backend/tests/test_openai_stream_usage.py +++ b/backend/tests/test_openai_stream_usage.py @@ -171,9 +171,8 @@ async def engine_up(): await run_manager.shutdown() -async def test_run_usage_counts_streamed_tokens(server, engine_up): - """整条链：兼容接入 → llm 节点流式调用 → llm.end → 运行的用量。修之前这里是 0 token、0 美元。""" - server() +async def _run_once(model: str) -> tuple[Run, list[RunEvent]]: + """兼容接入上用 model 跑一个 llm 节点，返回运行行和 llm.end 事件。""" provider = _provider() async with SessionLocal() as session: session.add(provider) @@ -185,7 +184,7 @@ async def test_run_usage_counts_streamed_tokens(server, engine_up): "nodes": [ {"id": "in", "type": "input", "data": {"config": {}}}, {"id": "ask", "type": "llm", "data": {"label": "问答", "config": { - "prompt": "上周到店多少人", "provider": provider.name, "model": PRICED}}}, + "prompt": "上周到店多少人", "provider": provider.name, "model": model}}}, {"id": "out", "type": "output", "data": {"config": { "fields": [{"name": "回答", "value": "{{ nodes.ask.text }}"}]}}}, ], @@ -202,12 +201,46 @@ async def test_run_usage_counts_streamed_tokens(server, engine_up): async with SessionLocal() as session: ends = list((await session.execute(select(RunEvent).where( RunEvent.run_id == started.id, RunEvent.type == "llm.end"))).scalars()) - assert [(e.data["input_tokens"], e.data["output_tokens"]) for e in ends] == [(1200, 300)] - assert row.usage["input_tokens"] == 1200 and row.usage["output_tokens"] == 300 - assert row.usage["total_tokens"] == 1500 - expected = round(catalog.estimate_cost(PRICED, 1200, 300), 6) - assert expected > 0 and row.usage["cost_usd"] == pytest.approx(expected) + return row, ends finally: async with SessionLocal() as session: await session.execute(delete(Provider).where(Provider.id == provider.id)) await session.commit() + + +async def test_run_usage_counts_streamed_tokens(server, engine_up): + """整条链：兼容接入 → llm 节点流式调用 → llm.end → 运行的用量。修之前这里是 0 token、0 美元。""" + server() + row, ends = await _run_once(PRICED) + assert [(e.data["input_tokens"], e.data["output_tokens"]) for e in ends] == [(1200, 300)] + assert row.usage["input_tokens"] == 1200 and row.usage["output_tokens"] == 300 + assert row.usage["total_tokens"] == 1500 + expected = round(catalog.estimate_cost(PRICED, 1200, 300), 6) + assert expected > 0 and row.usage["cost_usd"] == pytest.approx(expected) + assert "unpriced" not in ends[0].data and "unpriced_calls" not in row.usage + + +async def test_a_model_missing_from_the_price_catalog_is_marked_unpriced(server, engine_up): + """token 记到了，模型却不在价格目录里：金额估不出来，不能让运行页写 $0 像是没花钱。 + llm.end 标 unpriced（运行中的实时统计据此数），运行用量记 unpriced_calls（界面据此写「无法估算」）。""" + server() + row, ends = await _run_once("vendor-unlisted-model") + assert [(e.data["input_tokens"], e.data["output_tokens"]) for e in ends] == [(1200, 300)] + assert ends[0].data["unpriced"] is True and ends[0].data["cost_usd"] == 0 + assert row.usage["total_tokens"] == 1500 and row.usage["cost_usd"] == 0 + assert row.usage["unpriced_calls"] == 1 + + +@pytest.mark.parametrize(("data", "unpriced"), [ + ({"model": "vendor-unlisted-model", "input_tokens": 10, "output_tokens": 2, "cost_usd": 0}, True), + ({"model": PRICED, "input_tokens": 10, "output_tokens": 2, "cost_usd": 0.0001}, False), + # 目录里标 0 价的本地模型是真不花钱，不是估不出来 + ({"model": "qwen3:8b", "input_tokens": 10, "output_tokens": 2, "cost_usd": 0}, False), + # 没用 token 的调用（失败、被取消）谈不上金额 + ({"model": "vendor-unlisted-model", "input_tokens": 0, "output_tokens": 0}, False), + # 老事件没有 unpriced 键：按模型名补判；事件自己标了的以它为准 + ({"model": PRICED, "input_tokens": 10, "unpriced": True}, True), + (None, False), +]) +def test_unpriced_call(data, unpriced): + assert catalog.unpriced_call(data) is unpriced diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index cb37afe9..16ace5fa 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -46,7 +46,7 @@ import type { CatalogCardinality, CatalogMeasure, CatalogProfileNumberKey, CatalogSource, CatalogStatus, CatalogTableKind, NodeType, SnapshotReasonCode, SqlCheckCode, SqlCheckLevel, ToolTrust, } from '../types' -import { formatNumber } from './format' +import { formatCost, formatNumber } from './format' export const TERMS = { workflow: '工作流', @@ -651,6 +651,24 @@ export const JUDGE_ON_UNSUPPORTED_LABEL: Record = { } /** 设置页「证据裁判」一组（后端 engine/judge.py 的 JUDGE_DEFAULTS） */ +/** 运行的金额。有调用的模型不在价格目录中时金额估不出来，不写「$0」（看着像没花钱） */ +export const COST_TEXT = { + unpriced: '金额无法估算', + /** 悬停说明，n 是估不出金额的调用次数 */ + unpricedTitle: (n: number) => `${formatNumber(n)} 次调用的模型不在价格目录中，金额无法估算，未计入`, +} as const + +/** + * 运行级金额的写法。unpriced 是估不出金额的调用次数：一分都没估出来写「金额无法估算」，估出了一部分写「≥ $x」； + * 运行中（approx）只算已上报的调用，也写「≥」。没有金额返回 null，由调用方写占位 + */ +export function runCostText(usd: number | null | undefined, unpriced: number, approx = false): string | null { + const known = typeof usd === 'number' && Number.isFinite(usd) + if (unpriced > 0 && !(known && usd > 0)) return COST_TEXT.unpriced + if (!known) return null + return `${(approx || unpriced > 0) && usd > 0 ? '≥ ' : ''}${formatCost(usd)}` +} + export const JUDGE_SETTING_TEXT = { label: '证据裁判', hint: '报告中的结论句（如「增长主要来自新客首单」）由另一个模型按证据逐句判断：正式运行由报告撰写节点当场裁判，探索运行在点击时逐句裁判。判断由模型给出，不是系统核对，界面上一律标注「非确定」。', diff --git a/frontend/src/pages/runs/Telemetry.tsx b/frontend/src/pages/runs/Telemetry.tsx index 289e933c..01435e46 100644 --- a/frontend/src/pages/runs/Telemetry.tsx +++ b/frontend/src/pages/runs/Telemetry.tsx @@ -10,7 +10,7 @@ import { type StatusCode } from '../../lib/status' import { isExecuting, liveAt, project, type Projection, type RunPhase, type Trace } from '../../run/trace' import { useRunClock } from '../../run/useRunClock' import type { Approval, Run } from '../../types' -import { UNSAVED_HINT, isUnsaved, runName } from '../../lib/terms' +import { COST_TEXT, UNSAVED_HINT, isUnsaved, runCostText, runName } from '../../lib/terms' import { LONG_WAIT_MS, ageMs, cancelReason, runScope, runTiming, type RunTiming, } from './model' @@ -92,6 +92,9 @@ export function Telemetry({ run, trace, phase, code, streaming, pending, labelOf const tokens = settled || !streaming ? tokensSettled ?? (liveTokens || undefined) : liveTokens const cost = settled || !streaming ? (isNum(usage.cost_usd) ? usage.cost_usd : trace.costUsd || undefined) : trace.costUsd const approx = streaming && !settled + // 金额估不出来的调用（模型不在价格目录中）：有的话不写 $0 + const unpriced = (settled || !streaming ? (isNum(usage.unpriced_calls) ? usage.unpriced_calls : undefined) : undefined) + ?? trace.unpricedCalls const running = Object.values(trace.nodes).filter(isExecuting) const waitingId = trace.waitingNodeId ?? pending?.[0]?.node_id @@ -156,7 +159,7 @@ export function Telemetry({ run, trace, phase, code, streaming, pending, labelOf {tokens ? <>{approx && '≥ '}{formatTokens(tokens, { compact: true })} : NONE} - {isNum(cost) && cost > 0 ? `${approx ? '≥ ' : ''}${formatCost(cost)}` : isNum(cost) ? formatCost(cost) : ' '} + {runCostText(cost, unpriced, approx) ?? ' '}
    ) diff --git a/frontend/src/pages/runs/TracePane.tsx b/frontend/src/pages/runs/TracePane.tsx index 2e34fa0a..8c6523a0 100644 --- a/frontend/src/pages/runs/TracePane.tsx +++ b/frontend/src/pages/runs/TracePane.tsx @@ -11,7 +11,7 @@ import { explainRunError } from '../../lib/explain' import { NONE, formatCost, formatDateTime, formatNumber, formatOffset, formatSpan, formatTokens, shortId, } from '../../lib/format' -import { runClassLabel, nodeTypeLabel } from '../../lib/terms' +import { COST_TEXT, runClassLabel, nodeTypeLabel, runCostText } from '../../lib/terms' import { topology, type GraphLike } from '../../run/derive' import { isActivePhase, isSettled, liveAt, project, type NodeState, type Projection, type RunPhase, type Trace, @@ -245,7 +245,9 @@ function Readout({ sub={tokens ? `输入 ${formatNumber(proj.tokensIn)} · 输出 ${formatNumber(proj.tokensOut)}` : undefined}> {tokens == null ? NONE : formatTokens(tokens, { compact: true })} - {!metered || proj.costUsd == null ? NONE : formatCost(proj.costUsd)} + + {!metered || proj.costUsd == null ? NONE : runCostText(proj.costUsd, trace.unpricedCalls) ?? NONE} +
    diff --git a/frontend/src/run/RunHud.tsx b/frontend/src/run/RunHud.tsx index 807846bd..024c808e 100644 --- a/frontend/src/run/RunHud.tsx +++ b/frontend/src/run/RunHud.tsx @@ -8,7 +8,7 @@ import { confirmDialog, Kbd, Spinner, StatusBadge, toast } from '../components/u import { formatCost, formatDuration, formatLapse, formatOffset, formatSpan, formatTokens, NONE } from '../lib/format' import { isTypingTarget, matchShortcut } from '../lib/keys' import { statusMeta, STATUS, type StatusCode } from '../lib/status' -import { issuanceLabel, runClassLabel } from '../lib/terms' +import { COST_TEXT, issuanceLabel, runClassLabel, runCostText } from '../lib/terms' import { lastStampOf, projectSettled, useDock, waitedMs } from '../canvas/RunTimeline' import { structureSig, useStudio } from '../store/studio' import type { RunEvent } from '../types' @@ -91,6 +91,8 @@ export interface RunGlance { series: [number, number][] tokens: number | null cost: number | null + /** 金额估不出来的调用次数（模型不在价格目录中）；有的话金额不写 $0 */ + unpriced: number /** 这一段执行还剩多少时间；null = 不知道上限或眼下不在计时 */ remainingMs: number | null limitMs: number | null @@ -220,6 +222,7 @@ export function useRunGlance(active: boolean): RunGlance { const replayTok = proj.tokensIn != null && proj.tokensOut != null ? proj.tokensIn + proj.tokensOut : null const tokens = replayAt != null ? replayTok : backendTok ?? (seen ? usage.tokensIn + usage.tokensOut : null) const cost = replayAt != null ? proj.costUsd ?? null : backendCost ?? (seen ? usage.costUsd : null) + const unpriced = (settledUsage ? num(runUsage.unpriced_calls) : null) ?? trace.unpricedCalls // 时限按"这一段执行"计：等人工审批时执行已经结束，那段不算（和后端一致） const drive = trace.drives[trace.drives.length - 1] @@ -253,7 +256,7 @@ export function useRunGlance(active: boolean): RunGlance { headline: headlineOf(trace, phase, states, at, label, runError, replayAt == null ? cancelled : ''), executing: phase === 'running' && cells.some((c) => c.state === 'running' && !trace.nodes[c.id]?.looping), cells, done, visited, parallelNow: proj.parallelNow, series: trace.parallelSeries, - tokens, cost, remainingMs, limitMs, attention, + tokens, cost, unpriced, remainingMs, limitMs, attention, failedNodeId: trace.failedNodeId, waitingNodeId: trace.waitingNodeId, } } @@ -591,7 +594,7 @@ export function RunCapsule() { {nodesText} - {tokens}{g.cost ? ` · ${formatCost(g.cost)}` : ''} + {tokens}{g.cost || g.unpriced ? ` · ${runCostText(g.cost, g.unpriced)}` : ''} )} @@ -654,7 +657,9 @@ export function RunCapsule() {
    用量
    {tokens}
    -
    {g.cost != null ? `≈ ${formatCost(g.cost)}` : NONE}
    +
    + {g.unpriced ? runCostText(g.cost, g.unpriced) : g.cost != null ? `≈ ${formatCost(g.cost)}` : NONE} +
    时限
    diff --git a/frontend/src/run/trace.ts b/frontend/src/run/trace.ts index da4281bc..d0aedd17 100644 --- a/frontend/src/run/trace.ts +++ b/frontend/src/run/trace.ts @@ -96,6 +96,8 @@ export interface Trace { tokensIn: number tokensOut: number costUsd: number + /** 金额估不出来的调用次数（llm.end 标了 unpriced：模型不在价格目录中）。这时 costUsd 的 0 不是「没花钱」 */ + unpricedCalls: number failedNodeId?: string waitingNodeId?: string issuance?: { @@ -238,7 +240,7 @@ export const lastEventAt = (t: Trace): number => t.book.clock export function emptyTrace(): Trace { return { - phase: 'idle', waitMs: 0, nodes: {}, tokensIn: 0, tokensOut: 0, costUsd: 0, + phase: 'idle', waitMs: 0, nodes: {}, tokensIn: 0, tokensOut: 0, costUsd: 0, unpricedCalls: 0, lastSeq: 0, parallelSeries: [], usageSeries: [], timed: true, drives: [], waits: [], phases: [], lastReplay: false, book: { clock: 0, awaiting: false, dispatchFrom: {}, exactDispatch: {}, asking: {} }, @@ -673,6 +675,7 @@ export function foldEvent(t: Trace, ev: RunEvent): Trace { next.tokensIn += inTok next.tokensOut += outTok next.costUsd += cost + if (d.unpriced === true) next.unpricedCalls += 1 next.usageSeries = [...next.usageSeries, [at, next.tokensIn, next.tokensOut, next.costUsd]] if (nodeId) { const n = node(nodeId) @@ -816,6 +819,7 @@ export function foldEvent(t: Trace, ev: RunEvent): Trace { next.tokensIn = num(u.input_tokens) ?? next.tokensIn next.tokensOut = num(u.output_tokens) ?? next.tokensOut next.costUsd = num(u.cost_usd) ?? next.costUsd + next.unpricedCalls = num(u.unpriced_calls) ?? next.unpricedCalls } const timing = d.timing if (timing && num(timing.wall_ms) != null) { diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 0b1c6c84..1a47d0af 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -110,6 +110,8 @@ export interface RunUsage { input_tokens?: number output_tokens?: number cost_usd?: number + /** 金额估不出来的调用次数（模型不在价格目录中），cost_usd 只是估得出的那部分。没有这种调用时不给 */ + unpriced_calls?: number [key: string]: any } diff --git a/scripts/check-trace.mjs b/scripts/check-trace.mjs index 71a03652..57ee3da2 100644 --- a/scripts/check-trace.mjs +++ b/scripts/check-trace.mjs @@ -540,6 +540,35 @@ await section('右栏收尾（decodeRun）', async () => { fold([start('甲', 1), start('乙', 2), end('甲', 3, 100), end('乙', 4, 300)]).savedMs === 100) }) +await section('金额估不出来的调用', async () => { + // 模型不在价格目录中：token 记到了，金额只能是 0。运行页写 $0 像是没花钱，改写「金额无法估算」 + const ev = (seq, type, node_id, data = {}) => ({ seq, type, node_id, ts: 5500 + seq, data }) + const head = [ + ev(1, 'run.started', null, { nodes: 1 }), + ev(2, 'node.started', 'agent'), + ev(3, 'llm.end', 'agent', { model: 'vendor-unlisted', input_tokens: 900, output_tokens: 90, cost_usd: 0, unpriced: true }), + ev(4, 'llm.end', 'agent', { model: 'vendor-unlisted', input_tokens: 700, output_tokens: 70, cost_usd: 0, unpriced: true }), + ] + const gr = g(['agent'], []) + const live = run(head, gr) + check('运行中：llm.end 标了 unpriced 的逐次数进来', live.unpricedCalls === 2 && live.costUsd === 0, `${live.unpricedCalls}`) + const done = run([...head, ev(5, 'node.finished', 'agent', { duration_ms: 10 }), + ev(6, 'run.finished', null, { usage: { input_tokens: 1600, output_tokens: 160, cost_usd: 0, unpriced_calls: 3 } })], gr) + check('终态以后端总数为准（老后端少发的也算上）', done.unpricedCalls === 3, `${done.unpricedCalls}`) + const plain = run([ev(1, 'run.started', null, { nodes: 1 }), ev(2, 'node.started', 'agent'), + ev(3, 'llm.end', 'agent', { input_tokens: 10, output_tokens: 1, cost_usd: 0.001 })], gr) + check('目录里有价的调用不算', plain.unpricedCalls === 0) + + const { runCostText, COST_TEXT } = await import(await load('/src/lib/terms.ts')) + check('一分都没估出来：写「金额无法估算」，不写 $0', runCostText(0, 3) === COST_TEXT.unpriced, runCostText(0, 3)) + check('……没拿到金额时也一样', runCostText(null, 1) === COST_TEXT.unpriced) + check('估出了一部分：写「≥」', runCostText(0.012, 2) === '≥ $0.012', runCostText(0.012, 2)) + check('全都有价：照常写', runCostText(0.012, 0) === '$0.012' && runCostText(0, 0) === '$0', `${runCostText(0.012, 0)} / ${runCostText(0, 0)}`) + check('运行中只算已上报的：写「≥」', runCostText(0.012, 0, true) === '≥ $0.012') + check('没有金额也没有估不出的：交给调用方写占位', runCostText(undefined, 0) === null) + check('悬停说明写清几次调用、为什么', COST_TEXT.unpricedTitle(3).includes('3 次调用') && COST_TEXT.unpricedTitle(3).includes('价格目录')) +}) + await section('回放时的用量和工具数（逐刻度）', async () => { // 回放到半路，卡片和 HUD 要写「那一刻」的 tokens 和工具数，而不是拿终值冒充 const ev = (seq, type, node_id, data = {}) => ({ seq, type, node_id, ts: 5000 + seq, data }) From 536c9de1167ef5c150690bb0c21cdeca5c01b70a Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 4 Oct 2026 12:28:07 +0800 Subject: [PATCH 15/15] =?UTF-8?q?=E8=AF=81=E6=8D=AE=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E7=9A=84=E7=9C=81=E7=95=A5=E8=AD=A6=E5=91=8A=E7=85=A7=E5=AE=9E?= =?UTF-8?q?=E8=AF=B4=E7=9C=81=E7=95=A5=E4=BA=86=E4=BB=80=E4=B9=88=EF=BC=9A?= =?UTF-8?q?=E6=94=BE=E4=B8=8D=E4=B8=8B=E4=B8=8D=E4=B8=80=E5=AE=9A=E7=9C=81?= =?UTF-8?q?=E7=95=A5=E4=BA=86=E6=9F=A5=E8=AF=A2=E3=80=82=E7=9C=9F=E5=AE=9E?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E9=87=8C=E6=9F=A5=E8=AF=A2=E5=85=A8=E9=83=A8?= =?UTF-8?q?=E5=88=97=E5=87=BA=E3=80=81=E5=8F=AA=E6=98=AF=E6=9C=89=E7=9A=84?= =?UTF-8?q?=E5=8F=AA=E5=88=97=E5=89=8D=E5=87=A0=E8=A1=8C=EF=BC=8C=E8=AD=A6?= =?UTF-8?q?=E5=91=8A=E5=8D=B4=E5=86=99=E3=80=8C0=20=E6=AC=A1=E5=8F=AA?= =?UTF-8?q?=E5=88=97=E7=BC=96=E5=8F=B7=E5=92=8C=E6=91=98=E8=A6=81=EF=BC=88?= =?UTF-8?q?=EF=BC=89=E3=80=82=E5=86=99=E4=BD=9C=E6=A8=A1=E5=9E=8B=E7=9C=8B?= =?UTF-8?q?=E4=B8=8D=E5=88=B0=E8=BF=99=E4=BA=9B=E6=9F=A5=E8=AF=A2=E7=9A=84?= =?UTF-8?q?=E6=98=8E=E7=BB=86=E3=80=8D=E3=80=82=E7=8E=B0=E5=9C=A8=E5=88=86?= =?UTF-8?q?=E4=B8=89=E7=A7=8D=E5=86=99=EF=BC=9A=E6=9C=89=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E5=8F=AA=E5=88=97=E7=BC=96=E5=8F=B7=E5=92=8C=E6=91=98=E8=A6=81?= =?UTF-8?q?=E7=9A=84=E7=85=A7=E6=97=A7=EF=BC=9B=E6=9F=A5=E8=AF=A2=E9=83=BD?= =?UTF-8?q?=E5=88=97=E5=87=BA=E3=80=81=E6=9C=89=E7=9A=84=E5=8F=AA=E5=88=97?= =?UTF-8?q?=E5=89=8D=E5=87=A0=E8=A1=8C=E7=9A=84=E5=86=99=E3=80=8CN=20?= =?UTF-8?q?=E6=AC=A1=E6=9F=A5=E8=AF=A2=E9=83=BD=E5=88=97=E5=87=BA=E4=BA=86?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=EF=BC=8C=E5=85=B6=E4=B8=AD=20M=20=E6=AC=A1?= =?UTF-8?q?=E5=8F=AA=E5=88=97=E5=89=8D=203=20=E8=A1=8C=E3=80=8D=E5=B9=B6?= =?UTF-8?q?=E6=8C=89=E6=9D=A5=E6=BA=90=E5=88=97=E6=AC=A1=E6=95=B0=EF=BC=9B?= =?UTF-8?q?=E5=8F=AA=E7=9C=81=E7=95=A5=E4=BA=86=E8=A1=A8=E5=92=8C=E5=AD=97?= =?UTF-8?q?=E6=AE=B5=E8=AF=B4=E6=98=8E=E7=9A=84=E7=85=A7=E5=AE=9E=E8=AF=B4?= =?UTF-8?q?=E3=80=82=E4=BA=8B=E4=BB=B6=E5=A4=9A=E5=B8=A6=20compact?= =?UTF-8?q?=E3=80=81compact=5Frows=EF=BC=8C=E5=90=84=E6=9D=A5=E6=BA=90?= =?UTF-8?q?=E5=B8=A6=20compact=EF=BC=9B=E6=97=B6=E9=97=B4=E7=BA=BF?= =?UTF-8?q?=E6=8C=89=E5=90=8C=E6=A0=B7=E4=B8=89=E7=A7=8D=E5=86=99=E6=A0=87?= =?UTF-8?q?=E9=A2=98=E3=80=82=E5=8F=A6=E5=A4=96=E6=8A=8A=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E5=A4=B9=E5=85=B7=E3=80=81=E6=B3=A8=E9=87=8A=E5=92=8C=20check-?= =?UTF-8?q?decode=20=E9=87=8C=E6=B2=BF=E7=94=A8=E7=9A=84=E9=82=A3=E6=AC=A1?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E7=9A=84=E7=9C=9F=E5=AE=9E=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E6=AC=A1=E6=95=B0=E6=8D=A2=E6=88=90=E8=99=9A=E6=9E=84=E7=9A=84?= =?UTF-8?q?=E6=AC=A1=E6=95=B0=E6=88=96=E6=A6=82=E6=8B=AC=E8=AF=B4=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/engine/evidence.py | 7 ++++ backend/app/engine/nodes/report.py | 24 ++++++++---- backend/tests/test_report_catalog_budget.py | 42 +++++++++++++++++++++ frontend/src/run/decode.ts | 15 ++++++++ scripts/check-decode.mjs | 13 +++++++ 5 files changed, 94 insertions(+), 7 deletions(-) diff --git a/backend/app/engine/evidence.py b/backend/app/engine/evidence.py index e7fa5237..407ac600 100644 --- a/backend/app/engine/evidence.py +++ b/backend/app/engine/evidence.py @@ -3277,6 +3277,7 @@ class WriterCatalog: - trimmed：放不下、有省略；nodes 是各来源的查询数 [{node_id, title, total, full, compact, omitted}]， full 列出前 20 行、compact 只列前几行、omitted 只列编号和摘要；omitted 是只列摘要的查询编号 - empty_sources：结构化结论为空的来源（Agent 的名字） + - compact_rows：只列前几行的查询列了几行 清单有省略、或有来源的结构化结论为空时，报告节点加写作规则（ABSENCE_RULE）；有省略时组装文档另查缺失断言 """ @@ -3287,6 +3288,12 @@ class WriterCatalog: nodes: list[dict[str, Any]] = field(default_factory=list) omitted: list[str] = field(default_factory=list) empty_sources: list[str] = field(default_factory=list) + compact_rows: int = _PROMPT_COMPACT_ROWS + + @property + def compact(self) -> int: + """列出了数据、但只列了前 compact_rows 行的查询次数。""" + return sum(int(r.get("compact") or 0) for r in self.nodes) @property def guarded(self) -> bool: diff --git a/backend/app/engine/nodes/report.py b/backend/app/engine/nodes/report.py index 920930bc..37be9efa 100644 --- a/backend/app/engine/nodes/report.py +++ b/backend/app/engine/nodes/report.py @@ -189,12 +189,21 @@ async def model_window(session: Any, spec: ModelSpec, model_id: str) -> int | No def _trimmed_message(writer: WriterCatalog) -> str: - """写作目录有省略时的那条警告（给人看）：各来源省略了几次，写作模型看不到哪些明细。""" - per = "、".join(f"{r['title']}省略 {r['omitted']} 次" for r in writer.nodes if r["omitted"]) - shown = writer.queries - len(writer.omitted) - return (f"证据目录超出篇幅（约 {writer.budget} 字，按写作模型的上下文窗口确定）：{writer.queries} 次查询中 " - f"{shown} 次列出了数据，{len(writer.omitted)} 次只列编号和摘要（{per}）。写作模型看不到这些查询的明细，" - "报告不得据此断言没有数据") + """写作目录有省略时的那条警告（给人看）：省略的是什么、各来源省略了几次，写作模型看不到哪些明细。 + + 放不下不一定省略了查询：查询全都列出、只是有的只列前几行，或者只省略了表和字段的说明。以前一律按「省略查询」写， + 全部列出时成了「0 次只列编号和摘要（）。写作模型看不到这些查询的明细」，说的不是实情。""" + head = f"证据目录超出篇幅（约 {writer.budget} 字，按写作模型的上下文窗口确定）：" + if writer.omitted: + per = "、".join(f"{r['title']}省略 {r['omitted']} 次" for r in writer.nodes if r["omitted"]) + shown = writer.queries - len(writer.omitted) + return (f"{head}{writer.queries} 次查询中 {shown} 次列出了数据，{len(writer.omitted)} 次只列编号和摘要（{per}）。" + "写作模型看不到这些查询的明细，报告不得据此断言没有数据") + if writer.compact: + per = "、".join(f"{r['title']} {r['compact']} 次" for r in writer.nodes if r.get("compact")) + return (f"{head}{writer.queries} 次查询都列出了数据，其中 {writer.compact} 次只列前 {writer.compact_rows} 行" + f"（{per}）。写作模型看不到这些查询其余的行，报告不得据此断言没有数据") + return f"{head}{writer.queries} 次查询都列出了数据，表和字段、知识库检索的说明有省略" def report_entities(spec: GraphSpec, node: GraphNode) -> str: @@ -475,7 +484,8 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]: if writer.trimmed: ctx.emit(EventType.LOG, level="warn", code="report_catalog_trimmed", message=_trimmed_message(writer), budget=writer.budget, window=window, queries=writer.queries, omitted=len(writer.omitted), - nodes=[{k: r[k] for k in ("node_id", "title", "total", "omitted")} for r in writer.nodes]) + compact=writer.compact, compact_rows=writer.compact_rows, + nodes=[{k: r[k] for k in ("node_id", "title", "total", "omitted", "compact")} for r in writer.nodes]) messages = _messages(ctx, state, writer, claims) started = time.perf_counter() spent: list[dict[str, Any]] = [] diff --git a/backend/tests/test_report_catalog_budget.py b/backend/tests/test_report_catalog_budget.py index 3ea3da91..b02241a9 100644 --- a/backend/tests/test_report_catalog_budget.py +++ b/backend/tests/test_report_catalog_budget.py @@ -207,3 +207,45 @@ async def test_the_window_comes_from_the_provider_entry_then_the_catalog(): finally: await session.execute(delete(Provider).where(Provider.id == row.id)) await session.commit() + + +# ---------------------------------------------------------------- 警告说的是实情 + + +def _writer(nodes, omitted=()): + from app.engine.evidence import WriterCatalog + + return WriterCatalog(text="", budget=51200, queries=sum(r["total"] for r in nodes), trimmed=True, nodes=nodes, + omitted=list(omitted)) + + +def _row(title, total, *, full=0, compact=0, omitted=0): + return {"node_id": title, "title": title, "total": total, "full": full, "compact": compact, "omitted": omitted} + + +def test_the_warning_names_omitted_queries_when_there_are_some(): + from app.engine.nodes.report import _trimmed_message + + text = _trimmed_message(_writer([_row("会员专项", 26, full=26), _row("卡券核销", 79, full=20, omitted=59)], + omitted=[f"Q{i}" for i in range(1, 60)])) + assert "105 次查询中 46 次列出了数据，59 次只列编号和摘要（卡券核销省略 59 次）" in text + assert "看不到这些查询的明细" in text + + +def test_all_queries_listed_but_some_only_first_rows_is_not_called_omitted(): + """放不下但查询全都列出，只是有的只列前几行：不写「0 次只列编号和摘要（）」，写清是行数有省略。""" + from app.engine.nodes.report import _trimmed_message + + writer = _writer([_row("会员专项", 26, full=26), _row("卡券核销", 79, full=30, compact=49)]) + text = _trimmed_message(writer) + assert writer.compact == 49 + assert f"105 次查询都列出了数据，其中 49 次只列前 {writer.compact_rows} 行（卡券核销 49 次）" in text + assert "看不到这些查询其余的行" in text + assert "只列编号和摘要" not in text and "（）" not in text + + +def test_only_the_table_notes_were_cut(): + from app.engine.nodes.report import _trimmed_message + + text = _trimmed_message(_writer([_row("会员专项", 26, full=26)])) + assert "26 次查询都列出了数据，表和字段、知识库检索的说明有省略" in text and "（）" not in text diff --git a/frontend/src/run/decode.ts b/frontend/src/run/decode.ts index 51a83b4c..051a01ed 100644 --- a/frontend/src/run/decode.ts +++ b/frontend/src/run/decode.ts @@ -773,9 +773,24 @@ function explainLog(code: string | undefined, message: string, d: Record typeof r?.omitted === 'number' && r.omitted > 0 && str(r?.title)) .map((r) => `${r.title} 省略 ${formatNumber(r.omitted)}/${formatNumber(r.total)} 次`) + const all = total != null ? `${formatNumber(total)} 次查询都列出了数据` : '查询都列出了数据' + if (omitted === 0) { + // 放不下但一次查询都没省略：有的只列了前几行，或者只省略了表和字段的说明。不写「0 次只列了编号和摘要」 + const rowsPer = rows.filter((r) => typeof r?.compact === 'number' && r.compact > 0 && str(r?.title)) + .map((r) => `${r.title} ${formatNumber(r.compact)}/${formatNumber(r.total)} 次只列前几行`) + return { + title: compact ? `证据目录超出篇幅，${all}，${formatNumber(compact)} 次只列了前几行` + : compact === 0 ? `证据目录超出篇幅，${all}，表和字段的说明有省略` + : `证据目录超出篇幅，${all}，部分明细有省略`, + sub: rowsPer.length ? rowsPer.join('、') : undefined, + next: '写作模型看到了每次查询，但有的只看到前几行；其余的行仍在证据台账里，报告不得据此断言没有数据。' + + '需要完整明细时，请换用上下文窗口更大的写作模型，或让 Agent 在 SQL 中先汇总再交出结果', + } + } return { title: omitted != null ? `证据目录超出篇幅，${total != null ? `${formatNumber(total)} 次查询中 ` : ''}${formatNumber(omitted)} 次只列了编号和摘要` diff --git a/scripts/check-decode.mjs b/scripts/check-decode.mjs index f6fdaaaf..d6f1a459 100644 --- a/scripts/check-decode.mjs +++ b/scripts/check-decode.mjs @@ -1336,6 +1336,19 @@ await section('报告核对 report.checked、口径卡的台账（可点击证 { node_id: 'c', title: '活动内容', total: 42, omitted: 22 }] })])).find((s) => s.code === 'report_catalog_trimmed') check('report_catalog_trimmed：标题说几次查询只列了编号和摘要', trimmed?.title === '证据目录超出篇幅，164 次查询中 74 次只列了编号和摘要', trimmed?.title) check('……副标题按来源列省略了几次（没省略的来源不列）', trimmed?.sub === '卡券核销 省略 52/79 次、活动内容 省略 22/42 次', trimmed?.sub) + // 放不下但查询全都列出、只是有的只列前几行：不写「0 次只列了编号和摘要」 + const rowsOnly = flatten(mod.decodeRun([E(1, 'node.started', 'w', { node_type: 'report', label: '写报告' }), + E(2, 'log', 'w', { level: 'warn', code: 'report_catalog_trimmed', message: '证据目录超出篇幅', budget: 51200, window: null, + queries: 140, omitted: 0, compact: 38, compact_rows: 3, nodes: [{ node_id: 'a', title: '会员专项', total: 60, omitted: 0, compact: 0 }, + { node_id: 'b', title: '卡券核销', total: 80, omitted: 0, compact: 38 }] })])).find((s) => s.code === 'report_catalog_trimmed') + check('……查询全都列出、有的只列前几行：标题照实说，不写「0 次只列了编号和摘要」', + rowsOnly?.title === '证据目录超出篇幅，140 次查询都列出了数据，38 次只列了前几行' && !rowsOnly?.title?.includes('编号和摘要'), rowsOnly?.title) + check('……副标题按来源列只列了前几行的次数', rowsOnly?.sub === '卡券核销 38/80 次只列前几行', rowsOnly?.sub) + check('……下一步说其余的行仍在台账里', !!rowsOnly?.next?.includes('只看到前几行') && !!rowsOnly?.next?.includes('不得据此断言没有数据'), rowsOnly?.next) + const notesOnly = flatten(mod.decodeRun([E(1, 'node.started', 'w', { node_type: 'report', label: '写报告' }), + E(2, 'log', 'w', { level: 'warn', code: 'report_catalog_trimmed', message: '证据目录超出篇幅', queries: 12, omitted: 0, compact: 0, + nodes: [{ node_id: 'a', title: '会员专项', total: 12, omitted: 0, compact: 0 }] })])).find((s) => s.code === 'report_catalog_trimmed') + check('……只省略了表和字段的说明：照实说', notesOnly?.title === '证据目录超出篇幅，12 次查询都列出了数据，表和字段的说明有省略', notesOnly?.title) check('……下一步说被省略的仍在台账里、不得据此断言没有数据，以及怎么看全', !!trimmed?.next?.includes('不得据此断言没有数据') && !!trimmed?.next?.includes('上下文窗口更大') && trimmed?.level === 'warn' && trimmed?.detail === TRIMMED, trimmed?.next) const trimmedBare = flatten(mod.decodeRun([E(1, 'log', 'w', { level: 'warn', code: 'report_catalog_trimmed', message: '目录有省略' })]))