From 132d126c6f8e3054e5d5acde872bfbc710cc3342 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 01:46:55 +0800 Subject: [PATCH] =?UTF-8?q?=E6=A0=B8=E6=9F=A5=20agent=20=E5=A4=8D=E6=A0=B8?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=EF=BC=9A=E6=95=B0=E5=80=BC=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E7=9A=84=E5=90=84=E7=A7=8D=E5=86=99=E6=B3=95=E9=83=BD=E6=8D=A2?= =?UTF-8?q?=E6=88=90=E6=A0=BC=E5=AD=90=E5=BC=95=E7=94=A8=E3=80=81=E8=AE=A4?= =?UTF-8?q?=E4=B8=8D=E5=87=BA=E7=9A=84=E7=BB=93=E6=9E=84=E4=B8=8B=E7=9A=84?= =?UTF-8?q?=E6=95=B0=E4=B8=8D=E9=87=87=E4=BF=A1=EF=BC=9B=E9=80=80=E5=9B=9E?= =?UTF-8?q?=E6=99=AE=E9=80=9A=E8=B0=83=E7=94=A8=E6=97=B6=E6=8A=8A=E7=BB=93?= =?UTF-8?q?=E6=9E=84=E5=86=99=E8=BF=9B=E6=B6=88=E6=81=AF=EF=BC=8C=E6=B2=A1?= =?UTF-8?q?=E4=BA=A4=E7=9A=84=E5=AD=97=E6=AE=B5=E4=B8=8D=E5=86=8D=E7=AE=97?= =?UTF-8?q?=E3=80=8C=E6=9F=A5=E4=B8=8D=E5=88=B0=E3=80=8D=EF=BC=9B=E6=94=B6?= =?UTF-8?q?=E5=B0=BE=E7=9A=84=E9=92=B1=E6=8C=89=E6=9C=80=E8=B4=B5=E7=9A=84?= =?UTF-8?q?=E6=A0=B7=E5=AD=90=E7=95=99=E3=80=81=E7=95=99=E8=BF=87=E7=9A=84?= =?UTF-8?q?=E4=B8=8D=E5=86=8D=E8=A2=AB=E4=B8=8A=E9=99=90=E6=8B=A6=EF=BC=9B?= =?UTF-8?q?=E6=B0=B8=E8=BF=9C=E4=B8=8D=E6=8A=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 数只认格子:原来只认 type: number / integer,可为 null 的数(anyOf,Pydantic 的 Optional[float] 就这么写)、$ref 引用进来的、allOf 包着的数值字段都没换成格子引用,模型写的数原样收下。现在先展开本地 $ref、并掉 allOf,anyOf / oneOf 里有一支能放数就整个算数值字段;取值时按值的形状挑分支。认不出的结构(没写类型、文本字段、没写 items 的数组、没写属性的对象)下模型写的数一律丢掉、照记下来,码值里的数照收。嵌套的引用、数组写成一段 JSON 文本的照样认,还不是数组的照实记下格式不对。数值字段引用到日期、名称这类不是数的格子,记成 unresolved,不拿去和原流程的数比。 退回普通调用:原来消息里只有「按给定结构填写」,结构却不在请求里,模型猜不到字段名,交回来对不上的字段全成了 missing,双盲复算会说盲算什么也没查到。现在把结构(JSON Schema)写进消息;没交的字段记 unresolved(模型没有交这个字段),交了 null 才是 missing;一个要求的字段都没交就是没按结构交结果,照错误交回。模型建结构化输出就报错的,直接走普通调用,不白算一次调用。 上限与金额:一条查询自己的时限到了(ToolTimeout 也是 TimeoutError)原来被当成整个核查的时长到了,现在先认 ToolTimeout,原因交回模型、核查照常往下走。收尾要把查到的结果交给模型,原来留钱按眼下的消息估,最后一条查询的大段结果一进来就不够,查完了付不起收尾、结果白查;现在按收尾最贵的样子留(查询结果按 _digest 的上限、最后的说明截到 2000 字、结构也算),留过钱的收尾不再被上限拦下,实际用量超出估算或并行的核查先用掉当天余额时,最多超出一次收尾调用的钱。每日上限缺省按设置里的「核查每日金额上限」,明确传 None 才是不限,质询接口不必各自记得去读。 永远不抛:每日账写不进去、建工具出错这类意外原来会直接抛出,节点失败、质询接口回 500;现在整段包住,按已经攒下的用量、查询照实交回。 --- backend/app/engine/verify_agent.py | 543 +++++++++++++++++++++++------ backend/tests/test_verify_agent.py | 320 ++++++++++++++++- 2 files changed, 748 insertions(+), 115 deletions(-) diff --git a/backend/app/engine/verify_agent.py b/backend/app/engine/verify_agent.py index 4294f043..ad0a39d2 100644 --- a/backend/app/engine/verify_agent.py +++ b/backend/app/engine/verify_agent.py @@ -8,8 +8,10 @@ **数从快照取。** 每次成功的查询是一份查询快照(工具自己落的 put_json),按这个 agent 自己的顺序编 Q1、Q2…。 最后交结果时,调用方结构里的数值字段一律改成格子引用 {alias, row, column},由代码按引用从快照里取值:模型自己 -写的数字不采信,取不到就是 None,不兜底成 0。理由文字里的数字要写成 [[v:Qn.rX.列]],用本 agent 自己的台账 -(evidence.build_catalog)组装成文档(evidence.compose_doc),裸数字会被标出来。 +写的数字不采信,取不到就是 None,不兜底成 0。数值字段的各种写法都认(type 列表、anyOf 里的数、$ref 引用进来的、 +allOf 包着的);认不出的结构下模型写的数一律丢掉、照记下来,宁可少一个数,不让它绕过格子引用。理由文字里的数字 +要写成 [[v:Qn.rX.列]],用本 agent 自己的台账(evidence.build_catalog)组装成文档(evidence.compose_doc),裸数字 +会被标出来。 **不进写作目录。** 节点里调用时(给了 emit)每次工具调用发 tool.start / tool.end,查询的 tool.end 带 query_artifact 和 role: verify——随运行封存、证据面板能下钻。但这里绝不往 state["evidence"] 写 kind: query: @@ -17,11 +19,14 @@ query_artifact 和 role: verify——随运行封存、证据面板能下钻。 台账条目由它按自己的种类记(dual_check / challenge / inquiry)。 **上限。** 查询次数、时长、金额(这一次、全局每日)任何一项到顶就停止查询,按已经查到的结果收尾:收尾那次 -调用总是要做的(时长到顶时再给 SETTLE_GRACE_S 的宽限),所以金额上限每一轮都给它留出钱。模型不在价格目录里 -时估不出金额,金额上限对它无效,照裁判的说法记一条说明。每次调用都记进核查的每日账(engine/spend.py)。 +调用总是要做的(时长到顶时再给 SETTLE_GRACE_S 的宽限),所以每一轮都按收尾最贵的样子给它留出钱——收尾时要把 +查到的结果交给模型,结果有多长事先不知道,按 _digest 的上限算。留过钱的收尾不再被上限拦下:某一轮的实际用量 +超出了估算、或者并行的核查先用掉了当天的余额时,宁可超出上限一次收尾调用的钱,也不让已经查到的结果白查。 +每日上限不传就按设置里的「核查每日金额上限」,明确传 None 才是不限。模型不在价格目录里时估不出金额,金额上限 +对它无效,照裁判的说法记一条说明。每次调用都记进核查的每日账(engine/spend.py)。 -永远不抛:模型没配好、调用失败、格式不对、触顶,都落在返回值里(error / limit)。模型调用都经 get_chat_model, -测试换成剧本。 +永远不抛:模型没配好、调用失败、格式不对、触顶、库写不进去,都落在返回值里(error / limit)。模型调用都经 +get_chat_model,测试换成剧本。 """ from __future__ import annotations @@ -32,6 +37,7 @@ import logging import time from collections.abc import Callable, Iterable from dataclasses import asdict, dataclass, field +from enum import Enum from typing import Any from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage, ToolMessage @@ -42,9 +48,9 @@ from app.core.events import EventType from app.db.base import SessionLocal from app.engine import spend from app.engine.evidence import build_catalog, cell_eid, compose_doc, query_entry_fields, schema_ledger_entry -from app.engine.expressions import CellError, column_kind, locate_cell +from app.engine.expressions import CellError, column_kind, locate_cell, numeric_text from app.engine.judge import _json_in, _tokens, _usd -from app.engine.nodes.llm import _digest, _split_structured, _typed, explain_model_error +from app.engine.nodes.llm import _DIGEST_BUDGET, _digest, _split_structured, _typed, explain_model_error from app.engine.state import message_text from app.engine.toolcalls import ToolTimeout, limit_fields, limit_of, run_bounded from app.providers import catalog as pricing @@ -61,6 +67,14 @@ _OUT_STEP = 512 _OUT_FINAL = 1024 #: 工具结果交给模型时最多这么多字:完整结果在查询快照里,收尾时另按 _digest 列出 _TOOL_CHARS = 12_000 +#: 收尾时带上的模型最后那段说明最多这么多字。收尾那次调用的长度有了上限,循环里才能按它把钱留够 +_SAID_CHARS = 2_000 +#: 收尾那次调用里查询结果最多占多少字:_digest 的上限,加上它截断时末尾那句说明 +_DIGEST_ROOM = _DIGEST_BUDGET + 64 +#: 留钱时给「已达上限(…)」那句话留的字数 +_LIMIT_ROOM = 40 +#: 本地 $ref 最多展开这么多层 +_REF_DEPTH = 8 #: 结果里自动加上的理由字段(调用方的结构里已经有同名的字符串字段就沿用它) RATIONALE = "rationale" #: 触顶原因 @@ -96,14 +110,23 @@ _RATIONALE_FIELD: dict[str, Any] = { Emit = Callable[..., Any] +class Default(Enum): + """run_verify_agent 的 daily_max_usd 缺省值:按设置里的「核查每日金额上限」(api/settings.py 的 + verify_settings)。节点、质询接口不必各自记得去读;明确传 None 才是不限。""" + + SETTINGS = "settings" + + @dataclass class VerifyOutcome: """一次核查的结果。 result:调用方结构的形状,数值字段是从快照取到的值(取不到为 None),另有 rationale(理由渲染后的正文); 没交成结果时为 None,error 说明为什么。cells:每个数值字段的出处 {path, status, ref, alias, row, column, - artifact, eid, value},status 是 resolved / missing(模型说查不到)/ unresolved(reason 写原因,模型直接写了 - 数字的记 model_value)。queries:成功的查询 {alias, artifact, sql, row_count, truncated, source, tool, call_id}。 + artifact, eid, value},status 是 resolved(value 是那一格的值,那一格本身是 NULL 时为 None)/ missing(模型 + 交了 null:查不到)/ unresolved(reason 写原因:没交这个字段、引用指不到、指到的不是数;模型直接写了数的记 + model_value,非数值字段里写的数也照此记下)。queries:成功的查询 {alias, artifact, sql, row_count, truncated, + source, tool, call_id}。 rationale:理由组装成的报告文档(evidence.compose_doc 的形状,violations 里有裸数字)。usage:{input_tokens, output_tokens, total_tokens, cost_usd, calls, unpriced_calls}。limit:触顶原因(LIMITS 之一),没触顶为 None。 answered:模型是不是自己停下来的(没被上限掐断)。 @@ -139,30 +162,132 @@ def _is_object(schema: Any) -> bool: return isinstance(schema, dict) and ("object" in _types(schema) or "properties" in schema) -def _is_number(schema: Any) -> bool: - return bool(_types(schema) & {"number", "integer"}) +def _is_array(schema: Any) -> bool: + return isinstance(schema, dict) and ("array" in _types(schema) or "items" in schema) + + +def _numeric_types(schema: Any) -> set[str] | None: + """数值字段允许的类型(number / integer,连同写在一起的 null、string);不是数值字段返回 None。 + + type 写成 "number"、["number", "null"],或者 anyOf / oneOf 里有一支是数(可为 null 的数一般这么写, + Pydantic 的 Optional[float] 就是)都算:有一支能放数,整个字段就得是格子引用,不然数从那一支漏过去。 + """ + if not isinstance(schema, dict): + return None + types = set(_types(schema)) + for key in ("anyOf", "oneOf"): + for branch in schema.get(key) or (): + if isinstance(branch, dict) and not _is_object(branch) and not _is_array(branch): + types |= _numeric_types(branch) or _types(branch) + return types if types & {"number", "integer"} else None + + +def _has_numbers(schema: Any, depth: int = 0) -> bool: + """结构里有没有数值字段(嵌套的也算)。""" + if not isinstance(schema, dict) or depth > 32: + return False + if _numeric_types(schema) is not None: + return True + subs = [*(schema.get("properties") or {}).values(), schema.get("items"), schema.get("additionalProperties"), + *(schema.get("anyOf") or ()), *(schema.get("oneOf") or ())] + return any(_has_numbers(sub, depth + 1) for sub in subs if isinstance(sub, dict)) + + +#: 值是「名字 → 结构」的键、值是一个结构的键、值是一串结构的键:展开引用时往里走;其余的键(enum、description…)原样 +_SCHEMA_MAPS = ("properties", "patternProperties") +_SCHEMA_ONE = ("items", "additionalProperties", "contains", "not") +_SCHEMA_LIST = ("anyOf", "oneOf", "prefixItems") + + +def _merge(a: dict[str, Any], b: dict[str, Any]) -> dict[str, Any]: + """allOf 的两支并成一个节点:属性合在一起,必填合在一起,类型取并集(只用来认是不是数、是不是对象)。""" + out = dict(a) + for key, value in b.items(): + if key == "properties" and isinstance(value, dict): + out["properties"] = {**(out.get("properties") or {}), **value} + elif key == "required" and isinstance(value, list): + out["required"] = list(dict.fromkeys([*(out.get("required") or []), *value])) + elif key == "type": + merged = _types(out) | _types(b) + out["type"] = next(iter(merged)) if len(merged) == 1 else sorted(merged) + else: + out.setdefault(key, value) + return out + + +def _inline(schema: Any) -> dict[str, Any]: + """展开调用方结构里的本地引用($ref: #/$defs/…、#/definitions/…),allOf 并成一个节点,去掉 $defs。 + + 后面换格子引用、取值只认 type / properties / items / anyOf 这几种写法:引用不展开,引用进来的数值字段就绕过了 + 格子引用这一关(Pydantic 生成的结构里,嵌套的对象全是 $ref)。引用不到的、绕回自己的(树形结构第二次遇到同一个 + 引用)就地断开,当作认不出的结构:模型在那里写的数照样不采信。 + """ + root = schema if isinstance(schema, dict) else {} + + def target(ref: Any) -> dict[str, Any] | None: + if not isinstance(ref, str) or not ref.startswith("#/"): + return None + node: Any = root + for part in ref[2:].split("/"): + node = node.get(part.replace("~1", "/").replace("~0", "~")) if isinstance(node, dict) else None + return node if isinstance(node, dict) else None + + def expand(node: Any, trail: tuple[str, ...]) -> Any: + while isinstance(node, dict) and "$ref" in node: + ref, rest = node["$ref"], {k: v for k, v in node.items() if k != "$ref"} + found = target(ref) + if found is None or ref in trail or len(trail) >= _REF_DEPTH: + node = rest + break + node, trail = {**found, **rest}, (*trail, ref) + if not isinstance(node, dict): + return node + out: dict[str, Any] = {} + for key, value in node.items(): + if key in ("$defs", "definitions", "allOf"): + continue + if key in _SCHEMA_MAPS and isinstance(value, dict): + out[key] = {k: expand(v, trail) for k, v in value.items()} + elif key in _SCHEMA_ONE and isinstance(value, dict): + out[key] = expand(value, trail) + elif key in _SCHEMA_LIST and isinstance(value, list): + out[key] = [expand(v, trail) for v in value] + else: + out[key] = value + for part in node.get("allOf") or (): + if isinstance(part, dict): + out = _merge(out, expand(part, trail)) + return out + + return expand(root, ()) def result_schema(schema: dict[str, Any]) -> dict[str, Any]: - """交给模型的结构:调用方结构里的数值字段(number / integer,嵌套的对象、数组里的也算)一律换成可为 null 的 - 格子引用,另加一个理由字段(已有同名的字符串字段就沿用)。其余字段原样。""" + """交给模型的结构:调用方结构里的数值字段(见 _numeric_types;嵌套的对象、数组、anyOf 的分支里的也算,引用先 + 展开)一律换成可为 null 的格子引用,另加一个理由字段(已有同名的字符串字段就沿用)。其余字段原样。""" def rewrite(node: Any) -> Any: if not isinstance(node, dict): return node - if _is_number(node): + if _numeric_types(node) is not None: out: dict[str, Any] = {"anyOf": [_CELL_REF, {"type": "null"}]} if node.get("description"): out["description"] = f"{node['description']}(写格子引用,查不到写 null)" return out - if _is_object(node): - props = node.get("properties") or {} - return {**node, "type": "object", "properties": {k: rewrite(v) for k, v in props.items()}} - if "array" in _types(node) and isinstance(node.get("items"), dict): - return {**node, "items": rewrite(node["items"])} - return node + out = dict(node) + if isinstance(node.get("properties"), dict): + out["properties"] = {k: rewrite(v) for k, v in node["properties"].items()} + out.setdefault("type", "object") + for key in ("items", "additionalProperties"): + if isinstance(node.get(key), dict): + out[key] = rewrite(node[key]) + for key in ("anyOf", "oneOf"): + if isinstance(node.get(key), list): + out[key] = [rewrite(b) for b in node[key]] + return out - root = rewrite(schema if _is_object(schema) else {"type": "object", "properties": {}}) + inlined = _inline(schema) + root = rewrite(inlined if _is_object(inlined) else {"type": "object", "properties": {}}) props = dict(root.get("properties") or {}) props.setdefault(RATIONALE, _RATIONALE_FIELD) required = list(dict.fromkeys([*(root.get("required") or []), RATIONALE])) @@ -170,6 +295,42 @@ def result_schema(schema: dict[str, Any]) -> dict[str, Any]: return {**root, "title": "verify_result", "type": "object", "properties": props, "required": required} +@dataclass(frozen=True) +class _Gap: + """模型交的结果里这个位置没有值。reason 说为什么:没交这个字段、上一层的格式不对。""" + + reason: str + + +_LEFT_OUT = _Gap("模型没有交这个字段") + + +def _fits(branch: dict[str, Any], value: Any) -> bool: + """anyOf / oneOf 的这一支收不收这个值(按形状认)。""" + if isinstance(value, _Gap): + return _has_numbers(branch) + if isinstance(value, dict): + return _is_object(branch) + if isinstance(value, list): + return _is_array(branch) + if value is None: + return "null" in _types(branch) + if isinstance(value, bool): + return "boolean" in _types(branch) + if isinstance(value, str): + return "string" in _types(branch) or "enum" in branch or "const" in branch + # 数:能放数的分支已经整个换成了格子引用(_numeric_types),这里只剩码值里的数。True == 1,真假不算数 + choices = [*(branch.get("enum") or ()), *([branch["const"]] if "const" in branch else [])] + return any(c == value and not isinstance(c, bool) for c in choices) + + +def _decoded(value: Any, kind: type) -> Any: + """有的模型把嵌套的对象、数组写成一段 JSON 文本(裁判见过同样的事,judge._items_of):解得开、形状对就用解开的。""" + if isinstance(value, str) and isinstance(found := _json_in(value), kind): + return found + return value + + class _Cells: """按格子引用从查询快照里取值:快照经工件库读(复验哈希),同一份只读一次。""" @@ -187,15 +348,22 @@ class _Cells: self.memo[artifact] = (None, "查询快照与哈希不一致,可能已被修改") return self.memo[artifact] - def take(self, value: Any, path: str, schema: dict[str, Any]) -> Any: - """一个数值字段:按引用取值,记下出处,返回值(取不到为 None)。""" + def take(self, value: Any, path: str, types: set[str]) -> Any: + """一个数值字段:按引用取值,记下出处,返回值(取不到为 None)。types 是字段允许的类型:不许文本的, + 引用的格子不是数(日期、名称、真假)就不采信——拿它和原流程的数比,比较本身就会出错。""" + if isinstance(value, _Gap): + self.cells.append({"path": path, "status": "unresolved", "reason": value.reason}) + return None if value is None: self.cells.append({"path": path, "status": "missing"}) return None + value = _decoded(value, dict) if not isinstance(value, dict): # 模型直接写了数:不采信。照记下来,界面上能看到它想填什么 + number = isinstance(value, (int, float)) or numeric_text(value, loose=True) is not None self.cells.append({"path": path, "status": "unresolved", "model_value": value, - "reason": "模型直接写了数值,没有给出格子引用,不采信"}) + "reason": "模型直接写了数值,没有给出格子引用,不采信" if number + else "格子引用的格式不对(应写成 {alias, row, column}),不采信"}) return None alias = str(value.get("alias") or value.get("call") or "").strip() row, column = value.get("row"), value.get("column") @@ -220,25 +388,99 @@ class _Cells: self.cells.append({"path": path, "status": "unresolved", "alias": alias, "artifact": query["artifact"], "reason": f"{alias} {e}"}) return None - truth = _typed(raw, schema, column_kind(content, name)) - self.cells.append({"path": path, "status": "resolved", "ref": f"{alias}.r{row}.{name}", "alias": alias, - "row": row, "column": name, "artifact": query["artifact"], - "eid": cell_eid(query["artifact"], row, name), "value": truth}) + truth = _typed(raw, {"type": sorted(types)}, column_kind(content, name)) + cited = {"ref": f"{alias}.r{row}.{name}", "alias": alias, "row": row, "column": name, + "artifact": query["artifact"], "eid": cell_eid(query["artifact"], row, name), "value": truth} + if truth is not None and "string" not in types and (isinstance(truth, bool) + or not isinstance(truth, (int, float))): + self.cells.append({"path": path, "status": "unresolved", **cited, + "reason": f"{alias} 第 {row} 行「{name}」的值是「{truth}」,不是数值,不采信"}) + return None + self.cells.append({"path": path, "status": "resolved", **cited}) return truth def walk(self, schema: Any, value: Any, path: str = "") -> Any: - """按调用方的结构走一遍模型交的结果:数值字段换成从快照取到的值,其余原样。""" - if _is_number(schema): - return self.take(value, path, schema) - if _is_object(schema): - given = value if isinstance(value, dict) else {} - return {k: self.walk(sub, given.get(k), f"{path}.{k}" if path else k) - for k, sub in (schema.get("properties") or {}).items()} - if "array" in _types(schema) and isinstance(schema.get("items"), dict): - if not isinstance(value, list): + """按调用方的结构(_inline 展开过的)走一遍模型交的结果:数值字段换成从快照取到的值,其余原样。""" + types = _numeric_types(schema) + if types is not None: + return self.take(value, path, types) + if not isinstance(schema, dict): + return self.scrub({}, value, path) + branches = [b for key in ("anyOf", "oneOf") for b in schema.get(key) or () if isinstance(b, dict)] + if branches: + rest = {k: v for k, v in schema.items() if k not in ("anyOf", "oneOf")} + if any(_is_object(b) for b in branches): + value = _decoded(value, dict) + if any(_is_array(b) for b in branches): + value = _decoded(value, list) + if value is None and any("null" in _types(b) for b in branches): return None - return [self.walk(schema["items"], v, f"{path}[{i}]") for i, v in enumerate(value)] - return value + chosen = next((b for b in branches if _fits(b, value)), None) + return self.walk({**rest, **chosen} if chosen is not None else rest, value, path) + if _is_object(schema): + return self._object(schema, value, path) + if _is_array(schema): + return self._array(schema, value, path) + return self.scrub(schema, value, path) + + def _object(self, schema: dict[str, Any], value: Any, path: str) -> dict[str, Any]: + given = _decoded(value, dict) + if isinstance(given, dict): + def child(key: str) -> Any: + return given[key] if key in given else _LEFT_OUT + else: + gap = given if given is None or isinstance(given, _Gap) else _Gap("模型交的格式不对:这里应是一个对象") + + def child(key: str) -> Any: + return gap + props = schema.get("properties") or {} + out = {k: self.walk(sub, child(k), f"{path}.{k}" if path else k) for k, sub in props.items()} + extra = schema.get("additionalProperties") + for k, v in (given.items() if isinstance(given, dict) else ()): + if k in props: + continue + if isinstance(extra, dict): + out[k] = self.walk(extra, v, f"{path}.{k}" if path else k) + elif extra is True or (not props and extra is not False): + # 没写属性的对象(任意键)照收,里面的数照样不采信;写了属性的,没声明的键不要 + out[k] = self.scrub({}, v, f"{path}.{k}" if path else k) + return out + + def _array(self, schema: dict[str, Any], value: Any, path: str) -> list[Any] | None: + items = _decoded(value, list) + if isinstance(items, list): + sub = schema.get("items") if isinstance(schema.get("items"), dict) else {} + return [self.walk(sub, v, f"{path}[{i}]") for i, v in enumerate(items)] + if _has_numbers(schema): + if items is None: + self.cells.append({"path": path, "status": "missing"}) + elif isinstance(items, _Gap): + self.cells.append({"path": path, "status": "unresolved", "reason": items.reason}) + else: + self.cells.append({"path": path, "status": "unresolved", "model_value": items, + "reason": "模型交的格式不对:这里应是一个数组"}) + elif items is not None and not isinstance(items, _Gap): + self.scrub({}, items, path) + return None + + def scrub(self, schema: Any, value: Any, path: str) -> Any: + """认不出的结构(没写类型、文本字段、没写 items 的数组……)下的值:文本、真假、null 原样,数一律不采信、 + 照记下来(码值里的数是调用方给的选项,不是查出来的数,照收),对象、数组逐项照此处理。""" + if isinstance(value, _Gap): + return None + if value is None or isinstance(value, (bool, str)): + return value + if isinstance(value, (int, float)): + if isinstance(schema, dict) and _fits(schema, value): + return value + self.cells.append({"path": path, "status": "unresolved", "model_value": value, + "reason": "这里不是数值字段,模型却写了数值,不采信"}) + return None + if isinstance(value, dict): + return {k: self.scrub({}, v, f"{path}.{k}" if path else k) for k, v in value.items()} + if isinstance(value, list): + return [self.scrub({}, v, f"{path}[{i}]") for i, v in enumerate(value)] + return None # -------------------------------------------------------------------------- @@ -256,16 +498,25 @@ class _Budget: self.daily_max_usd = daily_max_usd self.spent = 0.0 - def estimate(self, text: str, out: int) -> float: - return pricing.estimate_cost(self.model_id, _tokens(text), out) if self.priced else 0.0 + def estimate(self, text: str, out: int, *, room: int = 0) -> float: + """按文本估一次调用的金额。room 是还不知道内容、按最坏情况(一个字一个 token)算的输入字数。""" + return pricing.estimate_cost(self.model_id, _tokens(text) + room, out) if self.priced else 0.0 - async def reserve(self, estimate: float, *, keep: float = 0.0) -> tuple[str | None, str | None]: - """(记在哪天, None) 或 (None, 拦下的上限)。keep 是要额外留出的钱(收尾那次调用):只用来判断,不预留。""" - if self.priced and self.max_cost_usd is not None and self.spent + estimate + keep > self.max_cost_usd + 1e-12: - return None, "max_cost_usd" - cap = self.daily_max_usd if self.priced else None - if cap is not None and keep and (await spend.daily_spend())["usd"] + estimate + keep > cap + 1e-12: - return None, "daily_max_usd" + async def reserve(self, estimate: float, *, keep: float = 0.0, kept: bool = False + ) -> tuple[str | None, str | None]: + """(记在哪天, None) 或 (None, 拦下的上限)。 + + keep:要额外留出的钱(收尾那次调用最贵的样子),只用来判断、不预留。kept:这就是循环里一直给它留着钱的 + 那次收尾,不再按上限拦,照记账:每一轮都留够了,还超出只能是某一轮实际用得比估算多,或者并行的核查先用掉了 + 当天的余额(每日余额只是比了一下、没有真的预留)。这时宁可超出上限一次收尾调用的钱,也不让查到的结果白查。 + """ + cap = self.daily_max_usd if self.priced and not kept else None + if not kept: + if self.priced and self.max_cost_usd is not None and \ + self.spent + estimate + keep > self.max_cost_usd + 1e-12: + return None, "max_cost_usd" + if cap is not None and keep and (await spend.daily_spend())["usd"] + estimate + keep > cap + 1e-12: + return None, "daily_max_usd" day = await spend.charge(estimate, limit=cap, priced=self.priced) return (day, None) if day is not None else (None, "daily_max_usd") @@ -326,16 +577,28 @@ def _text_of(messages: list[BaseMessage]) -> str: def _final_messages(system: str, task: str, queries: list[dict[str, Any]], said: str, limit_words: str | None ) -> list[BaseMessage]: """收尾那次调用的消息:任务、查到的结果(每次查询列出行号)、模型最后说的话,以及交结果的规则。 - limit_words:触顶时是哪个上限(「最多查询 6 次」),告诉模型为什么不能再查。""" + limit_words:触顶时是哪个上限(「最多查询 6 次」),告诉模型为什么不能再查。 + + 长度有上限(任务和规则之外,查询结果按 _digest 的上限、说明按 _SAID_CHARS):循环里按这个上限给收尾留钱。 + """ parts = [f"任务:\n{task.strip()}"] if limit_words: parts.append(f"已达上限({limit_words}),不能再查询,按已经查到的结果交结果。") parts.append(f"你查到的结果:\n{_digest(queries) if queries else '(没有成功的查询)'}") - parts.append(f"你最后的说明(只作参考,数以查询结果为准):\n{said.strip() or '(没有)'}") + parts.append(f"你最后的说明(只作参考,数以查询结果为准):\n{said.strip()[:_SAID_CHARS] or '(没有)'}") parts.append(VERIFY_FINAL_PROMPT) return [SystemMessage(content=system), HumanMessage(content="\n\n".join(parts))] +def _plain_messages(final: list[BaseMessage], schema: dict[str, Any]) -> list[BaseMessage]: + """普通调用(不走结构化输出)的消息:结构不在请求里了,要写进消息,不然模型不知道要交哪些字段、字段叫什么, + 交回来的字段对不上,全成了没交。""" + *head, last = final + text = (f"{last.content}\n\n只交一个 JSON 对象,不要写别的内容。对象的结构(JSON Schema):\n" + f"```json\n{json.dumps(schema, ensure_ascii=False)}\n```") + return [*head, HumanMessage(content=text)] + + def _parsed(value: Any) -> dict[str, Any] | None: """结构化结果:对象;有的模型把整个对象写成一段 JSON 文本,也认。""" if isinstance(value, str): @@ -343,6 +606,30 @@ def _parsed(value: Any) -> dict[str, Any] | None: return value if isinstance(value, dict) else None +@dataclass +class _State: + """一次核查过程中攒下的东西:出意外时也照这些交回,用了多少、查了什么不丢。""" + + started: float + usage: dict[str, Any] = field(default_factory=lambda: { + "input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "cost_usd": 0.0, "calls": 0, "unpriced_calls": 0}) + notes: list[str] = field(default_factory=list) + queries: list[dict[str, Any]] = field(default_factory=list) + ledger: list[dict[str, Any]] = field(default_factory=list) + model: str | None = None + + def outcome(self, **fields: Any) -> VerifyOutcome: + usage = self.usage + usage["total_tokens"] = usage["input_tokens"] + usage["output_tokens"] + usage["cost_usd"] = round(usage["cost_usd"], 6) + shown = [{k: q[k] for k in ("alias", "artifact", "sql", "row_count", "truncated", "source", "tool", "call_id")} + for q in self.queries] + base: dict[str, Any] = {"result": None, "cells": [], "queries": shown, "rationale": None, "usage": usage, + "notes": self.notes, "model": self.model, + "duration_ms": int((time.monotonic() - self.started) * 1000)} + return VerifyOutcome(**{**base, **fields}) + + async def run_verify_agent( *, spec: ModelSpec, @@ -353,7 +640,7 @@ async def run_verify_agent( max_queries: int | None = 6, timeout_s: float | None = 120, max_cost_usd: float | None = None, - daily_max_usd: float | None = None, + daily_max_usd: float | None | Default = Default.SETTINGS, max_steps: int | None = None, emit: Emit | None = None, run_id: str = "", @@ -364,37 +651,41 @@ async def run_verify_agent( - spec:用哪个模型(一般是 api/settings.py 的 verify_model_spec)。system / task:交给模型的角色说明和任务。 - sources:可以查的数据源名;不存在、停用了的照实记进 notes。 - - output_schema:调用方要的结果结构(JSON Schema 对象)。数值字段会换成格子引用交给模型、由代码取值。 + - output_schema:调用方要的结果结构(JSON Schema 对象,可以带 $defs / $ref)。数值字段会换成格子引用交给模型、 + 由代码取值。 - 上限:max_queries(db_query 的执行次数,被拒的语句也算)、timeout_s(整个循环的时长,收尾另给宽限)、 - max_cost_usd(这一次)、daily_max_usd(全局每日,和裁判分开记账);None 表示不限。max_steps 是模型最多想 - 几轮,缺省按查询次数推。 + max_cost_usd(这一次)、daily_max_usd(全局每日,和裁判分开记账;缺省按设置里的「核查每日金额上限」); + None 表示不限。max_steps 是模型最多想几轮,缺省按查询次数推。 - emit:节点里调用时传 NodeContext.emit,每次模型调用发 llm.start / llm.end、每次工具调用发 tool.start / tool.end(role: verify,查询带 query_artifact);不在节点里(质询接口)就不传,什么都不发。 - run_id / node_id:查询快照、表结构快照记在谁名下。data_versions:上传表格用哪一版(运行固定的那份 data_versions;不在运行里传 Versions.CURRENT;缺省 UNSET 时上传源一律报错,不拿当前版本顶上)。 """ - started = time.monotonic() - deadline = started + timeout_s if timeout_s is not None else None - usage: dict[str, Any] = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "cost_usd": 0.0, "calls": 0, - "unpriced_calls": 0} - notes: list[str] = [] - queries: list[dict[str, Any]] = [] - ledger: list[dict[str, Any]] = [] + state = _State(started=time.monotonic()) + try: + return await _verify(state, spec=spec, system=system, task=task, sources=sources, + output_schema=output_schema, max_queries=max_queries, timeout_s=timeout_s, + max_cost_usd=max_cost_usd, daily_max_usd=daily_max_usd, max_steps=max_steps, emit=emit, + run_id=run_id, node_id=node_id, data_versions=data_versions) + except Exception as e: # noqa: BLE001 - 永远不抛:库写不进去、建工具出错这类意外也落在返回值里 + logger.warning("核查出错", exc_info=True) + return state.outcome(error=f"核查出错({describe_exception(e)})") + + +async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sources: Iterable[str], + output_schema: dict[str, Any], max_queries: int | None, timeout_s: float | None, + max_cost_usd: float | None, daily_max_usd: float | None | Default, max_steps: int | None, + emit: Emit | None, run_id: str, node_id: str, + data_versions: dict[str, Any] | Versions | None) -> VerifyOutcome: + deadline = state.started + timeout_s if timeout_s is not None else None + usage, notes, queries, ledger = state.usage, state.notes, state.queries, state.ledger + outcome = state.outcome steps = max_steps if max_steps is not None else max(4, (max_queries if max_queries is not None else 6) * 2 + 2) owner = node_id or "verify" def left() -> float | None: return None if deadline is None else deadline - time.monotonic() - def outcome(**fields: Any) -> VerifyOutcome: - usage["total_tokens"] = usage["input_tokens"] + usage["output_tokens"] - usage["cost_usd"] = round(usage["cost_usd"], 6) - shown = [{k: q[k] for k in ("alias", "artifact", "sql", "row_count", "truncated", "source", "tool", "call_id")} - for q in queries] - base: dict[str, Any] = {"result": None, "cells": [], "queries": shown, "rationale": None, "usage": usage, - "notes": notes, "duration_ms": int((time.monotonic() - started) * 1000)} - return VerifyOutcome(**{**base, **fields}) - # ---- 模型 try: async with SessionLocal() as session: @@ -402,6 +693,12 @@ async def run_verify_agent( except Exception as e: # noqa: BLE001 - 核查模型没配好:这回核不了,照实交回 why = not_configured(e) if isinstance(e, ProviderNotConfigured) else describe_exception(e) return outcome(error=f"核查模型不可用({why})") + state.model = model_id + if daily_max_usd is Default.SETTINGS: + from app.api.settings import verify_settings # 在这里导入:api.settings 导入了 engine 里的模块 + + async with SessionLocal() as session: + daily_max_usd = (await verify_settings(session))["daily_max_usd"] budget = _Budget(model_id, max_cost_usd, daily_max_usd) if not budget.priced and (max_cost_usd is not None or daily_max_usd is not None): note = (f"模型「{model_id}」不在价格目录中,无法按 token 数估算金额:金额上限(这次核查、每日)对其无效," @@ -423,6 +720,14 @@ async def run_verify_agent( head = "\n\n".join(p for p in (system.strip(), VERIFY_SYSTEM.format(limit=limit_note)) if p) messages: list[BaseMessage] = [SystemMessage(content=head), HumanMessage(content=task)] tool_text = "\n".join(f"{t.name}:{t.description}" for t in tools) + resolved = _inline(output_schema) + schema = result_schema(resolved) + # 结构也算输入:结构化输出时它是工具定义,退回普通调用时写在消息里 + schema_text = json.dumps(schema, ensure_ascii=False) + # 收尾那次最贵的样子:查询结果、最后的说明都按上限、按一个字一个 token 算。查询结果要到查完才知道多长, + # 按眼下的消息估,等最后一条查询的结果进来就不够了(大段中文的结果尤其如此) + keep = budget.estimate(_text_of(_final_messages(head, task, [], "", "上" * _LIMIT_ROOM)) + schema_text, + _OUT_FINAL, room=_DIGEST_ROOM + _SAID_CHARS) async def ask(payload: list[BaseMessage], runner: Any, estimate: float, day: str, purpose: str, seconds: float | None) -> tuple[Any, BaseMessage | None, BaseException | None]: @@ -457,6 +762,7 @@ async def run_verify_agent( # ---- 查库的循环 stopped: str | None = None answered = False + kept = False # 有一轮按收尾最贵的样子留过钱:收尾不再按金额上限拦 said = "" attempts = 0 done: dict[str, str] = {} # 成功过的查询(工具名 + 参数)→ 编号:一模一样的不再执行 @@ -466,17 +772,17 @@ async def run_verify_agent( stopped = "timeout_s" break estimate = budget.estimate(_text_of(messages) + tool_text, _OUT_STEP) - day, refused = await budget.reserve(estimate, keep=budget.estimate(_text_of(messages), _OUT_FINAL)) + day, refused = await budget.reserve(estimate, keep=keep) if refused: stopped = refused break + kept = True response, _, failure = await ask(messages, model.ainvoke, estimate, day or "", "verify", remaining) if isinstance(failure, asyncio.TimeoutError): stopped = "timeout_s" break if failure is not None or not isinstance(response, BaseMessage): - return outcome(model=model_id, error=explain_model_error(failure, model_id) if failure - else "核查模型没有给出回复") + return outcome(error=explain_model_error(failure, model_id) if failure else "核查模型没有给出回复") messages.append(response) calls = getattr(response, "tool_calls", None) or [] if not calls: @@ -494,10 +800,10 @@ async def run_verify_agent( messages.append(ToolMessage(content=f"错误:没有名为 {name} 的工具,可用的工具:" + ("、".join(tool_map) or "(无)"), tool_call_id=call_id)) continue - schema = args_model_of(tool) - if schema is not None: + args_schema = args_model_of(tool) + if args_schema is not None: try: - args, _ = prepare_args(schema, args) + args, _ = prepare_args(args_schema, args) except ToolArgsError as e: await _announce(emit, EventType.TOOL_ERROR, tool=name, call_id=call_id, error=str(e), role="verify") messages.append(ToolMessage(content=str(e), tool_call_id=call_id)) @@ -535,10 +841,12 @@ async def run_verify_agent( else: stopped = "max_steps" - words = None - if stopped: - words = _limit_words(stopped, max_queries=max_queries, timeout_s=timeout_s, max_cost_usd=max_cost_usd, - daily_max_usd=daily_max_usd, steps=steps) + def words_of(reason: str) -> str: + return _limit_words(reason, max_queries=max_queries, timeout_s=timeout_s, max_cost_usd=max_cost_usd, + daily_max_usd=daily_max_usd, steps=steps) + + words = words_of(stopped) if stopped else None + if words: await _announce(emit, EventType.LOG, level="warn", code="verify_limit", limit=stopped, message=f"核查已达上限({words}),已停止查询,按已有结果收尾") @@ -547,16 +855,12 @@ async def run_verify_agent( async with SessionLocal() as session: settler, _ = await get_chat_model(session, spec.model_copy(update={"thinking": "off"})) except Exception as e: # noqa: BLE001 - return outcome(model=model_id, limit=stopped, answered=answered, error=f"核查模型不可用({describe_exception(e)})") - schema = result_schema(output_schema) + return outcome(limit=stopped, answered=answered, error=f"核查模型不可用({describe_exception(e)})") final = _final_messages(head, task, queries, said, words) - estimate = budget.estimate(_text_of(final), _OUT_FINAL) - day, refused = await budget.reserve(estimate) - if refused: - limit_words = _limit_words(refused, max_queries=max_queries, timeout_s=timeout_s, max_cost_usd=max_cost_usd, - daily_max_usd=daily_max_usd, steps=steps) - return outcome(model=model_id, limit=stopped or refused, answered=answered, - error=f"已达{limit_words},没能交结果") + try: + structured = settler.with_structured_output(schema, include_raw=True) + except Exception: # noqa: BLE001 - 模型不支持结构化输出:直接走普通调用,不白算一次调用 + structured = None remaining = left() grace = None if remaining is None else max(remaining, SETTLE_GRACE_S) finish = None if grace is None else time.monotonic() + grace @@ -564,38 +868,55 @@ async def run_verify_agent( def until_finish() -> float | None: return None if finish is None else finish - time.monotonic() - got, raw, failure = await ask(final, settler.with_structured_output(schema, include_raw=True).ainvoke, estimate, - day or "", "verify_result", until_finish()) - parsed = None if failure is not None else _parsed(_split_structured(got)[1]) - if parsed is None and raw is not None: - # 有的网关把 JSON 写在正文里、或者塞进了一个没按结构解析出来的工具调用 - for candidate in [message_text(raw), *(c.get("args") for c in getattr(raw, "tool_calls", None) or [])]: - if (parsed := _parsed(candidate)) is not None: - break + parsed: dict[str, Any] | None = None + failure: BaseException | None = None + if structured is not None: + estimate = budget.estimate(_text_of(final) + schema_text, _OUT_FINAL) + day, refused = await budget.reserve(estimate, kept=kept) + if refused: + return outcome(limit=stopped or refused, answered=answered, error=f"已达{words_of(refused)},没能交结果") + got, raw, failure = await ask(final, structured.ainvoke, estimate, day or "", "verify_result", until_finish()) + parsed = None if failure is not None else _parsed(_split_structured(got)[1]) + if parsed is None and raw is not None: + # 有的网关把 JSON 写在正文里、或者塞进了一个没按结构解析出来的工具调用 + for candidate in [message_text(raw), *(c.get("args") for c in getattr(raw, "tool_calls", None) or [])]: + if (parsed := _parsed(candidate)) is not None: + break if parsed is None and not isinstance(failure, asyncio.TimeoutError): - # 不支持结构化输出的模型:普通调用一次,从回复里把 JSON 抠出来 + # 不支持结构化输出的模型:普通调用一次,结构写进消息,从回复里把 JSON 抠出来。结构化那次已经做过的, + # 这一次是补救,照常按上限拦;没做过的,这一次就是收尾,循环里留过钱 + plain = _plain_messages(final, schema) remaining = until_finish() if remaining is None or remaining > 0: - estimate = budget.estimate(_text_of(final), _OUT_FINAL) - day, refused = await budget.reserve(estimate) + estimate = budget.estimate(_text_of(plain), _OUT_FINAL) + day, refused = await budget.reserve(estimate, kept=kept and structured is None) + if refused and structured is None: + return outcome(limit=stopped or refused, answered=answered, + error=f"已达{words_of(refused)},没能交结果") if not refused: - got, raw, failure = await ask(final, settler.ainvoke, estimate, day or "", "verify_result", remaining) + got, _, failure = await ask(plain, settler.ainvoke, estimate, day or "", "verify_result", remaining) parsed = None if failure is not None else _parsed(message_text(got) if got is not None else None) if parsed is None: why = "收尾时超过时长上限,没能交结果" if isinstance(failure, asyncio.TimeoutError) else \ f"核查模型没有按结构交结果({explain_model_error(failure, model_id)})" if failure is not None else \ "核查模型没有按结构交结果" - return outcome(model=model_id, limit=stopped, answered=answered, error=why) + return outcome(limit=stopped, answered=answered, error=why) + declared = list((resolved.get("properties") or {}) if _is_object(resolved) else {}) + if declared and not any(k in parsed for k in declared): + # 一个要求的字段都没交:不是「查不到」,是没按结构交。不这么分,字段名对不上的回复会被当成什么都没查到 + return outcome(limit=stopped, answered=answered, + error="核查模型没有按结构交结果(交回的内容里没有要求的字段:" + + "、".join(declared[:8]) + ("等" if len(declared) > 8 else "") + ")") # ---- 取值、组装理由 cells = _Cells(queries) - result = cells.walk(output_schema, parsed) if _is_object(output_schema) else {} + result = cells.walk(resolved, parsed) if _is_object(resolved) else {} rationale = parsed.get(RATIONALE) doc = None if isinstance(rationale, str) and rationale.strip(): doc = compose_doc(rationale, build_catalog(nodes={}, ledger=ledger), node_id=owner, run_id=run_id or None) result[RATIONALE] = doc["markdown"] if doc else "" - return outcome(model=model_id, result=result, cells=cells.cells, rationale=doc, limit=stopped, answered=answered) + return outcome(result=result, cells=cells.cells, rationale=doc, limit=stopped, answered=answered) async def _run_tool(tool: Any, name: str, args: dict[str, Any], call_id: str, remaining: float | None, @@ -612,15 +933,17 @@ async def _run_tool(tool: Any, name: str, args: dict[str, Any], call_id: str, re try: result = await _bounded(run_bounded(tool.ainvoke(args), limit, name), remaining) content = result if isinstance(result, str) else json.dumps(result, ensure_ascii=False, default=str) + except ToolTimeout as e: + # 这条查询自己的时限到了:交回原因让模型改写,核查照常往下走。要排在下一条前面:ToolTimeout 也是 + # TimeoutError(asyncio.TimeoutError 就是它),排在后面就被当成整个核查的时长到了 + await _announce(emit, EventType.TOOL_ERROR, tool=name, call_id=call_id, role="verify", timed_out=True, + duration_ms=int((time.perf_counter() - began) * 1000), error=str(e)) + return str(e) except asyncio.TimeoutError: await _announce(emit, EventType.TOOL_ERROR, tool=name, call_id=call_id, role="verify", timed_out=True, duration_ms=int((time.perf_counter() - began) * 1000), error="查询因核查的时长上限被中断") return None - except ToolTimeout as e: - await _announce(emit, EventType.TOOL_ERROR, tool=name, call_id=call_id, role="verify", timed_out=True, - duration_ms=int((time.perf_counter() - began) * 1000), error=str(e)) - return str(e) except Exception as e: # noqa: BLE001 - 工具失败交回模型,让它自己改 message = f"工具执行失败:{describe_exception(e)}" await _announce(emit, EventType.TOOL_ERROR, tool=name, call_id=call_id, role="verify", @@ -656,4 +979,4 @@ async def _run_tool(tool: Any, name: str, args: dict[str, Any], call_id: str, re return content -__all__ = ["LIMITS", "RATIONALE", "SETTLE_GRACE_S", "VerifyOutcome", "result_schema", "run_verify_agent"] +__all__ = ["LIMITS", "RATIONALE", "SETTLE_GRACE_S", "Default", "VerifyOutcome", "result_schema", "run_verify_agent"] diff --git a/backend/tests/test_verify_agent.py b/backend/tests/test_verify_agent.py index 7e72083f..0a8c04d0 100644 --- a/backend/tests/test_verify_agent.py +++ b/backend/tests/test_verify_agent.py @@ -4,9 +4,11 @@ - 每次成功的查询是一份查询快照,按这个 agent 自己的顺序编 Q1、Q2…;节点里调用时发 tool.end(带 query_artifact、 role: verify),但不往证据台账里写 query:核查的查询进了报告的写作目录,写作者就会把它们当成可以引用的证据 - 最后按调用方的结构交结果:数值字段一律是格子引用 {alias, row, column},由代码从快照取值,模型自己写的数字 - 不采信;理由里的数字要写成 [[v:Qn.rX.列]],用本 agent 自己的台账组装成文档,裸数字会被标出 -- 查询次数、时长、金额(单次和每日)任何一项到顶就停止查询、按已有结果收尾;模型不在价格目录里时金额上限 - 无效,照裁判的说法记一条说明 + 不采信(type 列表、anyOf、$ref、allOf 里的数也是;认不出的结构下写的数一律丢掉);没交的字段和交了 null 分开记; + 理由里的数字要写成 [[v:Qn.rX.列]],用本 agent 自己的台账组装成文档,裸数字会被标出 +- 查询次数、时长、金额(单次和每日)任何一项到顶就停止查询、按已有结果收尾;收尾的钱在查询之前就按最贵的样子 + 留好;每日上限缺省按设置;模型不在价格目录里时金额上限无效,照裁判的说法记一条说明 +- 永远不抛:库写不进去、模型不支持结构化输出,都落在返回值里 - 上传表格按调用方给的数据版本查(运行固定的那一版),不拿当前版本顶上 模型一律是 MockChatModel 剧本,数据源是现造的 SQLite 文件,表名、数字都是虚构的。 @@ -31,7 +33,8 @@ from app.data.engine import engines from app.db.base import SessionLocal from app.db.models import DataSource, Setting from app.engine import spend, verify_agent -from app.engine.verify_agent import run_verify_agent +from app.engine.toolcalls import Limit, ToolTimeout +from app.engine.verify_agent import result_schema, run_verify_agent from app.providers import catalog as pricing from app.providers.factory import ModelSpec, ProviderNotConfigured from app.providers.mock_model import MockChatModel @@ -48,9 +51,12 @@ VALID_SUM = "SELECT SUM(amount) AS total FROM orders WHERE is_deleted = 0" @pytest.fixture(autouse=True) async def clean_spend(): async with SessionLocal() as session: - await session.execute(delete(Setting).where(Setting.key == spend.SPEND_KEY)) + await session.execute(delete(Setting).where(Setting.key.in_([spend.SPEND_KEY, "verify"]))) await session.commit() yield + async with SessionLocal() as session: + await session.execute(delete(Setting).where(Setting.key == "verify")) + await session.commit() @pytest.fixture @@ -272,6 +278,32 @@ async def test_a_model_call_that_runs_past_the_deadline_is_cut(monkeypatch, shop assert out.result is None and out.error, "收尾那次也没赶上:照实说没交成结果" +async def test_a_query_that_hits_its_own_time_limit_is_not_the_overall_deadline(monkeypatch, shop): + """一条查询超过它自己的时限(ToolTimeout,也是一种 TimeoutError):这条查询作罢、原因交回模型让它改,核查 + 照常往下走。不能当成整个核查的时长到了,把还剩的时间全扔掉。""" + name, _ = shop + real = verify_agent.run_bounded + seen = {"n": 0} + + async def stuck_once(call, limit, tool_name): + seen["n"] += 1 + if seen["n"] == 1: + call.close() + raise ToolTimeout(tool_name, Limit(seconds=30.0, enforce=True, kind="query")) + return await real(call, limit, tool_name) + + monkeypatch.setattr(verify_agent, "run_bounded", stuck_once) + script = Script(monkeypatch, [q(name, "SELECT COUNT(*) AS n FROM orders", "c1"), q(name, VALID_SUM, "c2")], + {"value": {"alias": "Q1", "row": 0, "column": "total"}, "verdict": "一致", "rationale": "x"}) + events: list[tuple] = [] + out = await run(name, timeout_s=600, emit=lambda kind, **data: events.append((kind, data))) + assert out.limit is None, "整个核查的时长远没到" + assert script.tool_results()["c1"].startswith("查询超过 30 秒没有返回") + assert out.result["value"] == 230 and [qd["alias"] for qd in out.queries] == ["Q1"] + [error] = [d for k, d in events if k == EventType.TOOL_ERROR] + assert error["call_id"] == "c1" and "核查的时长上限" not in error["error"] + + @pytest.fixture def priced(monkeypatch): """每次调用按 0.01 美元估算、按 0.01 美元结算:金额上限的算术好核对。""" @@ -279,6 +311,88 @@ def priced(monkeypatch): monkeypatch.setattr(pricing, "estimate_cost", lambda model, inp, out: 0.01) +@pytest.fixture +def by_tokens(monkeypatch): + """按 token 数估算(每个 token 0.000001 美元):消息越长估得越多,收尾时交给模型的查询结果也要算钱。""" + monkeypatch.setattr(pricing, "find_pricing", lambda model: {"input": 1.0, "output": 1.0}) + monkeypatch.setattr(pricing, "estimate_cost", lambda model, inp, out: (inp + out) / 1_000_000) + + +@pytest.fixture +async def wordy(tmp_path): + """200 条留言,每条一大段中文:查出来交给收尾那次调用时很占 token。返回源名。""" + path = tmp_path / f"{uuid.uuid4().hex[:8]}.db" + db = sqlite3.connect(path) + db.execute("CREATE TABLE notes (id INTEGER PRIMARY KEY, note TEXT)") + db.executemany("INSERT INTO notes VALUES (?, ?)", [(i, f"第{i}条留言" + "虚构的长段落文字" * 50) for i in range(200)]) + db.commit() + db.close() + name = f"notes_{uuid.uuid4().hex[:8]}" + async with SessionLocal() as session: + row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True) + session.add(row) + await session.flush() + row.schema_cache = await introspect.introspect(row) + await session.commit() + source_id = row.id + yield name + await engines.invalidate(source_id) + + +@pytest.mark.parametrize("cap", [0.003, 0.004, 0.005, 0.02, 0.05]) +async def test_a_query_never_runs_without_money_left_to_settle_it(monkeypatch, wordy, by_tokens, cap): + """收尾那次调用要把查到的结果交给模型:结果越长越贵。查询之前就要按收尾最贵的样子把钱留好,不然查完了却 + 付不起收尾,查到的结果白查。""" + count = {"type": "object", "properties": {"count": {"type": "integer"}}} + Script(monkeypatch, [q(wordy, "SELECT id, note FROM notes ORDER BY id", "c1")], + {"count": {"alias": "Q1", "row": 0, "column": "id"}, "rationale": "x"}, model_id="priced-model") + out = await run(wordy, output_schema=count, max_cost_usd=cap, max_queries=1) + assert out.error is None and out.result is not None, "不管上限多紧,都交得出结果" + assert out.usage["cost_usd"] <= cap, "收尾那次也在上限以内:钱是查询之前就留好的,不是收尾时硬超出去" + if out.queries: + assert out.result["count"] == 0, "查了就用上了" + else: + assert out.limit == "max_cost_usd", "钱不够查询加收尾,就不查" + + +async def test_a_final_answer_budgeted_in_the_loop_is_not_refused_by_the_daily_cap(monkeypatch, shop, priced): + """查询之后、收尾之前,别的核查把当天的余额用光了(几个核查并行):收尾那次已经在循环里留过钱,照常交结果, + 查到的结果不白查。""" + name, _ = shop + + async def crowded(kind, **data): + if kind == EventType.TOOL_END: + async with SessionLocal() as session: + row = await session.get(Setting, spend.SPEND_KEY) + row.value = {**row.value, "usd": 1.999} + await session.commit() + + Script(monkeypatch, [q(name, VALID_SUM, "c1"), q(name, "SELECT COUNT(*) AS n FROM orders", "c2")], + {"value": {"alias": "Q1", "row": 0, "column": "total"}, "verdict": "一致", "rationale": "x"}, + model_id="priced-model") + out = await run(name, daily_max_usd=2.0, emit=crowded) + assert out.limit == "daily_max_usd", "下一轮查询付不起了" + assert out.error is None and out.result["value"] == 230 + + +async def test_the_daily_cap_comes_from_the_settings_unless_given(monkeypatch, shop, priced): + """不传每日上限就按设置里的「核查每日金额上限」:节点、质询接口不必各自记得去读;明确传 None 才是不限。""" + name, _ = shop + async with SessionLocal() as session: + session.add(Setting(key=spend.SPEND_KEY, value={"date": spend._today(), "usd": 0.995, "calls": 3})) + session.add(Setting(key="verify", value={"daily_max_usd": 1.0})) + await session.commit() + final = {"value": {"alias": "Q1", "row": 0, "column": "total"}, "verdict": "一致", "rationale": "x"} + script = Script(monkeypatch, [q(name, VALID_SUM, "c1")], final, model_id="priced-model") + out = await run(name) + assert out.limit == "daily_max_usd" and script.calls == [] and out.result is None + assert "$1.00" in out.error + + Script(monkeypatch, [q(name, VALID_SUM, "c1")], final, model_id="priced-model") + unlimited = await run(name, daily_max_usd=None) + assert unlimited.limit is None and unlimited.result["value"] == 230 + + async def test_the_cost_limit_keeps_room_for_the_final_answer(monkeypatch, shop, priced): name, _ = shop endless = [q(name, f"SELECT COUNT(*) AS n FROM orders WHERE id > {i}", f"c{i}") for i in range(10)] @@ -394,6 +508,131 @@ async def test_nested_numbers_and_arrays_are_references_too(monkeypatch, shop): assert [c["path"] for c in out.cells] == ["totals[0]", "totals[1]", "detail.count"] +#: Pydantic 这类工具生成的结构:可为 null 的数写成 anyOf,嵌套的对象写成 $ref 到 $defs,带说明的引用包一层 allOf, +#: Optional[对象] 是 anyOf [$ref, null] +PYDANTIC_STYLE = { + "type": "object", + "$defs": {"Region": {"type": "object", "properties": {"name": {"type": "string"}, "total": {"type": "number"}}}}, + "properties": { + "value": {"anyOf": [{"type": "number"}, {"type": "null"}], "description": "未作废订单的金额合计"}, + "count": {"type": ["integer", "null"]}, + "top": {"$ref": "#/$defs/Region"}, + "backup": {"anyOf": [{"$ref": "#/$defs/Region"}, {"type": "null"}]}, + "wrapped": {"allOf": [{"$ref": "#/$defs/Region"}], "description": "带说明的引用"}, + "regions": {"type": "array", "items": {"$ref": "#/$defs/Region"}}, + }, +} + + +def test_every_way_of_writing_a_number_becomes_a_reference(): + """anyOf、type 列表、$ref、allOf 里的数也换成格子引用:认不出这些写法,模型在那里写的数就绕过了格子这一关。""" + shaped = result_schema(PYDANTIC_STYLE) + assert "$ref" not in json.dumps(shaped) and "$defs" not in shaped + + def is_ref(node): + return node["anyOf"][0]["required"] == ["alias", "row", "column"] and node["anyOf"][1] == {"type": "null"} + + props = shaped["properties"] + assert is_ref(props["value"]) and "未作废订单的金额合计" in props["value"]["description"] + assert is_ref(props["count"]) + assert is_ref(props["top"]["properties"]["total"]) and props["top"]["properties"]["name"] == {"type": "string"} + assert is_ref(props["wrapped"]["properties"]["total"]) + assert is_ref(props["regions"]["items"]["properties"]["total"]) + [region, null] = props["backup"]["anyOf"] + assert is_ref(region["properties"]["total"]) and null == {"type": "null"} + + +async def test_numbers_written_directly_in_any_of_those_shapes_are_ignored(monkeypatch, shop): + name, _ = shop + Script(monkeypatch, [q(name, VALID_SUM, "c1")], + {"value": 999, "count": 7, "top": {"name": "分区甲", "total": 999}, "backup": {"name": "分区乙", "total": 1}, + "wrapped": {"name": "分区丙", "total": 2}, "regions": [{"name": "分区丁", "total": 3}], "rationale": "x"}) + out = await run(name, output_schema=PYDANTIC_STYLE) + assert out.result == {"value": None, "count": None, "top": {"name": "分区甲", "total": None}, + "backup": {"name": "分区乙", "total": None}, "wrapped": {"name": "分区丙", "total": None}, + "regions": [{"name": "分区丁", "total": None}], "rationale": "x"} + assert {c["path"]: c["model_value"] for c in out.cells} == { + "value": 999, "count": 7, "top.total": 999, "backup.total": 1, "wrapped.total": 2, "regions[0].total": 3} + assert all(c["status"] == "unresolved" and "没有给出格子引用" in c["reason"] for c in out.cells), \ + "都认成了数值字段(该写格子引用),不是碰巧被当成认不出的结构丢掉" + + +async def test_references_in_those_shapes_are_resolved(monkeypatch, shop): + name, _ = shop + ref = {"alias": "Q1", "row": 0, "column": "total"} + Script(monkeypatch, [q(name, VALID_SUM, "c1")], + {"value": ref, "count": None, "top": {"name": "分区甲", "total": ref}, "backup": None, + "wrapped": {"name": "分区丙", "total": ref}, "regions": [{"name": "分区丁", "total": ref}], "rationale": "x"}) + out = await run(name, output_schema=PYDANTIC_STYLE) + assert out.result["value"] == 230 and out.result["top"]["total"] == 230 and out.result["wrapped"]["total"] == 230 + assert out.result["regions"] == [{"name": "分区丁", "total": 230}] + assert out.result["count"] is None and out.result["backup"] is None, "可以为 null 的照实是 null" + + +async def test_numbers_under_a_shape_that_is_not_recognised_are_dropped(monkeypatch, shop): + """认不出的结构(没写类型、没写 items 的数组、没写属性的对象)、文本字段里冒出来的数:一律不采信,照记下来。 + 码值(enum)里的数是调用方给的选项,不是查出来的数,照收。""" + name, _ = shop + schema = {"type": "object", "properties": { + "anything": {}, "notes": {"type": "array"}, "label": {"type": "string"}, "level": {"enum": [1, 2, 3]}, + "extra": {"type": "object"}}} + Script(monkeypatch, [q(name, VALID_SUM, "c1")], + {"anything": 999, "notes": ["见 Q1", 120], "label": 5, "level": 2, + "extra": {"note": "按支付时间算", "share": 0.3}, "rationale": "x"}) + out = await run(name, output_schema=schema) + assert out.result == {"anything": None, "notes": ["见 Q1", None], "label": None, "level": 2, + "extra": {"note": "按支付时间算", "share": None}, "rationale": "x"} + assert {c["path"]: c["model_value"] for c in out.cells} == {"anything": 999, "notes[1]": 120, "label": 5, + "extra.share": 0.3} + assert all(c["status"] == "unresolved" for c in out.cells) + + +async def test_references_written_as_json_text_are_read(monkeypatch, shop): + """有的模型把嵌套的对象、数组写成一段 JSON 文本:照样认(裁判也是这么认的)。还不是数组的照实记下格式不对。""" + name, _ = shop + schema = {"type": "object", "properties": { + "value": {"type": "number"}, "totals": {"type": "array", "items": {"type": "number"}}, + "broken": {"type": "array", "items": {"type": "number"}}}} + Script(monkeypatch, [q(name, VALID_SUM, "c1")], + {"value": '{"alias": "Q1", "row": 0, "column": "total"}', + "totals": '[{"alias": "Q1", "row": 0, "column": "total"}]', "broken": "没有这一项", "rationale": "x"}) + out = await run(name, output_schema=schema) + assert out.result == {"value": 230, "totals": [230], "broken": None, "rationale": "x"} + broken = next(c for c in out.cells if c["path"] == "broken") + assert broken["status"] == "unresolved" and "数组" in broken["reason"] + + +async def test_a_number_field_pointing_at_a_text_cell_is_unresolved(monkeypatch, shop): + name, _ = shop + Script(monkeypatch, [q(name, "SELECT paid_at, amount FROM orders ORDER BY id", "c1")], + {"value": {"alias": "Q1", "row": 0, "column": "paid_at"}, "verdict": "一致", "rationale": "x"}) + out = await run(name) + assert out.result["value"] is None, "数值字段只收数:日期文本拿去和原流程的数比,比较会出错" + [cell] = out.cells + assert cell["status"] == "unresolved" and "不是数值" in cell["reason"] and cell["ref"] == "Q1.r0.paid_at" + assert cell["value"] == "2026-09-02" + + +async def test_a_field_left_out_is_not_a_field_reported_missing(monkeypatch, shop): + """没交这个字段和交了 null(模型说查不到)是两回事:前者是没按结构交,后者才是 missing。""" + name, _ = shop + Script(monkeypatch, [q(name, VALID_SUM, "c1")], {"verdict": "无法判断", "rationale": "x"}) + left_out = await run(name) + [cell] = left_out.cells + assert cell["status"] == "unresolved" and "没有交" in cell["reason"] + + Script(monkeypatch, [q(name, VALID_SUM, "c1")], {"value": None, "verdict": "无法判断", "rationale": "x"}) + said_null = await run(name) + assert said_null.cells == [{"path": "value", "status": "missing"}] + + +async def test_an_answer_with_none_of_the_fields_is_an_error(monkeypatch, shop): + name, _ = shop + Script(monkeypatch, [q(name, VALID_SUM, "c1")], {"total": 230, "rationale": "合计 230"}) + out = await run(name) + assert out.result is None and out.cells == [] and "没有按结构交结果" in out.error + + # -------------------------------------------------------------------------- # 数据版本 # -------------------------------------------------------------------------- @@ -482,3 +721,74 @@ async def test_a_final_answer_in_plain_text_json_is_still_read(monkeypatch, shop monkeypatch.setattr(MockChatModel, "_decide", decide) out = await run(name) assert out.result["value"] == 230 and script.kinds.count("final") == 1 + + +def plain_reply(monkeypatch, reply): + """普通调用(退回的那一次)的回复:reply(最后一条消息的正文) → 回复正文。其余照剧本。""" + original = MockChatModel._decide + seen: list[str] = [] + + def decide(model, messages): + text = str(messages[-1].content) + if not model.response_format and "交结果" in text: + seen.append(text) + return AIMessage(content=reply(text)) + return original(model, messages) + + monkeypatch.setattr(MockChatModel, "_decide", decide) + return seen + + +async def test_the_plain_fallback_is_told_which_fields_to_fill(monkeypatch, shop): + """结构化输出不成、退回普通调用时,结构不在请求里了:要写进消息,不然模型不知道字段叫什么,交回来的字段 + 对不上,全成了「查不到」。""" + name, _ = shop + schema = {"type": "object", "properties": { + "blind_total_amount": {"type": "number", "description": "盲算的金额合计"}, + "consistency_verdict": {"type": "string", "enum": ["一致", "不一致"]}}} + Script(monkeypatch, [q(name, VALID_SUM, "c1")], "不是 JSON") + + def reply(text): + # 模型只能按消息里看得到的字段名交 + body = {"rationale": "x"} + if "blind_total_amount" in text: + body["blind_total_amount"] = {"alias": "Q1", "row": 0, "column": "total"} + if "consistency_verdict" in text: + body["consistency_verdict"] = "一致" + return json.dumps(body, ensure_ascii=False) + + seen = plain_reply(monkeypatch, reply) + out = await run(name, output_schema=schema) + assert len(seen) == 1 and "盲算的金额合计" in seen[0] and "alias" in seen[0] + assert out.error is None and out.result == {"blind_total_amount": 230, "consistency_verdict": "一致", + "rationale": "x"} + + +async def test_a_model_without_structured_output_goes_straight_to_the_plain_call(monkeypatch, shop): + """模型不支持结构化输出(建的时候就报错):直接走普通调用,不算一次调用,也不抛。""" + name, _ = shop + + def unsupported(self, schema, **kwargs): + raise NotImplementedError("这个模型不支持结构化输出") + + monkeypatch.setattr(MockChatModel, "with_structured_output", unsupported) + script = Script(monkeypatch, [q(name, VALID_SUM, "c1")], "不会用到") + plain_reply(monkeypatch, lambda text: '{"value": {"alias": "Q1", "row": 0, "column": "total"}, ' + '"verdict": "一致", "rationale": "x"}') + out = await run(name) + assert out.error is None and out.result["value"] == 230 + assert out.usage["calls"] == len(script.calls) + 1 == 3, "两轮查库加一次普通调用,建不起来的那次不算" + + +async def test_a_failing_ledger_is_reported_not_raised(monkeypatch, shop): + """每日账写不进去(库被锁住):照实交回,不抛。节点因此降成「核不了」,质询接口不会回 500。""" + name, _ = shop + + async def locked(*args, **kwargs): + raise RuntimeError("database is locked") + + monkeypatch.setattr(spend, "charge", locked) + Script(monkeypatch, [q(name, VALID_SUM, "c1")], + {"value": {"alias": "Q1", "row": 0, "column": "total"}, "verdict": "一致", "rationale": "x"}) + out = await run(name) + assert out.result is None and out.error