合入 nr6/b:评测「期望没有数据」只认数值格和叙述里的数、逐题 E1 比产出答案的那条查询、判错时给配对上的那一格、估不出金额时不写 0、没做完的模型调用也计一次、助手只给思考不给操作时自动重来一次(问数据与评测同一个决定)、目录两种计数照实说、自由形应用入口与岗位显示名;评分规则升到第 2 版

This commit is contained in:
ThearchyHelios
2026-10-17 12:28:04 +08:00
35 changed files with 838 additions and 99 deletions
+3 -1
View File
@@ -844,7 +844,9 @@ async def review_catalog_batch(source_id: str, payload: CatalogBatchReviewIn,
async def catalog_table_runs(source_id: str, table: str,
limit: int = Query(20, ge=1, le=catalog_review.RUNS_LIMIT_MAX),
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
"""用过这张表的运行(total 是几次运行),新的在前;取法同使用次数(catalog.table_usage)。"""
"""用过这张表的运行,新的在前。按运行计:total 是几次运行,一次运行查了这张表几次都只算一次。和表清单的使用次数
(catalog.table_usage)数的是同一批查询快照,但使用次数按查询计(同一次运行里查了两次算两次),所以使用次数可能比 total
大。"""
_, source = await _resolved(session, source_id)
key, _ = await _table_key(session, source, table)
return await catalog_review.table_runs(session, source, key, limit=limit)
+9 -3
View File
@@ -1751,14 +1751,20 @@ async def answer_ops(payload: GenerateIn, *, catalog_view: Any = None):
# 流正常结束,却没有 done、也没有一个操作能认(格式跑偏,每一行都被跳过):
# 往下走就是拿原图去自查、交付,界面上写「无需修改」——而模型什么都没说
if gen.asked is None and all(o.get("op") == "plan" for o in applier.seen):
# 只写了思考(deepseek 这类推理模型偶发)、或者格式跑偏:带机读码,问数据页和评测按 chat/turn 的规矩自动
# 重来一次(error.next = retry),还不行再照实报
raise CopilotIncomplete("模型没有给出任何修改,也没有说明原因",
hint="画布未修改。可以重试,或把需求描述得更具体")
hint="画布未修改。可以重试,或把需求描述得更具体", code=turn.EMPTY_GENERATION)
except CopilotIncomplete as e:
yield {"op": "error", "message": f"助手本轮未完成:{e}", "hint": e.hint, "detail": e.detail}
failed = {"op": "error", "message": f"助手本轮未完成:{e}", "hint": e.hint, "detail": e.detail,
"reason": str(e), **({"code": e.code} if e.code else {})}
yield {**failed, "next": turn.next_step(failed)}
return
except Exception as e: # noqa: BLE001
reason, hint = explain_error(e)
yield {"op": "error", "message": f"助手本轮未完成:{reason}", "hint": hint, "detail": raw_error(e)}
failed = {"op": "error", "message": f"助手本轮未完成:{reason}", "hint": hint, "detail": raw_error(e),
"reason": reason}
yield {**failed, "next": turn.next_step(failed)}
return
# 兜底:一个操作都没落地,而上一轮有图。
+4 -2
View File
@@ -69,12 +69,14 @@ def _parse_op_line(line: str) -> dict[str, Any] | None:
class CopilotIncomplete(Exception):
"""模型这一轮没说完:输出被截断,或者一个操作都没给。画布不能按「无需修改」交付。"""
"""模型这一轮没说完:输出被截断,或者一个操作都没给。画布不能按「无需修改」交付。code 是机读码:一个操作都没给的是
chat/turn.EMPTY_GENERATION(问数据一轮按它自动重来一次),截断的不给。"""
def __init__(self, reason: str, *, hint: str = "", detail: str = "") -> None:
def __init__(self, reason: str, *, hint: str = "", detail: str = "", code: str = "") -> None:
super().__init__(reason)
self.hint = hint or "画布未修改。可以重试,或把需求拆成几步分别提出"
self.detail = detail
self.code = code
async def _drain(stream: Any, already: int) -> None:
+12 -4
View File
@@ -17,7 +17,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.apps.forms import input_schema
from app.apps.spec import APP_FORMAT, parse_spec
from app.apps.tree import input_fields, node_map, slot_refs
from app.apps.tree import input_fields, node_map, slot_label, slot_refs
PIN = "main"
VIEW = "latest"
@@ -39,6 +39,13 @@ class DeriveRefused(Exception):
self.code = code
def start_label(name: str) -> str:
"""流程类应用入口按钮的说法「发起…」。名字以英文字母、数字开头时中间空一格(「发起 accept 报销写回」),和界面上中英文
之间留空格的写法一致;以前直接拼成「发起accept 报销写回」。派生、蓝图模板(审批、自由形)都用它。"""
text = str(name or "").strip()
return f"发起 {text}" if text[:1].isascii() and text[:1].isalnum() else f"发起{text}"
def _safe(text: str) -> str:
return re.sub(r"[^A-Za-z0-9_-]", "_", text)
@@ -102,8 +109,9 @@ async def _derive(session: AsyncSession, graph: dict[str, Any], *, title: str, n
"pins": [{**pin, "use": "process" if process else "view"}],
"slots": {}, "entries": [], "forms": {}, "task_ui": {}, "views": {}, "pages": [], "hooks": [], "notify": [],
}
for ref in slot_refs(graph, writes):
spec["slots"].setdefault(ref["slot"], {"label": ref["slot"]})
refs = slot_refs(graph, writes)
for ref in refs:
spec["slots"].setdefault(ref["slot"], {"label": slot_label(ref["slot"], refs)})
for nid, node in humans.items():
spec["task_ui"][nid] = {"title": node["title"]}
if writes and not lookups:
@@ -113,7 +121,7 @@ async def _derive(session: AsyncSession, graph: dict[str, Any], *, title: str, n
names = list(spec["forms"][ENTRY]["properties"])
first = next((n for n in names if n in spec["forms"][ENTRY]["required"]), names[0] if names else None)
spec["entries"].append({
"id": ENTRY, "label": f"发起{title}" if process else "按条件运行", "pin": PIN,
"id": ENTRY, "label": start_label(title) if process else "按条件运行", "pin": PIN,
"form": {"$ref": f"#/forms/{ENTRY}"}, "map": {n: f"form.{n}" for n in names},
**({"instance_title": {"pattern": f"{title}·{{{first}}}", "fields": [first]}}
if first and re.fullmatch(r"[A-Za-z0-9_㐀-鿿]+", first) else {}),
+16 -1
View File
@@ -2,7 +2,7 @@
from __future__ import annotations
import hashlib
from collections.abc import Collection, Iterator
from collections.abc import Collection, Iterable, Iterator
from typing import Any
from app.core.artifact_store import canonical_json
@@ -121,6 +121,21 @@ def slot_refs(graph: Any, writes: Collection[str] = ()) -> list[dict[str, str]]:
return out
#: 按用到它的节点给岗位起显示名(slot_label):派给它的写节点名,超时升级给它的写「…超时升级」(同蓝图模板 slot_labels 的
#: 叫法),动作确认派给它的写「…发送前确认」
_SLOT_USE_LABEL = {"assignee": "{title}", "escalation": "{title}超时升级", "confirm": "{title}发送前确认"}
def slot_label(slot: str, refs: Iterable[dict[str, str]]) -> str:
"""岗位给人看的名字(应用里岗位的显示名)。图里只写了岗位名——画布上随手写的 id,常是英文(finance):以前显示名就是
它,看着像起好的名字。现在按用到它的节点在图上的名字拼(refs 是 slot_refs 的结果,只看这个岗位的那几条,几处用到用顿号
连);节点没起名字(名字就是节点 id)、或者没有节点用到它的,照实用岗位名——界面上显示名和岗位名一样时提示还没起名字。"""
names = [_SLOT_USE_LABEL[r["role"]].format(title=r["title"]) for r in refs
if r.get("slot") == slot and r.get("role") in _SLOT_USE_LABEL and r.get("title")
and r["title"] != r.get("node_id")]
return "、".join(dict.fromkeys(names)) or slot
def input_fields(graph: Any) -> list[dict[str, Any]]:
"""入口节点声明的字段(原样,含新增的 type / enum / format / description)。"""
out: list[dict[str, Any]] = []
+8 -3
View File
@@ -659,7 +659,7 @@ def app_base(bp: dict[str, Any], graph: dict[str, Any], *, use: str, name: str,
writes 是写外部系统的调用工具节点 → 能力名(模板知道哪几个是写入,不用查库)。页面、视图、入口由各模板排。"""
from app.apps.derive import PIN, RECONCILER
from app.apps.spec import APP_FORMAT
from app.apps.tree import node_map, slot_refs
from app.apps.tree import node_map, slot_label, slot_refs
from app.core.artifact_store import graph_hash
ctx = context(bp)
@@ -673,9 +673,14 @@ def app_base(bp: dict[str, Any], graph: dict[str, Any], *, use: str, name: str,
"format": APP_FORMAT, "name": name, "description": description, "icon": "app", "pins": [pin], "slots": {},
"entries": [], "forms": {}, "task_ui": {}, "views": {}, "pages": [], "hooks": [], "notify": [],
}
# 岗位的显示名:卡片上起了名字的用它(审批模板的步骤名);卡片上就是岗位名本身的(处理人写的是岗位、自由形导入的图)按
# 用到它的节点名拼(apps/tree.slot_label,同派生)
labels = slot_labels(bp)
for ref in slot_refs(graph, writes or {}):
spec["slots"].setdefault(ref["slot"], {"label": labels.get(ref["slot"], ref["slot"])})
refs = slot_refs(graph, writes or {})
for ref in refs:
label = labels.get(ref["slot"])
spec["slots"].setdefault(ref["slot"], {"label": label if label and label != ref["slot"]
else slot_label(ref["slot"], refs)})
for nid, n in node_map(graph).items():
if n["type"] == "human":
spec["task_ui"][nid] = {"title": n["title"]}
+3 -1
View File
@@ -50,7 +50,9 @@ def app_skeleton(bp: dict[str, Any], graph: dict[str, Any]) -> dict[str, Any]:
writes = {WRITE: capability} if capability else {}
spec = c.app_base(bp, graph, use="process", name=name, description=str(c.value(bp.get("purpose")) or ""),
writes=writes)
c.entry_and_pages(spec, bp, graph, label=f"发起{name}", writes=writes)
from app.apps.derive import start_label
c.entry_and_pages(spec, bp, graph, label=start_label(name), writes=writes)
return c.finish_app(spec)
+2 -2
View File
@@ -104,7 +104,7 @@ def needs_reconciler(bp: dict[str, Any]) -> bool:
def app_skeleton(bp: dict[str, Any], graph: dict[str, Any]) -> dict[str, Any]:
from app.apps.derive import ENTRY, INSTANCE_VIEW, PIN, VIEW, _entries, _results
from app.apps.derive import ENTRY, INSTANCE_VIEW, PIN, VIEW, _entries, _results, start_label
from app.apps.forms import input_schema
from app.apps.tree import input_fields, node_map
@@ -118,7 +118,7 @@ def app_skeleton(bp: dict[str, Any], graph: dict[str, Any]) -> dict[str, Any]:
if fields:
spec["forms"][ENTRY] = input_schema(fields)
names = list(spec["forms"][ENTRY]["properties"])
spec["entries"].append({"id": ENTRY, "label": f"发起{name}" if process else "按条件运行", "pin": PIN,
spec["entries"].append({"id": ENTRY, "label": start_label(name) if process else "按条件运行", "pin": PIN,
"form": {"$ref": f"#/forms/{ENTRY}"}, "map": {n: f"form.{n}" for n in names}})
spec["views"][INSTANCE_VIEW] = {"source": {"kind": "instance"}}
if not process:
+32 -8
View File
@@ -5,13 +5,13 @@
后端跑不了一轮问数据:评测要么照着抄一份(迟早走样,量出来的不是用户拿到的那一轮),要么做不了;这也违反「界面层不放
逻辑」。现在决定写在这里,是纯函数:
- next_step:生成过程交出结局之后做什么;
- next_step:生成过程交出结局之后做什么(含生成过程一个操作都没给时自动重来一次);
- fill_input:用户的问题填进这张图的入口字段;
- retry_after_review:复核之后要不要带着原图重来一次;
- retry_instruction:重来时交给助手的那句话(两种措辞)。
接口把决定交给问数据页:/copilot/generate-stream 的 final 带 next(run | ready | fail),/copilot/review 的结果带
retry_instruction(要重来时是那句话,不重来是 null)。问数据页照后端给的走;老后端没有这两个字段时退回它自己那份同样的
接口把决定交给问数据页:/copilot/generate-stream 的 final 带 next(run | ready | fail),error 带 next(retry | fail),
/copilot/review 的结果带 retry_instruction(要重来时是那句话,不重来是 null)。问数据页照后端给的走;老后端没有这两个字段时退回它自己那份同样的
规则(tests/test_chat_turn_policy.py 逐条对照两边的常量和措辞)。评测(第 3 波)用同一组函数把一轮串起来。
只搬「决定」:重连、断点续看、取消、复核等多久这些仍在问数据页(把整轮编排搬到后端要重做这些,单独立项)。
@@ -28,9 +28,19 @@ MAX_ATTEMPTS = 2
#: final 之后的三种走法(final.next):运行、只建不跑(摆着等用户点运行)、判失败(没过检查的图不拿去跑)
NEXT_RUN, NEXT_READY, NEXT_FAIL = "run", "ready", "fail"
NEXT_STEPS = (NEXT_RUN, NEXT_READY, NEXT_FAIL)
#: 别的结局:直接回答了(reply)、助手反问了等用户答(asked)、没自查过的老服务端交来没过检查的图时重来一次(retry)
#: 别的结局:直接回答了(reply)、助手反问了等用户答(asked)、重来一次(retry:没自查过的老服务端交来没过检查的图,或者
#: 生成过程一个操作都没给)
NEXT_REPLY, NEXT_ASKED, NEXT_RETRY = "reply", "asked", "retry"
#: 生成过程一个操作都没给就结束了(error 操作的 code):模型只写了思考(deepseek 这类推理模型偶发)、或者格式跑偏每一行都被
#: 跳过。这是偶发的,重来一次多半就好,按「重来一次」的规矩自动重来(最多 MAX_ATTEMPTS 次,含第一次);被截断的不重来——额度
#: 不变,重来还是截断
EMPTY_GENERATION = "copilot_empty"
#: 自动重来的 error(code)
RETRY_ERRORS = frozenset({EMPTY_GENERATION})
#: error 操作的 message 前缀(api/copilot.answer_ops):重来时交给助手的原因不带它
_ERROR_PREFIX = "助手本轮未完成:"
#: 问题往哪个入口字段放:名字像「问题」的优先(同 chat.ts fillInput 的正则,大小写不敏感)
QUESTION_FIELD = re.compile(r"question|query|topic|input|问题|需求|主题", re.IGNORECASE)
@@ -51,7 +61,9 @@ def next_step(terminal: Mapping[str, Any] | None, *, self_checked: bool = True)
"""生成过程交出结局之后做什么。terminal 是最后那条结局操作(final / reply / error);流结束时只有反问(ask)、没有
别的结局的给 None。
- reply:直接回答,这一轮到此为止;error:判失败;只有反问:等用户回答;
- reply:直接回答,这一轮到此为止;只有反问:等用户回答;
- error:生成过程一个操作都没给(code 是 EMPTY_GENERATION)时重来一次(retry,次数由调用方按 MAX_ATTEMPTS 管,原因
取 error_reason),别的判失败;
- final:有 error 级问题时,服务端已经自查过(收到过 check 的结论)就判失败——没过的问题已经交回模型在同一轮里改过
了,再整轮重来是再烧一次完整生成、改的还是同一批问题;没自查过(老服务端)才重来一次(retry);模型说不运行
(autorun=false:用户要的是流程本身、或者问了用户)只建不跑;其余运行。
@@ -61,6 +73,8 @@ def next_step(terminal: Mapping[str, Any] | None, *, self_checked: bool = True)
kind = terminal.get("op")
if kind == "reply":
return NEXT_REPLY
if kind == "error":
return NEXT_RETRY if terminal.get("code") in RETRY_ERRORS else NEXT_FAIL
if kind != "final":
return NEXT_FAIL
if _blockers(terminal):
@@ -70,6 +84,15 @@ def next_step(terminal: Mapping[str, Any] | None, *, self_checked: bool = True)
return NEXT_RUN
def error_reason(error: Mapping[str, Any]) -> str:
"""生成过程交出 error 时重来的原因(交给助手的那句话里用):error 带的 reason,没有就是 message 去掉「助手本轮未完成:」。"""
reason = str(error.get("reason") or "").strip()
if reason:
return reason
message = str(error.get("message") or "").strip()
return message[len(_ERROR_PREFIX):] if message.startswith(_ERROR_PREFIX) else message
def blocked_reason(final: Mapping[str, Any]) -> str:
"""没过检查的图重来时的原因(只有老服务端那条路用得到)。"""
return "生成的工作流没有通过校验:" + ";".join(_blockers(final))
@@ -124,6 +147,7 @@ def has_nodes(graph: Any) -> bool:
return isinstance(graph, Mapping) and bool(graph.get("nodes"))
__all__ = ["MAX_ATTEMPTS", "NEXT_ASKED", "NEXT_FAIL", "NEXT_READY", "NEXT_REPLY", "NEXT_RETRY", "NEXT_RUN", "NEXT_STEPS",
"QUESTION_FIELD", "RETRY_FROM_SCRATCH", "RETRY_WITH_BASE", "blocked_reason", "fill_input", "has_nodes",
"next_step", "retry_after_review", "retry_instruction", "review_reason"]
__all__ = ["EMPTY_GENERATION", "MAX_ATTEMPTS", "NEXT_ASKED", "NEXT_FAIL", "NEXT_READY", "NEXT_REPLY", "NEXT_RETRY",
"NEXT_RUN", "NEXT_STEPS", "QUESTION_FIELD", "RETRY_ERRORS", "RETRY_FROM_SCRATCH", "RETRY_WITH_BASE",
"blocked_reason", "error_reason", "fill_input", "has_nodes", "next_step", "retry_after_review",
"retry_instruction", "review_reason"]
+4 -3
View File
@@ -24,8 +24,8 @@
一张表冲突、写不进去不影响别的表;路径不对、表不存在逐条报(results 和请求一一对应)。审阅的规则同单项审阅
(catalog.review_item)。写进去之后影响数字的项被驳回了的,口径账本在后台标依据了它的答案「依据有变」
(catalog.after_review → ledger/recall,next-round 02 做法 7 二)。
- **被哪些运行用过**(table_runs,GET /catalog/{table}/usage):取法同使用次数(catalog.query_snapshot_tables),按运行
去重,新的在前;total 是运行次数。
- **被哪些运行用过**(table_runs,GET /catalog/{table}/usage):和使用次数数的是同一批查询快照(catalog.query_snapshot_tables),
但按运行去重,新的在前;total 是运行次数。使用次数按查询计,同一次运行里查了几次这张表,使用次数就比 total 多几。
- **进度**(review_progress,GET /catalog/review-progress):常用表(使用次数前 TOP_N 张,和起草对话框「使用次数最多的
20 张表」是同一批:catalog.default_tables 的取法,用到的不够 TOP_N 张时按表结构的顺序补齐)、全部、影响数字的项各自的
确认情况。项的范围和队列一样:表结构里没有的表、列不算。
@@ -391,7 +391,8 @@ async def review_batch(session: AsyncSession, source: Any, items: Iterable[Batch
async def table_runs(session: AsyncSession, source: Any, table: str, *, limit: int = 20) -> dict[str, Any]:
"""用过这张表的运行:{total, runs},新的在前(按开始时间,没开始的按建立时间)。取法同使用次数。"""
"""用过这张表的运行:{total, runs},新的在前(按开始时间,没开始的按建立时间)。数的是和使用次数同一批查询快照
(catalog.query_snapshot_tables),但按运行去重:total 是运行次数,使用次数按查询计、可能比它大。"""
run_ids = {run for run, hit in await catalog.query_snapshot_tables(session, source) if table in hit}
if not run_ids:
return {"total": 0, "runs": []}
+6
View File
@@ -17,6 +17,11 @@ agent 看到的目录少了遮罩列那部分(README 写明)。
文档是表名、注释、目录里的中文名 / 粒度 / 说明、列名、列的中文名(遮罩列只用列名);探查范围外的对象只按名字和注释参与,
权重低一档。切词用 memory/embeddings._tokens(英文按词、中文按字补二元组)加 catalog.name_words(下划线、驼峰拆词)。
**限制:范围外、又没有注释的对象,中文需求命不中。** 它只剩英文名可检索,英文名拆出的词没有中文说法:仓库里没有现成的英中
业务词对照(命名推断只拆词、不翻译,catalog.name_words 也一样),这里不临时编一份词表(编出来的覆盖不全、还会把不相干的表
抬上来),所以「上周的退款」找不到只叫 refund_orders、没写注释的表。真库多半有注释,注释照常参与检索,影响小;范围内的表另有
目录里的中文名、列名可配。要用这张表时按英文名点名查看(会按需加入),或者请管理员把它加进探查范围、补上注释或目录的中文名。
**三种「不在」**(absent_detail):表已不在、在探查范围外、在范围里却读不到结构,说法分开;只有真删除才说「表已不在」。
只读:只看表结构缓存(schema_cache)和调用方读好的数据目录、剖析事实、对象索引,不碰业务库。这里的文字都是写给模型看的,
@@ -342,6 +347,7 @@ def _doc_text(name: str, meta: Mapping[str, Any] | None, notes: Mapping[str, Any
def _outside_text(name: str, comment: str | None) -> str:
"""一个范围外对象的检索文档:名字、名字拆出的词、注释。没有注释的只有英文(见模块说明的限制)。"""
from app.data.catalog import name_words
return " ".join([name, " ".join(name_words(name)), comment or ""])
+25 -8
View File
@@ -10,7 +10,8 @@
复核跟着换);读哪份目录按变体(answer_ops 的 catalog_view,查看的表详情、自查的 SQL 检查器都按它)。
- 结局:reply 直接回答、asked 反问、fail 判失败(建图失败)、ready 只建不跑、retry(老服务端那条路:没自查过的图带着原因
重来一次)、run 运行。生成过程交出 error(助手本轮未完成)、开流前就拒绝(模型没配好、数据源不在)都是建图失败,另记
「助手调用失败」——连续几题都这样就止损(runner.py)。
「助手调用失败」——连续几题都这样就止损(runner.py);只有生成过程一个操作都没给(只写了思考,error.next=retry)时按
同一个规矩先带着原因重来一次,还不行才记建图失败。每次重来的原因照实记在 retries。
- 运行:fill_input 填入口字段(单轮,history 为空),来源 eval,名字「评测:题集名·第 n 题」,选项带目录视图(Run.options
的 catalog_view:运行时的 db_schema、冻结快照、自查都按它)、只读和遮罩(同使用者的问数据运行:评测页会显示答案里的数,
遮罩值不能从这里露出来;评测不写业务库)。写作模型按变体写进图的 defaults:节点自己点了模型的不覆盖,照实记在
@@ -79,6 +80,8 @@ class Turn:
#: 这一题发起过的全部作答运行(重来的话两次),失败的也在
runs: list[str] = field(default_factory=list)
attempts: int = 0
#: 每次自动重来的原因(和 attempts 一一对应):没过检查、复核说不可信、生成过程一个操作都没给
retries: list[str] = field(default_factory=list)
reason: str | None = None
#: 助手调用失败(生成过程交出 error、开流前就被拒):runner 据此止损
copilot_failed: bool = False
@@ -101,25 +104,27 @@ class Turn:
def to_dict(self) -> dict[str, Any]:
return {"path": self.path, "run_id": self.run_id, "runs": list(self.runs), "attempts": self.attempts,
"reason": self.reason, "copilot_failed": self.copilot_failed, "timed_out": self.timed_out,
"retries": list(self.retries), "reason": self.reason, "copilot_failed": self.copilot_failed, "timed_out": self.timed_out,
"reply": self.reply, "asked": self.asked, "review": self.review, "looks": list(self.looks),
"assistant_model": self.assistant_model, "usage": copy.deepcopy(self.usage),
"timings": dict(self.timings), "own_models": list(self.own_models)}
#: 用量汇总里相加的计数(providers/usage 的 summary):unfinished_calls 是没做完的调用(被取消、超时,token 只计了已报的)
_COUNTED = ("input_tokens", "output_tokens", "calls", "unpriced_calls", "unfinished_calls")
def _add_usage(into: dict[str, Any] | None, more: Mapping[str, Any] | None) -> dict[str, Any]:
"""两份 providers/usage 汇总相加(by_model 逐个模型加)。"""
out = copy.deepcopy(into) if into else {"by_model": {}, "input_tokens": 0, "output_tokens": 0, "calls": 0,
"cost_usd": 0.0, "unpriced_calls": 0}
out = copy.deepcopy(into) if into else {"by_model": {}, **dict.fromkeys(_COUNTED, 0), "cost_usd": 0.0}
if not more:
return out
for key in ("input_tokens", "output_tokens", "calls", "unpriced_calls"):
for key in _COUNTED:
out[key] = int(out.get(key) or 0) + int(more.get(key) or 0)
out["cost_usd"] = round(float(out.get("cost_usd") or 0) + float(more.get("cost_usd") or 0), 6)
for model, row in (more.get("by_model") or {}).items():
mine = out["by_model"].setdefault(model, {"input_tokens": 0, "output_tokens": 0, "calls": 0, "cost_usd": 0.0,
"unpriced_calls": 0})
for key in ("input_tokens", "output_tokens", "calls", "unpriced_calls"):
mine = out["by_model"].setdefault(model, {**dict.fromkeys(_COUNTED, 0), "cost_usd": 0.0})
for key in _COUNTED:
mine[key] = int(mine.get(key) or 0) + int(row.get(key) or 0)
mine["cost_usd"] = round(float(mine.get("cost_usd") or 0) + float(row.get("cost_usd") or 0), 6)
return out
@@ -379,6 +384,14 @@ class Driver:
return t
kind = terminal.get("op")
if kind == "error":
# 生成过程一个操作都没给(只写了思考):按重来一次的规矩带着原因重来(同问数据页,chat/turn.next_step)
step = terminal.get("next") or policy.next_step(terminal)
if step == policy.NEXT_RETRY and t.attempts + 1 < policy.MAX_ATTEMPTS:
reason = policy.error_reason(terminal)
t.attempts += 1
t.retries.append(reason)
instruction = policy.retry_instruction(self.question, reason, policy.has_nodes(base_graph))
continue
t.path, t.reason, t.copilot_failed = "build_failed", str(terminal.get("message") or "助手本轮未完成"), True
return t
if kind == "reply":
@@ -393,6 +406,7 @@ class Driver:
if step == policy.NEXT_RETRY:
if t.attempts + 1 < policy.MAX_ATTEMPTS:
t.attempts += 1
t.retries.append(policy.blocked_reason(terminal))
instruction = policy.retry_instruction(self.question, policy.blocked_reason(terminal),
policy.has_nodes(graph))
base_graph = graph if policy.has_nodes(graph) else None
@@ -421,7 +435,10 @@ class Driver:
t.review = {k: v for k, v in review.items() if k != "usage"}
t.usage["review"] = _add_usage(t.usage.get("review"), review.get("usage"))
if review.get("retry_instruction"):
from app.chat.turn import review_reason
t.attempts += 1
t.retries.append(review_reason(review) or "复核判答案不可用")
instruction = str(review["retry_instruction"])
base_graph = row.graph if policy.has_nodes(row.graph) else None
continue
+35 -9
View File
@@ -124,16 +124,44 @@ async def source_versions(session: AsyncSession, source_ids: Sequence[str]) -> d
async def estimate(session: AsyncSession, suite_id: str, items: int) -> dict[str, Any]:
"""开跑前的估算:按这个题集上一次结束的评测的每题平均花费 × 这次的项数。第一次评测估不出,照实写。"""
"""开跑前的估算:按这个题集上一次结束的评测的每题平均花费 × 这次的项数。第一次评测估不出,照实写。
上一次有估不出金额的调用(模型不在价格目录里、用量没报全)时,平均花费只是估得出的那部分:一分都没估出来的照实写估不出
(以前写「约 $0」,像不花钱),估出了一部分的写「至少」(at_least)、带上另有几次估不出(unpriced_calls)。老总表里每题
平均记成 0、却有估不出的调用的,也按估不出算。"""
row = (await session.execute(select(EvalRun).where(
EvalRun.suite_id == suite_id, EvalRun.status.in_(("succeeded", "partial")))
.order_by(EvalRun.created_at.desc()).limit(1))).scalar_one_or_none()
per_item = ((row.summary or {}).get("per_case_cost_usd") if row is not None else None)
if per_item is None:
if row is None:
return {"items": items, "per_item_usd": None, "total_usd": None, "basis": None,
"note": "这个题集还没有评完过,估不出花费:按每题开评前的实际花费累计,到了上限就停"}
return {"items": items, "per_item_usd": round(float(per_item), 6), "total_usd": round(float(per_item) * items, 4),
"basis": row.id, "note": "按这个题集上一次评测的每题平均花费估算(不含估不出金额的模型)"}
summary = row.summary or {}
per_item = summary.get("per_case_cost_usd")
unpriced = int(summary.get("unpriced_calls") or 0)
if per_item is None or (unpriced and not float(per_item)):
if not unpriced:
return {"items": items, "per_item_usd": None, "total_usd": None, "basis": None,
"note": "这个题集上一次评测没有评完一道题,估不出花费:按每题开评前的实际花费累计,到了上限就停"}
return {"items": items, "per_item_usd": None, "total_usd": None, "basis": row.id, "unpriced_calls": unpriced,
"note": f"上一次评测的 {unpriced} 次模型调用金额都估不出(模型不在价格目录中,或用量没有报全):"
"金额上限只管得住估得出金额的调用"}
out = {"items": items, "per_item_usd": round(float(per_item), 6), "total_usd": round(float(per_item) * items, 4),
"basis": row.id, "note": "按这个题集上一次评测的每题平均花费估算"}
if unpriced:
out.update(at_least=True, unpriced_calls=unpriced,
note=f"按这个题集上一次评测的每题平均花费估算,实际至少这么多:上一次另有 {unpriced} 次调用金额估不出,没有算进去")
return out
def judge_usage(judge: Mapping[str, Any], cost_usd: float) -> dict[str, Any]:
"""按角色的用量里裁判那一份:金额、次数,和估不出金额的次数(结论裁判的模型不在价格目录里时它的每次调用都估不出;
文字要点按计量汇总)。以前只有金额和次数,裁判模型不在价格目录里时写成 $0。"""
claims = judge.get("claims") or {}
text = judge.get("text") or {}
calls = int(claims.get("calls") or 0) + int(text.get("calls") or 0)
unpriced = (int(claims.get("calls") or 0) if claims.get("priced") is False else 0) \
+ int((text.get("usage") or {}).get("unpriced_calls") or 0)
return {"cost_usd": round(cost_usd, 6), "calls": calls, "unpriced_calls": unpriced}
# ==========================================================================
@@ -552,9 +580,7 @@ class EvalExecution(runtime.JobRun):
writer = await writer_usage(turn.runs)
assistant = add_usage(turn.usage.get("assistant"), turn.usage.get("review"))
usage = {"assistant": assistant, "writer": writer, "review": turn.usage.get("review"),
"judge": {"cost_usd": round(judge_cost, 6),
"calls": int((judge.get("claims") or {}).get("calls") or 0)
+ int((judge.get("text") or {}).get("calls") or 0)}}
"judge": judge_usage(judge, judge_cost)}
cost = float(assistant.get("cost_usd") or 0) + float(writer.get("cost_usd") or 0)
self.spent += cost + judge_cost
looks = {"model": sum(1 for x in turn.looks if x.get("by") == "model"),
@@ -1022,6 +1048,6 @@ async def shutdown() -> None:
__all__ = ["EVAL_BUSY", "EVAL_FINISHED", "EVAL_INVALID", "EvalExecution", "EvalRefused", "INTERRUPTED",
"NOTHING_TO_RETRY", "STOPPED", "active_run", "estimate", "gold_key", "leader_of", "manager",
"NOTHING_TO_RETRY", "STOPPED", "active_run", "estimate", "gold_key", "judge_usage", "leader_of", "manager",
"prompt_fingerprint", "reconcile_stale", "refresh_summary", "result_dict", "result_rows", "retry", "running",
"shutdown", "source_versions", "start", "startup", "stop"]
+96 -19
View File
@@ -17,11 +17,13 @@ wrong 错、scale 量纲不同(差 10 的整数次幂,比如元和万元、
- scalar:数值按「允许的相对误差」比(engine/tolerance.agree,和双盲复算同一个函数);期望没写容差时,引用的原值按
DEFAULT_TOLERANCE,裸写的数按它写出的精度(3.5 万 ±500)。维度配对:期望写了 where(或期望那一行本身有维度)时,答案
那一格所在行的非数值列要有这个取值;答案那一格没有维度(按条件筛过、只剩一个数)不算对不上。期望值是 0–10 的小整数
又没有维度可配时标「弱匹配」(巧合的数:3 家门店正好等于期望的 3)。
又没有维度可配时标「弱匹配」(巧合的数:3 家门店正好等于期望的 3)。答错时「答案里的数」给和这一项配对上的那一格(列名、
指标、说明同名,或者维度对得上的只有一个数),配不上就不给、原因里照实写配不上,列出答案引用了哪几个数。
- table:逐行按 keys 的取值配对,每个要比的值同 scalar;ordered 时按答案里各行键值出现的先后核对顺序。
- set:名单里每一项都要在答案里出现(ordered 时按顺序);每一项都出现在答案查过的结果里算可追溯。
- none:期望答「没有数据」:答案里没有数,而且查过的查询里有返回 0 行的(可追溯);没有数、但也没有一条查询返回 0 行的
算「对但没引用」。
算「对但没引用」。「给了数」只认引用的数值格和叙述里写的数:引用文字格(地区名这种维度)、空值格(NULL)不算——「华中区
没有数据」引用了地区那一格,说的仍是没有数据。
- ask:期望反问:作答路径是反问才对。
- text:文字要点,由裁判判(covered / partial / missing / unjudged),单列「模型判断」,不进准确率(评测-D4)。
@@ -30,9 +32,10 @@ wrong 错、scale 量纲不同(差 10 的整数次幂,比如元和万元、
没有错的)不能评分。期望没算出来按 gold 的结论:口径升版待处置是跳过,别的是不能评分——不混进「答错」。被驳回(总方案 X9)
记路径「被驳回」,不算失败:不能评分。只有文字要点的题不能评分(单列,不计入准确率)。
**逐题的 E1**(sql_match):答案里的查询(先看报告引用了的)和期望的取数(SQL 题是期望 SQL,口径题是口径卡的取数节点)
用口径账本同一个比对函数(ledger/match.compare_queries:先绑参数、再由 explain_sql_diff 判等价)比一次,给出等价与否和差在
哪个方面(总方案 X2:E1 只有一份实现)。
**逐题的 E1**(sql_match):产出答案的那条查询和期望的取数(SQL 题是期望 SQL,口径题是口径卡的取数节点)用口径账本同一个
比对函数(ledger/match.compare_queries:先绑参数、再由 explain_sql_diff 判等价)比一次,给出等价与否和差在哪个方面(总方案
X2:E1 只有一份实现)。哪条是产出答案的:答案引用了就是被引用的;没引用、只查了一条就是那一条(写明依据);没引用、查了
几条的不硬比(照实写没法对上是哪条,不计入 E1)。
"""
from __future__ import annotations
@@ -46,8 +49,10 @@ from app.engine.tolerance import agree, is_finite_number
logger = logging.getLogger(__name__)
#: 评分规则的版本:规则改了就加一,结果里记着按哪一版评的,可以按新规则重评
SCORER_VERSION = 1
#: 评分规则的版本:规则改了就加一,结果里记着按哪一版评的,可以按新规则重评。
#: 2:期望「没有数据」时引用文字格、空值格不算给了数;判错时「答案里的数」给配对上的那一格;答案没引用查询时逐题 E1
#: 不硬比第一条查询
SCORER_VERSION = 2
OUTCOMES = ("traced", "untraced", "wrong", "scale", "ungradable")
OUTCOME_LABEL = {"traced": "对且可追溯", "untraced": "对但没引用", "wrong": "错", "scale": "量纲不同",
@@ -64,6 +69,8 @@ DEFAULT_TOLERANCE = 1e-6
SCALE_FACTORS = (10.0, 100.0, 1000.0, 1e4, 1e5, 1e6, 1e8)
#: 弱匹配:期望是这么小的整数、又没有维度可配
WEAK_MAX = 10
#: 判错、又配不上时原因里列出答案引用的前几个数
PAIR_SHOWN = 3
#: 路径不对时给人看的原因
WRONG_PATH = {
@@ -316,9 +323,48 @@ def _score_scalar(exp: Mapping[str, Any], gold: Mapping[str, Any], answer: Answe
for token in answer.bare:
if _scaled(expected, float(token["value"])):
return {**out, "outcome": "scale", "got": token["value"], "raw": token["raw"]}
got = next((v.get("value") for v in clean), None)
return {**out, "outcome": "wrong", **({"got": got} if got is not None else {}),
"reason": "答案里没有对得上的数" if not answer.values and not answer.bare else "答案里的数和期望对不上"}
numeric = [v for v in clean if is_finite_number(v.get("value"))]
if not numeric and not answer.bare:
return {**out, "outcome": "wrong", "reason": "答案里没有对得上的数"}
paired = _paired(numeric, exp, gold, key)
if paired is not None:
return {**out, "outcome": "wrong", "got": paired.get("value"), "cell": _cell_ref(paired),
"reason": "答案里的数和期望对不上"}
if numeric:
shown = "、".join(f"{_value_name(v)} {_plain(v.get('value'))}" for v in numeric[:PAIR_SHOWN])
more = f" 等 {len(numeric)} 个" if len(numeric) > PAIR_SHOWN else ""
return {**out, "outcome": "wrong",
"reason": f"答案里的数和期望对不上:答案引用的数按列名、维度都配不上这一项(引用了 {shown}{more})"}
return {**out, "outcome": "wrong", "reason": "答案里的数和期望对不上"}
def _norm(name: Any) -> str:
return str(name).strip().strip('"`[]').lower() if name is not None else ""
def _paired(numeric: Sequence[Mapping[str, Any]], exp: Mapping[str, Any], gold: Mapping[str, Any],
key: Mapping[str, Any]) -> Mapping[str, Any] | None:
"""答错时答案里和这一项配对上的那一格(「答案里的数」给它):按名字配——列名(或口径卡的指标)是期望取的那一列、
那个指标,或者和期望的说明(label)同名;名字配不上时按维度配——期望写了维度、答案里维度对得上的数恰好只有一个。
都配不上为 None:以前取答案引用的第一个值,退款金额那一项显示成了答案里的订单数,甚至显示成一格文字。"""
names = {_norm(n) for n in (exp.get("column"), gold.get("column"), exp.get("metric"), gold.get("metric"),
gold.get("name"), exp.get("label")) if n not in (None, "")}
named = [v for v in numeric if _norm(v.get("column")) in names or _norm(v.get("metric")) in names]
if named:
return named[0]
if key:
on_dims = [v for v in numeric if v.get("dims") and _dims_ok(key, v["dims"])]
if len(on_dims) == 1:
return on_dims[0]
return None
def _value_name(value: Mapping[str, Any]) -> str:
return str(value.get("column") or value.get("metric") or value.get("alias") or "(没有列名)")
def _plain(value: Any) -> str:
return str(int(value)) if is_finite_number(value) and float(value).is_integer() else str(value)
def _cell_ref(value: Mapping[str, Any]) -> dict[str, Any]:
@@ -371,8 +417,15 @@ def _score_set(exp: Mapping[str, Any], gold: Mapping[str, Any], answer: Answer,
def _score_none(answer: Answer) -> dict[str, Any]:
if answer.values or answer.bare:
return {"outcome": "wrong", "reason": "期望答「没有数据」,答案里却给了数"}
"""期望答「没有数据」。「给了数」只认引用的数值格、叙述里写的数:引用的文字格(地区名这种维度)、空值格(NULL)不是数——
「华中区([[v:Q1.r0.地区]])没有数据」说的正是没有数据,以前因为它引用了一格就判成「给了数」。"""
numeric = next((v for v in answer.values if is_finite_number(v.get("value"))), None)
if numeric is not None:
return {"outcome": "wrong", "got": numeric.get("value"), "cell": _cell_ref(numeric),
"reason": "期望答「没有数据」,答案里却引用了数"}
if answer.bare:
return {"outcome": "wrong", "got": answer.bare[0]["value"], "raw": answer.bare[0]["raw"],
"reason": "期望答「没有数据」,答案的叙述里却写了数"}
if any(int(q.get("row_count") or 0) == 0 and q.get("role") != "verify" for q in answer.queries):
return {"outcome": "traced"}
return {"outcome": "untraced", "reason": "答案没有给数,但没有一条查询返回 0 行可以说明「没有数据」"}
@@ -561,7 +614,8 @@ async def gold_calibers(case: Mapping[str, Any], gold: Mapping[str, Any] | None)
async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, answer: Answer, *,
timeout: float = 8.0) -> dict[str, Any] | None:
"""逐题的 E1:{status: proven|related|none|skipped, query, aspects, diff, reason}。没有作答运行、没有期望取数的为 None。
"""逐题的 E1:{status: proven|related|none|skipped, query, aspects, diff, reason, cited, basis}。没有作答运行、没有期望
取数的为 None。比的是哪条查询见 _answering:basis 是 cited(答案引用的)或 only_query(答案没引用、运行只查了这一条)。
比对在线程里跑(纯计算,和账本一样限时)。"""
import asyncio
import time
@@ -594,13 +648,12 @@ async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, ans
except Exception: # noqa: BLE001 - 比对是附加的一步,出错照实记没比
logger.exception("评测:逐题比 SQL 出错")
return {"status": "skipped", "reason": "比对出错"}
cited = [r for r in rows if r.get("artifact") in set(answer.cited)]
pool = cited or rows
pool, basis = _answering(rows, answer)
if isinstance(pool, dict):
return pool
proven = next((r for r in pool if r.get("status") == "proven"), None)
chosen = proven or next((r for r in pool if r.get("status") == "related"), None) or \
next((r for r in pool if r.get("status") == "skipped"), None) or (pool[0] if pool else None)
if chosen is None:
return {"status": "none", "reason": "答案没有查询"}
next((r for r in pool if r.get("status") == "skipped"), None) or pool[0]
candidate = next((c for c in chosen.get("candidates") or [] if c.get("status") == chosen.get("status")), None) or \
next(iter(chosen.get("candidates") or []), None) or {}
diff = candidate.get("diff") or {}
@@ -608,7 +661,31 @@ async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, ans
sql = next((q.get("sql") for q in answer.queries if q.get("artifact") == chosen.get("artifact")), None)
return {"status": chosen.get("status"), "query": chosen.get("artifact"), "sql": sql, "aspects": aspects,
"diff": diff or None, "via": candidate.get("via"), "binding": candidate.get("binding"),
"reason": chosen.get("reason") or candidate.get("reason"), "cited": bool(cited)}
"reason": chosen.get("reason") or candidate.get("reason"), "cited": basis == "cited", "basis": basis}
def _answering(rows: list[dict[str, Any]], answer: Answer) -> tuple[list[dict[str, Any]] | dict[str, Any], str]:
"""逐题 E1 拿哪几条查询比:产出答案的那几条。交回 (查询, 依据),确定不了时交回 (照实的结论, "")。
- 答案引用了查询(报告里引用的格、指标所在的查询):比被引用的那几条(依据 cited);
- 没引用、这次运行(核查用的不算)只查了一条:答案的数只可能来自它,按它比(依据 only_query);
- 没引用、查了几条:不比——以前取第一条相关的,取到的往往是探索查询(看看有哪些地区那种),说「差在筛选条件」和答案的数
毫无关系。照实写没法对上是哪条,不计入 E1。"""
cited = set(answer.cited)
if cited:
pool = [r for r in rows if r.get("artifact") in cited]
if pool:
return pool, "cited"
return {"status": "skipped", "cited": True,
"reason": "答案引用的查询不在可比对的查询里(合并查询不比:它的数都来自它的输入)"}, ""
roles = {q.get("artifact"): q.get("role") for q in answer.queries}
pool = [r for r in rows if roles.get(r.get("artifact")) != "verify"]
if not pool:
return {"status": "none", "reason": "答案没有查询"}, ""
if len(pool) > 1:
return {"status": "skipped", "cited": False,
"reason": f"答案没有引用查询,这次运行执行了 {len(pool)} 条查询,没法对上是哪条产出了答案,不比"}, ""
return pool, "only_query"
__all__ = ["Answer", "CaseScore", "DEFAULT_TOLERANCE", "OUTCOMES", "OUTCOME_LABEL", "PATH_LABEL", "SCORER_VERSION",
+4 -2
View File
@@ -145,8 +145,10 @@ def summarize(results: Sequence[Mapping[str, Any]], *, judge_on: bool = False) -
"timings": {k: {"p50": percentile(v, 50), "p90": percentile(v, 90), "n": len(v)} for k, v in timings.items()},
"usage": {**usage, "gold": {"cost_usd": 0.0, "calls": 0}},
"cost_usd": cost, "unpriced_calls": unpriced,
# 每题平均花费(下一次开跑前的估算按它):只按评完了的题平均,没开评的不摊
"per_case_cost_usd": round(cost / len(evaluated), 6) if evaluated else None,
# 每题平均花费(下一次开跑前的估算按它):只按评完了的题平均,没开评的不摊。有估不出金额的调用时 cost_usd 只是估得出
# 的那部分:一分都没估出来(模型都不在价格目录里)的为 None,不写成 0(像不花钱);估出了一部分的照给,unpriced_calls
# 说明它只是下限
"per_case_cost_usd": round(cost / len(evaluated), 6) if evaluated and (cost > 0 or not unpriced) else None,
"citations": {"numbers": numbers, "numbers_cited": cited,
"uncited_numbers": sum(int(s.get("uncited_numbers") or 0) for s in stats),
"unresolved": sum(int(s.get("unresolved") or 0) for s in stats),
+50 -13
View File
@@ -12,12 +12,21 @@ async generator:with 块跨着 yield 的话,变量在两次 yield 之间留
在创建时拷走上下文,照样计到;调用方自己的代码一行都不在计量里。
**金额。** 按 providers/catalog.estimate_cost 算,和运行的 llm.end 一个口径。估不出的调用记 unpriced_calls:模型不在价格
目录里,或者用量没报全而目录价又不是 0。估不出的不当成 0 元,调用方据此写「≥」或「金额无法估算」。用量没报全有两种:
目录里,或者用量没报全而目录价又不是 0。估不出的不当成 0 元,调用方据此写「≥」或「金额无法估算」。汇总的 cost_usd 永远是
估得出的那部分(同运行的 usage,全系统一个口径),有 unpriced_calls 时它只是下限:写金额、算平均、做估算的地方都要连着
unpriced_calls 一起看,一分都没估出来的写估不出、不写 0(评测的开跑估算、每题平均花费以前就漏了这一条,见 eval/runner.estimate、
eval/stats.summarize)。用量没报全有两种:
回复没带用量(有的网关流式不给);读到一半就不读了——用量在流的末尾才报全(Anthropic 的输出 token 在最后一条,OpenAI
兼容系开了 stream_usage 也在最后)。生成过程读到 done、look、ask 那一行之后照样把流读完再关(api/copilot_gen._drain),
流末尾那一段用量收得到;只有模型在那一行之后还往下写超过 DRAIN_CHARS 字的闲话时才当场关流(不为闲话付钱),连接断了也一样,
这一次只拿得到已经报了的部分:照样计次数、计已报的 token,记成估不出,不悄悄少算。
**没做完的调用**(unfinished_calls)。被取消(挑表超时被 asyncio.wait_for 取消、调用方停了)、超时、流读到一半被关掉的调用
都记一次调用、标「未完成」:token 只计已经报了的(一段都没收到的记 0,当作不知道,记成估不出),调用次数照实。以前挑表超时
的那一次一点都不计,问数据一轮、评测按角色的调用次数就少一次。LangChain 的 ainvoke 被取消时不一定回调 on_llm_error
(agenerate 卡在 gather 上被取消,后面的回调走不到):开始了、到汇总时还没有结束回调的调用(_pending 里剩下的)一并算未完成。
请求刚发出去就被拒了的(鉴权错、参数不对,一段都没收到)不算一次调用:模型没干活。
模型名按发起调用时的参数取(invocation_params 的 model / model_name,就是接入里配的模型 id,价格目录按它查);调用参数里
没有的(以前的演示模型就没报,评测汇总里助手用量的模型因此记成 unknown),看 LangChain 照模型对象给回调的 metadata 里的
ls_model_name;再取不到看回复里的 model_name,都没有才记 UNKNOWN_MODEL。OpenAI 兼容、Anthropic 的模型在调用参数里就带着
@@ -37,6 +46,17 @@ from langchain_core.tracers.context import register_configure_hook
#: 取不到模型名的调用记在这个名下
UNKNOWN_MODEL = "unknown"
#: 每个模型累计的几项(unreported 用量没报全的次数、unfinished 没做完的次数)
_COUNTERS = ("input_tokens", "output_tokens", "calls", "unreported", "unfinished")
def _abandoned(error: BaseException) -> bool:
"""调用是半路放弃的(被取消、超时),不是一发出去就被拒:模型可能已经在干活、已经计费。"""
import asyncio
if isinstance(error, (asyncio.CancelledError, TimeoutError)):
return True
return any(cls.__name__.endswith(("Timeout", "TimeoutError")) for cls in type(error).__mro__)
class Meter(BaseCallbackHandler):
@@ -51,6 +71,10 @@ class Meter(BaseCallbackHandler):
self._pending: dict[UUID, str] = {}
self._models: dict[str, dict[str, int]] = {}
def _row(self, model: str) -> dict[str, int]:
"""这个模型的累计(调用方持锁)。unreported:用量没报全的次数;unfinished:没做完的次数(见模块开头)。"""
return self._models.setdefault(model, dict.fromkeys(_COUNTERS, 0))
# ------------------------------------------------------------------ 回调
def _started(self, run_id: UUID, serialized: dict[str, Any] | None, kwargs: dict[str, Any]) -> None:
@@ -83,7 +107,7 @@ class Meter(BaseCallbackHandler):
with self._lock:
model = self._pending.pop(run_id, "") or str(meta.get("model_name") or meta.get("model") or "") \
or UNKNOWN_MODEL
row = self._models.setdefault(model, {"input_tokens": 0, "output_tokens": 0, "calls": 0, "unreported": 0})
row = self._row(model)
row["calls"] += 1
if usage:
row["input_tokens"] += int(usage.get("input_tokens") or 0)
@@ -92,8 +116,9 @@ class Meter(BaseCallbackHandler):
row["unreported"] += 1
def on_llm_error(self, error: BaseException, *, run_id: UUID, **kwargs: Any) -> None:
"""流读到一半停了(生成过程在最后一行之后读到太多闲话就关流,或者连接断了):LangChain 照样交来已经收到的那几段。
有收到东西的照计一次、记已报的 token,算用量没报全;一段都没收到的(请求刚发出去就被拒了)不计。"""
"""调用没做完:流读到一半停了(生成过程在最后一行之后读到太多闲话就关流,或者连接断了)、被取消、超时。LangChain
照样交来已经收到的那几段。有收到东西的、被取消或超时的照计一次、记已报的 token,算用量没报全、未完成;一段都没收到
的报错(请求刚发出去就被拒了)不计。"""
from langchain_core.outputs import ChatGenerationChunk
response = kwargs.get("response")
@@ -105,36 +130,48 @@ class Meter(BaseCallbackHandler):
partial = None
with self._lock:
model = self._pending.pop(run_id, "")
if partial is None:
if partial is None and not _abandoned(error):
return
meta = getattr(partial, "response_metadata", None) or {}
model = model or str(meta.get("model_name") or meta.get("model") or "") or UNKNOWN_MODEL
row = self._models.setdefault(model, {"input_tokens": 0, "output_tokens": 0, "calls": 0, "unreported": 0})
row = self._row(model)
row["calls"] += 1
row["unreported"] += 1
row["unfinished"] += 1
reported = getattr(partial, "usage_metadata", None) or {}
row["input_tokens"] += int(reported.get("input_tokens") or 0)
row["output_tokens"] += int(reported.get("output_tokens") or 0)
def _rows(self) -> dict[str, dict[str, int]]:
"""累计的一份拷贝,开始了还没结束回调的调用算成没做完的(调用方持锁)。"""
rows = {k: dict(v) for k, v in self._models.items()}
for model in self._pending.values():
row = rows.setdefault(model or UNKNOWN_MODEL, dict.fromkeys(_COUNTERS, 0))
row["calls"] += 1
row["unreported"] += 1
row["unfinished"] += 1
return rows
# ------------------------------------------------------------------ 合并、汇总
def merge(self, other: Meter) -> None:
"""把另一次计量并进来(嵌套的计量出块时并进外层)。"""
with other._lock:
rows = {k: dict(v) for k, v in other._models.items()}
rows = other._rows()
with self._lock:
for model, row in rows.items():
mine = self._models.setdefault(model, dict.fromkeys(row, 0))
mine = self._row(model)
for key, value in row.items():
mine[key] = mine.get(key, 0) + value
def summary(self) -> dict[str, Any]:
"""{by_model: {模型: {input_tokens, output_tokens, calls, cost_usd, unpriced_calls}}, input_tokens, output_tokens,
calls, cost_usd, unpriced_calls}。cost_usd 只是估得出的那部分。"""
"""{by_model: {模型: {input_tokens, output_tokens, calls, cost_usd, unpriced_calls, unfinished_calls}}, input_tokens,
output_tokens, calls, cost_usd, unpriced_calls, unfinished_calls}。cost_usd 只是估得出的那部分;unfinished_calls 次
没做完(token 只计了已报的)。"""
from app.providers.catalog import estimate_cost, find_pricing
with self._lock:
rows = {k: dict(v) for k, v in self._models.items()}
rows = self._rows()
by_model: dict[str, dict[str, Any]] = {}
for model, row in sorted(rows.items()):
pricing = find_pricing(model)
@@ -146,9 +183,9 @@ class Meter(BaseCallbackHandler):
by_model[model] = {"input_tokens": row["input_tokens"], "output_tokens": row["output_tokens"],
"calls": row["calls"],
"cost_usd": round(estimate_cost(model, row["input_tokens"], row["output_tokens"]), 6),
"unpriced_calls": unpriced}
"unpriced_calls": unpriced, "unfinished_calls": row["unfinished"]}
total = {key: sum(r[key] for r in by_model.values())
for key in ("input_tokens", "output_tokens", "calls", "unpriced_calls")}
for key in ("input_tokens", "output_tokens", "calls", "unpriced_calls", "unfinished_calls")}
return {"by_model": by_model, **total, "cost_usd": round(sum(r["cost_usd"] for r in by_model.values()), 6)}
+2 -1
View File
@@ -344,7 +344,8 @@ async def test_derive_a_process_leaves_slots_empty(monkeypatch):
async with SessionLocal() as session:
spec = await derive_app(session, wf, 1, name="报销审批")
assert spec["name"] == "报销审批" and spec["pins"][0]["use"] == "process"
assert spec["slots"] == {"finance": {"label": "finance"}, "reconciler": {"label": "动作核实"}}
# 岗位的显示名按用到它的节点名起(apps/tree.slot_label),不再拿岗位名 finance 充当显示名
assert spec["slots"] == {"finance": {"label": "财务审批"}, "reconciler": {"label": "动作核实"}}
# 流程的首页只放发起和我发起的:结果在实例页看,不派生「最近封存运行」(派生出来只会挂一块永远空着的印章)
assert "latest" not in spec["views"] and spec["views"]["inst"]["source"]["kind"] == "instance"
home = next(p for p in spec["pages"] if p["id"] == "home")
+49
View File
@@ -299,3 +299,52 @@ def test_freeform_people_rows_follow_the_imported_graph():
stale = [i for i in issues if i["code"] == "bp.template_mismatch" and "gone" in i["message"]]
assert stale and stale[0]["level"] == "warning"
assert freeform.people_rows(c.with_graph(get_template("freeform").example(), freeform.example_graph())) == []
def _people_graph() -> dict:
"""两步审核派给岗位 finance(第一步超时升级给 boss),还有一步派给没起名字的节点用的岗位 ops。名字都是虚构的。"""
return {"nodes": [
c.node("start", "input", "发起", fields=[{"name": "amount", "type": "number"}]),
c.node("a", "human", "初审", mode="approve", assignee={"kind": "slot", "slot": "finance"},
on_timeout={"action": "escalate", "to_slot": "boss"}, due="PT1H"),
c.node("b", "human", "复审", mode="approve", assignee={"kind": "slot", "slot": "finance"}),
{"id": "h3", "type": "human", "position": {"x": 0, "y": 0},
"data": {"config": {"mode": "approve", "assignee": {"kind": "slot", "slot": "ops"}}}},
c.node("done", "output", "完成", fields=[])],
"edges": [c.edge("start", "a"), c.edge("a", "b"), c.edge("b", "h3"), c.edge("h3", "done")]}
def test_entry_labels_keep_a_space_before_a_latin_name():
"""入口按钮「发起…」:名字以英文开头时中间空一格(验收时写成了「发起accept 报销写回应用」)。"""
from app.apps.derive import start_label
assert start_label("accept 报销写回应用") == "发起 accept 报销写回应用"
assert start_label("报销") == "发起报销" and start_label("2026 年报") == "发起 2026 年报"
graph = _people_graph()
inst = instantiate({**get_template("freeform").example()}, name="accept 报销写回应用", graph=graph)
assert inst.app_spec["entries"][0]["label"] == "发起 accept 报销写回应用", inst.app_spec["entries"]
approval = instantiate(get_template("approval").example(), name="erp 报销")
assert approval.app_spec["entries"][0]["label"] == "发起 erp 报销"
def test_slot_labels_use_the_node_names_and_say_so_when_there_is_none():
"""应用里岗位的显示名:图里只写了岗位名(画布上随手写的 id,常是英文 finance),以前显示名就是它,看着像起好的名字。
现在按用到它的节点在图上的名字拼(几处用到用顿号连、超时升级的写「…超时升级」);节点没起名字的照实用岗位名。蓝图卡片
上起了名字的(审批模板的步骤名)照旧用它。"""
inst = instantiate({**get_template("freeform").example()}, name="报销", graph=_people_graph())
slots = inst.app_spec["slots"]
assert slots["finance"]["label"] == "初审、复审", slots
assert slots["boss"]["label"] == "初审超时升级" and slots["ops"]["label"] == "ops", slots
approval = instantiate(get_template("approval").example())
labels = {k: v["label"] for k, v in approval.app_spec["slots"].items()}
assert all(labels[k] != k for k in ("lead", "finance", "finance_up")), labels
async def test_derived_apps_use_the_same_slot_labels_and_entry_label():
from app.apps.derive import derive_from_graph
from app.db.base import SessionLocal
async with SessionLocal() as session:
spec = await derive_from_graph(session, _people_graph(), name="accept 报销", blueprint_id="bp1")
assert {k: v["label"] for k, v in spec["slots"].items()} == {"finance": "初审、复审", "boss": "初审超时升级", "ops": "ops"}
assert spec["entries"][0]["label"] == "发起 accept 报销"
@@ -359,6 +359,27 @@ async def test_table_usage_lists_runs_newest_first(client):
assert (await client.get(f"{_base(sid)}/ghost/usage")).status_code == 404
async def test_usage_counts_queries_and_the_runs_list_counts_runs(client):
"""两个数口径不同、说明照实:表的「使用次数」按查询计(同一次运行里查了两次算两次),「用过这张表的运行」按运行计。
接口说明以前写「取法同使用次数」,看着该是同一个数,验收时 4 对 3 对不上。"""
sid = await _source()
async with SessionLocal() as session:
name = (await session.get(DataSource, sid)).name
runs = await _use(sid, name, {"orders": 3})
async with SessionLocal() as session:
session.add(Artifact(id=uuid.uuid4().hex, run_id=runs[0], kind=catalog.QUERY_SNAPSHOT_KIND,
meta={"source_id": sid, "source": name, "tables": ["orders"]}))
await session.commit()
listed = (await client.get(f"/api/datasources/{sid}/catalog")).json()
assert next(t for t in listed["tables"] if t["table_name"] == "orders")["usage"] == 4
assert (await client.get(f"{_base(sid)}/orders/usage")).json()["total"] == 3
doc = _fastapi_app.openapi()["paths"]["/api/datasources/{source_id}/catalog/{table}/usage"]["get"]["description"]
assert "取法同使用次数" not in doc and "按运行" in doc and "按查询" in doc, doc
from app.data import catalog_review
assert "取法同使用次数" not in (catalog_review.table_runs.__doc__ or "")
# --------------------------------------------------------------------------
# 进度
# --------------------------------------------------------------------------
+73
View File
@@ -48,6 +48,11 @@ def test_next_step_branches():
assert turn.next_step(_final(issues=[ERROR], autorun=False)) == "fail", "没过检查比只建不跑优先"
assert turn.next_step({"op": "reply", "text": "已经查过了"}) == "reply"
assert turn.next_step({"op": "error", "message": "x"}) == "fail"
empty = {"op": "error", "message": "助手本轮未完成:模型没有给出任何修改,也没有说明原因",
"code": turn.EMPTY_GENERATION, "reason": "模型没有给出任何修改,也没有说明原因"}
assert turn.next_step(empty) == "retry", "只写了思考、一个操作都没给:按重来一次的规矩自动重来"
assert turn.error_reason(empty) == "模型没有给出任何修改,也没有说明原因"
assert turn.error_reason({"op": "error", "message": "助手本轮未完成:接入返回了 502"}) == "接入返回了 502"
assert turn.next_step(None) == "asked", "流结束时只有反问"
assert set(turn.NEXT_STEPS) == {"run", "ready", "fail"}
@@ -167,6 +172,74 @@ async def _run(output: dict, graph: dict, code: str) -> str:
return run.id
def test_the_frontend_retries_an_error_the_server_says_to_retry():
"""问数据页照服务端给的 next 走:error 操作带 next=retry(只写了思考、一个操作都没给)时带着原因重来一次,不再直接报
「助手本轮未完成」。读 chat.ts 里 error 那一支(行为另由 scripts/check-chat.mjs 的 emptyretry 一节在浏览器里守)。"""
branch = re.search(r"case 'error':(.*?)break\n", _ts(), re.S)
assert branch, "chat.ts 里找不到 error 那一支"
assert re.search(r"op\.next === 'retry'", branch.group(1)) and "retry(" in branch.group(1), branch.group(1)
async def test_answer_ops_says_to_retry_an_empty_generation(client, monkeypatch):
"""生成过程一个操作都没给(deepseek 偶发只出思考):error 操作带机读码 copilot_empty、原因、next=retry。"""
import json
import app.api.copilot as copilot
from tests.test_usage_meter import Scripted
model = Scripted(model="claude-opus-5", replies=[[""]])
async def _model(*_a, **_k):
return model, "claude-opus-5"
monkeypatch.setattr(copilot, "get_chat_model", _model)
events = []
async with client.stream("POST", "/api/copilot/generate-stream",
json={"instruction": "上周各地区销售额", "intent": "answer"}) as r:
async for line in r.aiter_lines():
if line.startswith("data: "):
events.append(json.loads(line[6:]))
[err] = [e for e in events if e["op"] == "error"]
assert err["code"] == turn.EMPTY_GENERATION and err["next"] == "retry", err
assert err["reason"] == "模型没有给出任何修改,也没有说明原因" and err["message"].startswith("助手本轮未完成")
async def test_the_eval_driver_retries_an_empty_generation_once(monkeypatch):
"""评测的作答(app/eval/answer.Driver)走同一个决定:第一次一个操作都没给就带着原因重来一次,原因照实记在 retries;
重来还不行才照实报建图失败(助手调用失败),不再试第三次。"""
import json
import app.api.copilot as copilot
from app.eval import answer
from tests.test_usage_meter import Scripted
reply = json.dumps({"op": "reply", "text": "上周没有华中区的数据"}, ensure_ascii=False)
for replies, path in (([[""], [reply]], "reply"), ([[""]], "build_failed")):
model = Scripted(model="claude-opus-5", replies=replies)
seen: list[str] = []
async def _model(*_a, _m=model, **_k):
return _m, "claude-opus-5"
monkeypatch.setattr(copilot, "get_chat_model", _model)
real = answer.Driver._generate
async def spy(self, instruction, base_graph, _seen=seen, _real=real):
_seen.append(instruction)
return await _real(self, instruction, base_graph)
monkeypatch.setattr(answer.Driver, "_generate", spy)
driver = answer.Driver("上周华中区销售额", datasource_ids=[], ctx=answer.TurnContext(
suite_id="s", suite_name="虚构题集", eval_run_id="e", case_id=None, case_seq=1, actor=None))
t = await driver.drive()
assert t.path == path and t.attempts == 1 and model.calls == 2, (t.path, t.attempts, model.calls, t.reason)
assert t.retries == ["模型没有给出任何修改,也没有说明原因"], t.retries
assert seen[1] == turn.retry_instruction("上周华中区销售额", t.retries[0], False), seen
if path == "build_failed":
assert t.copilot_failed and t.reason.startswith("助手本轮未完成"), t.reason
assert t.usage["assistant"]["calls"] == 2, "两次生成都是这一题花的"
async def test_review_says_whether_and_how_to_retry(client, monkeypatch):
import app.api.copilot as copilot
from app.providers.factory import ProviderNotConfigured
+9 -1
View File
@@ -5,7 +5,8 @@
- 几种结局(建图交付、直接回答、只问不改、自查改不好、模型出错、没给操作、输出被截断)的 SSE 输出,和抽出来之前
逐条一样(剧本见 SCENARIOS,抽出来之前录下的输出在 fixtures/copilot_answer_ops_baseline.json),末尾只多一条 usage,
final 只多一个 next(问数据一轮的决定,chat/turn.py);
final 只多一个 next(问数据一轮的决定,chat/turn.py);error 多一个 next 和原因 reason(一个操作都没给的另带机读码 code,
next 是 retry:问数据页和评测据此自动重来一次);
- 直接迭代 answer_ops 和 SSE 收到的是同一串;
- 开流前的拒绝(模型没配好、限定的数据源都不在)照旧是 HTTP 错误,不进流。
@@ -133,6 +134,8 @@ BODY = {"instruction": "数一数门店", "intent": "answer"}
#: final 带的 next(chat/turn.next_step):问数据页照它走
NEXT = {"graph_ok": "run", "graph_fixed": "run", "check_failed": "fail", "ask_only": "ready"}
#: error 带的 (next, code):只有一个操作都没给的重来一次
ERROR_NEXT = {"error": ("fail", None), "no_ops": ("retry", "copilot_empty"), "truncated": ("fail", None)}
@pytest.mark.parametrize("name", list(SCENARIOS))
@@ -144,6 +147,11 @@ async def test_the_sse_output_is_unchanged_but_for_a_trailing_usage(client, monk
# final 多了 next(问数据一轮的决定下沉到后端),别的一个字不变
finals = [op for op in got if op["op"] == "final"]
assert [op.pop("next") for op in finals] == ([NEXT[name]] if name in NEXT else [])
errors = [op for op in got if op["op"] == "error"]
assert [(op.pop("next"), op.pop("code", None)) for op in errors] == \
([ERROR_NEXT[name]] if name in ERROR_NEXT else [])
for op in errors:
assert op["message"] == f"助手本轮未完成:{op.pop('reason')}", op
assert got[:-1] == expected
+38
View File
@@ -347,6 +347,44 @@ async def test_the_api_starts_estimates_and_reports_progress(client, monkeypatch
assert bad.status_code == 422 and bad.json()["code"] == "eval_variant_invalid"
async def test_the_estimate_says_it_cannot_price_instead_of_zero():
"""上一次评测的模型都不在价格目录里(价格目录空):估算以前写「约 $0」,像不花钱。现在一分都没估出来的写估不出;估出了
一部分的写「至少」、另有几次估不出(老总表里每题平均记成 0、却有估不出的调用的,也按估不出算)。"""
from datetime import timedelta
suite_id = await es.make_suite("none", cases=ask_cases(1))
now = utcnow()
async with SessionLocal() as session:
session.add(EvalRun(suite_id=suite_id, status="succeeded", created_at=now - timedelta(minutes=2),
summary={"per_case_cost_usd": 0.0, "unpriced_calls": 6}))
await session.commit()
got = await runner.estimate(session, suite_id, 3)
assert got["total_usd"] is None and got["per_item_usd"] is None and got["unpriced_calls"] == 6, got
assert "估不出" in got["note"] and got["basis"], got
async with SessionLocal() as session:
session.add(EvalRun(suite_id=suite_id, status="succeeded", created_at=now - timedelta(minutes=1),
summary={"per_case_cost_usd": 0.02, "unpriced_calls": 2}))
await session.commit()
got = await runner.estimate(session, suite_id, 3)
assert got["total_usd"] == 0.06 and got["at_least"] is True and got["unpriced_calls"] == 2, got
assert "至少" in got["note"] and "2 次" in got["note"], got["note"]
async with SessionLocal() as session:
session.add(EvalRun(suite_id=suite_id, status="succeeded", created_at=now,
summary={"per_case_cost_usd": 0.0, "unpriced_calls": 0}))
await session.commit()
got = await runner.estimate(session, suite_id, 3)
assert got["total_usd"] == 0.0 and not got.get("at_least") and not got.get("unpriced_calls"), "目录价是 0 的才是真不花钱"
def test_the_judge_role_keeps_its_unpriced_calls():
"""按角色的用量里裁判那一份以前只有金额和次数:裁判模型不在价格目录里时写成 $0。"""
usage = runner.judge_usage({"claims": {"cost_usd": 0.0, "calls": 2, "priced": False},
"text": {"cost_usd": 0.0, "calls": 1, "usage": {"unpriced_calls": 1}}}, 0.0)
assert usage == {"cost_usd": 0.0, "calls": 3, "unpriced_calls": 3}
priced = runner.judge_usage({"claims": {"cost_usd": 0.01, "calls": 2, "priced": True}}, 0.01)
assert priced == {"cost_usd": 0.01, "calls": 2, "unpriced_calls": 0}
async def test_results_give_the_question_actually_asked_and_the_template_apart(client, monkeypatch):
"""逐题结果的 question 是这一题实际问的(变量按这次评测的截至时间渲染好了,和作答时 Driver 收到的一字不差),模板原文
另放 question_template(下一轮验收:以前 question 是带 {{ schedule.last_week_start }} 的原始模板)。没有变量的题两个一样。"""
+96
View File
@@ -116,6 +116,59 @@ def test_tables_sets_none_and_ask():
assert score_case(ask, {"status": "ok"}, Answer(run_id="r"), path="ran").verdict == "fail"
def test_a_wrong_answer_shows_the_paired_cell():
"""判错时「答案里的数」给评分时配对上的那一格(按列名、指标、维度配),不是答案引用的第一个值;配不上照实写配不上。"""
case = _case({**EXP, "column": "refund", "label": "退款金额"})
gold = _gold({"e1": _scalar(1234.5, column="refund")})
def cell(value, column, **dims):
return {"value": value, "kind": "cell", "artifact": "a1", "row": 0, "column": column, "dims": dims}
question = cell("上周退款金额是多少", "问题")
both = score_case(case, gold, Answer(run_id="r", values=[question, cell(36, "单数"), cell(1200.0, "refund")]),
path="ran")
assert both.verdict == "fail" and both.scores[0]["got"] == 1200.0, both.scores
assert both.scores[0]["cell"]["column"] == "refund"
labelled = score_case(case, gold, Answer(run_id="r", values=[cell(36, "单数"), cell(1200.0, "退款金额")]), path="ran")
assert labelled.scores[0]["got"] == 1200.0, "期望的说明(label)和答案的列名一样也算配上"
lone = score_case(case, gold, Answer(run_id="r", values=[question, cell(36, "单数")]), path="ran")
[s] = lone.scores
assert s["outcome"] == "wrong" and "got" not in s, "单数不是退款金额:配不上就不给「答案里的数」"
assert "配不上" in s["reason"] and "单数 36" in s["reason"] and "上周退款" not in s["reason"], s["reason"]
# 按维度配:华东那一行只有一个数时就是它;同一行有几个数、列名又都配不上时不猜
by_dims = _case({**EXP, "column": "refund", "where": {"region": "华东"}})
gold = _gold({"e1": _scalar(1234.5, column="refund", where={"region": "华东"}, dims={"region": "华东"})})
one = score_case(by_dims, gold, Answer(run_id="r", values=[cell(1200.0, "total", region="华东")]), path="ran")
assert one.scores[0]["got"] == 1200.0, one.scores
two = score_case(by_dims, gold, Answer(run_id="r", values=[cell(36, "单数", region="华东"),
cell(1200.0, "total", region="华东")]), path="ran")
assert "got" not in two.scores[0] and "配不上" in two.scores[0]["reason"], two.scores
text_only = score_case(case, _gold({"e1": _scalar(1234.5, column="refund")}), Answer(run_id="r", values=[question]),
path="ran")
assert "got" not in text_only.scores[0] and text_only.scores[0]["reason"] == "答案里没有对得上的数"
def test_none_counts_only_numbers_as_giving_data():
"""期望「没有数据」:只有引用了数值格、或叙述里写了数,才算答案给了数。引用的是文字格(地区名)、空值格(NULL)的,
答案明说没有数据就按没有数据判对。"""
none = _case({"id": "n", "kind": "none", "required": True})
gold = _gold({"n": {"kind": "none", "rows": 0}})
empty = [{"artifact": "q", "row_count": 0, "sql": "SELECT 1"}]
text_cell = {"value": "华中", "kind": "cell", "artifact": "q", "row": 0, "column": "region", "dims": {}}
named = score_case(none, gold, Answer(run_id="r", text="华中区没有数据", values=[text_cell], queries=empty), path="ran")
assert named.verdict == "pass" and named.scores[0]["outcome"] == "traced", named.scores
null_cell = {"value": None, "kind": "cell", "artifact": "q", "row": 0, "column": "gmv", "dims": {}}
null = score_case(none, gold, Answer(run_id="r", text="华中区的销售额为空,没有数据", values=[null_cell],
queries=empty), path="ran")
assert null.verdict == "pass", null.scores
number = score_case(none, gold, Answer(run_id="r", text="华中区 12 元", values=[_cell(12.0)], queries=empty),
path="ran")
assert number.verdict == "fail" and number.scores[0]["got"] == 12.0, number.scores
said = score_case(none, gold, Answer(run_id="r", text="华中区 12 元", values=[text_cell], bare=_bare("12 元"),
queries=empty), path="ran")
assert said.verdict == "fail" and said.scores[0]["raw"] == "12", said.scores
def test_the_path_and_the_gold_decide_before_numbers():
case = _case(EXP)
gold = _gold({"e1": _scalar(330.0)})
@@ -195,6 +248,49 @@ async def test_reading_the_sealed_answer_and_the_per_case_sql_match(engine_on, m
assert (await score.sql_match(case, gold, answer))["status"] == "proven"
async def test_the_per_case_sql_match_compares_the_query_behind_the_answer(engine_on, monkeypatch, tmp_path):
"""逐题 E1 比的是产出答案的那条查询:答案引用了就比被引用的;没引用、这次运行又查了几条时不硬比第一条(往往是探索
查询),照实写没法对上是哪条;没引用、只查了一条时它就是唯一可能的那条,按它比、写明依据。"""
from app.db.base import SessionLocal
from app.db.models import DataSource
from app.engine.runner import run_manager
from app.eval import score
from app.ledger import match as ledger_match
from sqlalchemy import select
from tests import ledger_samples as ls
await ls.fresh_source(tmp_path / "shop.db")
async with SessionLocal() as session:
sid = (await session.execute(select(DataSource.id).where(DataSource.name == ls.CURRENT["source"]))).scalar()
case = {"gold": {"kind": "sql", "datasource_id": sid, "sql": "x"}, "expectations": []}
gold = {"status": "ok", "before": {"rendered": {"sql": (
"SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' AND paid_at >= '2026-09-01' "
"AND paid_at < '2026-09-08'")}, "sets": []}}
# 先探了一下有哪些地区,再查销售额;答案没引用查询(数写在叙述里)
graph = ls.chain(ls.node("start", "input"),
ls.node("peek", "tool", tool=ls.tool(), args={"sql": "SELECT DISTINCT region FROM orders"}),
ls.node("q", "tool", tool=ls.tool(), args={"sql": ls.WITH_REFUNDS}),
ls.node("write", "report", instructions="写答案"),
ls.node("out", "output", fields=[{"name": "答案", "value": "{{ nodes.write.text }}"}]))
ls.script(monkeypatch, "本周销售额约 200 元。")
started = await run_manager.start(graph=graph, input_payload={}, run_class="exploratory", origin_kind="canvas")
await ls.wait(started.id)
await ledger_match.drain()
answer = await score.read_answer(started.id)
assert answer.cited == [] and len(answer.queries) == 2, answer.queries
match = await score.sql_match(case, gold, answer)
assert match["status"] == "skipped" and match["cited"] is False and "没有引用查询" in match["reason"], match
assert match.get("query") is None and match.get("aspects") in (None, []), "没法对上是哪条,就不给差在哪"
# 只查了一条、答案没引用:唯一的那条就是产出答案的,按它比,依据写明
run = await ls.explore(monkeypatch, ls.EQUIVALENT, text="本周销售额 170.5 元。")
only = await score.read_answer(run.id)
assert only.cited == [] and len(only.queries) == 1
match = await score.sql_match(case, gold, only)
assert match["status"] == "proven" and match["basis"] == "only_query" and not match["cited"], match
async def test_rescoring_reads_the_seal_and_never_calls_a_model(engine_on, monkeypatch, tmp_path):
from app.db.base import SessionLocal
from app.db.models_eval import EvalResult, EvalRun
+12
View File
@@ -69,6 +69,18 @@ def test_summary_counts_only_gradable_items_and_splits_roles():
assert summary["e1"]["passed"] == 1 and summary["e1"]["n"] == 2 and summary["e1"]["aspects"] == {"time": 1}
def test_the_per_case_cost_is_not_zero_when_calls_could_not_be_priced():
"""模型不在价格目录里(价格目录空):金额 0 只是「估得出的那部分」,每题平均花费不能写成 0(像不花钱)——一分都没估出来
的为 None(估不出),估出了一部分的照给、unpriced_calls 说明它只是下限。"""
unpriced = {"assistant": {"by_model": {}, "calls": 4, "cost_usd": 0.0, "input_tokens": 900, "output_tokens": 90,
"unpriced_calls": 4}}
summary = stats.summarize([_row("c1", "pass", usage=unpriced), _row("c2", "fail", usage=unpriced)])
assert summary["unpriced_calls"] == 8 and summary["per_case_cost_usd"] is None, summary["per_case_cost_usd"]
partly = {"assistant": {**unpriced["assistant"], "cost_usd": 0.02, "unpriced_calls": 1}}
summary = stats.summarize([_row("c1", "pass", usage=partly), _row("c2", "fail", usage=partly)])
assert summary["per_case_cost_usd"] == 0.02 and summary["unpriced_calls"] == 2
def test_repeats_give_a_pass_rate_per_case():
rows = [_row("c1", "pass", repeat=0), _row("c1", "fail", repeat=1), _row("c1", "pass", repeat=2)]
summary = stats.summarize(rows)
+17
View File
@@ -236,3 +236,20 @@ async def test_out_of_scope_parent_is_not_called_missing(big):
checks = catalog_profile.plan_checks("ord_hdr_east", tables["ord_hdr_east"], {"relations": [rel]},
tables=tables, masked=set(), outside=outside)
assert [c for c in checks if c.kind == "relation"][0].skip[0] == "missing"
def test_out_of_scope_objects_without_a_comment_match_only_their_english_name():
"""范围外、又没有注释的对象只有英文名可检索(data/schema_view 模块说明写明的限制):英文名拆出的词(catalog.name_words)
没有中文说法——仓库里没有现成的英中业务词对照,不临时编词表,中文需求命不中只叫 refund_orders 的表;有注释的照常命中。
以后接了词表,这条要跟着改。"""
from types import SimpleNamespace
from app.data import schema_view
source = SimpleNamespace(schema_cache={"tables": {}}, options={})
outside = [SimpleNamespace(name="refund_orders", comment=None),
SimpleNamespace(name="cust_complaint", comment="客户投诉单")]
search = schema_view.TableSearch(source, {}, outside)
assert [h.name for h in search.search("refund", limit=5)] == ["refund_orders"], "英文名、拆出的词照常命中"
assert "refund_orders" not in [h.name for h in search.search("上周的退款", limit=5)], "没有注释、没有词表:中文需求命不中英文名"
assert [h.name for h in search.search("投诉", limit=5)] == ["cust_complaint"], "有注释的按注释命中"
+60 -1
View File
@@ -96,7 +96,8 @@ async def test_metering_counts_by_model_and_prices_from_the_catalog():
await Scripted(model=FREE, report=False).ainvoke("你好")
got = m.summary()
assert got["by_model"][PRICED] == {"input_tokens": 2000, "output_tokens": 400, "calls": 2,
"cost_usd": round(estimate_cost(PRICED, 2000, 400), 6), "unpriced_calls": 0}
"cost_usd": round(estimate_cost(PRICED, 2000, 400), 6), "unpriced_calls": 0,
"unfinished_calls": 0}
assert got["by_model"][UNPRICED]["unpriced_calls"] == 1 and got["by_model"][UNPRICED]["cost_usd"] == 0
# 目录外的模型名(多了后缀)照样算估不出;目录价是 0 的本地模型没报用量也不算估不出
assert got["by_model"][PRICED + "-silent"]["unpriced_calls"] == 1
@@ -184,6 +185,64 @@ async def test_the_eval_driver_on_the_demo_model_records_its_name(monkeypatch):
assert set(got) == {"mock-fast"} and got["mock-fast"]["calls"] >= 1, got
class Slow(Scripted):
"""回得很慢的模型:调用方等不及(挑表超时被 asyncio.wait_for 取消)。partial=True 时先报一段输入用量再卡住。"""
delay: float = 5.0
partial: bool = False
async def _agenerate(self, messages, stop=None, run_manager=None, **kwargs) -> ChatResult:
self.calls += 1
await asyncio.sleep(self.delay)
return ChatResult(generations=[ChatGeneration(message=AIMessage(content="晚了"))])
async def _astream(self, messages, stop=None, run_manager=None, **kwargs):
self.calls += 1
if self.partial:
yield ChatGenerationChunk(message=AIMessageChunk(content="{", usage_metadata={
"input_tokens": 900, "output_tokens": 0, "total_tokens": 900}))
await asyncio.sleep(self.delay)
yield ChatGenerationChunk(message=AIMessageChunk(content="晚了"))
class Refused(Scripted):
"""请求刚发出去就被拒了(鉴权错、参数不对):模型没干活,不算一次调用。"""
async def _agenerate(self, messages, stop=None, run_manager=None, **kwargs) -> ChatResult:
raise ValueError("401 鉴权失败")
async def test_a_call_cancelled_before_any_result_still_counts_as_an_unfinished_call():
"""挑表超时被取消的那次调用以前一点都不计(LangChain 取消时连 on_llm_error 都不一定回调):调用次数少了一次。现在
记一次调用、token 不知道(没有部分结果的记 0、算没报全),标「未完成」;有部分用量的照实记已报的;被拒的仍不计。"""
from app.look import pick
m = usage.meter()
with usage.metering(m):
with pytest.raises(asyncio.TimeoutError):
await asyncio.wait_for(Slow(model=PRICED).ainvoke("挑表"), timeout=0.05)
with pytest.raises(asyncio.TimeoutError):
await asyncio.wait_for(pick._ask(Slow(model=UNPRICED), [("human", "挑表")]), timeout=0.05)
async def read():
async for _ in Slow(model=PRICED + "-partial", partial=True).astream("挑表"):
pass
with pytest.raises(asyncio.TimeoutError):
await asyncio.wait_for(read(), timeout=0.05)
with pytest.raises(ValueError):
await Refused(model=PRICED + "-refused").ainvoke("挑表")
got = m.summary()
rows = got["by_model"]
assert (rows[PRICED]["calls"], rows[PRICED]["unfinished_calls"], rows[PRICED]["input_tokens"]) == (1, 1, 0), rows
assert rows[PRICED]["unpriced_calls"] == 1, "token 不知道,金额也就估不出"
assert (rows[UNPRICED]["calls"], rows[UNPRICED]["unfinished_calls"]) == (1, 1), "挑表(look/pick._ask)超时同样计"
partial = rows[PRICED + "-partial"]
assert (partial["calls"], partial["unfinished_calls"], partial["input_tokens"]) == (1, 1, 900), partial
assert PRICED + "-refused" not in rows, "被拒的不算一次调用"
assert got["calls"] == 3 and got["unfinished_calls"] == 3
async def test_outside_the_meter_nothing_is_counted():
m = usage.meter()
with usage.metering(m):
+9 -1
View File
@@ -783,6 +783,12 @@ export const TURN_COST_TEXT = {
`其中 ${formatNumber(n)} 次调用的金额无法估算(模型不在价格目录中,或用量没有报全),未计入,实际花费不低于此数`,
} as const
/** 问数据一轮自动重来一次时的进度说明(服务端 chat/turn.next_step 判的 retry) */
export const CHAT_RETRY_TEXT = {
/** 助手上一次只写了思考、一个操作都没给(error.next = retry) */
empty: '助手上一次没有给出任何操作,正在重新生成工作流…',
} as const
export const JUDGE_SETTING_TEXT = {
label: '证据裁判',
hint: '报告中的结论句(如「增长主要来自新客首单」)由另一个模型按证据逐句判断:正式运行由报告撰写节点当场裁判,探索运行在点击时逐句裁判。判断由模型给出,不是系统核对,界面上一律标注「非确定」。',
@@ -2314,7 +2320,7 @@ export const CATALOG_TEXT = {
missing: '表结构中已没有这张表',
noCatalog: '没有目录',
usage: (n: number) => `使用 ${formatNumber(n)} 次`,
usageHint: '被运行查询过的次数,同一次运行中结果相同的重复查询只算一次',
usageHint: '按查询算:被运行查询过的次数,同一次运行里查了几次算几次(结果相同的重复查询只算一次);「用过这张表的运行」按运行算,两个数可能不一样',
relationCount: (n: number) => `${formatNumber(n)} 个关联`,
countTitle: (label: string, n: number) => `${label} ${formatNumber(n)} 项`,
noMatch: '没有符合条件的表',
@@ -6092,6 +6098,8 @@ export const APP_EDITOR_TEXT = {
slotAdd: '新增岗位',
slotName: '岗位名',
slotLabel: '显示名',
/** 显示名和岗位名一样:派生时没有节点名可用(服务端 apps/tree.slot_label),照实用的是岗位名,不是起好的名字 */
slotLabelRaw: '还没有起显示名,现在用的是图里的岗位名,可以改成一个好认的名字',
slotUnbound: '未绑定:发布检查会拦下流程应用',
slotRemove: (name: string) => `删除岗位「${name}」`,
grants: '授权',
+13 -1
View File
@@ -356,6 +356,8 @@ export const EVAL_TEXT = {
estimateText: (items: number, cases: number, repeats: number, arms: number) =>
`${formatNumber(cases)} 道题 × 每题 ${repeats} 次${arms > 1 ? ` × ${arms} 臂` : ''},共 ${formatNumber(items)} 项`,
cost: (total: string, per: string) => `约 ${total}(每项约 ${per})`,
/** 上一次有估不出金额的调用:估出来的只是下限 */
costAtLeast: (total: string, per: string) => `至少约 ${total}(每项至少约 ${per})`,
costNone: '估不出花费',
budget: '生效的预算',
parallel: (n: number) => `同时评 ${n} 题`,
@@ -416,6 +418,8 @@ export const EVAL_TEXT = {
cost: '花费',
costHint: '助手(建图、挑表、复核)+ 写作(答案运行里的模型调用)+ 结论裁判;算期望不调模型',
perCase: (v: string) => `每题约 ${v}`,
/** 有估不出金额的调用:每题平均只是下限 */
perCaseAtLeast: (v: string) => `每题至少约 ${v}`,
e1: '逐题的口径命中(E1)',
e1Hint: '答案执行的 SQL 和期望的取数被证明等价的比例(口径账本同一个比对函数)',
citations: '报告引用率',
@@ -441,7 +445,9 @@ export const EVAL_TEXT = {
timing: { build_ms: '建图', run_wall_ms: '运行(墙钟)', run_active_ms: '运行(执行)', review_ms: '复核', total_ms: '整题' } as Record<string, string>,
usage: '按角色的用量',
tokens: (input: string, output: string) => `输入 ${input} · 输出 ${output}`,
callsN: (n: number) => `${formatNumber(n)} 次调用`,
/** unfinished:没做完的调用(被取消、超时,token 只计了已报的;服务端 providers/usage 的 unfinished_calls) */
callsN: (n: number, unfinished = 0) =>
`${formatNumber(n)} 次调用${unfinished > 0 ? `(其中 ${formatNumber(unfinished)} 次未完成:被取消或超时,token 只计了已报的部分)` : ''}`,
role: { assistant: '助手', writer: '写作', judge: '结论裁判', gold: '算期望' } as Record<string, string>,
passRates: '重复评的题的通过率',
outcomes: '各期望的结论',
@@ -464,6 +470,10 @@ export const EVAL_TEXT = {
sqlMatch: {
proven: '等价已证明', related: '不等价', none: '答案没有查询', skipped: '没比对',
} as Record<string, string>,
/** 逐题 SQL 比对比的是哪条查询(服务端 score.sql_match 的 basis)。答案引用了查询的不用多说 */
sqlMatchBasis: {
only_query: '答案没有引用查询;这次运行只执行了这一条查询(核查用的不算),按它比',
} as Record<string, string>,
aspect: {
time: '时间口径', time_range: '时间范围', filter: '筛选条件', aggregate: '聚合', expression: '计算式', join: '关联',
tables: '用到的表', group_by: '分组', limit: '行数限制', other: '其他', parse: '解析不了',
@@ -495,6 +505,8 @@ export const EVAL_TEXT = {
filter: '只看',
repeat: (n: number) => `第 ${n + 1} 次`,
attempts: (n: number) => `这一题自动重来了 ${n} 次`,
/** 每次自动重来的原因(服务端 eval/answer 的 turn.retries) */
retriesWhy: (reasons: string[]) => `:${reasons.join(';')}`,
turnReason: '作答的经过',
answer: '答案原文',
expand: '展开',
@@ -60,6 +60,7 @@ export function AccessSection() {
setBindings((b) => { const next = { ...b.slots }; delete next[name]; return { ...b, slots: next } })
}} />
</div>
{slot.label.trim() === name && <p className="text-2xs text-faint" data-app-slot-raw="">{T.slotLabelRaw}</p>}
<PrincipalPicker label={T.slots} value={bound as Picked[]}
onChange={(v) => setBindings((b) => ({
...b, slots: { ...b.slots, [name]: v.filter((x) => x.kind !== 'everyone') as { kind: 'user' | 'group'; name: string }[] },
+21 -4
View File
@@ -215,8 +215,7 @@ function Overview({ run }: { run: EvalRun }) {
<div className="grid gap-2 sm:grid-cols-2 lg:grid-cols-4">
<RateCard title={O.accuracy} hint={O.accuracyHint} rate={s.accuracy} attr="accuracy" />
<RateCard title={O.traced} hint={O.tracedHint} rate={s.traced_accuracy} attr="traced" />
<StatCard title={O.cost} hint={O.costHint} value={cost} attr="cost"
sub={s.per_case_cost_usd != null ? O.perCase(formatCost(s.per_case_cost_usd)) : undefined} />
<StatCard title={O.cost} hint={O.costHint} value={cost} attr="cost" sub={perCaseText(s)} />
<StatCard title={O.e1} hint={O.e1Hint} value={s.e1?.n ? pct(s.e1.rate) : '—'} sub={rateText(s.e1)} attr="e1" />
</div>
<Section title={O.verdicts} attr="verdicts">
@@ -278,7 +277,7 @@ function Overview({ run }: { run: EvalRun }) {
<li key={role} data-eval-usage-role={role}>
{O.role[role]}:{usageCost(u)}
{u.input_tokens != null && <span className="text-faint"> · {O.tokens(formatTokens(u.input_tokens, { compact: true }), formatTokens(u.output_tokens ?? 0, { compact: true }))}</span>}
{u.calls != null && <span className="text-faint"> · {O.callsN(u.calls)}</span>}
{u.calls != null && <span className="text-faint"> · {O.callsN(u.calls, u.unfinished_calls ?? 0)}</span>}
</li>
)
})}
@@ -296,6 +295,17 @@ function Overview({ run }: { run: EvalRun }) {
)
}
/**
* 每题平均花费:有估不出金额的调用时只是下限(写「至少」);一分都没估出来的不写(老总表记成 0 的也一样,金额那一格已经写了
* 「金额无法估算」)
*/
function perCaseText(s: EvalSummary): string | undefined {
const per = s.per_case_cost_usd
if (per == null) return undefined
if (!s.unpriced_calls) return T.overview.perCase(formatCost(per))
return per > 0 ? T.overview.perCaseAtLeast(formatCost(per)) : undefined
}
function Section({ title, children, attr }: { title: string; children: ReactNode; attr?: string }) {
return (
<section className="space-y-1 rounded-lg border bg-panel p-3 text-xs" aria-label={title} data-eval-section={attr}>
@@ -417,6 +427,9 @@ function Results({ id, live, focusCase }: { id: string; live: boolean; focusCase
{s.reason && <span> · {s.reason}</span>}
</p>
))}
{r.sql_match?.basis && T.sqlMatchBasis[r.sql_match.basis] && (
<p className="text-faint" data-eval-sql-basis={r.sql_match.basis}>{T.sqlMatchBasis[r.sql_match.basis]}</p>
)}
{r.sql_match?.reason && <p className="text-faint">{r.sql_match.reason}</p>}
<ResultDetail evaluationId={id} resultId={r.id} />
<div className="flex flex-wrap gap-2">
@@ -450,7 +463,11 @@ function ResultDetail({ evaluationId, resultId }: { evaluationId: string; result
const R = T.results
return (
<div className="space-y-1" data-eval-result-details="">
{(turn.attempts ?? 0) > 0 && <p className="text-faint">{R.attempts(turn.attempts)}</p>}
{(turn.attempts ?? 0) > 0 && (
<p className="text-faint" data-eval-retries={turn.retries?.length ?? 0}>
{R.attempts(turn.attempts)}{turn.retries?.length ? R.retriesWhy(turn.retries) : ''}
</p>
)}
{turn.reason && <p className="text-dim"><span className="text-faint">{R.turnReason}:</span>{turn.reason}</p>}
{answer && (
<details>
+2 -3
View File
@@ -263,9 +263,8 @@ export function StartDialog({ open, onClose, suite, evaluations, onStarted }: {
<>
<p data-eval-estimate-items={est.estimate.items}>{T.estimateText(est.estimate.items, est.cases, est.repeats, est.arms.length)}</p>
<p data-eval-estimate-cost="">
{est.estimate.total_usd != null
? T.cost(formatCost(est.estimate.total_usd), formatCost(est.estimate.per_item_usd))
: T.costNone}
{est.estimate.total_usd == null ? T.costNone
: (est.estimate.at_least ? T.costAtLeast : T.cost)(formatCost(est.estimate.total_usd), formatCost(est.estimate.per_item_usd))}
<span className="text-faint"> · {est.estimate.note}</span>
</p>
{est.arms.map((a, i) => (
+10 -5
View File
@@ -2,7 +2,7 @@ import { create } from 'zustand'
import { ApiError, api, streamCopilot, streamRun } from '../api/client'
import type { CopilotOp, FixKind, RunFinal } from '../run/decode'
import { copilotContext, copilotLook, decodeRun, looksOf } from '../run/decode'
import { CHAT_BLUEPRINT_TEXT, LEDGER_TEXT, LOOK_TEXT, REJECTED_TEXT } from '../lib/terms'
import { CHAT_BLUEPRINT_TEXT, CHAT_RETRY_TEXT, LEDGER_TEXT, LOOK_TEXT, REJECTED_TEXT } from '../lib/terms'
import type { LookEvent } from '../typesApp'
import { errorMessage, humanizeError, isNetworkError } from '../lib/errors'
import { guardReview, hasEvidence } from '../lib/evidence'
@@ -174,6 +174,8 @@ export interface UsageTotal {
cost_usd?: number | null
unpriced_calls?: number
calls?: number
/** 没做完的调用(被取消、超时):算在 calls 里,token 只计了已报的(服务端 providers/usage 的 unfinished_calls) */
unfinished_calls?: number
input_tokens?: number
output_tokens?: number
}
@@ -498,7 +500,7 @@ function buildMeta(t: ChatTurn): TurnMeta {
}
}
const USAGE_KEYS = ['cost_usd', 'unpriced_calls', 'calls', 'input_tokens', 'output_tokens'] as const
const USAGE_KEYS = ['cost_usd', 'unpriced_calls', 'calls', 'unfinished_calls', 'input_tokens', 'output_tokens'] as const
/** 两份用量相加(缺的当 0)。b 不是对象时原样交回 a */
export function addUsage(a: UsageTotal | undefined, b: UsageTotal | null | undefined): UsageTotal | undefined {
@@ -1417,14 +1419,14 @@ function plan(flow: Flow, args: PlanArgs) {
* 这里会清空这一次的过程和答案——它是自动补救,用户还没拿到过答案。用户主动
* 点的重试走 retryTurn,上一次要留档。
*/
const retry = (reason: string, instruction?: string) => {
const retry = (reason: string, instruction?: string, status?: string) => {
// 服务端给了那句话(复核的 retry_instruction)就是它已经按次数判过可以重来;老服务端由这里按次数判
if ((!instruction && attempt + 1 >= MAX_ATTEMPTS) || !flow.alive()) return false
const base = built && (built.nodes?.length ?? 0) > 0 ? built : null
patch((t) => ({
attempt: attempt + 1,
phase: 'planning',
status: base ? '上次运行未成功,正在调整工作流…' : '上次运行未成功,正在重新生成工作流…',
status: status ?? (base ? '上次运行未成功,正在调整工作流…' : '上次运行未成功,正在重新生成工作流…'),
ops: [], events: [], run: null, output: null, final: undefined, runStatus: undefined,
review: null, reviewRaw: null, rawOutput: null, lastSeq: 0, failure: null, error: '',
// 换下来的那次运行花的钱还是这一轮的
@@ -1584,9 +1586,12 @@ function plan(flow: Flow, args: PlanArgs) {
void flow.persist()
break
case 'error':
settled = true
// 生成过程一个操作都没给(模型只写了思考):服务端说重来一次(error.next = retry,backend/app/chat/turn.py 的
// next_step),照它带着原因重来,次数照 MAX_ATTEMPTS;试满了才照实报。这一次建图的用量随后那条 usage 照样记进这一轮
if (op.next === 'retry' && retry(String(op.reason ?? op.message ?? ''), undefined, CHAT_RETRY_TEXT.empty)) break
// 后端给的是三段:message 人话、hint 怎么办、detail 原始异常。
// 原文不再截成一行甩给用户,收进可展开、可复制的技术细节
settled = true
fail({
...failureOf(op.message ?? '生成失败'),
...(op.hint ? { hint: String(op.hint) } : {}),
+11 -2
View File
@@ -365,7 +365,11 @@ export interface EvalEstimate {
arms: { label: string | null; variant: EvalVariant; models: EvalResolvedModels; text: string }[]
cases: number
repeats: number
estimate: { items: number; per_item_usd: number | null; total_usd: number | null; basis: string | null; note: string }
/** at_least:上一次另有 unpriced_calls 次调用金额估不出,估算只是下限;total_usd 为空、unpriced_calls 有值:一分都没估出来 */
estimate: {
items: number; per_item_usd: number | null; total_usd: number | null; basis: string | null; note: string
at_least?: boolean; unpriced_calls?: number
}
budget: EvalBudget
parallel: number
spend: EvalSpend
@@ -415,7 +419,10 @@ export interface EvalUsage {
calls?: number
cost_usd?: number | null
unpriced_calls?: number
by_model?: Record<string, { input_tokens: number; output_tokens: number; calls: number; cost_usd: number; unpriced_calls: number }>
/** 没做完的调用(被取消、超时):算在 calls 里,token 只计了已报的 */
unfinished_calls?: number
by_model?: Record<string, { input_tokens: number; output_tokens: number; calls: number; cost_usd: number; unpriced_calls: number
unfinished_calls?: number }>
}
export interface EvalTiming {
@@ -531,6 +538,8 @@ export interface EvalSqlMatch {
aspects?: string[]
reason?: string | null
cited?: boolean
/** 比的是哪条查询:cited 答案引用的;only_query 答案没引用、这次运行只查了这一条 */
basis?: 'cited' | 'only_query' | string
}
export interface EvalResult {
+49 -1
View File
@@ -29,7 +29,8 @@
// 14. 助手反问(normal-user-apps N11):只问不改的一轮记成「助手反问」,问题作为这一轮的内容、落库进历史,不再一直停在
// 「正在理解需求」;输入框提示回答助手的问题;刷新后照样标「助手反问」,不当成没查库的回答;
// 15. 回答上的「做成应用」带着这次运行进蓝图(建运行起点的蓝图),不再提取工作流、跳画布的发布弹窗;
// 16. 服务端这一轮已经自查修正过(收到 check 的结论)还没过的:不再整轮重新生成一遍,直接说没过、给补救。
// 16. 服务端这一轮已经自查修正过(收到 check 的结论)还没过的:不再整轮重新生成一遍,直接说没过、给补救;助手只写了思考、
// 一个操作都没给(error.next=retry)的照服务端的决定自动重来一次,还不行再照实报。
// 17. 口径账本(next-round 02):交付了的轮次底部写「正在比对口径」,按退避再取到比对出结果、比完不再取;一行摘要
// (几个数来自哪份口径)、按口径正式出具和对账按钮照服务端给的动作摆;出具、对账的写请求在浏览器层拦下并核对请求体,
// 出具过的正式运行挂在轮次下面(档位、打开运行)。
@@ -532,6 +533,26 @@ async function fakeApi(route) {
return route.fulfill({ status: 200, contentType: 'text/event-stream',
body: ops.map((o) => `data: ${JSON.stringify(o)}\n\n`).join('') })
}
if (String(b.instruction ?? '').includes('只写了思考')) {
// 生成过程一个操作都没给(deepseek 偶发只出思考):服务端的 error 带 next=retry(backend/app/chat/turn.py 的 next_step)。
// 重来那一次(指令里缀了「上一次没有成功」)照常直接回答;「两次都」那一问每次都只写思考
const again = String(b.instruction).includes('上一次没有成功') && !String(b.instruction).includes('两次都')
const spent = { by_model: { m: { input_tokens: 500, output_tokens: 40, calls: 1, cost_usd: 0.001, unpriced_calls: 0 } },
input_tokens: 500, output_tokens: 40, calls: 1, cost_usd: 0.001, unpriced_calls: 0 }
const ops = again ? [
{ op: 'heartbeat', phase: 'planning', elapsed_ms: 200 },
{ op: 'reply', text: '已经查过了:一共 3 家门店。' },
{ op: 'usage', ...spent },
] : [
{ op: 'heartbeat', phase: 'planning', elapsed_ms: 200 },
{ op: 'thinking', delta: '先想一想要查哪张表……' },
{ op: 'error', message: '助手本轮未完成:模型没有给出任何修改,也没有说明原因', hint: '画布未修改。可以重试,或把需求描述得更具体',
detail: '', reason: '模型没有给出任何修改,也没有说明原因', code: 'copilot_empty', next: 'retry' },
{ op: 'usage', ...spent },
]
return route.fulfill({ status: 200, contentType: 'text/event-stream',
body: ops.map((o) => `data: ${JSON.stringify(o)}\n\n`).join('') })
}
if (String(b.instruction ?? '').includes('照服务端的决定') || String(b.instruction ?? '').includes(DECIDE_RETRY)) {
// 新服务端:final 带 next(backend/app/chat/turn.py),最后一条是这一轮建图的用量(answer_ops 的 usage)。
// 「只建不跑」那一问:autorun 没写 false,next 却是 ready——前端得照 next 走
@@ -1858,6 +1879,33 @@ for (const theme of THEMES) {
await ctx.close()
})
await section('emptyretry', '助手只写了思考、一个操作都没给:照服务端的 next=retry 自动重来一次,还不行再照实报', async () => {
const { page, ctx, errors } = await open(theme)
await goto(page, 'c0emptg')
let gens = log.gens.length
await send(page, '只写了思考:有几家门店')
const done = await page.waitForFunction(() => window.__chat.getState().byConversation.c0emptg?.at(-1)?.phase === 'done',
null, { timeout: 6000 }).then(() => true, () => false)
await page.waitForTimeout(300)
const turn = (await chatState(page)).byConversation.c0emptg.at(-1)
check('第一次没给操作:自动重来一次,第二次照常回答', done && log.gens.length - gens === 2
&& String(turn.output?.answer ?? '').includes('3 家'), `${log.gens.length - gens} 次生成 · ${JSON.stringify(turn.output)}`)
check('重来时交回去的话带着原因(同服务端 retry_instruction 的从零重建措辞)',
log.gens.at(-1)?.instruction === '只写了思考:有几家门店\n\n(上一次没有成功:模型没有给出任何修改,也没有说明原因。请针对这个原因调整,重新给出完整的工作流。)',
log.gens.at(-1)?.instruction)
check('两次建图的用量都记进这一轮', Math.abs((turn.usage?.copilot?.cost_usd ?? 0) - 0.002) < 1e-9, JSON.stringify(turn.usage))
gens = log.gens.length
await send(page, '两次都只写了思考:有几家门店')
const failed = await page.waitForFunction(() => window.__chat.getState().byConversation.c0emptg?.at(-1)?.phase === 'error',
null, { timeout: 6000 }).then(() => true, () => false)
await page.waitForTimeout(500)
const last = (await chatState(page)).byConversation.c0emptg.at(-1)
check('重来还不行:照实报「助手本轮未完成」,不再试第三次', failed && log.gens.length - gens === 2
&& JSON.stringify(last.failure ?? {}).includes('助手本轮未完成'), `${log.gens.length - gens} 次生成 · ${JSON.stringify(last.failure)}`)
check('没有运行时报错', errors.length === 0, errors[0] ?? '')
await ctx.close()
})
await section('decide', '新服务端:照 final.next 和复核的 retry_instruction 走,这一轮的花费 = 助手 + 运行 + 复核', async () => {
const { page, ctx, errors } = await open(theme)
await goto(page, 'c0emptg')
+36
View File
@@ -9,6 +9,7 @@
// 数值期望不许人填(请求体里不出现 value 这类键);
// - 口径升版待处置:顶部提示、处置三选一(叫法由服务端给,没有缺省);启用开关、导出;
// - 发起评测:估算和发起是同一份请求体,评一次 / A/B 对照 / 目录贡献对照(预设)各自的请求体;估算的项数、花费、今天已花照服务端写;
// 上一次的调用金额估不出时估算写「估不出花费」、估出一部分的写「至少约」,总览的每题平均不写成 $0;
// - 评测详情:进度、每题正在做什么;停止、只重试没做完的、重评、设为基线——摆不摆按钮照服务端的 actions;总览卡片和各项分布;
// 逐题表(结论、期望和答案里的数、SQL 差在哪、点开去运行详情)、?case= 定位那道题;和上一次比(指纹差异、转移、归因、换基准);
// A/B 配对表(都对、只 A 对、只 B 对、p 值、看不出差别);
@@ -695,6 +696,41 @@ await section('加题:选口径、从运行认定、期望反问', async () =>
// ---------------------------------------------------------------- 发起评测
await section('发起评测:估不出金额时估算和总览不写成 $0', async () => {
// 服务端 eval/runner.estimate:上一次的调用金额都估不出 → total_usd 为空、带次数;估出一部分 → at_least
const shaped = (patch) => ({ ...ESTIMATE(['当前目录']), estimate: { ...ESTIMATE(['当前目录']).estimate, ...patch } })
const none = shaped({ per_item_usd: null, total_usd: null, unpriced_calls: 6,
note: '上一次评测的 6 次模型调用金额都估不出(模型不在价格目录中,或用量没有报全):金额上限只管得住估得出金额的调用' })
const some = shaped({ at_least: true, unpriced_calls: 2,
note: '按这个题集上一次评测的每题平均花费估算,实际至少这么多:上一次另有 2 次调用金额估不出,没有算进去' })
for (const [label, body] of [['都估不出', none], ['估出一部分', some]]) {
const state = freshState({ override: (method, path) => (method === 'POST' && path.endsWith('/evaluations/estimate')
? { json: body } : null) })
const { page, errors } = await open('/evals/suites/suite-a', { state })
await page.locator('[data-eval-suite-start]').click()
await page.locator('[data-eval-estimate-items]').waitFor()
const cost = await text(page, '[data-eval-estimate-cost]')
if (label === '都估不出') {
check('估算:上一次都估不出时写「估不出花费」和原因,不写 $0', cost.includes('估不出花费') && cost.includes('6 次') && !cost.includes('$0'), cost)
} else {
check('估算:估出一部分时写「至少约」', cost.includes('至少约 $0.042') && cost.includes('每项至少约 $0.014') && cost.includes('2 次'), cost)
}
check(`没有页面报错(${label})`, !errors.length, errors.join(' | '))
}
const state = freshState()
state.evaluations['ev-1'].summary = { ...state.evaluations['ev-1'].summary, cost_usd: 0, unpriced_calls: 6, per_case_cost_usd: null }
const { page, errors } = await open('/evals/runs/ev-1', { state })
await page.locator('[data-eval-overview]').waitFor()
const card = await text(page, '[data-eval-card="cost"]')
check('总览:都估不出时花费写「金额无法估算」,不写每题 $0', card.includes('金额无法估算') && !card.includes('每题'), card)
state.evaluations['ev-1'].summary = { ...state.evaluations['ev-1'].summary, cost_usd: 0.042, unpriced_calls: 2, per_case_cost_usd: 0.014 }
await page.reload({ waitUntil: 'networkidle' })
await page.locator('[data-eval-overview]').waitFor()
const partly = await text(page, '[data-eval-card="cost"]')
check('总览:估出一部分时写「≥」和「每题至少约」', partly.includes('≥ $0.042') && partly.includes('每题至少约 $0.014'), partly)
check('没有页面报错(总览)', !errors.length, errors.join(' | '))
})
await section('发起评测:估算和发起同一份请求体;评一次、A/B、目录贡献对照', async () => {
const { page, state, errors } = await open('/evals/suites/suite-a')
await page.locator('[data-eval-suite-start]').click()