diff --git a/backend/app/api/catalog.py b/backend/app/api/catalog.py index 8810486c..250a48ad 100644 --- a/backend/app/api/catalog.py +++ b/backend/app/api/catalog.py @@ -844,7 +844,9 @@ async def review_catalog_batch(source_id: str, payload: CatalogBatchReviewIn, async def catalog_table_runs(source_id: str, table: str, limit: int = Query(20, ge=1, le=catalog_review.RUNS_LIMIT_MAX), session: AsyncSession = Depends(get_session)) -> dict[str, Any]: - """用过这张表的运行（total 是几次运行），新的在前；取法同使用次数（catalog.table_usage）。""" + """用过这张表的运行，新的在前。按运行计：total 是几次运行，一次运行查了这张表几次都只算一次。和表清单的使用次数 + （catalog.table_usage）数的是同一批查询快照，但使用次数按查询计（同一次运行里查了两次算两次），所以使用次数可能比 total + 大。""" _, source = await _resolved(session, source_id) key, _ = await _table_key(session, source, table) return await catalog_review.table_runs(session, source, key, limit=limit) diff --git a/backend/app/api/copilot.py b/backend/app/api/copilot.py index b10f8944..63ee57f6 100644 --- a/backend/app/api/copilot.py +++ b/backend/app/api/copilot.py @@ -1751,14 +1751,20 @@ async def answer_ops(payload: GenerateIn, *, catalog_view: Any = None): # 流正常结束，却没有 done、也没有一个操作能认（格式跑偏，每一行都被跳过）： # 往下走就是拿原图去自查、交付，界面上写「无需修改」——而模型什么都没说 if gen.asked is None and all(o.get("op") == "plan" for o in applier.seen): + # 只写了思考（deepseek 这类推理模型偶发）、或者格式跑偏：带机读码，问数据页和评测按 chat/turn 的规矩自动 + # 重来一次（error.next = retry），还不行再照实报 raise CopilotIncomplete("模型没有给出任何修改，也没有说明原因", - hint="画布未修改。可以重试，或把需求描述得更具体") + hint="画布未修改。可以重试，或把需求描述得更具体", code=turn.EMPTY_GENERATION) except CopilotIncomplete as e: - yield {"op": "error", "message": f"助手本轮未完成：{e}", "hint": e.hint, "detail": e.detail} + failed = {"op": "error", "message": f"助手本轮未完成：{e}", "hint": e.hint, "detail": e.detail, + "reason": str(e), **({"code": e.code} if e.code else {})} + yield {**failed, "next": turn.next_step(failed)} return except Exception as e: # noqa: BLE001 reason, hint = explain_error(e) - yield {"op": "error", "message": f"助手本轮未完成：{reason}", "hint": hint, "detail": raw_error(e)} + failed = {"op": "error", "message": f"助手本轮未完成：{reason}", "hint": hint, "detail": raw_error(e), + "reason": reason} + yield {**failed, "next": turn.next_step(failed)} return # 兜底：一个操作都没落地，而上一轮有图。 diff --git a/backend/app/api/copilot_gen.py b/backend/app/api/copilot_gen.py index bd49edee..465ff222 100644 --- a/backend/app/api/copilot_gen.py +++ b/backend/app/api/copilot_gen.py @@ -69,12 +69,14 @@ def _parse_op_line(line: str) -> dict[str, Any] | None: class CopilotIncomplete(Exception): - """模型这一轮没说完：输出被截断，或者一个操作都没给。画布不能按「无需修改」交付。""" + """模型这一轮没说完：输出被截断，或者一个操作都没给。画布不能按「无需修改」交付。code 是机读码：一个操作都没给的是 + chat/turn.EMPTY_GENERATION（问数据一轮按它自动重来一次），截断的不给。""" - def __init__(self, reason: str, *, hint: str = "", detail: str = "") -> None: + def __init__(self, reason: str, *, hint: str = "", detail: str = "", code: str = "") -> None: super().__init__(reason) self.hint = hint or "画布未修改。可以重试，或把需求拆成几步分别提出" self.detail = detail + self.code = code async def _drain(stream: Any, already: int) -> None: diff --git a/backend/app/apps/derive.py b/backend/app/apps/derive.py index 7fbb84d4..831bffe6 100644 --- a/backend/app/apps/derive.py +++ b/backend/app/apps/derive.py @@ -17,7 +17,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.apps.forms import input_schema from app.apps.spec import APP_FORMAT, parse_spec -from app.apps.tree import input_fields, node_map, slot_refs +from app.apps.tree import input_fields, node_map, slot_label, slot_refs PIN = "main" VIEW = "latest" @@ -39,6 +39,13 @@ class DeriveRefused(Exception): self.code = code +def start_label(name: str) -> str: + """流程类应用入口按钮的说法「发起…」。名字以英文字母、数字开头时中间空一格（「发起 accept 报销写回」），和界面上中英文 + 之间留空格的写法一致；以前直接拼成「发起accept 报销写回」。派生、蓝图模板（审批、自由形）都用它。""" + text = str(name or "").strip() + return f"发起 {text}" if text[:1].isascii() and text[:1].isalnum() else f"发起{text}" + + def _safe(text: str) -> str: return re.sub(r"[^A-Za-z0-9_-]", "_", text) @@ -102,8 +109,9 @@ async def _derive(session: AsyncSession, graph: dict[str, Any], *, title: str, n "pins": [{**pin, "use": "process" if process else "view"}], "slots": {}, "entries": [], "forms": {}, "task_ui": {}, "views": {}, "pages": [], "hooks": [], "notify": [], } - for ref in slot_refs(graph, writes): - spec["slots"].setdefault(ref["slot"], {"label": ref["slot"]}) + refs = slot_refs(graph, writes) + for ref in refs: + spec["slots"].setdefault(ref["slot"], {"label": slot_label(ref["slot"], refs)}) for nid, node in humans.items(): spec["task_ui"][nid] = {"title": node["title"]} if writes and not lookups: @@ -113,7 +121,7 @@ async def _derive(session: AsyncSession, graph: dict[str, Any], *, title: str, n names = list(spec["forms"][ENTRY]["properties"]) first = next((n for n in names if n in spec["forms"][ENTRY]["required"]), names[0] if names else None) spec["entries"].append({ - "id": ENTRY, "label": f"发起{title}" if process else "按条件运行", "pin": PIN, + "id": ENTRY, "label": start_label(title) if process else "按条件运行", "pin": PIN, "form": {"$ref": f"#/forms/{ENTRY}"}, "map": {n: f"form.{n}" for n in names}, **({"instance_title": {"pattern": f"{title}·{{{first}}}", "fields": [first]}} if first and re.fullmatch(r"[A-Za-z0-9_㐀-鿿]+", first) else {}), diff --git a/backend/app/apps/tree.py b/backend/app/apps/tree.py index c1314d1d..580ac335 100644 --- a/backend/app/apps/tree.py +++ b/backend/app/apps/tree.py @@ -2,7 +2,7 @@ from __future__ import annotations import hashlib -from collections.abc import Collection, Iterator +from collections.abc import Collection, Iterable, Iterator from typing import Any from app.core.artifact_store import canonical_json @@ -121,6 +121,21 @@ def slot_refs(graph: Any, writes: Collection[str] = ()) -> list[dict[str, str]]: return out +#: 按用到它的节点给岗位起显示名（slot_label）：派给它的写节点名，超时升级给它的写「…超时升级」（同蓝图模板 slot_labels 的 +#: 叫法），动作确认派给它的写「…发送前确认」 +_SLOT_USE_LABEL = {"assignee": "{title}", "escalation": "{title}超时升级", "confirm": "{title}发送前确认"} + + +def slot_label(slot: str, refs: Iterable[dict[str, str]]) -> str: + """岗位给人看的名字（应用里岗位的显示名）。图里只写了岗位名——画布上随手写的 id，常是英文（finance）：以前显示名就是 + 它，看着像起好的名字。现在按用到它的节点在图上的名字拼（refs 是 slot_refs 的结果，只看这个岗位的那几条，几处用到用顿号 + 连）；节点没起名字（名字就是节点 id）、或者没有节点用到它的，照实用岗位名——界面上显示名和岗位名一样时提示还没起名字。""" + names = [_SLOT_USE_LABEL[r["role"]].format(title=r["title"]) for r in refs + if r.get("slot") == slot and r.get("role") in _SLOT_USE_LABEL and r.get("title") + and r["title"] != r.get("node_id")] + return "、".join(dict.fromkeys(names)) or slot + + def input_fields(graph: Any) -> list[dict[str, Any]]: """入口节点声明的字段（原样，含新增的 type / enum / format / description）。""" out: list[dict[str, Any]] = [] diff --git a/backend/app/blueprints/templates/_common.py b/backend/app/blueprints/templates/_common.py index 20ee6e0c..1284def9 100644 --- a/backend/app/blueprints/templates/_common.py +++ b/backend/app/blueprints/templates/_common.py @@ -659,7 +659,7 @@ def app_base(bp: dict[str, Any], graph: dict[str, Any], *, use: str, name: str, writes 是写外部系统的调用工具节点 → 能力名（模板知道哪几个是写入，不用查库）。页面、视图、入口由各模板排。""" from app.apps.derive import PIN, RECONCILER from app.apps.spec import APP_FORMAT - from app.apps.tree import node_map, slot_refs + from app.apps.tree import node_map, slot_label, slot_refs from app.core.artifact_store import graph_hash ctx = context(bp) @@ -673,9 +673,14 @@ def app_base(bp: dict[str, Any], graph: dict[str, Any], *, use: str, name: str, "format": APP_FORMAT, "name": name, "description": description, "icon": "app", "pins": [pin], "slots": {}, "entries": [], "forms": {}, "task_ui": {}, "views": {}, "pages": [], "hooks": [], "notify": [], } + # 岗位的显示名：卡片上起了名字的用它（审批模板的步骤名）；卡片上就是岗位名本身的（处理人写的是岗位、自由形导入的图）按 + # 用到它的节点名拼（apps/tree.slot_label，同派生） labels = slot_labels(bp) - for ref in slot_refs(graph, writes or {}): - spec["slots"].setdefault(ref["slot"], {"label": labels.get(ref["slot"], ref["slot"])}) + refs = slot_refs(graph, writes or {}) + for ref in refs: + label = labels.get(ref["slot"]) + spec["slots"].setdefault(ref["slot"], {"label": label if label and label != ref["slot"] + else slot_label(ref["slot"], refs)}) for nid, n in node_map(graph).items(): if n["type"] == "human": spec["task_ui"][nid] = {"title": n["title"]} diff --git a/backend/app/blueprints/templates/approval.py b/backend/app/blueprints/templates/approval.py index a0e058ba..80bdadff 100644 --- a/backend/app/blueprints/templates/approval.py +++ b/backend/app/blueprints/templates/approval.py @@ -50,7 +50,9 @@ def app_skeleton(bp: dict[str, Any], graph: dict[str, Any]) -> dict[str, Any]: writes = {WRITE: capability} if capability else {} spec = c.app_base(bp, graph, use="process", name=name, description=str(c.value(bp.get("purpose")) or ""), writes=writes) - c.entry_and_pages(spec, bp, graph, label=f"发起{name}", writes=writes) + from app.apps.derive import start_label + + c.entry_and_pages(spec, bp, graph, label=start_label(name), writes=writes) return c.finish_app(spec) diff --git a/backend/app/blueprints/templates/freeform.py b/backend/app/blueprints/templates/freeform.py index db886bb4..18a9d2dc 100644 --- a/backend/app/blueprints/templates/freeform.py +++ b/backend/app/blueprints/templates/freeform.py @@ -104,7 +104,7 @@ def needs_reconciler(bp: dict[str, Any]) -> bool: def app_skeleton(bp: dict[str, Any], graph: dict[str, Any]) -> dict[str, Any]: - from app.apps.derive import ENTRY, INSTANCE_VIEW, PIN, VIEW, _entries, _results + from app.apps.derive import ENTRY, INSTANCE_VIEW, PIN, VIEW, _entries, _results, start_label from app.apps.forms import input_schema from app.apps.tree import input_fields, node_map @@ -118,7 +118,7 @@ def app_skeleton(bp: dict[str, Any], graph: dict[str, Any]) -> dict[str, Any]: if fields: spec["forms"][ENTRY] = input_schema(fields) names = list(spec["forms"][ENTRY]["properties"]) - spec["entries"].append({"id": ENTRY, "label": f"发起{name}" if process else "按条件运行", "pin": PIN, + spec["entries"].append({"id": ENTRY, "label": start_label(name) if process else "按条件运行", "pin": PIN, "form": {"$ref": f"#/forms/{ENTRY}"}, "map": {n: f"form.{n}" for n in names}}) spec["views"][INSTANCE_VIEW] = {"source": {"kind": "instance"}} if not process: diff --git a/backend/app/chat/turn.py b/backend/app/chat/turn.py index 993c0ad5..f416d95a 100644 --- a/backend/app/chat/turn.py +++ b/backend/app/chat/turn.py @@ -5,13 +5,13 @@ 后端跑不了一轮问数据：评测要么照着抄一份（迟早走样，量出来的不是用户拿到的那一轮），要么做不了；这也违反「界面层不放 逻辑」。现在决定写在这里，是纯函数： -- next_step：生成过程交出结局之后做什么； +- next_step：生成过程交出结局之后做什么（含生成过程一个操作都没给时自动重来一次）； - fill_input：用户的问题填进这张图的入口字段； - retry_after_review：复核之后要不要带着原图重来一次； - retry_instruction：重来时交给助手的那句话（两种措辞）。 -接口把决定交给问数据页：/copilot/generate-stream 的 final 带 next（run | ready | fail），/copilot/review 的结果带 -retry_instruction（要重来时是那句话，不重来是 null）。问数据页照后端给的走；老后端没有这两个字段时退回它自己那份同样的 +接口把决定交给问数据页：/copilot/generate-stream 的 final 带 next（run | ready | fail），error 带 next（retry | fail）， +/copilot/review 的结果带 retry_instruction（要重来时是那句话，不重来是 null）。问数据页照后端给的走；老后端没有这两个字段时退回它自己那份同样的 规则（tests/test_chat_turn_policy.py 逐条对照两边的常量和措辞）。评测（第 3 波）用同一组函数把一轮串起来。 只搬「决定」：重连、断点续看、取消、复核等多久这些仍在问数据页（把整轮编排搬到后端要重做这些，单独立项）。 @@ -28,9 +28,19 @@ MAX_ATTEMPTS = 2 #: final 之后的三种走法（final.next）：运行、只建不跑（摆着等用户点运行）、判失败（没过检查的图不拿去跑） NEXT_RUN, NEXT_READY, NEXT_FAIL = "run", "ready", "fail" NEXT_STEPS = (NEXT_RUN, NEXT_READY, NEXT_FAIL) -#: 别的结局：直接回答了（reply）、助手反问了等用户答（asked）、没自查过的老服务端交来没过检查的图时重来一次（retry） +#: 别的结局：直接回答了（reply）、助手反问了等用户答（asked）、重来一次（retry：没自查过的老服务端交来没过检查的图，或者 +#: 生成过程一个操作都没给） NEXT_REPLY, NEXT_ASKED, NEXT_RETRY = "reply", "asked", "retry" +#: 生成过程一个操作都没给就结束了（error 操作的 code）：模型只写了思考（deepseek 这类推理模型偶发）、或者格式跑偏每一行都被 +#: 跳过。这是偶发的，重来一次多半就好，按「重来一次」的规矩自动重来（最多 MAX_ATTEMPTS 次，含第一次）；被截断的不重来——额度 +#: 不变，重来还是截断 +EMPTY_GENERATION = "copilot_empty" +#: 自动重来的 error（code） +RETRY_ERRORS = frozenset({EMPTY_GENERATION}) +#: error 操作的 message 前缀（api/copilot.answer_ops）：重来时交给助手的原因不带它 +_ERROR_PREFIX = "助手本轮未完成：" + #: 问题往哪个入口字段放：名字像「问题」的优先（同 chat.ts fillInput 的正则，大小写不敏感） QUESTION_FIELD = re.compile(r"question|query|topic|input|问题|需求|主题", re.IGNORECASE) @@ -51,7 +61,9 @@ def next_step(terminal: Mapping[str, Any] | None, *, self_checked: bool = True) """生成过程交出结局之后做什么。terminal 是最后那条结局操作（final / reply / error）；流结束时只有反问（ask）、没有 别的结局的给 None。 - - reply：直接回答，这一轮到此为止；error：判失败；只有反问：等用户回答； + - reply：直接回答，这一轮到此为止；只有反问：等用户回答； + - error：生成过程一个操作都没给（code 是 EMPTY_GENERATION）时重来一次（retry，次数由调用方按 MAX_ATTEMPTS 管，原因 + 取 error_reason），别的判失败； - final：有 error 级问题时，服务端已经自查过（收到过 check 的结论）就判失败——没过的问题已经交回模型在同一轮里改过 了，再整轮重来是再烧一次完整生成、改的还是同一批问题；没自查过（老服务端）才重来一次（retry）；模型说不运行 （autorun=false：用户要的是流程本身、或者问了用户）只建不跑；其余运行。 @@ -61,6 +73,8 @@ def next_step(terminal: Mapping[str, Any] | None, *, self_checked: bool = True) kind = terminal.get("op") if kind == "reply": return NEXT_REPLY + if kind == "error": + return NEXT_RETRY if terminal.get("code") in RETRY_ERRORS else NEXT_FAIL if kind != "final": return NEXT_FAIL if _blockers(terminal): @@ -70,6 +84,15 @@ def next_step(terminal: Mapping[str, Any] | None, *, self_checked: bool = True) return NEXT_RUN +def error_reason(error: Mapping[str, Any]) -> str: + """生成过程交出 error 时重来的原因（交给助手的那句话里用）：error 带的 reason，没有就是 message 去掉「助手本轮未完成：」。""" + reason = str(error.get("reason") or "").strip() + if reason: + return reason + message = str(error.get("message") or "").strip() + return message[len(_ERROR_PREFIX):] if message.startswith(_ERROR_PREFIX) else message + + def blocked_reason(final: Mapping[str, Any]) -> str: """没过检查的图重来时的原因（只有老服务端那条路用得到）。""" return "生成的工作流没有通过校验：" + "；".join(_blockers(final)) @@ -124,6 +147,7 @@ def has_nodes(graph: Any) -> bool: return isinstance(graph, Mapping) and bool(graph.get("nodes")) -__all__ = ["MAX_ATTEMPTS", "NEXT_ASKED", "NEXT_FAIL", "NEXT_READY", "NEXT_REPLY", "NEXT_RETRY", "NEXT_RUN", "NEXT_STEPS", - "QUESTION_FIELD", "RETRY_FROM_SCRATCH", "RETRY_WITH_BASE", "blocked_reason", "fill_input", "has_nodes", - "next_step", "retry_after_review", "retry_instruction", "review_reason"] +__all__ = ["EMPTY_GENERATION", "MAX_ATTEMPTS", "NEXT_ASKED", "NEXT_FAIL", "NEXT_READY", "NEXT_REPLY", "NEXT_RETRY", + "NEXT_RUN", "NEXT_STEPS", "QUESTION_FIELD", "RETRY_ERRORS", "RETRY_FROM_SCRATCH", "RETRY_WITH_BASE", + "blocked_reason", "error_reason", "fill_input", "has_nodes", "next_step", "retry_after_review", + "retry_instruction", "review_reason"] diff --git a/backend/app/data/catalog_review.py b/backend/app/data/catalog_review.py index 78eb3438..e9d47172 100644 --- a/backend/app/data/catalog_review.py +++ b/backend/app/data/catalog_review.py @@ -24,8 +24,8 @@ 一张表冲突、写不进去不影响别的表；路径不对、表不存在逐条报（results 和请求一一对应）。审阅的规则同单项审阅 （catalog.review_item）。写进去之后影响数字的项被驳回了的，口径账本在后台标依据了它的答案「依据有变」 （catalog.after_review → ledger/recall，next-round 02 做法 7 二）。 -- **被哪些运行用过**（table_runs，GET /catalog/{table}/usage）：取法同使用次数（catalog.query_snapshot_tables），按运行 - 去重，新的在前；total 是运行次数。 +- **被哪些运行用过**（table_runs，GET /catalog/{table}/usage）：和使用次数数的是同一批查询快照（catalog.query_snapshot_tables）， + 但按运行去重，新的在前；total 是运行次数。使用次数按查询计，同一次运行里查了几次这张表，使用次数就比 total 多几。 - **进度**（review_progress，GET /catalog/review-progress）：常用表（使用次数前 TOP_N 张，和起草对话框「使用次数最多的 20 张表」是同一批：catalog.default_tables 的取法，用到的不够 TOP_N 张时按表结构的顺序补齐）、全部、影响数字的项各自的 确认情况。项的范围和队列一样：表结构里没有的表、列不算。 @@ -391,7 +391,8 @@ async def review_batch(session: AsyncSession, source: Any, items: Iterable[Batch async def table_runs(session: AsyncSession, source: Any, table: str, *, limit: int = 20) -> dict[str, Any]: - """用过这张表的运行：{total, runs}，新的在前（按开始时间，没开始的按建立时间）。取法同使用次数。""" + """用过这张表的运行：{total, runs}，新的在前（按开始时间，没开始的按建立时间）。数的是和使用次数同一批查询快照 + （catalog.query_snapshot_tables），但按运行去重：total 是运行次数，使用次数按查询计、可能比它大。""" run_ids = {run for run, hit in await catalog.query_snapshot_tables(session, source) if table in hit} if not run_ids: return {"total": 0, "runs": []} diff --git a/backend/app/data/schema_view.py b/backend/app/data/schema_view.py index 9f95bd66..1b3ef7cb 100644 --- a/backend/app/data/schema_view.py +++ b/backend/app/data/schema_view.py @@ -17,6 +17,11 @@ agent 看到的目录少了遮罩列那部分（README 写明）。 文档是表名、注释、目录里的中文名 / 粒度 / 说明、列名、列的中文名（遮罩列只用列名）；探查范围外的对象只按名字和注释参与， 权重低一档。切词用 memory/embeddings._tokens（英文按词、中文按字补二元组）加 catalog.name_words（下划线、驼峰拆词）。 +**限制：范围外、又没有注释的对象，中文需求命不中。** 它只剩英文名可检索，英文名拆出的词没有中文说法：仓库里没有现成的英中 +业务词对照（命名推断只拆词、不翻译，catalog.name_words 也一样），这里不临时编一份词表（编出来的覆盖不全、还会把不相干的表 +抬上来），所以「上周的退款」找不到只叫 refund_orders、没写注释的表。真库多半有注释，注释照常参与检索，影响小；范围内的表另有 +目录里的中文名、列名可配。要用这张表时按英文名点名查看（会按需加入），或者请管理员把它加进探查范围、补上注释或目录的中文名。 + **三种「不在」**（absent_detail）：表已不在、在探查范围外、在范围里却读不到结构，说法分开；只有真删除才说「表已不在」。 只读：只看表结构缓存（schema_cache）和调用方读好的数据目录、剖析事实、对象索引，不碰业务库。这里的文字都是写给模型看的， @@ -342,6 +347,7 @@ def _doc_text(name: str, meta: Mapping[str, Any] | None, notes: Mapping[str, Any def _outside_text(name: str, comment: str | None) -> str: + """一个范围外对象的检索文档：名字、名字拆出的词、注释。没有注释的只有英文（见模块说明的限制）。""" from app.data.catalog import name_words return " ".join([name, " ".join(name_words(name)), comment or ""]) diff --git a/backend/app/eval/answer.py b/backend/app/eval/answer.py index 9d281947..d9e66982 100644 --- a/backend/app/eval/answer.py +++ b/backend/app/eval/answer.py @@ -10,7 +10,8 @@ 复核跟着换）；读哪份目录按变体（answer_ops 的 catalog_view，查看的表详情、自查的 SQL 检查器都按它）。 - 结局：reply 直接回答、asked 反问、fail 判失败（建图失败）、ready 只建不跑、retry（老服务端那条路：没自查过的图带着原因 重来一次）、run 运行。生成过程交出 error（助手本轮未完成）、开流前就拒绝（模型没配好、数据源不在）都是建图失败，另记 - 「助手调用失败」——连续几题都这样就止损（runner.py）。 + 「助手调用失败」——连续几题都这样就止损（runner.py）；只有生成过程一个操作都没给（只写了思考，error.next=retry）时按 + 同一个规矩先带着原因重来一次，还不行才记建图失败。每次重来的原因照实记在 retries。 - 运行：fill_input 填入口字段（单轮，history 为空），来源 eval，名字「评测：题集名·第 n 题」，选项带目录视图（Run.options 的 catalog_view：运行时的 db_schema、冻结快照、自查都按它）、只读和遮罩（同使用者的问数据运行：评测页会显示答案里的数， 遮罩值不能从这里露出来；评测不写业务库）。写作模型按变体写进图的 defaults：节点自己点了模型的不覆盖，照实记在 @@ -79,6 +80,8 @@ class Turn: #: 这一题发起过的全部作答运行（重来的话两次），失败的也在 runs: list[str] = field(default_factory=list) attempts: int = 0 + #: 每次自动重来的原因（和 attempts 一一对应）：没过检查、复核说不可信、生成过程一个操作都没给 + retries: list[str] = field(default_factory=list) reason: str | None = None #: 助手调用失败（生成过程交出 error、开流前就被拒）：runner 据此止损 copilot_failed: bool = False @@ -101,25 +104,27 @@ class Turn: def to_dict(self) -> dict[str, Any]: return {"path": self.path, "run_id": self.run_id, "runs": list(self.runs), "attempts": self.attempts, - "reason": self.reason, "copilot_failed": self.copilot_failed, "timed_out": self.timed_out, + "retries": list(self.retries), "reason": self.reason, "copilot_failed": self.copilot_failed, "timed_out": self.timed_out, "reply": self.reply, "asked": self.asked, "review": self.review, "looks": list(self.looks), "assistant_model": self.assistant_model, "usage": copy.deepcopy(self.usage), "timings": dict(self.timings), "own_models": list(self.own_models)} +#: 用量汇总里相加的计数（providers/usage 的 summary）：unfinished_calls 是没做完的调用（被取消、超时，token 只计了已报的） +_COUNTED = ("input_tokens", "output_tokens", "calls", "unpriced_calls", "unfinished_calls") + + def _add_usage(into: dict[str, Any] | None, more: Mapping[str, Any] | None) -> dict[str, Any]: """两份 providers/usage 汇总相加（by_model 逐个模型加）。""" - out = copy.deepcopy(into) if into else {"by_model": {}, "input_tokens": 0, "output_tokens": 0, "calls": 0, - "cost_usd": 0.0, "unpriced_calls": 0} + out = copy.deepcopy(into) if into else {"by_model": {}, **dict.fromkeys(_COUNTED, 0), "cost_usd": 0.0} if not more: return out - for key in ("input_tokens", "output_tokens", "calls", "unpriced_calls"): + for key in _COUNTED: out[key] = int(out.get(key) or 0) + int(more.get(key) or 0) out["cost_usd"] = round(float(out.get("cost_usd") or 0) + float(more.get("cost_usd") or 0), 6) for model, row in (more.get("by_model") or {}).items(): - mine = out["by_model"].setdefault(model, {"input_tokens": 0, "output_tokens": 0, "calls": 0, "cost_usd": 0.0, - "unpriced_calls": 0}) - for key in ("input_tokens", "output_tokens", "calls", "unpriced_calls"): + mine = out["by_model"].setdefault(model, {**dict.fromkeys(_COUNTED, 0), "cost_usd": 0.0}) + for key in _COUNTED: mine[key] = int(mine.get(key) or 0) + int(row.get(key) or 0) mine["cost_usd"] = round(float(mine.get("cost_usd") or 0) + float(row.get("cost_usd") or 0), 6) return out @@ -379,6 +384,14 @@ class Driver: return t kind = terminal.get("op") if kind == "error": + # 生成过程一个操作都没给（只写了思考）：按重来一次的规矩带着原因重来（同问数据页，chat/turn.next_step） + step = terminal.get("next") or policy.next_step(terminal) + if step == policy.NEXT_RETRY and t.attempts + 1 < policy.MAX_ATTEMPTS: + reason = policy.error_reason(terminal) + t.attempts += 1 + t.retries.append(reason) + instruction = policy.retry_instruction(self.question, reason, policy.has_nodes(base_graph)) + continue t.path, t.reason, t.copilot_failed = "build_failed", str(terminal.get("message") or "助手本轮未完成"), True return t if kind == "reply": @@ -393,6 +406,7 @@ class Driver: if step == policy.NEXT_RETRY: if t.attempts + 1 < policy.MAX_ATTEMPTS: t.attempts += 1 + t.retries.append(policy.blocked_reason(terminal)) instruction = policy.retry_instruction(self.question, policy.blocked_reason(terminal), policy.has_nodes(graph)) base_graph = graph if policy.has_nodes(graph) else None @@ -421,7 +435,10 @@ class Driver: t.review = {k: v for k, v in review.items() if k != "usage"} t.usage["review"] = _add_usage(t.usage.get("review"), review.get("usage")) if review.get("retry_instruction"): + from app.chat.turn import review_reason + t.attempts += 1 + t.retries.append(review_reason(review) or "复核判答案不可用") instruction = str(review["retry_instruction"]) base_graph = row.graph if policy.has_nodes(row.graph) else None continue diff --git a/backend/app/eval/runner.py b/backend/app/eval/runner.py index 3412c005..83c21730 100644 --- a/backend/app/eval/runner.py +++ b/backend/app/eval/runner.py @@ -124,16 +124,44 @@ async def source_versions(session: AsyncSession, source_ids: Sequence[str]) -> d async def estimate(session: AsyncSession, suite_id: str, items: int) -> dict[str, Any]: - """开跑前的估算：按这个题集上一次结束的评测的每题平均花费 × 这次的项数。第一次评测估不出，照实写。""" + """开跑前的估算：按这个题集上一次结束的评测的每题平均花费 × 这次的项数。第一次评测估不出，照实写。 + + 上一次有估不出金额的调用（模型不在价格目录里、用量没报全）时，平均花费只是估得出的那部分：一分都没估出来的照实写估不出 + （以前写「约 $0」，像不花钱），估出了一部分的写「至少」（at_least）、带上另有几次估不出（unpriced_calls）。老总表里每题 + 平均记成 0、却有估不出的调用的，也按估不出算。""" row = (await session.execute(select(EvalRun).where( EvalRun.suite_id == suite_id, EvalRun.status.in_(("succeeded", "partial"))) .order_by(EvalRun.created_at.desc()).limit(1))).scalar_one_or_none() - per_item = ((row.summary or {}).get("per_case_cost_usd") if row is not None else None) - if per_item is None: + if row is None: return {"items": items, "per_item_usd": None, "total_usd": None, "basis": None, "note": "这个题集还没有评完过，估不出花费：按每题开评前的实际花费累计，到了上限就停"} - return {"items": items, "per_item_usd": round(float(per_item), 6), "total_usd": round(float(per_item) * items, 4), - "basis": row.id, "note": "按这个题集上一次评测的每题平均花费估算（不含估不出金额的模型）"} + summary = row.summary or {} + per_item = summary.get("per_case_cost_usd") + unpriced = int(summary.get("unpriced_calls") or 0) + if per_item is None or (unpriced and not float(per_item)): + if not unpriced: + return {"items": items, "per_item_usd": None, "total_usd": None, "basis": None, + "note": "这个题集上一次评测没有评完一道题，估不出花费：按每题开评前的实际花费累计，到了上限就停"} + return {"items": items, "per_item_usd": None, "total_usd": None, "basis": row.id, "unpriced_calls": unpriced, + "note": f"上一次评测的 {unpriced} 次模型调用金额都估不出（模型不在价格目录中，或用量没有报全）：" + "金额上限只管得住估得出金额的调用"} + out = {"items": items, "per_item_usd": round(float(per_item), 6), "total_usd": round(float(per_item) * items, 4), + "basis": row.id, "note": "按这个题集上一次评测的每题平均花费估算"} + if unpriced: + out.update(at_least=True, unpriced_calls=unpriced, + note=f"按这个题集上一次评测的每题平均花费估算，实际至少这么多：上一次另有 {unpriced} 次调用金额估不出，没有算进去") + return out + + +def judge_usage(judge: Mapping[str, Any], cost_usd: float) -> dict[str, Any]: + """按角色的用量里裁判那一份：金额、次数，和估不出金额的次数（结论裁判的模型不在价格目录里时它的每次调用都估不出； + 文字要点按计量汇总）。以前只有金额和次数，裁判模型不在价格目录里时写成 $0。""" + claims = judge.get("claims") or {} + text = judge.get("text") or {} + calls = int(claims.get("calls") or 0) + int(text.get("calls") or 0) + unpriced = (int(claims.get("calls") or 0) if claims.get("priced") is False else 0) \ + + int((text.get("usage") or {}).get("unpriced_calls") or 0) + return {"cost_usd": round(cost_usd, 6), "calls": calls, "unpriced_calls": unpriced} # ========================================================================== @@ -552,9 +580,7 @@ class EvalExecution(runtime.JobRun): writer = await writer_usage(turn.runs) assistant = add_usage(turn.usage.get("assistant"), turn.usage.get("review")) usage = {"assistant": assistant, "writer": writer, "review": turn.usage.get("review"), - "judge": {"cost_usd": round(judge_cost, 6), - "calls": int((judge.get("claims") or {}).get("calls") or 0) - + int((judge.get("text") or {}).get("calls") or 0)}} + "judge": judge_usage(judge, judge_cost)} cost = float(assistant.get("cost_usd") or 0) + float(writer.get("cost_usd") or 0) self.spent += cost + judge_cost looks = {"model": sum(1 for x in turn.looks if x.get("by") == "model"), @@ -1022,6 +1048,6 @@ async def shutdown() -> None: __all__ = ["EVAL_BUSY", "EVAL_FINISHED", "EVAL_INVALID", "EvalExecution", "EvalRefused", "INTERRUPTED", - "NOTHING_TO_RETRY", "STOPPED", "active_run", "estimate", "gold_key", "leader_of", "manager", + "NOTHING_TO_RETRY", "STOPPED", "active_run", "estimate", "gold_key", "judge_usage", "leader_of", "manager", "prompt_fingerprint", "reconcile_stale", "refresh_summary", "result_dict", "result_rows", "retry", "running", "shutdown", "source_versions", "start", "startup", "stop"] diff --git a/backend/app/eval/score.py b/backend/app/eval/score.py index f471c750..e1c916ec 100644 --- a/backend/app/eval/score.py +++ b/backend/app/eval/score.py @@ -17,11 +17,13 @@ wrong 错、scale 量纲不同（差 10 的整数次幂，比如元和万元、 - scalar：数值按「允许的相对误差」比（engine/tolerance.agree，和双盲复算同一个函数）；期望没写容差时，引用的原值按 DEFAULT_TOLERANCE，裸写的数按它写出的精度（3.5 万 ±500）。维度配对：期望写了 where（或期望那一行本身有维度）时，答案 那一格所在行的非数值列要有这个取值；答案那一格没有维度（按条件筛过、只剩一个数）不算对不上。期望值是 0–10 的小整数 - 又没有维度可配时标「弱匹配」（巧合的数：3 家门店正好等于期望的 3）。 + 又没有维度可配时标「弱匹配」（巧合的数：3 家门店正好等于期望的 3）。答错时「答案里的数」给和这一项配对上的那一格（列名、 + 指标、说明同名，或者维度对得上的只有一个数），配不上就不给、原因里照实写配不上，列出答案引用了哪几个数。 - table：逐行按 keys 的取值配对，每个要比的值同 scalar；ordered 时按答案里各行键值出现的先后核对顺序。 - set：名单里每一项都要在答案里出现（ordered 时按顺序）；每一项都出现在答案查过的结果里算可追溯。 - none：期望答「没有数据」：答案里没有数，而且查过的查询里有返回 0 行的（可追溯）；没有数、但也没有一条查询返回 0 行的 - 算「对但没引用」。 + 算「对但没引用」。「给了数」只认引用的数值格和叙述里写的数：引用文字格（地区名这种维度）、空值格（NULL）不算——「华中区 + 没有数据」引用了地区那一格，说的仍是没有数据。 - ask：期望反问：作答路径是反问才对。 - text：文字要点，由裁判判（covered / partial / missing / unjudged），单列「模型判断」，不进准确率（评测-D4）。 @@ -30,9 +32,10 @@ wrong 错、scale 量纲不同（差 10 的整数次幂，比如元和万元、 没有错的）不能评分。期望没算出来按 gold 的结论：口径升版待处置是跳过，别的是不能评分——不混进「答错」。被驳回（总方案 X9） 记路径「被驳回」，不算失败：不能评分。只有文字要点的题不能评分（单列，不计入准确率）。 -**逐题的 E1**（sql_match）：答案里的查询（先看报告引用了的）和期望的取数（SQL 题是期望 SQL，口径题是口径卡的取数节点） -用口径账本同一个比对函数（ledger/match.compare_queries：先绑参数、再由 explain_sql_diff 判等价）比一次，给出等价与否和差在 -哪个方面（总方案 X2：E1 只有一份实现）。 +**逐题的 E1**（sql_match）：产出答案的那条查询和期望的取数（SQL 题是期望 SQL，口径题是口径卡的取数节点）用口径账本同一个 +比对函数（ledger/match.compare_queries：先绑参数、再由 explain_sql_diff 判等价）比一次，给出等价与否和差在哪个方面（总方案 +X2：E1 只有一份实现）。哪条是产出答案的：答案引用了就是被引用的；没引用、只查了一条就是那一条（写明依据）；没引用、查了 +几条的不硬比（照实写没法对上是哪条，不计入 E1）。 """ from __future__ import annotations @@ -46,8 +49,10 @@ from app.engine.tolerance import agree, is_finite_number logger = logging.getLogger(__name__) -#: 评分规则的版本：规则改了就加一，结果里记着按哪一版评的，可以按新规则重评 -SCORER_VERSION = 1 +#: 评分规则的版本：规则改了就加一，结果里记着按哪一版评的，可以按新规则重评。 +#: 2：期望「没有数据」时引用文字格、空值格不算给了数；判错时「答案里的数」给配对上的那一格；答案没引用查询时逐题 E1 +#: 不硬比第一条查询 +SCORER_VERSION = 2 OUTCOMES = ("traced", "untraced", "wrong", "scale", "ungradable") OUTCOME_LABEL = {"traced": "对且可追溯", "untraced": "对但没引用", "wrong": "错", "scale": "量纲不同", @@ -64,6 +69,8 @@ DEFAULT_TOLERANCE = 1e-6 SCALE_FACTORS = (10.0, 100.0, 1000.0, 1e4, 1e5, 1e6, 1e8) #: 弱匹配：期望是这么小的整数、又没有维度可配 WEAK_MAX = 10 +#: 判错、又配不上时原因里列出答案引用的前几个数 +PAIR_SHOWN = 3 #: 路径不对时给人看的原因 WRONG_PATH = { @@ -316,9 +323,48 @@ def _score_scalar(exp: Mapping[str, Any], gold: Mapping[str, Any], answer: Answe for token in answer.bare: if _scaled(expected, float(token["value"])): return {**out, "outcome": "scale", "got": token["value"], "raw": token["raw"]} - got = next((v.get("value") for v in clean), None) - return {**out, "outcome": "wrong", **({"got": got} if got is not None else {}), - "reason": "答案里没有对得上的数" if not answer.values and not answer.bare else "答案里的数和期望对不上"} + numeric = [v for v in clean if is_finite_number(v.get("value"))] + if not numeric and not answer.bare: + return {**out, "outcome": "wrong", "reason": "答案里没有对得上的数"} + paired = _paired(numeric, exp, gold, key) + if paired is not None: + return {**out, "outcome": "wrong", "got": paired.get("value"), "cell": _cell_ref(paired), + "reason": "答案里的数和期望对不上"} + if numeric: + shown = "、".join(f"{_value_name(v)} {_plain(v.get('value'))}" for v in numeric[:PAIR_SHOWN]) + more = f" 等 {len(numeric)} 个" if len(numeric) > PAIR_SHOWN else "" + return {**out, "outcome": "wrong", + "reason": f"答案里的数和期望对不上：答案引用的数按列名、维度都配不上这一项（引用了 {shown}{more}）"} + return {**out, "outcome": "wrong", "reason": "答案里的数和期望对不上"} + + +def _norm(name: Any) -> str: + return str(name).strip().strip('"`[]').lower() if name is not None else "" + + +def _paired(numeric: Sequence[Mapping[str, Any]], exp: Mapping[str, Any], gold: Mapping[str, Any], + key: Mapping[str, Any]) -> Mapping[str, Any] | None: + """答错时答案里和这一项配对上的那一格（「答案里的数」给它）：按名字配——列名（或口径卡的指标）是期望取的那一列、 + 那个指标，或者和期望的说明（label）同名；名字配不上时按维度配——期望写了维度、答案里维度对得上的数恰好只有一个。 + 都配不上为 None：以前取答案引用的第一个值，退款金额那一项显示成了答案里的订单数，甚至显示成一格文字。""" + names = {_norm(n) for n in (exp.get("column"), gold.get("column"), exp.get("metric"), gold.get("metric"), + gold.get("name"), exp.get("label")) if n not in (None, "")} + named = [v for v in numeric if _norm(v.get("column")) in names or _norm(v.get("metric")) in names] + if named: + return named[0] + if key: + on_dims = [v for v in numeric if v.get("dims") and _dims_ok(key, v["dims"])] + if len(on_dims) == 1: + return on_dims[0] + return None + + +def _value_name(value: Mapping[str, Any]) -> str: + return str(value.get("column") or value.get("metric") or value.get("alias") or "（没有列名）") + + +def _plain(value: Any) -> str: + return str(int(value)) if is_finite_number(value) and float(value).is_integer() else str(value) def _cell_ref(value: Mapping[str, Any]) -> dict[str, Any]: @@ -371,8 +417,15 @@ def _score_set(exp: Mapping[str, Any], gold: Mapping[str, Any], answer: Answer, def _score_none(answer: Answer) -> dict[str, Any]: - if answer.values or answer.bare: - return {"outcome": "wrong", "reason": "期望答「没有数据」，答案里却给了数"} + """期望答「没有数据」。「给了数」只认引用的数值格、叙述里写的数：引用的文字格（地区名这种维度）、空值格（NULL）不是数—— + 「华中区（[[v:Q1.r0.地区]]）没有数据」说的正是没有数据，以前因为它引用了一格就判成「给了数」。""" + numeric = next((v for v in answer.values if is_finite_number(v.get("value"))), None) + if numeric is not None: + return {"outcome": "wrong", "got": numeric.get("value"), "cell": _cell_ref(numeric), + "reason": "期望答「没有数据」，答案里却引用了数"} + if answer.bare: + return {"outcome": "wrong", "got": answer.bare[0]["value"], "raw": answer.bare[0]["raw"], + "reason": "期望答「没有数据」，答案的叙述里却写了数"} if any(int(q.get("row_count") or 0) == 0 and q.get("role") != "verify" for q in answer.queries): return {"outcome": "traced"} return {"outcome": "untraced", "reason": "答案没有给数，但没有一条查询返回 0 行可以说明「没有数据」"} @@ -561,7 +614,8 @@ async def gold_calibers(case: Mapping[str, Any], gold: Mapping[str, Any] | None) async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, answer: Answer, *, timeout: float = 8.0) -> dict[str, Any] | None: - """逐题的 E1：{status: proven|related|none|skipped, query, aspects, diff, reason}。没有作答运行、没有期望取数的为 None。 + """逐题的 E1：{status: proven|related|none|skipped, query, aspects, diff, reason, cited, basis}。没有作答运行、没有期望 + 取数的为 None。比的是哪条查询见 _answering：basis 是 cited（答案引用的）或 only_query（答案没引用、运行只查了这一条）。 比对在线程里跑（纯计算，和账本一样限时）。""" import asyncio import time @@ -594,13 +648,12 @@ async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, ans except Exception: # noqa: BLE001 - 比对是附加的一步，出错照实记没比 logger.exception("评测：逐题比 SQL 出错") return {"status": "skipped", "reason": "比对出错"} - cited = [r for r in rows if r.get("artifact") in set(answer.cited)] - pool = cited or rows + pool, basis = _answering(rows, answer) + if isinstance(pool, dict): + return pool proven = next((r for r in pool if r.get("status") == "proven"), None) chosen = proven or next((r for r in pool if r.get("status") == "related"), None) or \ - next((r for r in pool if r.get("status") == "skipped"), None) or (pool[0] if pool else None) - if chosen is None: - return {"status": "none", "reason": "答案没有查询"} + next((r for r in pool if r.get("status") == "skipped"), None) or pool[0] candidate = next((c for c in chosen.get("candidates") or [] if c.get("status") == chosen.get("status")), None) or \ next(iter(chosen.get("candidates") or []), None) or {} diff = candidate.get("diff") or {} @@ -608,7 +661,31 @@ async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, ans sql = next((q.get("sql") for q in answer.queries if q.get("artifact") == chosen.get("artifact")), None) return {"status": chosen.get("status"), "query": chosen.get("artifact"), "sql": sql, "aspects": aspects, "diff": diff or None, "via": candidate.get("via"), "binding": candidate.get("binding"), - "reason": chosen.get("reason") or candidate.get("reason"), "cited": bool(cited)} + "reason": chosen.get("reason") or candidate.get("reason"), "cited": basis == "cited", "basis": basis} + + +def _answering(rows: list[dict[str, Any]], answer: Answer) -> tuple[list[dict[str, Any]] | dict[str, Any], str]: + """逐题 E1 拿哪几条查询比：产出答案的那几条。交回 (查询, 依据)，确定不了时交回 (照实的结论, "")。 + + - 答案引用了查询（报告里引用的格、指标所在的查询）：比被引用的那几条（依据 cited）； + - 没引用、这次运行（核查用的不算）只查了一条：答案的数只可能来自它，按它比（依据 only_query）； + - 没引用、查了几条：不比——以前取第一条相关的，取到的往往是探索查询（看看有哪些地区那种），说「差在筛选条件」和答案的数 + 毫无关系。照实写没法对上是哪条，不计入 E1。""" + cited = set(answer.cited) + if cited: + pool = [r for r in rows if r.get("artifact") in cited] + if pool: + return pool, "cited" + return {"status": "skipped", "cited": True, + "reason": "答案引用的查询不在可比对的查询里（合并查询不比：它的数都来自它的输入）"}, "" + roles = {q.get("artifact"): q.get("role") for q in answer.queries} + pool = [r for r in rows if roles.get(r.get("artifact")) != "verify"] + if not pool: + return {"status": "none", "reason": "答案没有查询"}, "" + if len(pool) > 1: + return {"status": "skipped", "cited": False, + "reason": f"答案没有引用查询，这次运行执行了 {len(pool)} 条查询，没法对上是哪条产出了答案，不比"}, "" + return pool, "only_query" __all__ = ["Answer", "CaseScore", "DEFAULT_TOLERANCE", "OUTCOMES", "OUTCOME_LABEL", "PATH_LABEL", "SCORER_VERSION", diff --git a/backend/app/eval/stats.py b/backend/app/eval/stats.py index 7e4f6a8e..77f9dd22 100644 --- a/backend/app/eval/stats.py +++ b/backend/app/eval/stats.py @@ -145,8 +145,10 @@ def summarize(results: Sequence[Mapping[str, Any]], *, judge_on: bool = False) - "timings": {k: {"p50": percentile(v, 50), "p90": percentile(v, 90), "n": len(v)} for k, v in timings.items()}, "usage": {**usage, "gold": {"cost_usd": 0.0, "calls": 0}}, "cost_usd": cost, "unpriced_calls": unpriced, - # 每题平均花费（下一次开跑前的估算按它）：只按评完了的题平均，没开评的不摊 - "per_case_cost_usd": round(cost / len(evaluated), 6) if evaluated else None, + # 每题平均花费（下一次开跑前的估算按它）：只按评完了的题平均，没开评的不摊。有估不出金额的调用时 cost_usd 只是估得出 + # 的那部分：一分都没估出来（模型都不在价格目录里）的为 None，不写成 0（像不花钱）；估出了一部分的照给，unpriced_calls + # 说明它只是下限 + "per_case_cost_usd": round(cost / len(evaluated), 6) if evaluated and (cost > 0 or not unpriced) else None, "citations": {"numbers": numbers, "numbers_cited": cited, "uncited_numbers": sum(int(s.get("uncited_numbers") or 0) for s in stats), "unresolved": sum(int(s.get("unresolved") or 0) for s in stats), diff --git a/backend/app/providers/usage.py b/backend/app/providers/usage.py index 56298a2c..dbf029d6 100644 --- a/backend/app/providers/usage.py +++ b/backend/app/providers/usage.py @@ -12,12 +12,21 @@ async generator：with 块跨着 yield 的话，变量在两次 yield 之间留 在创建时拷走上下文，照样计到；调用方自己的代码一行都不在计量里。 **金额。** 按 providers/catalog.estimate_cost 算，和运行的 llm.end 一个口径。估不出的调用记 unpriced_calls：模型不在价格 -目录里，或者用量没报全而目录价又不是 0。估不出的不当成 0 元，调用方据此写「≥」或「金额无法估算」。用量没报全有两种： +目录里，或者用量没报全而目录价又不是 0。估不出的不当成 0 元，调用方据此写「≥」或「金额无法估算」。汇总的 cost_usd 永远是 +估得出的那部分（同运行的 usage，全系统一个口径），有 unpriced_calls 时它只是下限：写金额、算平均、做估算的地方都要连着 +unpriced_calls 一起看，一分都没估出来的写估不出、不写 0（评测的开跑估算、每题平均花费以前就漏了这一条，见 eval/runner.estimate、 +eval/stats.summarize）。用量没报全有两种： 回复没带用量（有的网关流式不给）；读到一半就不读了——用量在流的末尾才报全（Anthropic 的输出 token 在最后一条，OpenAI 兼容系开了 stream_usage 也在最后）。生成过程读到 done、look、ask 那一行之后照样把流读完再关（api/copilot_gen._drain）， 流末尾那一段用量收得到；只有模型在那一行之后还往下写超过 DRAIN_CHARS 字的闲话时才当场关流（不为闲话付钱），连接断了也一样， 这一次只拿得到已经报了的部分：照样计次数、计已报的 token，记成估不出，不悄悄少算。 +**没做完的调用**（unfinished_calls）。被取消（挑表超时被 asyncio.wait_for 取消、调用方停了）、超时、流读到一半被关掉的调用 +都记一次调用、标「未完成」：token 只计已经报了的（一段都没收到的记 0，当作不知道，记成估不出），调用次数照实。以前挑表超时 +的那一次一点都不计，问数据一轮、评测按角色的调用次数就少一次。LangChain 的 ainvoke 被取消时不一定回调 on_llm_error +（agenerate 卡在 gather 上被取消，后面的回调走不到）：开始了、到汇总时还没有结束回调的调用（_pending 里剩下的）一并算未完成。 +请求刚发出去就被拒了的（鉴权错、参数不对，一段都没收到）不算一次调用：模型没干活。 + 模型名按发起调用时的参数取（invocation_params 的 model / model_name，就是接入里配的模型 id，价格目录按它查）；调用参数里 没有的（以前的演示模型就没报，评测汇总里助手用量的模型因此记成 unknown），看 LangChain 照模型对象给回调的 metadata 里的 ls_model_name；再取不到看回复里的 model_name，都没有才记 UNKNOWN_MODEL。OpenAI 兼容、Anthropic 的模型在调用参数里就带着 @@ -37,6 +46,17 @@ from langchain_core.tracers.context import register_configure_hook #: 取不到模型名的调用记在这个名下 UNKNOWN_MODEL = "unknown" +#: 每个模型累计的几项（unreported 用量没报全的次数、unfinished 没做完的次数） +_COUNTERS = ("input_tokens", "output_tokens", "calls", "unreported", "unfinished") + + +def _abandoned(error: BaseException) -> bool: + """调用是半路放弃的（被取消、超时），不是一发出去就被拒：模型可能已经在干活、已经计费。""" + import asyncio + + if isinstance(error, (asyncio.CancelledError, TimeoutError)): + return True + return any(cls.__name__.endswith(("Timeout", "TimeoutError")) for cls in type(error).__mro__) class Meter(BaseCallbackHandler): @@ -51,6 +71,10 @@ class Meter(BaseCallbackHandler): self._pending: dict[UUID, str] = {} self._models: dict[str, dict[str, int]] = {} + def _row(self, model: str) -> dict[str, int]: + """这个模型的累计（调用方持锁）。unreported：用量没报全的次数；unfinished：没做完的次数（见模块开头）。""" + return self._models.setdefault(model, dict.fromkeys(_COUNTERS, 0)) + # ------------------------------------------------------------------ 回调 def _started(self, run_id: UUID, serialized: dict[str, Any] | None, kwargs: dict[str, Any]) -> None: @@ -83,7 +107,7 @@ class Meter(BaseCallbackHandler): with self._lock: model = self._pending.pop(run_id, "") or str(meta.get("model_name") or meta.get("model") or "") \ or UNKNOWN_MODEL - row = self._models.setdefault(model, {"input_tokens": 0, "output_tokens": 0, "calls": 0, "unreported": 0}) + row = self._row(model) row["calls"] += 1 if usage: row["input_tokens"] += int(usage.get("input_tokens") or 0) @@ -92,8 +116,9 @@ class Meter(BaseCallbackHandler): row["unreported"] += 1 def on_llm_error(self, error: BaseException, *, run_id: UUID, **kwargs: Any) -> None: - """流读到一半停了（生成过程在最后一行之后读到太多闲话就关流，或者连接断了）：LangChain 照样交来已经收到的那几段。 - 有收到东西的照计一次、记已报的 token，算用量没报全；一段都没收到的（请求刚发出去就被拒了）不计。""" + """调用没做完：流读到一半停了（生成过程在最后一行之后读到太多闲话就关流，或者连接断了）、被取消、超时。LangChain + 照样交来已经收到的那几段。有收到东西的、被取消或超时的照计一次、记已报的 token，算用量没报全、未完成；一段都没收到 + 的报错（请求刚发出去就被拒了）不计。""" from langchain_core.outputs import ChatGenerationChunk response = kwargs.get("response") @@ -105,36 +130,48 @@ class Meter(BaseCallbackHandler): partial = None with self._lock: model = self._pending.pop(run_id, "") - if partial is None: + if partial is None and not _abandoned(error): return meta = getattr(partial, "response_metadata", None) or {} model = model or str(meta.get("model_name") or meta.get("model") or "") or UNKNOWN_MODEL - row = self._models.setdefault(model, {"input_tokens": 0, "output_tokens": 0, "calls": 0, "unreported": 0}) + row = self._row(model) row["calls"] += 1 row["unreported"] += 1 + row["unfinished"] += 1 reported = getattr(partial, "usage_metadata", None) or {} row["input_tokens"] += int(reported.get("input_tokens") or 0) row["output_tokens"] += int(reported.get("output_tokens") or 0) + def _rows(self) -> dict[str, dict[str, int]]: + """累计的一份拷贝，开始了还没结束回调的调用算成没做完的（调用方持锁）。""" + rows = {k: dict(v) for k, v in self._models.items()} + for model in self._pending.values(): + row = rows.setdefault(model or UNKNOWN_MODEL, dict.fromkeys(_COUNTERS, 0)) + row["calls"] += 1 + row["unreported"] += 1 + row["unfinished"] += 1 + return rows + # ------------------------------------------------------------------ 合并、汇总 def merge(self, other: Meter) -> None: """把另一次计量并进来（嵌套的计量出块时并进外层）。""" with other._lock: - rows = {k: dict(v) for k, v in other._models.items()} + rows = other._rows() with self._lock: for model, row in rows.items(): - mine = self._models.setdefault(model, dict.fromkeys(row, 0)) + mine = self._row(model) for key, value in row.items(): mine[key] = mine.get(key, 0) + value def summary(self) -> dict[str, Any]: - """{by_model: {模型: {input_tokens, output_tokens, calls, cost_usd, unpriced_calls}}, input_tokens, output_tokens, - calls, cost_usd, unpriced_calls}。cost_usd 只是估得出的那部分。""" + """{by_model: {模型: {input_tokens, output_tokens, calls, cost_usd, unpriced_calls, unfinished_calls}}, input_tokens, + output_tokens, calls, cost_usd, unpriced_calls, unfinished_calls}。cost_usd 只是估得出的那部分；unfinished_calls 次 + 没做完（token 只计了已报的）。""" from app.providers.catalog import estimate_cost, find_pricing with self._lock: - rows = {k: dict(v) for k, v in self._models.items()} + rows = self._rows() by_model: dict[str, dict[str, Any]] = {} for model, row in sorted(rows.items()): pricing = find_pricing(model) @@ -146,9 +183,9 @@ class Meter(BaseCallbackHandler): by_model[model] = {"input_tokens": row["input_tokens"], "output_tokens": row["output_tokens"], "calls": row["calls"], "cost_usd": round(estimate_cost(model, row["input_tokens"], row["output_tokens"]), 6), - "unpriced_calls": unpriced} + "unpriced_calls": unpriced, "unfinished_calls": row["unfinished"]} total = {key: sum(r[key] for r in by_model.values()) - for key in ("input_tokens", "output_tokens", "calls", "unpriced_calls")} + for key in ("input_tokens", "output_tokens", "calls", "unpriced_calls", "unfinished_calls")} return {"by_model": by_model, **total, "cost_usd": round(sum(r["cost_usd"] for r in by_model.values()), 6)} diff --git a/backend/tests/test_app_spec_tools.py b/backend/tests/test_app_spec_tools.py index ace89d43..d1234746 100644 --- a/backend/tests/test_app_spec_tools.py +++ b/backend/tests/test_app_spec_tools.py @@ -344,7 +344,8 @@ async def test_derive_a_process_leaves_slots_empty(monkeypatch): async with SessionLocal() as session: spec = await derive_app(session, wf, 1, name="报销审批") assert spec["name"] == "报销审批" and spec["pins"][0]["use"] == "process" - assert spec["slots"] == {"finance": {"label": "finance"}, "reconciler": {"label": "动作核实"}} + # 岗位的显示名按用到它的节点名起（apps/tree.slot_label），不再拿岗位名 finance 充当显示名 + assert spec["slots"] == {"finance": {"label": "财务审批"}, "reconciler": {"label": "动作核实"}} # 流程的首页只放发起和我发起的：结果在实例页看，不派生「最近封存运行」（派生出来只会挂一块永远空着的印章） assert "latest" not in spec["views"] and spec["views"]["inst"]["source"]["kind"] == "instance" home = next(p for p in spec["pages"] if p["id"] == "home") diff --git a/backend/tests/test_blueprint_templates.py b/backend/tests/test_blueprint_templates.py index 888d890f..7b2804ee 100644 --- a/backend/tests/test_blueprint_templates.py +++ b/backend/tests/test_blueprint_templates.py @@ -299,3 +299,52 @@ def test_freeform_people_rows_follow_the_imported_graph(): stale = [i for i in issues if i["code"] == "bp.template_mismatch" and "gone" in i["message"]] assert stale and stale[0]["level"] == "warning" assert freeform.people_rows(c.with_graph(get_template("freeform").example(), freeform.example_graph())) == [] + + +def _people_graph() -> dict: + """两步审核派给岗位 finance（第一步超时升级给 boss），还有一步派给没起名字的节点用的岗位 ops。名字都是虚构的。""" + return {"nodes": [ + c.node("start", "input", "发起", fields=[{"name": "amount", "type": "number"}]), + c.node("a", "human", "初审", mode="approve", assignee={"kind": "slot", "slot": "finance"}, + on_timeout={"action": "escalate", "to_slot": "boss"}, due="PT1H"), + c.node("b", "human", "复审", mode="approve", assignee={"kind": "slot", "slot": "finance"}), + {"id": "h3", "type": "human", "position": {"x": 0, "y": 0}, + "data": {"config": {"mode": "approve", "assignee": {"kind": "slot", "slot": "ops"}}}}, + c.node("done", "output", "完成", fields=[])], + "edges": [c.edge("start", "a"), c.edge("a", "b"), c.edge("b", "h3"), c.edge("h3", "done")]} + + +def test_entry_labels_keep_a_space_before_a_latin_name(): + """入口按钮「发起…」：名字以英文开头时中间空一格（验收时写成了「发起accept 报销写回应用」）。""" + from app.apps.derive import start_label + + assert start_label("accept 报销写回应用") == "发起 accept 报销写回应用" + assert start_label("报销") == "发起报销" and start_label("2026 年报") == "发起 2026 年报" + graph = _people_graph() + inst = instantiate({**get_template("freeform").example()}, name="accept 报销写回应用", graph=graph) + assert inst.app_spec["entries"][0]["label"] == "发起 accept 报销写回应用", inst.app_spec["entries"] + approval = instantiate(get_template("approval").example(), name="erp 报销") + assert approval.app_spec["entries"][0]["label"] == "发起 erp 报销" + + +def test_slot_labels_use_the_node_names_and_say_so_when_there_is_none(): + """应用里岗位的显示名：图里只写了岗位名（画布上随手写的 id，常是英文 finance），以前显示名就是它，看着像起好的名字。 + 现在按用到它的节点在图上的名字拼（几处用到用顿号连、超时升级的写「…超时升级」）；节点没起名字的照实用岗位名。蓝图卡片 + 上起了名字的（审批模板的步骤名）照旧用它。""" + inst = instantiate({**get_template("freeform").example()}, name="报销", graph=_people_graph()) + slots = inst.app_spec["slots"] + assert slots["finance"]["label"] == "初审、复审", slots + assert slots["boss"]["label"] == "初审超时升级" and slots["ops"]["label"] == "ops", slots + approval = instantiate(get_template("approval").example()) + labels = {k: v["label"] for k, v in approval.app_spec["slots"].items()} + assert all(labels[k] != k for k in ("lead", "finance", "finance_up")), labels + + +async def test_derived_apps_use_the_same_slot_labels_and_entry_label(): + from app.apps.derive import derive_from_graph + from app.db.base import SessionLocal + + async with SessionLocal() as session: + spec = await derive_from_graph(session, _people_graph(), name="accept 报销", blueprint_id="bp1") + assert {k: v["label"] for k, v in spec["slots"].items()} == {"finance": "初审、复审", "boss": "初审超时升级", "ops": "ops"} + assert spec["entries"][0]["label"] == "发起 accept 报销" diff --git a/backend/tests/test_catalog_review_queue.py b/backend/tests/test_catalog_review_queue.py index 239694d2..107528d7 100644 --- a/backend/tests/test_catalog_review_queue.py +++ b/backend/tests/test_catalog_review_queue.py @@ -359,6 +359,27 @@ async def test_table_usage_lists_runs_newest_first(client): assert (await client.get(f"{_base(sid)}/ghost/usage")).status_code == 404 +async def test_usage_counts_queries_and_the_runs_list_counts_runs(client): + """两个数口径不同、说明照实：表的「使用次数」按查询计（同一次运行里查了两次算两次），「用过这张表的运行」按运行计。 + 接口说明以前写「取法同使用次数」，看着该是同一个数，验收时 4 对 3 对不上。""" + sid = await _source() + async with SessionLocal() as session: + name = (await session.get(DataSource, sid)).name + runs = await _use(sid, name, {"orders": 3}) + async with SessionLocal() as session: + session.add(Artifact(id=uuid.uuid4().hex, run_id=runs[0], kind=catalog.QUERY_SNAPSHOT_KIND, + meta={"source_id": sid, "source": name, "tables": ["orders"]})) + await session.commit() + listed = (await client.get(f"/api/datasources/{sid}/catalog")).json() + assert next(t for t in listed["tables"] if t["table_name"] == "orders")["usage"] == 4 + assert (await client.get(f"{_base(sid)}/orders/usage")).json()["total"] == 3 + doc = _fastapi_app.openapi()["paths"]["/api/datasources/{source_id}/catalog/{table}/usage"]["get"]["description"] + assert "取法同使用次数" not in doc and "按运行" in doc and "按查询" in doc, doc + from app.data import catalog_review + + assert "取法同使用次数" not in (catalog_review.table_runs.__doc__ or "") + + # -------------------------------------------------------------------------- # 进度 # -------------------------------------------------------------------------- diff --git a/backend/tests/test_chat_turn_policy.py b/backend/tests/test_chat_turn_policy.py index 2f3bd8d1..f21f7a1c 100644 --- a/backend/tests/test_chat_turn_policy.py +++ b/backend/tests/test_chat_turn_policy.py @@ -48,6 +48,11 @@ def test_next_step_branches(): assert turn.next_step(_final(issues=[ERROR], autorun=False)) == "fail", "没过检查比只建不跑优先" assert turn.next_step({"op": "reply", "text": "已经查过了"}) == "reply" assert turn.next_step({"op": "error", "message": "x"}) == "fail" + empty = {"op": "error", "message": "助手本轮未完成：模型没有给出任何修改，也没有说明原因", + "code": turn.EMPTY_GENERATION, "reason": "模型没有给出任何修改，也没有说明原因"} + assert turn.next_step(empty) == "retry", "只写了思考、一个操作都没给：按重来一次的规矩自动重来" + assert turn.error_reason(empty) == "模型没有给出任何修改，也没有说明原因" + assert turn.error_reason({"op": "error", "message": "助手本轮未完成：接入返回了 502"}) == "接入返回了 502" assert turn.next_step(None) == "asked", "流结束时只有反问" assert set(turn.NEXT_STEPS) == {"run", "ready", "fail"} @@ -167,6 +172,74 @@ async def _run(output: dict, graph: dict, code: str) -> str: return run.id +def test_the_frontend_retries_an_error_the_server_says_to_retry(): + """问数据页照服务端给的 next 走：error 操作带 next=retry（只写了思考、一个操作都没给）时带着原因重来一次，不再直接报 + 「助手本轮未完成」。读 chat.ts 里 error 那一支（行为另由 scripts/check-chat.mjs 的 emptyretry 一节在浏览器里守）。""" + branch = re.search(r"case 'error':(.*?)break\n", _ts(), re.S) + assert branch, "chat.ts 里找不到 error 那一支" + assert re.search(r"op\.next === 'retry'", branch.group(1)) and "retry(" in branch.group(1), branch.group(1) + + +async def test_answer_ops_says_to_retry_an_empty_generation(client, monkeypatch): + """生成过程一个操作都没给（deepseek 偶发只出思考）：error 操作带机读码 copilot_empty、原因、next=retry。""" + import json + + import app.api.copilot as copilot + from tests.test_usage_meter import Scripted + + model = Scripted(model="claude-opus-5", replies=[[""]]) + + async def _model(*_a, **_k): + return model, "claude-opus-5" + + monkeypatch.setattr(copilot, "get_chat_model", _model) + events = [] + async with client.stream("POST", "/api/copilot/generate-stream", + json={"instruction": "上周各地区销售额", "intent": "answer"}) as r: + async for line in r.aiter_lines(): + if line.startswith("data: "): + events.append(json.loads(line[6:])) + [err] = [e for e in events if e["op"] == "error"] + assert err["code"] == turn.EMPTY_GENERATION and err["next"] == "retry", err + assert err["reason"] == "模型没有给出任何修改，也没有说明原因" and err["message"].startswith("助手本轮未完成") + + +async def test_the_eval_driver_retries_an_empty_generation_once(monkeypatch): + """评测的作答（app/eval/answer.Driver）走同一个决定：第一次一个操作都没给就带着原因重来一次，原因照实记在 retries； + 重来还不行才照实报建图失败（助手调用失败），不再试第三次。""" + import json + + import app.api.copilot as copilot + from app.eval import answer + from tests.test_usage_meter import Scripted + + reply = json.dumps({"op": "reply", "text": "上周没有华中区的数据"}, ensure_ascii=False) + for replies, path in (([[""], [reply]], "reply"), ([[""]], "build_failed")): + model = Scripted(model="claude-opus-5", replies=replies) + seen: list[str] = [] + + async def _model(*_a, _m=model, **_k): + return _m, "claude-opus-5" + + monkeypatch.setattr(copilot, "get_chat_model", _model) + real = answer.Driver._generate + + async def spy(self, instruction, base_graph, _seen=seen, _real=real): + _seen.append(instruction) + return await _real(self, instruction, base_graph) + + monkeypatch.setattr(answer.Driver, "_generate", spy) + driver = answer.Driver("上周华中区销售额", datasource_ids=[], ctx=answer.TurnContext( + suite_id="s", suite_name="虚构题集", eval_run_id="e", case_id=None, case_seq=1, actor=None)) + t = await driver.drive() + assert t.path == path and t.attempts == 1 and model.calls == 2, (t.path, t.attempts, model.calls, t.reason) + assert t.retries == ["模型没有给出任何修改，也没有说明原因"], t.retries + assert seen[1] == turn.retry_instruction("上周华中区销售额", t.retries[0], False), seen + if path == "build_failed": + assert t.copilot_failed and t.reason.startswith("助手本轮未完成"), t.reason + assert t.usage["assistant"]["calls"] == 2, "两次生成都是这一题花的" + + async def test_review_says_whether_and_how_to_retry(client, monkeypatch): import app.api.copilot as copilot from app.providers.factory import ProviderNotConfigured diff --git a/backend/tests/test_copilot_answer_ops.py b/backend/tests/test_copilot_answer_ops.py index c100b15c..e65d8d30 100644 --- a/backend/tests/test_copilot_answer_ops.py +++ b/backend/tests/test_copilot_answer_ops.py @@ -5,7 +5,8 @@ - 几种结局（建图交付、直接回答、只问不改、自查改不好、模型出错、没给操作、输出被截断）的 SSE 输出，和抽出来之前 逐条一样（剧本见 SCENARIOS，抽出来之前录下的输出在 fixtures/copilot_answer_ops_baseline.json），末尾只多一条 usage， - final 只多一个 next（问数据一轮的决定，chat/turn.py）； + final 只多一个 next（问数据一轮的决定，chat/turn.py）；error 多一个 next 和原因 reason（一个操作都没给的另带机读码 code， + next 是 retry：问数据页和评测据此自动重来一次）； - 直接迭代 answer_ops 和 SSE 收到的是同一串； - 开流前的拒绝（模型没配好、限定的数据源都不在）照旧是 HTTP 错误，不进流。 @@ -133,6 +134,8 @@ BODY = {"instruction": "数一数门店", "intent": "answer"} #: final 带的 next（chat/turn.next_step）：问数据页照它走 NEXT = {"graph_ok": "run", "graph_fixed": "run", "check_failed": "fail", "ask_only": "ready"} +#: error 带的 (next, code)：只有一个操作都没给的重来一次 +ERROR_NEXT = {"error": ("fail", None), "no_ops": ("retry", "copilot_empty"), "truncated": ("fail", None)} @pytest.mark.parametrize("name", list(SCENARIOS)) @@ -144,6 +147,11 @@ async def test_the_sse_output_is_unchanged_but_for_a_trailing_usage(client, monk # final 多了 next（问数据一轮的决定下沉到后端），别的一个字不变 finals = [op for op in got if op["op"] == "final"] assert [op.pop("next") for op in finals] == ([NEXT[name]] if name in NEXT else []) + errors = [op for op in got if op["op"] == "error"] + assert [(op.pop("next"), op.pop("code", None)) for op in errors] == \ + ([ERROR_NEXT[name]] if name in ERROR_NEXT else []) + for op in errors: + assert op["message"] == f"助手本轮未完成：{op.pop('reason')}", op assert got[:-1] == expected diff --git a/backend/tests/test_eval_runner.py b/backend/tests/test_eval_runner.py index c6e0ee79..a9685e62 100644 --- a/backend/tests/test_eval_runner.py +++ b/backend/tests/test_eval_runner.py @@ -347,6 +347,44 @@ async def test_the_api_starts_estimates_and_reports_progress(client, monkeypatch assert bad.status_code == 422 and bad.json()["code"] == "eval_variant_invalid" +async def test_the_estimate_says_it_cannot_price_instead_of_zero(): + """上一次评测的模型都不在价格目录里（价格目录空）：估算以前写「约 $0」，像不花钱。现在一分都没估出来的写估不出；估出了 + 一部分的写「至少」、另有几次估不出（老总表里每题平均记成 0、却有估不出的调用的，也按估不出算）。""" + from datetime import timedelta + + suite_id = await es.make_suite("none", cases=ask_cases(1)) + now = utcnow() + async with SessionLocal() as session: + session.add(EvalRun(suite_id=suite_id, status="succeeded", created_at=now - timedelta(minutes=2), + summary={"per_case_cost_usd": 0.0, "unpriced_calls": 6})) + await session.commit() + got = await runner.estimate(session, suite_id, 3) + assert got["total_usd"] is None and got["per_item_usd"] is None and got["unpriced_calls"] == 6, got + assert "估不出" in got["note"] and got["basis"], got + async with SessionLocal() as session: + session.add(EvalRun(suite_id=suite_id, status="succeeded", created_at=now - timedelta(minutes=1), + summary={"per_case_cost_usd": 0.02, "unpriced_calls": 2})) + await session.commit() + got = await runner.estimate(session, suite_id, 3) + assert got["total_usd"] == 0.06 and got["at_least"] is True and got["unpriced_calls"] == 2, got + assert "至少" in got["note"] and "2 次" in got["note"], got["note"] + async with SessionLocal() as session: + session.add(EvalRun(suite_id=suite_id, status="succeeded", created_at=now, + summary={"per_case_cost_usd": 0.0, "unpriced_calls": 0})) + await session.commit() + got = await runner.estimate(session, suite_id, 3) + assert got["total_usd"] == 0.0 and not got.get("at_least") and not got.get("unpriced_calls"), "目录价是 0 的才是真不花钱" + + +def test_the_judge_role_keeps_its_unpriced_calls(): + """按角色的用量里裁判那一份以前只有金额和次数：裁判模型不在价格目录里时写成 $0。""" + usage = runner.judge_usage({"claims": {"cost_usd": 0.0, "calls": 2, "priced": False}, + "text": {"cost_usd": 0.0, "calls": 1, "usage": {"unpriced_calls": 1}}}, 0.0) + assert usage == {"cost_usd": 0.0, "calls": 3, "unpriced_calls": 3} + priced = runner.judge_usage({"claims": {"cost_usd": 0.01, "calls": 2, "priced": True}}, 0.01) + assert priced == {"cost_usd": 0.01, "calls": 2, "unpriced_calls": 0} + + async def test_results_give_the_question_actually_asked_and_the_template_apart(client, monkeypatch): """逐题结果的 question 是这一题实际问的（变量按这次评测的截至时间渲染好了，和作答时 Driver 收到的一字不差），模板原文 另放 question_template（下一轮验收：以前 question 是带 {{ schedule.last_week_start }} 的原始模板）。没有变量的题两个一样。""" diff --git a/backend/tests/test_eval_score.py b/backend/tests/test_eval_score.py index b9578a20..a65560ab 100644 --- a/backend/tests/test_eval_score.py +++ b/backend/tests/test_eval_score.py @@ -116,6 +116,59 @@ def test_tables_sets_none_and_ask(): assert score_case(ask, {"status": "ok"}, Answer(run_id="r"), path="ran").verdict == "fail" +def test_a_wrong_answer_shows_the_paired_cell(): + """判错时「答案里的数」给评分时配对上的那一格（按列名、指标、维度配），不是答案引用的第一个值；配不上照实写配不上。""" + case = _case({**EXP, "column": "refund", "label": "退款金额"}) + gold = _gold({"e1": _scalar(1234.5, column="refund")}) + + def cell(value, column, **dims): + return {"value": value, "kind": "cell", "artifact": "a1", "row": 0, "column": column, "dims": dims} + + question = cell("上周退款金额是多少", "问题") + both = score_case(case, gold, Answer(run_id="r", values=[question, cell(36, "单数"), cell(1200.0, "refund")]), + path="ran") + assert both.verdict == "fail" and both.scores[0]["got"] == 1200.0, both.scores + assert both.scores[0]["cell"]["column"] == "refund" + labelled = score_case(case, gold, Answer(run_id="r", values=[cell(36, "单数"), cell(1200.0, "退款金额")]), path="ran") + assert labelled.scores[0]["got"] == 1200.0, "期望的说明（label）和答案的列名一样也算配上" + lone = score_case(case, gold, Answer(run_id="r", values=[question, cell(36, "单数")]), path="ran") + [s] = lone.scores + assert s["outcome"] == "wrong" and "got" not in s, "单数不是退款金额：配不上就不给「答案里的数」" + assert "配不上" in s["reason"] and "单数 36" in s["reason"] and "上周退款" not in s["reason"], s["reason"] + # 按维度配：华东那一行只有一个数时就是它；同一行有几个数、列名又都配不上时不猜 + by_dims = _case({**EXP, "column": "refund", "where": {"region": "华东"}}) + gold = _gold({"e1": _scalar(1234.5, column="refund", where={"region": "华东"}, dims={"region": "华东"})}) + one = score_case(by_dims, gold, Answer(run_id="r", values=[cell(1200.0, "total", region="华东")]), path="ran") + assert one.scores[0]["got"] == 1200.0, one.scores + two = score_case(by_dims, gold, Answer(run_id="r", values=[cell(36, "单数", region="华东"), + cell(1200.0, "total", region="华东")]), path="ran") + assert "got" not in two.scores[0] and "配不上" in two.scores[0]["reason"], two.scores + text_only = score_case(case, _gold({"e1": _scalar(1234.5, column="refund")}), Answer(run_id="r", values=[question]), + path="ran") + assert "got" not in text_only.scores[0] and text_only.scores[0]["reason"] == "答案里没有对得上的数" + + +def test_none_counts_only_numbers_as_giving_data(): + """期望「没有数据」：只有引用了数值格、或叙述里写了数，才算答案给了数。引用的是文字格（地区名）、空值格（NULL）的， + 答案明说没有数据就按没有数据判对。""" + none = _case({"id": "n", "kind": "none", "required": True}) + gold = _gold({"n": {"kind": "none", "rows": 0}}) + empty = [{"artifact": "q", "row_count": 0, "sql": "SELECT 1"}] + text_cell = {"value": "华中", "kind": "cell", "artifact": "q", "row": 0, "column": "region", "dims": {}} + named = score_case(none, gold, Answer(run_id="r", text="华中区没有数据", values=[text_cell], queries=empty), path="ran") + assert named.verdict == "pass" and named.scores[0]["outcome"] == "traced", named.scores + null_cell = {"value": None, "kind": "cell", "artifact": "q", "row": 0, "column": "gmv", "dims": {}} + null = score_case(none, gold, Answer(run_id="r", text="华中区的销售额为空，没有数据", values=[null_cell], + queries=empty), path="ran") + assert null.verdict == "pass", null.scores + number = score_case(none, gold, Answer(run_id="r", text="华中区 12 元", values=[_cell(12.0)], queries=empty), + path="ran") + assert number.verdict == "fail" and number.scores[0]["got"] == 12.0, number.scores + said = score_case(none, gold, Answer(run_id="r", text="华中区 12 元", values=[text_cell], bare=_bare("12 元"), + queries=empty), path="ran") + assert said.verdict == "fail" and said.scores[0]["raw"] == "12", said.scores + + def test_the_path_and_the_gold_decide_before_numbers(): case = _case(EXP) gold = _gold({"e1": _scalar(330.0)}) @@ -195,6 +248,49 @@ async def test_reading_the_sealed_answer_and_the_per_case_sql_match(engine_on, m assert (await score.sql_match(case, gold, answer))["status"] == "proven" +async def test_the_per_case_sql_match_compares_the_query_behind_the_answer(engine_on, monkeypatch, tmp_path): + """逐题 E1 比的是产出答案的那条查询：答案引用了就比被引用的；没引用、这次运行又查了几条时不硬比第一条（往往是探索 + 查询），照实写没法对上是哪条；没引用、只查了一条时它就是唯一可能的那条，按它比、写明依据。""" + from app.db.base import SessionLocal + from app.db.models import DataSource + from app.engine.runner import run_manager + from app.eval import score + from app.ledger import match as ledger_match + from sqlalchemy import select + from tests import ledger_samples as ls + + await ls.fresh_source(tmp_path / "shop.db") + async with SessionLocal() as session: + sid = (await session.execute(select(DataSource.id).where(DataSource.name == ls.CURRENT["source"]))).scalar() + case = {"gold": {"kind": "sql", "datasource_id": sid, "sql": "x"}, "expectations": []} + gold = {"status": "ok", "before": {"rendered": {"sql": ( + "SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' AND paid_at >= '2026-09-01' " + "AND paid_at < '2026-09-08'")}, "sets": []}} + + # 先探了一下有哪些地区，再查销售额；答案没引用查询（数写在叙述里） + graph = ls.chain(ls.node("start", "input"), + ls.node("peek", "tool", tool=ls.tool(), args={"sql": "SELECT DISTINCT region FROM orders"}), + ls.node("q", "tool", tool=ls.tool(), args={"sql": ls.WITH_REFUNDS}), + ls.node("write", "report", instructions="写答案"), + ls.node("out", "output", fields=[{"name": "答案", "value": "{{ nodes.write.text }}"}])) + ls.script(monkeypatch, "本周销售额约 200 元。") + started = await run_manager.start(graph=graph, input_payload={}, run_class="exploratory", origin_kind="canvas") + await ls.wait(started.id) + await ledger_match.drain() + answer = await score.read_answer(started.id) + assert answer.cited == [] and len(answer.queries) == 2, answer.queries + match = await score.sql_match(case, gold, answer) + assert match["status"] == "skipped" and match["cited"] is False and "没有引用查询" in match["reason"], match + assert match.get("query") is None and match.get("aspects") in (None, []), "没法对上是哪条，就不给差在哪" + + # 只查了一条、答案没引用：唯一的那条就是产出答案的，按它比，依据写明 + run = await ls.explore(monkeypatch, ls.EQUIVALENT, text="本周销售额 170.5 元。") + only = await score.read_answer(run.id) + assert only.cited == [] and len(only.queries) == 1 + match = await score.sql_match(case, gold, only) + assert match["status"] == "proven" and match["basis"] == "only_query" and not match["cited"], match + + async def test_rescoring_reads_the_seal_and_never_calls_a_model(engine_on, monkeypatch, tmp_path): from app.db.base import SessionLocal from app.db.models_eval import EvalResult, EvalRun diff --git a/backend/tests/test_eval_stats.py b/backend/tests/test_eval_stats.py index 1b809646..eb997cba 100644 --- a/backend/tests/test_eval_stats.py +++ b/backend/tests/test_eval_stats.py @@ -69,6 +69,18 @@ def test_summary_counts_only_gradable_items_and_splits_roles(): assert summary["e1"]["passed"] == 1 and summary["e1"]["n"] == 2 and summary["e1"]["aspects"] == {"time": 1} +def test_the_per_case_cost_is_not_zero_when_calls_could_not_be_priced(): + """模型不在价格目录里（价格目录空）：金额 0 只是「估得出的那部分」，每题平均花费不能写成 0（像不花钱）——一分都没估出来 + 的为 None（估不出），估出了一部分的照给、unpriced_calls 说明它只是下限。""" + unpriced = {"assistant": {"by_model": {}, "calls": 4, "cost_usd": 0.0, "input_tokens": 900, "output_tokens": 90, + "unpriced_calls": 4}} + summary = stats.summarize([_row("c1", "pass", usage=unpriced), _row("c2", "fail", usage=unpriced)]) + assert summary["unpriced_calls"] == 8 and summary["per_case_cost_usd"] is None, summary["per_case_cost_usd"] + partly = {"assistant": {**unpriced["assistant"], "cost_usd": 0.02, "unpriced_calls": 1}} + summary = stats.summarize([_row("c1", "pass", usage=partly), _row("c2", "fail", usage=partly)]) + assert summary["per_case_cost_usd"] == 0.02 and summary["unpriced_calls"] == 2 + + def test_repeats_give_a_pass_rate_per_case(): rows = [_row("c1", "pass", repeat=0), _row("c1", "fail", repeat=1), _row("c1", "pass", repeat=2)] summary = stats.summarize(rows) diff --git a/backend/tests/test_look_large.py b/backend/tests/test_look_large.py index aaad0f73..584435e8 100644 --- a/backend/tests/test_look_large.py +++ b/backend/tests/test_look_large.py @@ -236,3 +236,20 @@ async def test_out_of_scope_parent_is_not_called_missing(big): checks = catalog_profile.plan_checks("ord_hdr_east", tables["ord_hdr_east"], {"relations": [rel]}, tables=tables, masked=set(), outside=outside) assert [c for c in checks if c.kind == "relation"][0].skip[0] == "missing" + + +def test_out_of_scope_objects_without_a_comment_match_only_their_english_name(): + """范围外、又没有注释的对象只有英文名可检索（data/schema_view 模块说明写明的限制）：英文名拆出的词（catalog.name_words） + 没有中文说法——仓库里没有现成的英中业务词对照，不临时编词表，中文需求命不中只叫 refund_orders 的表；有注释的照常命中。 + 以后接了词表，这条要跟着改。""" + from types import SimpleNamespace + + from app.data import schema_view + + source = SimpleNamespace(schema_cache={"tables": {}}, options={}) + outside = [SimpleNamespace(name="refund_orders", comment=None), + SimpleNamespace(name="cust_complaint", comment="客户投诉单")] + search = schema_view.TableSearch(source, {}, outside) + assert [h.name for h in search.search("refund", limit=5)] == ["refund_orders"], "英文名、拆出的词照常命中" + assert "refund_orders" not in [h.name for h in search.search("上周的退款", limit=5)], "没有注释、没有词表：中文需求命不中英文名" + assert [h.name for h in search.search("投诉", limit=5)] == ["cust_complaint"], "有注释的按注释命中" diff --git a/backend/tests/test_usage_meter.py b/backend/tests/test_usage_meter.py index 828a8130..894a63cf 100644 --- a/backend/tests/test_usage_meter.py +++ b/backend/tests/test_usage_meter.py @@ -96,7 +96,8 @@ async def test_metering_counts_by_model_and_prices_from_the_catalog(): await Scripted(model=FREE, report=False).ainvoke("你好") got = m.summary() assert got["by_model"][PRICED] == {"input_tokens": 2000, "output_tokens": 400, "calls": 2, - "cost_usd": round(estimate_cost(PRICED, 2000, 400), 6), "unpriced_calls": 0} + "cost_usd": round(estimate_cost(PRICED, 2000, 400), 6), "unpriced_calls": 0, + "unfinished_calls": 0} assert got["by_model"][UNPRICED]["unpriced_calls"] == 1 and got["by_model"][UNPRICED]["cost_usd"] == 0 # 目录外的模型名（多了后缀）照样算估不出；目录价是 0 的本地模型没报用量也不算估不出 assert got["by_model"][PRICED + "-silent"]["unpriced_calls"] == 1 @@ -184,6 +185,64 @@ async def test_the_eval_driver_on_the_demo_model_records_its_name(monkeypatch): assert set(got) == {"mock-fast"} and got["mock-fast"]["calls"] >= 1, got +class Slow(Scripted): + """回得很慢的模型：调用方等不及（挑表超时被 asyncio.wait_for 取消）。partial=True 时先报一段输入用量再卡住。""" + + delay: float = 5.0 + partial: bool = False + + async def _agenerate(self, messages, stop=None, run_manager=None, **kwargs) -> ChatResult: + self.calls += 1 + await asyncio.sleep(self.delay) + return ChatResult(generations=[ChatGeneration(message=AIMessage(content="晚了"))]) + + async def _astream(self, messages, stop=None, run_manager=None, **kwargs): + self.calls += 1 + if self.partial: + yield ChatGenerationChunk(message=AIMessageChunk(content="{", usage_metadata={ + "input_tokens": 900, "output_tokens": 0, "total_tokens": 900})) + await asyncio.sleep(self.delay) + yield ChatGenerationChunk(message=AIMessageChunk(content="晚了")) + + +class Refused(Scripted): + """请求刚发出去就被拒了（鉴权错、参数不对）：模型没干活，不算一次调用。""" + + async def _agenerate(self, messages, stop=None, run_manager=None, **kwargs) -> ChatResult: + raise ValueError("401 鉴权失败") + + +async def test_a_call_cancelled_before_any_result_still_counts_as_an_unfinished_call(): + """挑表超时被取消的那次调用以前一点都不计（LangChain 取消时连 on_llm_error 都不一定回调）：调用次数少了一次。现在 + 记一次调用、token 不知道（没有部分结果的记 0、算没报全），标「未完成」；有部分用量的照实记已报的；被拒的仍不计。""" + from app.look import pick + + m = usage.meter() + with usage.metering(m): + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(Slow(model=PRICED).ainvoke("挑表"), timeout=0.05) + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(pick._ask(Slow(model=UNPRICED), [("human", "挑表")]), timeout=0.05) + + async def read(): + async for _ in Slow(model=PRICED + "-partial", partial=True).astream("挑表"): + pass + + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(read(), timeout=0.05) + with pytest.raises(ValueError): + await Refused(model=PRICED + "-refused").ainvoke("挑表") + got = m.summary() + rows = got["by_model"] + assert (rows[PRICED]["calls"], rows[PRICED]["unfinished_calls"], rows[PRICED]["input_tokens"]) == (1, 1, 0), rows + assert rows[PRICED]["unpriced_calls"] == 1, "token 不知道，金额也就估不出" + assert (rows[UNPRICED]["calls"], rows[UNPRICED]["unfinished_calls"]) == (1, 1), "挑表（look/pick._ask）超时同样计" + partial = rows[PRICED + "-partial"] + assert (partial["calls"], partial["unfinished_calls"], partial["input_tokens"]) == (1, 1, 900), partial + assert PRICED + "-refused" not in rows, "被拒的不算一次调用" + assert got["calls"] == 3 and got["unfinished_calls"] == 3 + + async def test_outside_the_meter_nothing_is_counted(): m = usage.meter() with usage.metering(m): diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index aa1d4356..8f01ebb6 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -783,6 +783,12 @@ export const TURN_COST_TEXT = { `其中 ${formatNumber(n)} 次调用的金额无法估算（模型不在价格目录中，或用量没有报全），未计入，实际花费不低于此数`, } as const +/** 问数据一轮自动重来一次时的进度说明（服务端 chat/turn.next_step 判的 retry） */ +export const CHAT_RETRY_TEXT = { + /** 助手上一次只写了思考、一个操作都没给（error.next = retry） */ + empty: '助手上一次没有给出任何操作，正在重新生成工作流…', +} as const + export const JUDGE_SETTING_TEXT = { label: '证据裁判', hint: '报告中的结论句（如「增长主要来自新客首单」）由另一个模型按证据逐句判断：正式运行由报告撰写节点当场裁判，探索运行在点击时逐句裁判。判断由模型给出，不是系统核对，界面上一律标注「非确定」。', @@ -2314,7 +2320,7 @@ export const CATALOG_TEXT = { missing: '表结构中已没有这张表', noCatalog: '没有目录', usage: (n: number) => `使用 ${formatNumber(n)} 次`, - usageHint: '被运行查询过的次数，同一次运行中结果相同的重复查询只算一次', + usageHint: '按查询算：被运行查询过的次数，同一次运行里查了几次算几次（结果相同的重复查询只算一次）；「用过这张表的运行」按运行算，两个数可能不一样', relationCount: (n: number) => `${formatNumber(n)} 个关联`, countTitle: (label: string, n: number) => `${label} ${formatNumber(n)} 项`, noMatch: '没有符合条件的表', @@ -6092,6 +6098,8 @@ export const APP_EDITOR_TEXT = { slotAdd: '新增岗位', slotName: '岗位名', slotLabel: '显示名', + /** 显示名和岗位名一样：派生时没有节点名可用（服务端 apps/tree.slot_label），照实用的是岗位名，不是起好的名字 */ + slotLabelRaw: '还没有起显示名，现在用的是图里的岗位名，可以改成一个好认的名字', slotUnbound: '未绑定：发布检查会拦下流程应用', slotRemove: (name: string) => `删除岗位「${name}」`, grants: '授权', diff --git a/frontend/src/lib/termsEval.ts b/frontend/src/lib/termsEval.ts index 348f9404..85bd5f31 100644 --- a/frontend/src/lib/termsEval.ts +++ b/frontend/src/lib/termsEval.ts @@ -356,6 +356,8 @@ export const EVAL_TEXT = { estimateText: (items: number, cases: number, repeats: number, arms: number) => `${formatNumber(cases)} 道题 × 每题 ${repeats} 次${arms > 1 ? ` × ${arms} 臂` : ''}，共 ${formatNumber(items)} 项`, cost: (total: string, per: string) => `约 ${total}（每项约 ${per}）`, + /** 上一次有估不出金额的调用：估出来的只是下限 */ + costAtLeast: (total: string, per: string) => `至少约 ${total}（每项至少约 ${per}）`, costNone: '估不出花费', budget: '生效的预算', parallel: (n: number) => `同时评 ${n} 题`, @@ -416,6 +418,8 @@ export const EVAL_TEXT = { cost: '花费', costHint: '助手（建图、挑表、复核）+ 写作（答案运行里的模型调用）+ 结论裁判；算期望不调模型', perCase: (v: string) => `每题约 ${v}`, + /** 有估不出金额的调用：每题平均只是下限 */ + perCaseAtLeast: (v: string) => `每题至少约 ${v}`, e1: '逐题的口径命中（E1）', e1Hint: '答案执行的 SQL 和期望的取数被证明等价的比例（口径账本同一个比对函数）', citations: '报告引用率', @@ -441,7 +445,9 @@ export const EVAL_TEXT = { timing: { build_ms: '建图', run_wall_ms: '运行（墙钟）', run_active_ms: '运行（执行）', review_ms: '复核', total_ms: '整题' } as Record, usage: '按角色的用量', tokens: (input: string, output: string) => `输入 ${input} · 输出 ${output}`, - callsN: (n: number) => `${formatNumber(n)} 次调用`, + /** unfinished：没做完的调用（被取消、超时，token 只计了已报的；服务端 providers/usage 的 unfinished_calls） */ + callsN: (n: number, unfinished = 0) => + `${formatNumber(n)} 次调用${unfinished > 0 ? `（其中 ${formatNumber(unfinished)} 次未完成：被取消或超时，token 只计了已报的部分）` : ''}`, role: { assistant: '助手', writer: '写作', judge: '结论裁判', gold: '算期望' } as Record, passRates: '重复评的题的通过率', outcomes: '各期望的结论', @@ -464,6 +470,10 @@ export const EVAL_TEXT = { sqlMatch: { proven: '等价已证明', related: '不等价', none: '答案没有查询', skipped: '没比对', } as Record, + /** 逐题 SQL 比对比的是哪条查询（服务端 score.sql_match 的 basis）。答案引用了查询的不用多说 */ + sqlMatchBasis: { + only_query: '答案没有引用查询；这次运行只执行了这一条查询（核查用的不算），按它比', + } as Record, aspect: { time: '时间口径', time_range: '时间范围', filter: '筛选条件', aggregate: '聚合', expression: '计算式', join: '关联', tables: '用到的表', group_by: '分组', limit: '行数限制', other: '其他', parse: '解析不了', @@ -495,6 +505,8 @@ export const EVAL_TEXT = { filter: '只看', repeat: (n: number) => `第 ${n + 1} 次`, attempts: (n: number) => `这一题自动重来了 ${n} 次`, + /** 每次自动重来的原因（服务端 eval/answer 的 turn.retries） */ + retriesWhy: (reasons: string[]) => `：${reasons.join('；')}`, turnReason: '作答的经过', answer: '答案原文', expand: '展开', diff --git a/frontend/src/pages/apps/edit/AccessSections.tsx b/frontend/src/pages/apps/edit/AccessSections.tsx index bd031221..4dbb764a 100644 --- a/frontend/src/pages/apps/edit/AccessSections.tsx +++ b/frontend/src/pages/apps/edit/AccessSections.tsx @@ -60,6 +60,7 @@ export function AccessSection() { setBindings((b) => { const next = { ...b.slots }; delete next[name]; return { ...b, slots: next } }) }} /> + {slot.label.trim() === name &&

{T.slotLabelRaw}

} setBindings((b) => ({ ...b, slots: { ...b.slots, [name]: v.filter((x) => x.kind !== 'everyone') as { kind: 'user' | 'group'; name: string }[] }, diff --git a/frontend/src/pages/evals/EvaluationPage.tsx b/frontend/src/pages/evals/EvaluationPage.tsx index bef39bc5..37d0e35a 100644 --- a/frontend/src/pages/evals/EvaluationPage.tsx +++ b/frontend/src/pages/evals/EvaluationPage.tsx @@ -215,8 +215,7 @@ function Overview({ run }: { run: EvalRun }) {
- +
@@ -278,7 +277,7 @@ function Overview({ run }: { run: EvalRun }) {
  • {O.role[role]}：{usageCost(u)} {u.input_tokens != null && · {O.tokens(formatTokens(u.input_tokens, { compact: true }), formatTokens(u.output_tokens ?? 0, { compact: true }))}} - {u.calls != null && · {O.callsN(u.calls)}} + {u.calls != null && · {O.callsN(u.calls, u.unfinished_calls ?? 0)}}
    • ) })} @@ -296,6 +295,17 @@ function Overview({ run }: { run: EvalRun }) { ) } +/** + * 每题平均花费：有估不出金额的调用时只是下限（写「至少」）；一分都没估出来的不写（老总表记成 0 的也一样，金额那一格已经写了 + * 「金额无法估算」） + */ +function perCaseText(s: EvalSummary): string | undefined { + const per = s.per_case_cost_usd + if (per == null) return undefined + if (!s.unpriced_calls) return T.overview.perCase(formatCost(per)) + return per > 0 ? T.overview.perCaseAtLeast(formatCost(per)) : undefined +} + function Section({ title, children, attr }: { title: string; children: ReactNode; attr?: string }) { return (
    @@ -417,6 +427,9 @@ function Results({ id, live, focusCase }: { id: string; live: boolean; focusCase {s.reason && · {s.reason}}

    ))} + {r.sql_match?.basis && T.sqlMatchBasis[r.sql_match.basis] && ( +

    {T.sqlMatchBasis[r.sql_match.basis]}

    + )} {r.sql_match?.reason &&

    {r.sql_match.reason}

    }
    @@ -450,7 +463,11 @@ function ResultDetail({ evaluationId, resultId }: { evaluationId: string; result const R = T.results return (
    - {(turn.attempts ?? 0) > 0 &&

    {R.attempts(turn.attempts)}

    } + {(turn.attempts ?? 0) > 0 && ( +

    + {R.attempts(turn.attempts)}{turn.retries?.length ? R.retriesWhy(turn.retries) : ''} +

    + )} {turn.reason &&

    {R.turnReason}：{turn.reason}

    } {answer && (
    diff --git a/frontend/src/pages/evals/StartDialog.tsx b/frontend/src/pages/evals/StartDialog.tsx index 62a30944..19386493 100644 --- a/frontend/src/pages/evals/StartDialog.tsx +++ b/frontend/src/pages/evals/StartDialog.tsx @@ -263,9 +263,8 @@ export function StartDialog({ open, onClose, suite, evaluations, onStarted }: { <>

    {T.estimateText(est.estimate.items, est.cases, est.repeats, est.arms.length)}

    - {est.estimate.total_usd != null - ? T.cost(formatCost(est.estimate.total_usd), formatCost(est.estimate.per_item_usd)) - : T.costNone} + {est.estimate.total_usd == null ? T.costNone + : (est.estimate.at_least ? T.costAtLeast : T.cost)(formatCost(est.estimate.total_usd), formatCost(est.estimate.per_item_usd))} · {est.estimate.note}

    {est.arms.map((a, i) => ( diff --git a/frontend/src/store/chat.ts b/frontend/src/store/chat.ts index 1f8f2bed..c50c3252 100644 --- a/frontend/src/store/chat.ts +++ b/frontend/src/store/chat.ts @@ -2,7 +2,7 @@ import { create } from 'zustand' import { ApiError, api, streamCopilot, streamRun } from '../api/client' import type { CopilotOp, FixKind, RunFinal } from '../run/decode' import { copilotContext, copilotLook, decodeRun, looksOf } from '../run/decode' -import { CHAT_BLUEPRINT_TEXT, LEDGER_TEXT, LOOK_TEXT, REJECTED_TEXT } from '../lib/terms' +import { CHAT_BLUEPRINT_TEXT, CHAT_RETRY_TEXT, LEDGER_TEXT, LOOK_TEXT, REJECTED_TEXT } from '../lib/terms' import type { LookEvent } from '../typesApp' import { errorMessage, humanizeError, isNetworkError } from '../lib/errors' import { guardReview, hasEvidence } from '../lib/evidence' @@ -174,6 +174,8 @@ export interface UsageTotal { cost_usd?: number | null unpriced_calls?: number calls?: number + /** 没做完的调用（被取消、超时）：算在 calls 里，token 只计了已报的（服务端 providers/usage 的 unfinished_calls） */ + unfinished_calls?: number input_tokens?: number output_tokens?: number } @@ -498,7 +500,7 @@ function buildMeta(t: ChatTurn): TurnMeta { } } -const USAGE_KEYS = ['cost_usd', 'unpriced_calls', 'calls', 'input_tokens', 'output_tokens'] as const +const USAGE_KEYS = ['cost_usd', 'unpriced_calls', 'calls', 'unfinished_calls', 'input_tokens', 'output_tokens'] as const /** 两份用量相加（缺的当 0）。b 不是对象时原样交回 a */ export function addUsage(a: UsageTotal | undefined, b: UsageTotal | null | undefined): UsageTotal | undefined { @@ -1417,14 +1419,14 @@ function plan(flow: Flow, args: PlanArgs) { * 这里会清空这一次的过程和答案——它是自动补救，用户还没拿到过答案。用户主动 * 点的重试走 retryTurn，上一次要留档。 */ - const retry = (reason: string, instruction?: string) => { + const retry = (reason: string, instruction?: string, status?: string) => { // 服务端给了那句话（复核的 retry_instruction）就是它已经按次数判过可以重来；老服务端由这里按次数判 if ((!instruction && attempt + 1 >= MAX_ATTEMPTS) || !flow.alive()) return false const base = built && (built.nodes?.length ?? 0) > 0 ? built : null patch((t) => ({ attempt: attempt + 1, phase: 'planning', - status: base ? '上次运行未成功，正在调整工作流…' : '上次运行未成功，正在重新生成工作流…', + status: status ?? (base ? '上次运行未成功，正在调整工作流…' : '上次运行未成功，正在重新生成工作流…'), ops: [], events: [], run: null, output: null, final: undefined, runStatus: undefined, review: null, reviewRaw: null, rawOutput: null, lastSeq: 0, failure: null, error: '', // 换下来的那次运行花的钱还是这一轮的 @@ -1584,9 +1586,12 @@ function plan(flow: Flow, args: PlanArgs) { void flow.persist() break case 'error': + settled = true + // 生成过程一个操作都没给（模型只写了思考）：服务端说重来一次（error.next = retry，backend/app/chat/turn.py 的 + // next_step），照它带着原因重来，次数照 MAX_ATTEMPTS；试满了才照实报。这一次建图的用量随后那条 usage 照样记进这一轮 + if (op.next === 'retry' && retry(String(op.reason ?? op.message ?? ''), undefined, CHAT_RETRY_TEXT.empty)) break // 后端给的是三段：message 人话、hint 怎么办、detail 原始异常。 // 原文不再截成一行甩给用户，收进可展开、可复制的技术细节 - settled = true fail({ ...failureOf(op.message ?? '生成失败'), ...(op.hint ? { hint: String(op.hint) } : {}), diff --git a/frontend/src/typesEval.ts b/frontend/src/typesEval.ts index 39d3e185..77f4b743 100644 --- a/frontend/src/typesEval.ts +++ b/frontend/src/typesEval.ts @@ -365,7 +365,11 @@ export interface EvalEstimate { arms: { label: string | null; variant: EvalVariant; models: EvalResolvedModels; text: string }[] cases: number repeats: number - estimate: { items: number; per_item_usd: number | null; total_usd: number | null; basis: string | null; note: string } + /** at_least：上一次另有 unpriced_calls 次调用金额估不出，估算只是下限；total_usd 为空、unpriced_calls 有值：一分都没估出来 */ + estimate: { + items: number; per_item_usd: number | null; total_usd: number | null; basis: string | null; note: string + at_least?: boolean; unpriced_calls?: number + } budget: EvalBudget parallel: number spend: EvalSpend @@ -415,7 +419,10 @@ export interface EvalUsage { calls?: number cost_usd?: number | null unpriced_calls?: number - by_model?: Record + /** 没做完的调用（被取消、超时）：算在 calls 里，token 只计了已报的 */ + unfinished_calls?: number + by_model?: Record } export interface EvalTiming { @@ -531,6 +538,8 @@ export interface EvalSqlMatch { aspects?: string[] reason?: string | null cited?: boolean + /** 比的是哪条查询：cited 答案引用的；only_query 答案没引用、这次运行只查了这一条 */ + basis?: 'cited' | 'only_query' | string } export interface EvalResult { diff --git a/scripts/check-chat.mjs b/scripts/check-chat.mjs index d69eafa9..a09dd895 100644 --- a/scripts/check-chat.mjs +++ b/scripts/check-chat.mjs @@ -29,7 +29,8 @@ // 14. 助手反问（normal-user-apps N11）：只问不改的一轮记成「助手反问」，问题作为这一轮的内容、落库进历史，不再一直停在 // 「正在理解需求」；输入框提示回答助手的问题；刷新后照样标「助手反问」，不当成没查库的回答； // 15. 回答上的「做成应用」带着这次运行进蓝图（建运行起点的蓝图），不再提取工作流、跳画布的发布弹窗； -// 16. 服务端这一轮已经自查修正过（收到 check 的结论）还没过的：不再整轮重新生成一遍，直接说没过、给补救。 +// 16. 服务端这一轮已经自查修正过（收到 check 的结论）还没过的：不再整轮重新生成一遍，直接说没过、给补救；助手只写了思考、 +// 一个操作都没给（error.next=retry）的照服务端的决定自动重来一次，还不行再照实报。 // 17. 口径账本（next-round 02）：交付了的轮次底部写「正在比对口径」，按退避再取到比对出结果、比完不再取；一行摘要 // （几个数来自哪份口径）、按口径正式出具和对账按钮照服务端给的动作摆；出具、对账的写请求在浏览器层拦下并核对请求体， // 出具过的正式运行挂在轮次下面（档位、打开运行）。 @@ -532,6 +533,26 @@ async function fakeApi(route) { return route.fulfill({ status: 200, contentType: 'text/event-stream', body: ops.map((o) => `data: ${JSON.stringify(o)}\n\n`).join('') }) } + if (String(b.instruction ?? '').includes('只写了思考')) { + // 生成过程一个操作都没给（deepseek 偶发只出思考）：服务端的 error 带 next=retry（backend/app/chat/turn.py 的 next_step）。 + // 重来那一次（指令里缀了「上一次没有成功」）照常直接回答；「两次都」那一问每次都只写思考 + const again = String(b.instruction).includes('上一次没有成功') && !String(b.instruction).includes('两次都') + const spent = { by_model: { m: { input_tokens: 500, output_tokens: 40, calls: 1, cost_usd: 0.001, unpriced_calls: 0 } }, + input_tokens: 500, output_tokens: 40, calls: 1, cost_usd: 0.001, unpriced_calls: 0 } + const ops = again ? [ + { op: 'heartbeat', phase: 'planning', elapsed_ms: 200 }, + { op: 'reply', text: '已经查过了：一共 3 家门店。' }, + { op: 'usage', ...spent }, + ] : [ + { op: 'heartbeat', phase: 'planning', elapsed_ms: 200 }, + { op: 'thinking', delta: '先想一想要查哪张表……' }, + { op: 'error', message: '助手本轮未完成：模型没有给出任何修改，也没有说明原因', hint: '画布未修改。可以重试，或把需求描述得更具体', + detail: '', reason: '模型没有给出任何修改，也没有说明原因', code: 'copilot_empty', next: 'retry' }, + { op: 'usage', ...spent }, + ] + return route.fulfill({ status: 200, contentType: 'text/event-stream', + body: ops.map((o) => `data: ${JSON.stringify(o)}\n\n`).join('') }) + } if (String(b.instruction ?? '').includes('照服务端的决定') || String(b.instruction ?? '').includes(DECIDE_RETRY)) { // 新服务端：final 带 next（backend/app/chat/turn.py），最后一条是这一轮建图的用量（answer_ops 的 usage）。 // 「只建不跑」那一问：autorun 没写 false，next 却是 ready——前端得照 next 走 @@ -1858,6 +1879,33 @@ for (const theme of THEMES) { await ctx.close() }) + await section('emptyretry', '助手只写了思考、一个操作都没给：照服务端的 next=retry 自动重来一次，还不行再照实报', async () => { + const { page, ctx, errors } = await open(theme) + await goto(page, 'c0emptg') + let gens = log.gens.length + await send(page, '只写了思考：有几家门店') + const done = await page.waitForFunction(() => window.__chat.getState().byConversation.c0emptg?.at(-1)?.phase === 'done', + null, { timeout: 6000 }).then(() => true, () => false) + await page.waitForTimeout(300) + const turn = (await chatState(page)).byConversation.c0emptg.at(-1) + check('第一次没给操作：自动重来一次，第二次照常回答', done && log.gens.length - gens === 2 + && String(turn.output?.answer ?? '').includes('3 家'), `${log.gens.length - gens} 次生成 · ${JSON.stringify(turn.output)}`) + check('重来时交回去的话带着原因（同服务端 retry_instruction 的从零重建措辞）', + log.gens.at(-1)?.instruction === '只写了思考：有几家门店\n\n（上一次没有成功：模型没有给出任何修改，也没有说明原因。请针对这个原因调整，重新给出完整的工作流。）', + log.gens.at(-1)?.instruction) + check('两次建图的用量都记进这一轮', Math.abs((turn.usage?.copilot?.cost_usd ?? 0) - 0.002) < 1e-9, JSON.stringify(turn.usage)) + gens = log.gens.length + await send(page, '两次都只写了思考：有几家门店') + const failed = await page.waitForFunction(() => window.__chat.getState().byConversation.c0emptg?.at(-1)?.phase === 'error', + null, { timeout: 6000 }).then(() => true, () => false) + await page.waitForTimeout(500) + const last = (await chatState(page)).byConversation.c0emptg.at(-1) + check('重来还不行：照实报「助手本轮未完成」，不再试第三次', failed && log.gens.length - gens === 2 + && JSON.stringify(last.failure ?? {}).includes('助手本轮未完成'), `${log.gens.length - gens} 次生成 · ${JSON.stringify(last.failure)}`) + check('没有运行时报错', errors.length === 0, errors[0] ?? '') + await ctx.close() + }) + await section('decide', '新服务端：照 final.next 和复核的 retry_instruction 走，这一轮的花费 = 助手 + 运行 + 复核', async () => { const { page, ctx, errors } = await open(theme) await goto(page, 'c0emptg') diff --git a/scripts/check-evals.mjs b/scripts/check-evals.mjs index db3f53b1..8f669ff9 100644 --- a/scripts/check-evals.mjs +++ b/scripts/check-evals.mjs @@ -9,6 +9,7 @@ // 数值期望不许人填（请求体里不出现 value 这类键）； // - 口径升版待处置：顶部提示、处置三选一（叫法由服务端给，没有缺省）；启用开关、导出； // - 发起评测：估算和发起是同一份请求体，评一次 / A/B 对照 / 目录贡献对照（预设）各自的请求体；估算的项数、花费、今天已花照服务端写； +// 上一次的调用金额估不出时估算写「估不出花费」、估出一部分的写「至少约」，总览的每题平均不写成 $0； // - 评测详情：进度、每题正在做什么；停止、只重试没做完的、重评、设为基线——摆不摆按钮照服务端的 actions；总览卡片和各项分布； // 逐题表（结论、期望和答案里的数、SQL 差在哪、点开去运行详情）、?case= 定位那道题；和上一次比（指纹差异、转移、归因、换基准）； // A/B 配对表（都对、只 A 对、只 B 对、p 值、看不出差别）； @@ -695,6 +696,41 @@ await section('加题：选口径、从运行认定、期望反问', async () => // ---------------------------------------------------------------- 发起评测 +await section('发起评测：估不出金额时估算和总览不写成 $0', async () => { + // 服务端 eval/runner.estimate：上一次的调用金额都估不出 → total_usd 为空、带次数；估出一部分 → at_least + const shaped = (patch) => ({ ...ESTIMATE(['当前目录']), estimate: { ...ESTIMATE(['当前目录']).estimate, ...patch } }) + const none = shaped({ per_item_usd: null, total_usd: null, unpriced_calls: 6, + note: '上一次评测的 6 次模型调用金额都估不出（模型不在价格目录中，或用量没有报全）：金额上限只管得住估得出金额的调用' }) + const some = shaped({ at_least: true, unpriced_calls: 2, + note: '按这个题集上一次评测的每题平均花费估算，实际至少这么多：上一次另有 2 次调用金额估不出，没有算进去' }) + for (const [label, body] of [['都估不出', none], ['估出一部分', some]]) { + const state = freshState({ override: (method, path) => (method === 'POST' && path.endsWith('/evaluations/estimate') + ? { json: body } : null) }) + const { page, errors } = await open('/evals/suites/suite-a', { state }) + await page.locator('[data-eval-suite-start]').click() + await page.locator('[data-eval-estimate-items]').waitFor() + const cost = await text(page, '[data-eval-estimate-cost]') + if (label === '都估不出') { + check('估算：上一次都估不出时写「估不出花费」和原因，不写 $0', cost.includes('估不出花费') && cost.includes('6 次') && !cost.includes('$0'), cost) + } else { + check('估算：估出一部分时写「至少约」', cost.includes('至少约 $0.042') && cost.includes('每项至少约 $0.014') && cost.includes('2 次'), cost) + } + check(`没有页面报错（${label}）`, !errors.length, errors.join(' ｜ ')) + } + const state = freshState() + state.evaluations['ev-1'].summary = { ...state.evaluations['ev-1'].summary, cost_usd: 0, unpriced_calls: 6, per_case_cost_usd: null } + const { page, errors } = await open('/evals/runs/ev-1', { state }) + await page.locator('[data-eval-overview]').waitFor() + const card = await text(page, '[data-eval-card="cost"]') + check('总览：都估不出时花费写「金额无法估算」，不写每题 $0', card.includes('金额无法估算') && !card.includes('每题'), card) + state.evaluations['ev-1'].summary = { ...state.evaluations['ev-1'].summary, cost_usd: 0.042, unpriced_calls: 2, per_case_cost_usd: 0.014 } + await page.reload({ waitUntil: 'networkidle' }) + await page.locator('[data-eval-overview]').waitFor() + const partly = await text(page, '[data-eval-card="cost"]') + check('总览：估出一部分时写「≥」和「每题至少约」', partly.includes('≥ $0.042') && partly.includes('每题至少约 $0.014'), partly) + check('没有页面报错（总览）', !errors.length, errors.join(' ｜ ')) +}) + await section('发起评测：估算和发起同一份请求体；评一次、A/B、目录贡献对照', async () => { const { page, state, errors } = await open('/evals/suites/suite-a') await page.locator('[data-eval-suite-start]').click()