mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 03:01:58 +00:00
合入 nr6/b:评测「期望没有数据」只认数值格和叙述里的数、逐题 E1 比产出答案的那条查询、判错时给配对上的那一格、估不出金额时不写 0、没做完的模型调用也计一次、助手只给思考不给操作时自动重来一次(问数据与评测同一个决定)、目录两种计数照实说、自由形应用入口与岗位显示名;评分规则升到第 2 版
This commit is contained in:
@@ -844,7 +844,9 @@ async def review_catalog_batch(source_id: str, payload: CatalogBatchReviewIn,
|
||||
async def catalog_table_runs(source_id: str, table: str,
|
||||
limit: int = Query(20, ge=1, le=catalog_review.RUNS_LIMIT_MAX),
|
||||
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
"""用过这张表的运行(total 是几次运行),新的在前;取法同使用次数(catalog.table_usage)。"""
|
||||
"""用过这张表的运行,新的在前。按运行计:total 是几次运行,一次运行查了这张表几次都只算一次。和表清单的使用次数
|
||||
(catalog.table_usage)数的是同一批查询快照,但使用次数按查询计(同一次运行里查了两次算两次),所以使用次数可能比 total
|
||||
大。"""
|
||||
_, source = await _resolved(session, source_id)
|
||||
key, _ = await _table_key(session, source, table)
|
||||
return await catalog_review.table_runs(session, source, key, limit=limit)
|
||||
|
||||
@@ -1751,14 +1751,20 @@ async def answer_ops(payload: GenerateIn, *, catalog_view: Any = None):
|
||||
# 流正常结束,却没有 done、也没有一个操作能认(格式跑偏,每一行都被跳过):
|
||||
# 往下走就是拿原图去自查、交付,界面上写「无需修改」——而模型什么都没说
|
||||
if gen.asked is None and all(o.get("op") == "plan" for o in applier.seen):
|
||||
# 只写了思考(deepseek 这类推理模型偶发)、或者格式跑偏:带机读码,问数据页和评测按 chat/turn 的规矩自动
|
||||
# 重来一次(error.next = retry),还不行再照实报
|
||||
raise CopilotIncomplete("模型没有给出任何修改,也没有说明原因",
|
||||
hint="画布未修改。可以重试,或把需求描述得更具体")
|
||||
hint="画布未修改。可以重试,或把需求描述得更具体", code=turn.EMPTY_GENERATION)
|
||||
except CopilotIncomplete as e:
|
||||
yield {"op": "error", "message": f"助手本轮未完成:{e}", "hint": e.hint, "detail": e.detail}
|
||||
failed = {"op": "error", "message": f"助手本轮未完成:{e}", "hint": e.hint, "detail": e.detail,
|
||||
"reason": str(e), **({"code": e.code} if e.code else {})}
|
||||
yield {**failed, "next": turn.next_step(failed)}
|
||||
return
|
||||
except Exception as e: # noqa: BLE001
|
||||
reason, hint = explain_error(e)
|
||||
yield {"op": "error", "message": f"助手本轮未完成:{reason}", "hint": hint, "detail": raw_error(e)}
|
||||
failed = {"op": "error", "message": f"助手本轮未完成:{reason}", "hint": hint, "detail": raw_error(e),
|
||||
"reason": reason}
|
||||
yield {**failed, "next": turn.next_step(failed)}
|
||||
return
|
||||
|
||||
# 兜底:一个操作都没落地,而上一轮有图。
|
||||
|
||||
@@ -69,12 +69,14 @@ def _parse_op_line(line: str) -> dict[str, Any] | None:
|
||||
|
||||
|
||||
class CopilotIncomplete(Exception):
|
||||
"""模型这一轮没说完:输出被截断,或者一个操作都没给。画布不能按「无需修改」交付。"""
|
||||
"""模型这一轮没说完:输出被截断,或者一个操作都没给。画布不能按「无需修改」交付。code 是机读码:一个操作都没给的是
|
||||
chat/turn.EMPTY_GENERATION(问数据一轮按它自动重来一次),截断的不给。"""
|
||||
|
||||
def __init__(self, reason: str, *, hint: str = "", detail: str = "") -> None:
|
||||
def __init__(self, reason: str, *, hint: str = "", detail: str = "", code: str = "") -> None:
|
||||
super().__init__(reason)
|
||||
self.hint = hint or "画布未修改。可以重试,或把需求拆成几步分别提出"
|
||||
self.detail = detail
|
||||
self.code = code
|
||||
|
||||
|
||||
async def _drain(stream: Any, already: int) -> None:
|
||||
|
||||
@@ -17,7 +17,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.apps.forms import input_schema
|
||||
from app.apps.spec import APP_FORMAT, parse_spec
|
||||
from app.apps.tree import input_fields, node_map, slot_refs
|
||||
from app.apps.tree import input_fields, node_map, slot_label, slot_refs
|
||||
|
||||
PIN = "main"
|
||||
VIEW = "latest"
|
||||
@@ -39,6 +39,13 @@ class DeriveRefused(Exception):
|
||||
self.code = code
|
||||
|
||||
|
||||
def start_label(name: str) -> str:
|
||||
"""流程类应用入口按钮的说法「发起…」。名字以英文字母、数字开头时中间空一格(「发起 accept 报销写回」),和界面上中英文
|
||||
之间留空格的写法一致;以前直接拼成「发起accept 报销写回」。派生、蓝图模板(审批、自由形)都用它。"""
|
||||
text = str(name or "").strip()
|
||||
return f"发起 {text}" if text[:1].isascii() and text[:1].isalnum() else f"发起{text}"
|
||||
|
||||
|
||||
def _safe(text: str) -> str:
|
||||
return re.sub(r"[^A-Za-z0-9_-]", "_", text)
|
||||
|
||||
@@ -102,8 +109,9 @@ async def _derive(session: AsyncSession, graph: dict[str, Any], *, title: str, n
|
||||
"pins": [{**pin, "use": "process" if process else "view"}],
|
||||
"slots": {}, "entries": [], "forms": {}, "task_ui": {}, "views": {}, "pages": [], "hooks": [], "notify": [],
|
||||
}
|
||||
for ref in slot_refs(graph, writes):
|
||||
spec["slots"].setdefault(ref["slot"], {"label": ref["slot"]})
|
||||
refs = slot_refs(graph, writes)
|
||||
for ref in refs:
|
||||
spec["slots"].setdefault(ref["slot"], {"label": slot_label(ref["slot"], refs)})
|
||||
for nid, node in humans.items():
|
||||
spec["task_ui"][nid] = {"title": node["title"]}
|
||||
if writes and not lookups:
|
||||
@@ -113,7 +121,7 @@ async def _derive(session: AsyncSession, graph: dict[str, Any], *, title: str, n
|
||||
names = list(spec["forms"][ENTRY]["properties"])
|
||||
first = next((n for n in names if n in spec["forms"][ENTRY]["required"]), names[0] if names else None)
|
||||
spec["entries"].append({
|
||||
"id": ENTRY, "label": f"发起{title}" if process else "按条件运行", "pin": PIN,
|
||||
"id": ENTRY, "label": start_label(title) if process else "按条件运行", "pin": PIN,
|
||||
"form": {"$ref": f"#/forms/{ENTRY}"}, "map": {n: f"form.{n}" for n in names},
|
||||
**({"instance_title": {"pattern": f"{title}·{{{first}}}", "fields": [first]}}
|
||||
if first and re.fullmatch(r"[A-Za-z0-9_㐀-鿿]+", first) else {}),
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from collections.abc import Collection, Iterator
|
||||
from collections.abc import Collection, Iterable, Iterator
|
||||
from typing import Any
|
||||
|
||||
from app.core.artifact_store import canonical_json
|
||||
@@ -121,6 +121,21 @@ def slot_refs(graph: Any, writes: Collection[str] = ()) -> list[dict[str, str]]:
|
||||
return out
|
||||
|
||||
|
||||
#: 按用到它的节点给岗位起显示名(slot_label):派给它的写节点名,超时升级给它的写「…超时升级」(同蓝图模板 slot_labels 的
|
||||
#: 叫法),动作确认派给它的写「…发送前确认」
|
||||
_SLOT_USE_LABEL = {"assignee": "{title}", "escalation": "{title}超时升级", "confirm": "{title}发送前确认"}
|
||||
|
||||
|
||||
def slot_label(slot: str, refs: Iterable[dict[str, str]]) -> str:
|
||||
"""岗位给人看的名字(应用里岗位的显示名)。图里只写了岗位名——画布上随手写的 id,常是英文(finance):以前显示名就是
|
||||
它,看着像起好的名字。现在按用到它的节点在图上的名字拼(refs 是 slot_refs 的结果,只看这个岗位的那几条,几处用到用顿号
|
||||
连);节点没起名字(名字就是节点 id)、或者没有节点用到它的,照实用岗位名——界面上显示名和岗位名一样时提示还没起名字。"""
|
||||
names = [_SLOT_USE_LABEL[r["role"]].format(title=r["title"]) for r in refs
|
||||
if r.get("slot") == slot and r.get("role") in _SLOT_USE_LABEL and r.get("title")
|
||||
and r["title"] != r.get("node_id")]
|
||||
return "、".join(dict.fromkeys(names)) or slot
|
||||
|
||||
|
||||
def input_fields(graph: Any) -> list[dict[str, Any]]:
|
||||
"""入口节点声明的字段(原样,含新增的 type / enum / format / description)。"""
|
||||
out: list[dict[str, Any]] = []
|
||||
|
||||
@@ -659,7 +659,7 @@ def app_base(bp: dict[str, Any], graph: dict[str, Any], *, use: str, name: str,
|
||||
writes 是写外部系统的调用工具节点 → 能力名(模板知道哪几个是写入,不用查库)。页面、视图、入口由各模板排。"""
|
||||
from app.apps.derive import PIN, RECONCILER
|
||||
from app.apps.spec import APP_FORMAT
|
||||
from app.apps.tree import node_map, slot_refs
|
||||
from app.apps.tree import node_map, slot_label, slot_refs
|
||||
from app.core.artifact_store import graph_hash
|
||||
|
||||
ctx = context(bp)
|
||||
@@ -673,9 +673,14 @@ def app_base(bp: dict[str, Any], graph: dict[str, Any], *, use: str, name: str,
|
||||
"format": APP_FORMAT, "name": name, "description": description, "icon": "app", "pins": [pin], "slots": {},
|
||||
"entries": [], "forms": {}, "task_ui": {}, "views": {}, "pages": [], "hooks": [], "notify": [],
|
||||
}
|
||||
# 岗位的显示名:卡片上起了名字的用它(审批模板的步骤名);卡片上就是岗位名本身的(处理人写的是岗位、自由形导入的图)按
|
||||
# 用到它的节点名拼(apps/tree.slot_label,同派生)
|
||||
labels = slot_labels(bp)
|
||||
for ref in slot_refs(graph, writes or {}):
|
||||
spec["slots"].setdefault(ref["slot"], {"label": labels.get(ref["slot"], ref["slot"])})
|
||||
refs = slot_refs(graph, writes or {})
|
||||
for ref in refs:
|
||||
label = labels.get(ref["slot"])
|
||||
spec["slots"].setdefault(ref["slot"], {"label": label if label and label != ref["slot"]
|
||||
else slot_label(ref["slot"], refs)})
|
||||
for nid, n in node_map(graph).items():
|
||||
if n["type"] == "human":
|
||||
spec["task_ui"][nid] = {"title": n["title"]}
|
||||
|
||||
@@ -50,7 +50,9 @@ def app_skeleton(bp: dict[str, Any], graph: dict[str, Any]) -> dict[str, Any]:
|
||||
writes = {WRITE: capability} if capability else {}
|
||||
spec = c.app_base(bp, graph, use="process", name=name, description=str(c.value(bp.get("purpose")) or ""),
|
||||
writes=writes)
|
||||
c.entry_and_pages(spec, bp, graph, label=f"发起{name}", writes=writes)
|
||||
from app.apps.derive import start_label
|
||||
|
||||
c.entry_and_pages(spec, bp, graph, label=start_label(name), writes=writes)
|
||||
return c.finish_app(spec)
|
||||
|
||||
|
||||
|
||||
@@ -104,7 +104,7 @@ def needs_reconciler(bp: dict[str, Any]) -> bool:
|
||||
|
||||
|
||||
def app_skeleton(bp: dict[str, Any], graph: dict[str, Any]) -> dict[str, Any]:
|
||||
from app.apps.derive import ENTRY, INSTANCE_VIEW, PIN, VIEW, _entries, _results
|
||||
from app.apps.derive import ENTRY, INSTANCE_VIEW, PIN, VIEW, _entries, _results, start_label
|
||||
from app.apps.forms import input_schema
|
||||
from app.apps.tree import input_fields, node_map
|
||||
|
||||
@@ -118,7 +118,7 @@ def app_skeleton(bp: dict[str, Any], graph: dict[str, Any]) -> dict[str, Any]:
|
||||
if fields:
|
||||
spec["forms"][ENTRY] = input_schema(fields)
|
||||
names = list(spec["forms"][ENTRY]["properties"])
|
||||
spec["entries"].append({"id": ENTRY, "label": f"发起{name}" if process else "按条件运行", "pin": PIN,
|
||||
spec["entries"].append({"id": ENTRY, "label": start_label(name) if process else "按条件运行", "pin": PIN,
|
||||
"form": {"$ref": f"#/forms/{ENTRY}"}, "map": {n: f"form.{n}" for n in names}})
|
||||
spec["views"][INSTANCE_VIEW] = {"source": {"kind": "instance"}}
|
||||
if not process:
|
||||
|
||||
@@ -5,13 +5,13 @@
|
||||
后端跑不了一轮问数据:评测要么照着抄一份(迟早走样,量出来的不是用户拿到的那一轮),要么做不了;这也违反「界面层不放
|
||||
逻辑」。现在决定写在这里,是纯函数:
|
||||
|
||||
- next_step:生成过程交出结局之后做什么;
|
||||
- next_step:生成过程交出结局之后做什么(含生成过程一个操作都没给时自动重来一次);
|
||||
- fill_input:用户的问题填进这张图的入口字段;
|
||||
- retry_after_review:复核之后要不要带着原图重来一次;
|
||||
- retry_instruction:重来时交给助手的那句话(两种措辞)。
|
||||
|
||||
接口把决定交给问数据页:/copilot/generate-stream 的 final 带 next(run | ready | fail),/copilot/review 的结果带
|
||||
retry_instruction(要重来时是那句话,不重来是 null)。问数据页照后端给的走;老后端没有这两个字段时退回它自己那份同样的
|
||||
接口把决定交给问数据页:/copilot/generate-stream 的 final 带 next(run | ready | fail),error 带 next(retry | fail),
|
||||
/copilot/review 的结果带 retry_instruction(要重来时是那句话,不重来是 null)。问数据页照后端给的走;老后端没有这两个字段时退回它自己那份同样的
|
||||
规则(tests/test_chat_turn_policy.py 逐条对照两边的常量和措辞)。评测(第 3 波)用同一组函数把一轮串起来。
|
||||
|
||||
只搬「决定」:重连、断点续看、取消、复核等多久这些仍在问数据页(把整轮编排搬到后端要重做这些,单独立项)。
|
||||
@@ -28,9 +28,19 @@ MAX_ATTEMPTS = 2
|
||||
#: final 之后的三种走法(final.next):运行、只建不跑(摆着等用户点运行)、判失败(没过检查的图不拿去跑)
|
||||
NEXT_RUN, NEXT_READY, NEXT_FAIL = "run", "ready", "fail"
|
||||
NEXT_STEPS = (NEXT_RUN, NEXT_READY, NEXT_FAIL)
|
||||
#: 别的结局:直接回答了(reply)、助手反问了等用户答(asked)、没自查过的老服务端交来没过检查的图时重来一次(retry)
|
||||
#: 别的结局:直接回答了(reply)、助手反问了等用户答(asked)、重来一次(retry:没自查过的老服务端交来没过检查的图,或者
|
||||
#: 生成过程一个操作都没给)
|
||||
NEXT_REPLY, NEXT_ASKED, NEXT_RETRY = "reply", "asked", "retry"
|
||||
|
||||
#: 生成过程一个操作都没给就结束了(error 操作的 code):模型只写了思考(deepseek 这类推理模型偶发)、或者格式跑偏每一行都被
|
||||
#: 跳过。这是偶发的,重来一次多半就好,按「重来一次」的规矩自动重来(最多 MAX_ATTEMPTS 次,含第一次);被截断的不重来——额度
|
||||
#: 不变,重来还是截断
|
||||
EMPTY_GENERATION = "copilot_empty"
|
||||
#: 自动重来的 error(code)
|
||||
RETRY_ERRORS = frozenset({EMPTY_GENERATION})
|
||||
#: error 操作的 message 前缀(api/copilot.answer_ops):重来时交给助手的原因不带它
|
||||
_ERROR_PREFIX = "助手本轮未完成:"
|
||||
|
||||
#: 问题往哪个入口字段放:名字像「问题」的优先(同 chat.ts fillInput 的正则,大小写不敏感)
|
||||
QUESTION_FIELD = re.compile(r"question|query|topic|input|问题|需求|主题", re.IGNORECASE)
|
||||
|
||||
@@ -51,7 +61,9 @@ def next_step(terminal: Mapping[str, Any] | None, *, self_checked: bool = True)
|
||||
"""生成过程交出结局之后做什么。terminal 是最后那条结局操作(final / reply / error);流结束时只有反问(ask)、没有
|
||||
别的结局的给 None。
|
||||
|
||||
- reply:直接回答,这一轮到此为止;error:判失败;只有反问:等用户回答;
|
||||
- reply:直接回答,这一轮到此为止;只有反问:等用户回答;
|
||||
- error:生成过程一个操作都没给(code 是 EMPTY_GENERATION)时重来一次(retry,次数由调用方按 MAX_ATTEMPTS 管,原因
|
||||
取 error_reason),别的判失败;
|
||||
- final:有 error 级问题时,服务端已经自查过(收到过 check 的结论)就判失败——没过的问题已经交回模型在同一轮里改过
|
||||
了,再整轮重来是再烧一次完整生成、改的还是同一批问题;没自查过(老服务端)才重来一次(retry);模型说不运行
|
||||
(autorun=false:用户要的是流程本身、或者问了用户)只建不跑;其余运行。
|
||||
@@ -61,6 +73,8 @@ def next_step(terminal: Mapping[str, Any] | None, *, self_checked: bool = True)
|
||||
kind = terminal.get("op")
|
||||
if kind == "reply":
|
||||
return NEXT_REPLY
|
||||
if kind == "error":
|
||||
return NEXT_RETRY if terminal.get("code") in RETRY_ERRORS else NEXT_FAIL
|
||||
if kind != "final":
|
||||
return NEXT_FAIL
|
||||
if _blockers(terminal):
|
||||
@@ -70,6 +84,15 @@ def next_step(terminal: Mapping[str, Any] | None, *, self_checked: bool = True)
|
||||
return NEXT_RUN
|
||||
|
||||
|
||||
def error_reason(error: Mapping[str, Any]) -> str:
|
||||
"""生成过程交出 error 时重来的原因(交给助手的那句话里用):error 带的 reason,没有就是 message 去掉「助手本轮未完成:」。"""
|
||||
reason = str(error.get("reason") or "").strip()
|
||||
if reason:
|
||||
return reason
|
||||
message = str(error.get("message") or "").strip()
|
||||
return message[len(_ERROR_PREFIX):] if message.startswith(_ERROR_PREFIX) else message
|
||||
|
||||
|
||||
def blocked_reason(final: Mapping[str, Any]) -> str:
|
||||
"""没过检查的图重来时的原因(只有老服务端那条路用得到)。"""
|
||||
return "生成的工作流没有通过校验:" + ";".join(_blockers(final))
|
||||
@@ -124,6 +147,7 @@ def has_nodes(graph: Any) -> bool:
|
||||
return isinstance(graph, Mapping) and bool(graph.get("nodes"))
|
||||
|
||||
|
||||
__all__ = ["MAX_ATTEMPTS", "NEXT_ASKED", "NEXT_FAIL", "NEXT_READY", "NEXT_REPLY", "NEXT_RETRY", "NEXT_RUN", "NEXT_STEPS",
|
||||
"QUESTION_FIELD", "RETRY_FROM_SCRATCH", "RETRY_WITH_BASE", "blocked_reason", "fill_input", "has_nodes",
|
||||
"next_step", "retry_after_review", "retry_instruction", "review_reason"]
|
||||
__all__ = ["EMPTY_GENERATION", "MAX_ATTEMPTS", "NEXT_ASKED", "NEXT_FAIL", "NEXT_READY", "NEXT_REPLY", "NEXT_RETRY",
|
||||
"NEXT_RUN", "NEXT_STEPS", "QUESTION_FIELD", "RETRY_ERRORS", "RETRY_FROM_SCRATCH", "RETRY_WITH_BASE",
|
||||
"blocked_reason", "error_reason", "fill_input", "has_nodes", "next_step", "retry_after_review",
|
||||
"retry_instruction", "review_reason"]
|
||||
|
||||
@@ -24,8 +24,8 @@
|
||||
一张表冲突、写不进去不影响别的表;路径不对、表不存在逐条报(results 和请求一一对应)。审阅的规则同单项审阅
|
||||
(catalog.review_item)。写进去之后影响数字的项被驳回了的,口径账本在后台标依据了它的答案「依据有变」
|
||||
(catalog.after_review → ledger/recall,next-round 02 做法 7 二)。
|
||||
- **被哪些运行用过**(table_runs,GET /catalog/{table}/usage):取法同使用次数(catalog.query_snapshot_tables),按运行
|
||||
去重,新的在前;total 是运行次数。
|
||||
- **被哪些运行用过**(table_runs,GET /catalog/{table}/usage):和使用次数数的是同一批查询快照(catalog.query_snapshot_tables),
|
||||
但按运行去重,新的在前;total 是运行次数。使用次数按查询计,同一次运行里查了几次这张表,使用次数就比 total 多几。
|
||||
- **进度**(review_progress,GET /catalog/review-progress):常用表(使用次数前 TOP_N 张,和起草对话框「使用次数最多的
|
||||
20 张表」是同一批:catalog.default_tables 的取法,用到的不够 TOP_N 张时按表结构的顺序补齐)、全部、影响数字的项各自的
|
||||
确认情况。项的范围和队列一样:表结构里没有的表、列不算。
|
||||
@@ -391,7 +391,8 @@ async def review_batch(session: AsyncSession, source: Any, items: Iterable[Batch
|
||||
|
||||
|
||||
async def table_runs(session: AsyncSession, source: Any, table: str, *, limit: int = 20) -> dict[str, Any]:
|
||||
"""用过这张表的运行:{total, runs},新的在前(按开始时间,没开始的按建立时间)。取法同使用次数。"""
|
||||
"""用过这张表的运行:{total, runs},新的在前(按开始时间,没开始的按建立时间)。数的是和使用次数同一批查询快照
|
||||
(catalog.query_snapshot_tables),但按运行去重:total 是运行次数,使用次数按查询计、可能比它大。"""
|
||||
run_ids = {run for run, hit in await catalog.query_snapshot_tables(session, source) if table in hit}
|
||||
if not run_ids:
|
||||
return {"total": 0, "runs": []}
|
||||
|
||||
@@ -17,6 +17,11 @@ agent 看到的目录少了遮罩列那部分(README 写明)。
|
||||
文档是表名、注释、目录里的中文名 / 粒度 / 说明、列名、列的中文名(遮罩列只用列名);探查范围外的对象只按名字和注释参与,
|
||||
权重低一档。切词用 memory/embeddings._tokens(英文按词、中文按字补二元组)加 catalog.name_words(下划线、驼峰拆词)。
|
||||
|
||||
**限制:范围外、又没有注释的对象,中文需求命不中。** 它只剩英文名可检索,英文名拆出的词没有中文说法:仓库里没有现成的英中
|
||||
业务词对照(命名推断只拆词、不翻译,catalog.name_words 也一样),这里不临时编一份词表(编出来的覆盖不全、还会把不相干的表
|
||||
抬上来),所以「上周的退款」找不到只叫 refund_orders、没写注释的表。真库多半有注释,注释照常参与检索,影响小;范围内的表另有
|
||||
目录里的中文名、列名可配。要用这张表时按英文名点名查看(会按需加入),或者请管理员把它加进探查范围、补上注释或目录的中文名。
|
||||
|
||||
**三种「不在」**(absent_detail):表已不在、在探查范围外、在范围里却读不到结构,说法分开;只有真删除才说「表已不在」。
|
||||
|
||||
只读:只看表结构缓存(schema_cache)和调用方读好的数据目录、剖析事实、对象索引,不碰业务库。这里的文字都是写给模型看的,
|
||||
@@ -342,6 +347,7 @@ def _doc_text(name: str, meta: Mapping[str, Any] | None, notes: Mapping[str, Any
|
||||
|
||||
|
||||
def _outside_text(name: str, comment: str | None) -> str:
|
||||
"""一个范围外对象的检索文档:名字、名字拆出的词、注释。没有注释的只有英文(见模块说明的限制)。"""
|
||||
from app.data.catalog import name_words
|
||||
|
||||
return " ".join([name, " ".join(name_words(name)), comment or ""])
|
||||
|
||||
@@ -10,7 +10,8 @@
|
||||
复核跟着换);读哪份目录按变体(answer_ops 的 catalog_view,查看的表详情、自查的 SQL 检查器都按它)。
|
||||
- 结局:reply 直接回答、asked 反问、fail 判失败(建图失败)、ready 只建不跑、retry(老服务端那条路:没自查过的图带着原因
|
||||
重来一次)、run 运行。生成过程交出 error(助手本轮未完成)、开流前就拒绝(模型没配好、数据源不在)都是建图失败,另记
|
||||
「助手调用失败」——连续几题都这样就止损(runner.py)。
|
||||
「助手调用失败」——连续几题都这样就止损(runner.py);只有生成过程一个操作都没给(只写了思考,error.next=retry)时按
|
||||
同一个规矩先带着原因重来一次,还不行才记建图失败。每次重来的原因照实记在 retries。
|
||||
- 运行:fill_input 填入口字段(单轮,history 为空),来源 eval,名字「评测:题集名·第 n 题」,选项带目录视图(Run.options
|
||||
的 catalog_view:运行时的 db_schema、冻结快照、自查都按它)、只读和遮罩(同使用者的问数据运行:评测页会显示答案里的数,
|
||||
遮罩值不能从这里露出来;评测不写业务库)。写作模型按变体写进图的 defaults:节点自己点了模型的不覆盖,照实记在
|
||||
@@ -79,6 +80,8 @@ class Turn:
|
||||
#: 这一题发起过的全部作答运行(重来的话两次),失败的也在
|
||||
runs: list[str] = field(default_factory=list)
|
||||
attempts: int = 0
|
||||
#: 每次自动重来的原因(和 attempts 一一对应):没过检查、复核说不可信、生成过程一个操作都没给
|
||||
retries: list[str] = field(default_factory=list)
|
||||
reason: str | None = None
|
||||
#: 助手调用失败(生成过程交出 error、开流前就被拒):runner 据此止损
|
||||
copilot_failed: bool = False
|
||||
@@ -101,25 +104,27 @@ class Turn:
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {"path": self.path, "run_id": self.run_id, "runs": list(self.runs), "attempts": self.attempts,
|
||||
"reason": self.reason, "copilot_failed": self.copilot_failed, "timed_out": self.timed_out,
|
||||
"retries": list(self.retries), "reason": self.reason, "copilot_failed": self.copilot_failed, "timed_out": self.timed_out,
|
||||
"reply": self.reply, "asked": self.asked, "review": self.review, "looks": list(self.looks),
|
||||
"assistant_model": self.assistant_model, "usage": copy.deepcopy(self.usage),
|
||||
"timings": dict(self.timings), "own_models": list(self.own_models)}
|
||||
|
||||
|
||||
#: 用量汇总里相加的计数(providers/usage 的 summary):unfinished_calls 是没做完的调用(被取消、超时,token 只计了已报的)
|
||||
_COUNTED = ("input_tokens", "output_tokens", "calls", "unpriced_calls", "unfinished_calls")
|
||||
|
||||
|
||||
def _add_usage(into: dict[str, Any] | None, more: Mapping[str, Any] | None) -> dict[str, Any]:
|
||||
"""两份 providers/usage 汇总相加(by_model 逐个模型加)。"""
|
||||
out = copy.deepcopy(into) if into else {"by_model": {}, "input_tokens": 0, "output_tokens": 0, "calls": 0,
|
||||
"cost_usd": 0.0, "unpriced_calls": 0}
|
||||
out = copy.deepcopy(into) if into else {"by_model": {}, **dict.fromkeys(_COUNTED, 0), "cost_usd": 0.0}
|
||||
if not more:
|
||||
return out
|
||||
for key in ("input_tokens", "output_tokens", "calls", "unpriced_calls"):
|
||||
for key in _COUNTED:
|
||||
out[key] = int(out.get(key) or 0) + int(more.get(key) or 0)
|
||||
out["cost_usd"] = round(float(out.get("cost_usd") or 0) + float(more.get("cost_usd") or 0), 6)
|
||||
for model, row in (more.get("by_model") or {}).items():
|
||||
mine = out["by_model"].setdefault(model, {"input_tokens": 0, "output_tokens": 0, "calls": 0, "cost_usd": 0.0,
|
||||
"unpriced_calls": 0})
|
||||
for key in ("input_tokens", "output_tokens", "calls", "unpriced_calls"):
|
||||
mine = out["by_model"].setdefault(model, {**dict.fromkeys(_COUNTED, 0), "cost_usd": 0.0})
|
||||
for key in _COUNTED:
|
||||
mine[key] = int(mine.get(key) or 0) + int(row.get(key) or 0)
|
||||
mine["cost_usd"] = round(float(mine.get("cost_usd") or 0) + float(row.get("cost_usd") or 0), 6)
|
||||
return out
|
||||
@@ -379,6 +384,14 @@ class Driver:
|
||||
return t
|
||||
kind = terminal.get("op")
|
||||
if kind == "error":
|
||||
# 生成过程一个操作都没给(只写了思考):按重来一次的规矩带着原因重来(同问数据页,chat/turn.next_step)
|
||||
step = terminal.get("next") or policy.next_step(terminal)
|
||||
if step == policy.NEXT_RETRY and t.attempts + 1 < policy.MAX_ATTEMPTS:
|
||||
reason = policy.error_reason(terminal)
|
||||
t.attempts += 1
|
||||
t.retries.append(reason)
|
||||
instruction = policy.retry_instruction(self.question, reason, policy.has_nodes(base_graph))
|
||||
continue
|
||||
t.path, t.reason, t.copilot_failed = "build_failed", str(terminal.get("message") or "助手本轮未完成"), True
|
||||
return t
|
||||
if kind == "reply":
|
||||
@@ -393,6 +406,7 @@ class Driver:
|
||||
if step == policy.NEXT_RETRY:
|
||||
if t.attempts + 1 < policy.MAX_ATTEMPTS:
|
||||
t.attempts += 1
|
||||
t.retries.append(policy.blocked_reason(terminal))
|
||||
instruction = policy.retry_instruction(self.question, policy.blocked_reason(terminal),
|
||||
policy.has_nodes(graph))
|
||||
base_graph = graph if policy.has_nodes(graph) else None
|
||||
@@ -421,7 +435,10 @@ class Driver:
|
||||
t.review = {k: v for k, v in review.items() if k != "usage"}
|
||||
t.usage["review"] = _add_usage(t.usage.get("review"), review.get("usage"))
|
||||
if review.get("retry_instruction"):
|
||||
from app.chat.turn import review_reason
|
||||
|
||||
t.attempts += 1
|
||||
t.retries.append(review_reason(review) or "复核判答案不可用")
|
||||
instruction = str(review["retry_instruction"])
|
||||
base_graph = row.graph if policy.has_nodes(row.graph) else None
|
||||
continue
|
||||
|
||||
@@ -124,16 +124,44 @@ async def source_versions(session: AsyncSession, source_ids: Sequence[str]) -> d
|
||||
|
||||
|
||||
async def estimate(session: AsyncSession, suite_id: str, items: int) -> dict[str, Any]:
|
||||
"""开跑前的估算:按这个题集上一次结束的评测的每题平均花费 × 这次的项数。第一次评测估不出,照实写。"""
|
||||
"""开跑前的估算:按这个题集上一次结束的评测的每题平均花费 × 这次的项数。第一次评测估不出,照实写。
|
||||
|
||||
上一次有估不出金额的调用(模型不在价格目录里、用量没报全)时,平均花费只是估得出的那部分:一分都没估出来的照实写估不出
|
||||
(以前写「约 $0」,像不花钱),估出了一部分的写「至少」(at_least)、带上另有几次估不出(unpriced_calls)。老总表里每题
|
||||
平均记成 0、却有估不出的调用的,也按估不出算。"""
|
||||
row = (await session.execute(select(EvalRun).where(
|
||||
EvalRun.suite_id == suite_id, EvalRun.status.in_(("succeeded", "partial")))
|
||||
.order_by(EvalRun.created_at.desc()).limit(1))).scalar_one_or_none()
|
||||
per_item = ((row.summary or {}).get("per_case_cost_usd") if row is not None else None)
|
||||
if per_item is None:
|
||||
if row is None:
|
||||
return {"items": items, "per_item_usd": None, "total_usd": None, "basis": None,
|
||||
"note": "这个题集还没有评完过,估不出花费:按每题开评前的实际花费累计,到了上限就停"}
|
||||
return {"items": items, "per_item_usd": round(float(per_item), 6), "total_usd": round(float(per_item) * items, 4),
|
||||
"basis": row.id, "note": "按这个题集上一次评测的每题平均花费估算(不含估不出金额的模型)"}
|
||||
summary = row.summary or {}
|
||||
per_item = summary.get("per_case_cost_usd")
|
||||
unpriced = int(summary.get("unpriced_calls") or 0)
|
||||
if per_item is None or (unpriced and not float(per_item)):
|
||||
if not unpriced:
|
||||
return {"items": items, "per_item_usd": None, "total_usd": None, "basis": None,
|
||||
"note": "这个题集上一次评测没有评完一道题,估不出花费:按每题开评前的实际花费累计,到了上限就停"}
|
||||
return {"items": items, "per_item_usd": None, "total_usd": None, "basis": row.id, "unpriced_calls": unpriced,
|
||||
"note": f"上一次评测的 {unpriced} 次模型调用金额都估不出(模型不在价格目录中,或用量没有报全):"
|
||||
"金额上限只管得住估得出金额的调用"}
|
||||
out = {"items": items, "per_item_usd": round(float(per_item), 6), "total_usd": round(float(per_item) * items, 4),
|
||||
"basis": row.id, "note": "按这个题集上一次评测的每题平均花费估算"}
|
||||
if unpriced:
|
||||
out.update(at_least=True, unpriced_calls=unpriced,
|
||||
note=f"按这个题集上一次评测的每题平均花费估算,实际至少这么多:上一次另有 {unpriced} 次调用金额估不出,没有算进去")
|
||||
return out
|
||||
|
||||
|
||||
def judge_usage(judge: Mapping[str, Any], cost_usd: float) -> dict[str, Any]:
|
||||
"""按角色的用量里裁判那一份:金额、次数,和估不出金额的次数(结论裁判的模型不在价格目录里时它的每次调用都估不出;
|
||||
文字要点按计量汇总)。以前只有金额和次数,裁判模型不在价格目录里时写成 $0。"""
|
||||
claims = judge.get("claims") or {}
|
||||
text = judge.get("text") or {}
|
||||
calls = int(claims.get("calls") or 0) + int(text.get("calls") or 0)
|
||||
unpriced = (int(claims.get("calls") or 0) if claims.get("priced") is False else 0) \
|
||||
+ int((text.get("usage") or {}).get("unpriced_calls") or 0)
|
||||
return {"cost_usd": round(cost_usd, 6), "calls": calls, "unpriced_calls": unpriced}
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
@@ -552,9 +580,7 @@ class EvalExecution(runtime.JobRun):
|
||||
writer = await writer_usage(turn.runs)
|
||||
assistant = add_usage(turn.usage.get("assistant"), turn.usage.get("review"))
|
||||
usage = {"assistant": assistant, "writer": writer, "review": turn.usage.get("review"),
|
||||
"judge": {"cost_usd": round(judge_cost, 6),
|
||||
"calls": int((judge.get("claims") or {}).get("calls") or 0)
|
||||
+ int((judge.get("text") or {}).get("calls") or 0)}}
|
||||
"judge": judge_usage(judge, judge_cost)}
|
||||
cost = float(assistant.get("cost_usd") or 0) + float(writer.get("cost_usd") or 0)
|
||||
self.spent += cost + judge_cost
|
||||
looks = {"model": sum(1 for x in turn.looks if x.get("by") == "model"),
|
||||
@@ -1022,6 +1048,6 @@ async def shutdown() -> None:
|
||||
|
||||
|
||||
__all__ = ["EVAL_BUSY", "EVAL_FINISHED", "EVAL_INVALID", "EvalExecution", "EvalRefused", "INTERRUPTED",
|
||||
"NOTHING_TO_RETRY", "STOPPED", "active_run", "estimate", "gold_key", "leader_of", "manager",
|
||||
"NOTHING_TO_RETRY", "STOPPED", "active_run", "estimate", "gold_key", "judge_usage", "leader_of", "manager",
|
||||
"prompt_fingerprint", "reconcile_stale", "refresh_summary", "result_dict", "result_rows", "retry", "running",
|
||||
"shutdown", "source_versions", "start", "startup", "stop"]
|
||||
|
||||
+96
-19
@@ -17,11 +17,13 @@ wrong 错、scale 量纲不同(差 10 的整数次幂,比如元和万元、
|
||||
- scalar:数值按「允许的相对误差」比(engine/tolerance.agree,和双盲复算同一个函数);期望没写容差时,引用的原值按
|
||||
DEFAULT_TOLERANCE,裸写的数按它写出的精度(3.5 万 ±500)。维度配对:期望写了 where(或期望那一行本身有维度)时,答案
|
||||
那一格所在行的非数值列要有这个取值;答案那一格没有维度(按条件筛过、只剩一个数)不算对不上。期望值是 0–10 的小整数
|
||||
又没有维度可配时标「弱匹配」(巧合的数:3 家门店正好等于期望的 3)。
|
||||
又没有维度可配时标「弱匹配」(巧合的数:3 家门店正好等于期望的 3)。答错时「答案里的数」给和这一项配对上的那一格(列名、
|
||||
指标、说明同名,或者维度对得上的只有一个数),配不上就不给、原因里照实写配不上,列出答案引用了哪几个数。
|
||||
- table:逐行按 keys 的取值配对,每个要比的值同 scalar;ordered 时按答案里各行键值出现的先后核对顺序。
|
||||
- set:名单里每一项都要在答案里出现(ordered 时按顺序);每一项都出现在答案查过的结果里算可追溯。
|
||||
- none:期望答「没有数据」:答案里没有数,而且查过的查询里有返回 0 行的(可追溯);没有数、但也没有一条查询返回 0 行的
|
||||
算「对但没引用」。
|
||||
算「对但没引用」。「给了数」只认引用的数值格和叙述里写的数:引用文字格(地区名这种维度)、空值格(NULL)不算——「华中区
|
||||
没有数据」引用了地区那一格,说的仍是没有数据。
|
||||
- ask:期望反问:作答路径是反问才对。
|
||||
- text:文字要点,由裁判判(covered / partial / missing / unjudged),单列「模型判断」,不进准确率(评测-D4)。
|
||||
|
||||
@@ -30,9 +32,10 @@ wrong 错、scale 量纲不同(差 10 的整数次幂,比如元和万元、
|
||||
没有错的)不能评分。期望没算出来按 gold 的结论:口径升版待处置是跳过,别的是不能评分——不混进「答错」。被驳回(总方案 X9)
|
||||
记路径「被驳回」,不算失败:不能评分。只有文字要点的题不能评分(单列,不计入准确率)。
|
||||
|
||||
**逐题的 E1**(sql_match):答案里的查询(先看报告引用了的)和期望的取数(SQL 题是期望 SQL,口径题是口径卡的取数节点)
|
||||
用口径账本同一个比对函数(ledger/match.compare_queries:先绑参数、再由 explain_sql_diff 判等价)比一次,给出等价与否和差在
|
||||
哪个方面(总方案 X2:E1 只有一份实现)。
|
||||
**逐题的 E1**(sql_match):产出答案的那条查询和期望的取数(SQL 题是期望 SQL,口径题是口径卡的取数节点)用口径账本同一个
|
||||
比对函数(ledger/match.compare_queries:先绑参数、再由 explain_sql_diff 判等价)比一次,给出等价与否和差在哪个方面(总方案
|
||||
X2:E1 只有一份实现)。哪条是产出答案的:答案引用了就是被引用的;没引用、只查了一条就是那一条(写明依据);没引用、查了
|
||||
几条的不硬比(照实写没法对上是哪条,不计入 E1)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -46,8 +49,10 @@ from app.engine.tolerance import agree, is_finite_number
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: 评分规则的版本:规则改了就加一,结果里记着按哪一版评的,可以按新规则重评
|
||||
SCORER_VERSION = 1
|
||||
#: 评分规则的版本:规则改了就加一,结果里记着按哪一版评的,可以按新规则重评。
|
||||
#: 2:期望「没有数据」时引用文字格、空值格不算给了数;判错时「答案里的数」给配对上的那一格;答案没引用查询时逐题 E1
|
||||
#: 不硬比第一条查询
|
||||
SCORER_VERSION = 2
|
||||
|
||||
OUTCOMES = ("traced", "untraced", "wrong", "scale", "ungradable")
|
||||
OUTCOME_LABEL = {"traced": "对且可追溯", "untraced": "对但没引用", "wrong": "错", "scale": "量纲不同",
|
||||
@@ -64,6 +69,8 @@ DEFAULT_TOLERANCE = 1e-6
|
||||
SCALE_FACTORS = (10.0, 100.0, 1000.0, 1e4, 1e5, 1e6, 1e8)
|
||||
#: 弱匹配:期望是这么小的整数、又没有维度可配
|
||||
WEAK_MAX = 10
|
||||
#: 判错、又配不上时原因里列出答案引用的前几个数
|
||||
PAIR_SHOWN = 3
|
||||
|
||||
#: 路径不对时给人看的原因
|
||||
WRONG_PATH = {
|
||||
@@ -316,9 +323,48 @@ def _score_scalar(exp: Mapping[str, Any], gold: Mapping[str, Any], answer: Answe
|
||||
for token in answer.bare:
|
||||
if _scaled(expected, float(token["value"])):
|
||||
return {**out, "outcome": "scale", "got": token["value"], "raw": token["raw"]}
|
||||
got = next((v.get("value") for v in clean), None)
|
||||
return {**out, "outcome": "wrong", **({"got": got} if got is not None else {}),
|
||||
"reason": "答案里没有对得上的数" if not answer.values and not answer.bare else "答案里的数和期望对不上"}
|
||||
numeric = [v for v in clean if is_finite_number(v.get("value"))]
|
||||
if not numeric and not answer.bare:
|
||||
return {**out, "outcome": "wrong", "reason": "答案里没有对得上的数"}
|
||||
paired = _paired(numeric, exp, gold, key)
|
||||
if paired is not None:
|
||||
return {**out, "outcome": "wrong", "got": paired.get("value"), "cell": _cell_ref(paired),
|
||||
"reason": "答案里的数和期望对不上"}
|
||||
if numeric:
|
||||
shown = "、".join(f"{_value_name(v)} {_plain(v.get('value'))}" for v in numeric[:PAIR_SHOWN])
|
||||
more = f" 等 {len(numeric)} 个" if len(numeric) > PAIR_SHOWN else ""
|
||||
return {**out, "outcome": "wrong",
|
||||
"reason": f"答案里的数和期望对不上:答案引用的数按列名、维度都配不上这一项(引用了 {shown}{more})"}
|
||||
return {**out, "outcome": "wrong", "reason": "答案里的数和期望对不上"}
|
||||
|
||||
|
||||
def _norm(name: Any) -> str:
|
||||
return str(name).strip().strip('"`[]').lower() if name is not None else ""
|
||||
|
||||
|
||||
def _paired(numeric: Sequence[Mapping[str, Any]], exp: Mapping[str, Any], gold: Mapping[str, Any],
|
||||
key: Mapping[str, Any]) -> Mapping[str, Any] | None:
|
||||
"""答错时答案里和这一项配对上的那一格(「答案里的数」给它):按名字配——列名(或口径卡的指标)是期望取的那一列、
|
||||
那个指标,或者和期望的说明(label)同名;名字配不上时按维度配——期望写了维度、答案里维度对得上的数恰好只有一个。
|
||||
都配不上为 None:以前取答案引用的第一个值,退款金额那一项显示成了答案里的订单数,甚至显示成一格文字。"""
|
||||
names = {_norm(n) for n in (exp.get("column"), gold.get("column"), exp.get("metric"), gold.get("metric"),
|
||||
gold.get("name"), exp.get("label")) if n not in (None, "")}
|
||||
named = [v for v in numeric if _norm(v.get("column")) in names or _norm(v.get("metric")) in names]
|
||||
if named:
|
||||
return named[0]
|
||||
if key:
|
||||
on_dims = [v for v in numeric if v.get("dims") and _dims_ok(key, v["dims"])]
|
||||
if len(on_dims) == 1:
|
||||
return on_dims[0]
|
||||
return None
|
||||
|
||||
|
||||
def _value_name(value: Mapping[str, Any]) -> str:
|
||||
return str(value.get("column") or value.get("metric") or value.get("alias") or "(没有列名)")
|
||||
|
||||
|
||||
def _plain(value: Any) -> str:
|
||||
return str(int(value)) if is_finite_number(value) and float(value).is_integer() else str(value)
|
||||
|
||||
|
||||
def _cell_ref(value: Mapping[str, Any]) -> dict[str, Any]:
|
||||
@@ -371,8 +417,15 @@ def _score_set(exp: Mapping[str, Any], gold: Mapping[str, Any], answer: Answer,
|
||||
|
||||
|
||||
def _score_none(answer: Answer) -> dict[str, Any]:
|
||||
if answer.values or answer.bare:
|
||||
return {"outcome": "wrong", "reason": "期望答「没有数据」,答案里却给了数"}
|
||||
"""期望答「没有数据」。「给了数」只认引用的数值格、叙述里写的数:引用的文字格(地区名这种维度)、空值格(NULL)不是数——
|
||||
「华中区([[v:Q1.r0.地区]])没有数据」说的正是没有数据,以前因为它引用了一格就判成「给了数」。"""
|
||||
numeric = next((v for v in answer.values if is_finite_number(v.get("value"))), None)
|
||||
if numeric is not None:
|
||||
return {"outcome": "wrong", "got": numeric.get("value"), "cell": _cell_ref(numeric),
|
||||
"reason": "期望答「没有数据」,答案里却引用了数"}
|
||||
if answer.bare:
|
||||
return {"outcome": "wrong", "got": answer.bare[0]["value"], "raw": answer.bare[0]["raw"],
|
||||
"reason": "期望答「没有数据」,答案的叙述里却写了数"}
|
||||
if any(int(q.get("row_count") or 0) == 0 and q.get("role") != "verify" for q in answer.queries):
|
||||
return {"outcome": "traced"}
|
||||
return {"outcome": "untraced", "reason": "答案没有给数,但没有一条查询返回 0 行可以说明「没有数据」"}
|
||||
@@ -561,7 +614,8 @@ async def gold_calibers(case: Mapping[str, Any], gold: Mapping[str, Any] | None)
|
||||
|
||||
async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, answer: Answer, *,
|
||||
timeout: float = 8.0) -> dict[str, Any] | None:
|
||||
"""逐题的 E1:{status: proven|related|none|skipped, query, aspects, diff, reason}。没有作答运行、没有期望取数的为 None。
|
||||
"""逐题的 E1:{status: proven|related|none|skipped, query, aspects, diff, reason, cited, basis}。没有作答运行、没有期望
|
||||
取数的为 None。比的是哪条查询见 _answering:basis 是 cited(答案引用的)或 only_query(答案没引用、运行只查了这一条)。
|
||||
比对在线程里跑(纯计算,和账本一样限时)。"""
|
||||
import asyncio
|
||||
import time
|
||||
@@ -594,13 +648,12 @@ async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, ans
|
||||
except Exception: # noqa: BLE001 - 比对是附加的一步,出错照实记没比
|
||||
logger.exception("评测:逐题比 SQL 出错")
|
||||
return {"status": "skipped", "reason": "比对出错"}
|
||||
cited = [r for r in rows if r.get("artifact") in set(answer.cited)]
|
||||
pool = cited or rows
|
||||
pool, basis = _answering(rows, answer)
|
||||
if isinstance(pool, dict):
|
||||
return pool
|
||||
proven = next((r for r in pool if r.get("status") == "proven"), None)
|
||||
chosen = proven or next((r for r in pool if r.get("status") == "related"), None) or \
|
||||
next((r for r in pool if r.get("status") == "skipped"), None) or (pool[0] if pool else None)
|
||||
if chosen is None:
|
||||
return {"status": "none", "reason": "答案没有查询"}
|
||||
next((r for r in pool if r.get("status") == "skipped"), None) or pool[0]
|
||||
candidate = next((c for c in chosen.get("candidates") or [] if c.get("status") == chosen.get("status")), None) or \
|
||||
next(iter(chosen.get("candidates") or []), None) or {}
|
||||
diff = candidate.get("diff") or {}
|
||||
@@ -608,7 +661,31 @@ async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, ans
|
||||
sql = next((q.get("sql") for q in answer.queries if q.get("artifact") == chosen.get("artifact")), None)
|
||||
return {"status": chosen.get("status"), "query": chosen.get("artifact"), "sql": sql, "aspects": aspects,
|
||||
"diff": diff or None, "via": candidate.get("via"), "binding": candidate.get("binding"),
|
||||
"reason": chosen.get("reason") or candidate.get("reason"), "cited": bool(cited)}
|
||||
"reason": chosen.get("reason") or candidate.get("reason"), "cited": basis == "cited", "basis": basis}
|
||||
|
||||
|
||||
def _answering(rows: list[dict[str, Any]], answer: Answer) -> tuple[list[dict[str, Any]] | dict[str, Any], str]:
|
||||
"""逐题 E1 拿哪几条查询比:产出答案的那几条。交回 (查询, 依据),确定不了时交回 (照实的结论, "")。
|
||||
|
||||
- 答案引用了查询(报告里引用的格、指标所在的查询):比被引用的那几条(依据 cited);
|
||||
- 没引用、这次运行(核查用的不算)只查了一条:答案的数只可能来自它,按它比(依据 only_query);
|
||||
- 没引用、查了几条:不比——以前取第一条相关的,取到的往往是探索查询(看看有哪些地区那种),说「差在筛选条件」和答案的数
|
||||
毫无关系。照实写没法对上是哪条,不计入 E1。"""
|
||||
cited = set(answer.cited)
|
||||
if cited:
|
||||
pool = [r for r in rows if r.get("artifact") in cited]
|
||||
if pool:
|
||||
return pool, "cited"
|
||||
return {"status": "skipped", "cited": True,
|
||||
"reason": "答案引用的查询不在可比对的查询里(合并查询不比:它的数都来自它的输入)"}, ""
|
||||
roles = {q.get("artifact"): q.get("role") for q in answer.queries}
|
||||
pool = [r for r in rows if roles.get(r.get("artifact")) != "verify"]
|
||||
if not pool:
|
||||
return {"status": "none", "reason": "答案没有查询"}, ""
|
||||
if len(pool) > 1:
|
||||
return {"status": "skipped", "cited": False,
|
||||
"reason": f"答案没有引用查询,这次运行执行了 {len(pool)} 条查询,没法对上是哪条产出了答案,不比"}, ""
|
||||
return pool, "only_query"
|
||||
|
||||
|
||||
__all__ = ["Answer", "CaseScore", "DEFAULT_TOLERANCE", "OUTCOMES", "OUTCOME_LABEL", "PATH_LABEL", "SCORER_VERSION",
|
||||
|
||||
@@ -145,8 +145,10 @@ def summarize(results: Sequence[Mapping[str, Any]], *, judge_on: bool = False) -
|
||||
"timings": {k: {"p50": percentile(v, 50), "p90": percentile(v, 90), "n": len(v)} for k, v in timings.items()},
|
||||
"usage": {**usage, "gold": {"cost_usd": 0.0, "calls": 0}},
|
||||
"cost_usd": cost, "unpriced_calls": unpriced,
|
||||
# 每题平均花费(下一次开跑前的估算按它):只按评完了的题平均,没开评的不摊
|
||||
"per_case_cost_usd": round(cost / len(evaluated), 6) if evaluated else None,
|
||||
# 每题平均花费(下一次开跑前的估算按它):只按评完了的题平均,没开评的不摊。有估不出金额的调用时 cost_usd 只是估得出
|
||||
# 的那部分:一分都没估出来(模型都不在价格目录里)的为 None,不写成 0(像不花钱);估出了一部分的照给,unpriced_calls
|
||||
# 说明它只是下限
|
||||
"per_case_cost_usd": round(cost / len(evaluated), 6) if evaluated and (cost > 0 or not unpriced) else None,
|
||||
"citations": {"numbers": numbers, "numbers_cited": cited,
|
||||
"uncited_numbers": sum(int(s.get("uncited_numbers") or 0) for s in stats),
|
||||
"unresolved": sum(int(s.get("unresolved") or 0) for s in stats),
|
||||
|
||||
@@ -12,12 +12,21 @@ async generator:with 块跨着 yield 的话,变量在两次 yield 之间留
|
||||
在创建时拷走上下文,照样计到;调用方自己的代码一行都不在计量里。
|
||||
|
||||
**金额。** 按 providers/catalog.estimate_cost 算,和运行的 llm.end 一个口径。估不出的调用记 unpriced_calls:模型不在价格
|
||||
目录里,或者用量没报全而目录价又不是 0。估不出的不当成 0 元,调用方据此写「≥」或「金额无法估算」。用量没报全有两种:
|
||||
目录里,或者用量没报全而目录价又不是 0。估不出的不当成 0 元,调用方据此写「≥」或「金额无法估算」。汇总的 cost_usd 永远是
|
||||
估得出的那部分(同运行的 usage,全系统一个口径),有 unpriced_calls 时它只是下限:写金额、算平均、做估算的地方都要连着
|
||||
unpriced_calls 一起看,一分都没估出来的写估不出、不写 0(评测的开跑估算、每题平均花费以前就漏了这一条,见 eval/runner.estimate、
|
||||
eval/stats.summarize)。用量没报全有两种:
|
||||
回复没带用量(有的网关流式不给);读到一半就不读了——用量在流的末尾才报全(Anthropic 的输出 token 在最后一条,OpenAI
|
||||
兼容系开了 stream_usage 也在最后)。生成过程读到 done、look、ask 那一行之后照样把流读完再关(api/copilot_gen._drain),
|
||||
流末尾那一段用量收得到;只有模型在那一行之后还往下写超过 DRAIN_CHARS 字的闲话时才当场关流(不为闲话付钱),连接断了也一样,
|
||||
这一次只拿得到已经报了的部分:照样计次数、计已报的 token,记成估不出,不悄悄少算。
|
||||
|
||||
**没做完的调用**(unfinished_calls)。被取消(挑表超时被 asyncio.wait_for 取消、调用方停了)、超时、流读到一半被关掉的调用
|
||||
都记一次调用、标「未完成」:token 只计已经报了的(一段都没收到的记 0,当作不知道,记成估不出),调用次数照实。以前挑表超时
|
||||
的那一次一点都不计,问数据一轮、评测按角色的调用次数就少一次。LangChain 的 ainvoke 被取消时不一定回调 on_llm_error
|
||||
(agenerate 卡在 gather 上被取消,后面的回调走不到):开始了、到汇总时还没有结束回调的调用(_pending 里剩下的)一并算未完成。
|
||||
请求刚发出去就被拒了的(鉴权错、参数不对,一段都没收到)不算一次调用:模型没干活。
|
||||
|
||||
模型名按发起调用时的参数取(invocation_params 的 model / model_name,就是接入里配的模型 id,价格目录按它查);调用参数里
|
||||
没有的(以前的演示模型就没报,评测汇总里助手用量的模型因此记成 unknown),看 LangChain 照模型对象给回调的 metadata 里的
|
||||
ls_model_name;再取不到看回复里的 model_name,都没有才记 UNKNOWN_MODEL。OpenAI 兼容、Anthropic 的模型在调用参数里就带着
|
||||
@@ -37,6 +46,17 @@ from langchain_core.tracers.context import register_configure_hook
|
||||
|
||||
#: 取不到模型名的调用记在这个名下
|
||||
UNKNOWN_MODEL = "unknown"
|
||||
#: 每个模型累计的几项(unreported 用量没报全的次数、unfinished 没做完的次数)
|
||||
_COUNTERS = ("input_tokens", "output_tokens", "calls", "unreported", "unfinished")
|
||||
|
||||
|
||||
def _abandoned(error: BaseException) -> bool:
|
||||
"""调用是半路放弃的(被取消、超时),不是一发出去就被拒:模型可能已经在干活、已经计费。"""
|
||||
import asyncio
|
||||
|
||||
if isinstance(error, (asyncio.CancelledError, TimeoutError)):
|
||||
return True
|
||||
return any(cls.__name__.endswith(("Timeout", "TimeoutError")) for cls in type(error).__mro__)
|
||||
|
||||
|
||||
class Meter(BaseCallbackHandler):
|
||||
@@ -51,6 +71,10 @@ class Meter(BaseCallbackHandler):
|
||||
self._pending: dict[UUID, str] = {}
|
||||
self._models: dict[str, dict[str, int]] = {}
|
||||
|
||||
def _row(self, model: str) -> dict[str, int]:
|
||||
"""这个模型的累计(调用方持锁)。unreported:用量没报全的次数;unfinished:没做完的次数(见模块开头)。"""
|
||||
return self._models.setdefault(model, dict.fromkeys(_COUNTERS, 0))
|
||||
|
||||
# ------------------------------------------------------------------ 回调
|
||||
|
||||
def _started(self, run_id: UUID, serialized: dict[str, Any] | None, kwargs: dict[str, Any]) -> None:
|
||||
@@ -83,7 +107,7 @@ class Meter(BaseCallbackHandler):
|
||||
with self._lock:
|
||||
model = self._pending.pop(run_id, "") or str(meta.get("model_name") or meta.get("model") or "") \
|
||||
or UNKNOWN_MODEL
|
||||
row = self._models.setdefault(model, {"input_tokens": 0, "output_tokens": 0, "calls": 0, "unreported": 0})
|
||||
row = self._row(model)
|
||||
row["calls"] += 1
|
||||
if usage:
|
||||
row["input_tokens"] += int(usage.get("input_tokens") or 0)
|
||||
@@ -92,8 +116,9 @@ class Meter(BaseCallbackHandler):
|
||||
row["unreported"] += 1
|
||||
|
||||
def on_llm_error(self, error: BaseException, *, run_id: UUID, **kwargs: Any) -> None:
|
||||
"""流读到一半停了(生成过程在最后一行之后读到太多闲话就关流,或者连接断了):LangChain 照样交来已经收到的那几段。
|
||||
有收到东西的照计一次、记已报的 token,算用量没报全;一段都没收到的(请求刚发出去就被拒了)不计。"""
|
||||
"""调用没做完:流读到一半停了(生成过程在最后一行之后读到太多闲话就关流,或者连接断了)、被取消、超时。LangChain
|
||||
照样交来已经收到的那几段。有收到东西的、被取消或超时的照计一次、记已报的 token,算用量没报全、未完成;一段都没收到
|
||||
的报错(请求刚发出去就被拒了)不计。"""
|
||||
from langchain_core.outputs import ChatGenerationChunk
|
||||
|
||||
response = kwargs.get("response")
|
||||
@@ -105,36 +130,48 @@ class Meter(BaseCallbackHandler):
|
||||
partial = None
|
||||
with self._lock:
|
||||
model = self._pending.pop(run_id, "")
|
||||
if partial is None:
|
||||
if partial is None and not _abandoned(error):
|
||||
return
|
||||
meta = getattr(partial, "response_metadata", None) or {}
|
||||
model = model or str(meta.get("model_name") or meta.get("model") or "") or UNKNOWN_MODEL
|
||||
row = self._models.setdefault(model, {"input_tokens": 0, "output_tokens": 0, "calls": 0, "unreported": 0})
|
||||
row = self._row(model)
|
||||
row["calls"] += 1
|
||||
row["unreported"] += 1
|
||||
row["unfinished"] += 1
|
||||
reported = getattr(partial, "usage_metadata", None) or {}
|
||||
row["input_tokens"] += int(reported.get("input_tokens") or 0)
|
||||
row["output_tokens"] += int(reported.get("output_tokens") or 0)
|
||||
|
||||
def _rows(self) -> dict[str, dict[str, int]]:
|
||||
"""累计的一份拷贝,开始了还没结束回调的调用算成没做完的(调用方持锁)。"""
|
||||
rows = {k: dict(v) for k, v in self._models.items()}
|
||||
for model in self._pending.values():
|
||||
row = rows.setdefault(model or UNKNOWN_MODEL, dict.fromkeys(_COUNTERS, 0))
|
||||
row["calls"] += 1
|
||||
row["unreported"] += 1
|
||||
row["unfinished"] += 1
|
||||
return rows
|
||||
|
||||
# ------------------------------------------------------------------ 合并、汇总
|
||||
|
||||
def merge(self, other: Meter) -> None:
|
||||
"""把另一次计量并进来(嵌套的计量出块时并进外层)。"""
|
||||
with other._lock:
|
||||
rows = {k: dict(v) for k, v in other._models.items()}
|
||||
rows = other._rows()
|
||||
with self._lock:
|
||||
for model, row in rows.items():
|
||||
mine = self._models.setdefault(model, dict.fromkeys(row, 0))
|
||||
mine = self._row(model)
|
||||
for key, value in row.items():
|
||||
mine[key] = mine.get(key, 0) + value
|
||||
|
||||
def summary(self) -> dict[str, Any]:
|
||||
"""{by_model: {模型: {input_tokens, output_tokens, calls, cost_usd, unpriced_calls}}, input_tokens, output_tokens,
|
||||
calls, cost_usd, unpriced_calls}。cost_usd 只是估得出的那部分。"""
|
||||
"""{by_model: {模型: {input_tokens, output_tokens, calls, cost_usd, unpriced_calls, unfinished_calls}}, input_tokens,
|
||||
output_tokens, calls, cost_usd, unpriced_calls, unfinished_calls}。cost_usd 只是估得出的那部分;unfinished_calls 次
|
||||
没做完(token 只计了已报的)。"""
|
||||
from app.providers.catalog import estimate_cost, find_pricing
|
||||
|
||||
with self._lock:
|
||||
rows = {k: dict(v) for k, v in self._models.items()}
|
||||
rows = self._rows()
|
||||
by_model: dict[str, dict[str, Any]] = {}
|
||||
for model, row in sorted(rows.items()):
|
||||
pricing = find_pricing(model)
|
||||
@@ -146,9 +183,9 @@ class Meter(BaseCallbackHandler):
|
||||
by_model[model] = {"input_tokens": row["input_tokens"], "output_tokens": row["output_tokens"],
|
||||
"calls": row["calls"],
|
||||
"cost_usd": round(estimate_cost(model, row["input_tokens"], row["output_tokens"]), 6),
|
||||
"unpriced_calls": unpriced}
|
||||
"unpriced_calls": unpriced, "unfinished_calls": row["unfinished"]}
|
||||
total = {key: sum(r[key] for r in by_model.values())
|
||||
for key in ("input_tokens", "output_tokens", "calls", "unpriced_calls")}
|
||||
for key in ("input_tokens", "output_tokens", "calls", "unpriced_calls", "unfinished_calls")}
|
||||
return {"by_model": by_model, **total, "cost_usd": round(sum(r["cost_usd"] for r in by_model.values()), 6)}
|
||||
|
||||
|
||||
|
||||
@@ -344,7 +344,8 @@ async def test_derive_a_process_leaves_slots_empty(monkeypatch):
|
||||
async with SessionLocal() as session:
|
||||
spec = await derive_app(session, wf, 1, name="报销审批")
|
||||
assert spec["name"] == "报销审批" and spec["pins"][0]["use"] == "process"
|
||||
assert spec["slots"] == {"finance": {"label": "finance"}, "reconciler": {"label": "动作核实"}}
|
||||
# 岗位的显示名按用到它的节点名起(apps/tree.slot_label),不再拿岗位名 finance 充当显示名
|
||||
assert spec["slots"] == {"finance": {"label": "财务审批"}, "reconciler": {"label": "动作核实"}}
|
||||
# 流程的首页只放发起和我发起的:结果在实例页看,不派生「最近封存运行」(派生出来只会挂一块永远空着的印章)
|
||||
assert "latest" not in spec["views"] and spec["views"]["inst"]["source"]["kind"] == "instance"
|
||||
home = next(p for p in spec["pages"] if p["id"] == "home")
|
||||
|
||||
@@ -299,3 +299,52 @@ def test_freeform_people_rows_follow_the_imported_graph():
|
||||
stale = [i for i in issues if i["code"] == "bp.template_mismatch" and "gone" in i["message"]]
|
||||
assert stale and stale[0]["level"] == "warning"
|
||||
assert freeform.people_rows(c.with_graph(get_template("freeform").example(), freeform.example_graph())) == []
|
||||
|
||||
|
||||
def _people_graph() -> dict:
|
||||
"""两步审核派给岗位 finance(第一步超时升级给 boss),还有一步派给没起名字的节点用的岗位 ops。名字都是虚构的。"""
|
||||
return {"nodes": [
|
||||
c.node("start", "input", "发起", fields=[{"name": "amount", "type": "number"}]),
|
||||
c.node("a", "human", "初审", mode="approve", assignee={"kind": "slot", "slot": "finance"},
|
||||
on_timeout={"action": "escalate", "to_slot": "boss"}, due="PT1H"),
|
||||
c.node("b", "human", "复审", mode="approve", assignee={"kind": "slot", "slot": "finance"}),
|
||||
{"id": "h3", "type": "human", "position": {"x": 0, "y": 0},
|
||||
"data": {"config": {"mode": "approve", "assignee": {"kind": "slot", "slot": "ops"}}}},
|
||||
c.node("done", "output", "完成", fields=[])],
|
||||
"edges": [c.edge("start", "a"), c.edge("a", "b"), c.edge("b", "h3"), c.edge("h3", "done")]}
|
||||
|
||||
|
||||
def test_entry_labels_keep_a_space_before_a_latin_name():
|
||||
"""入口按钮「发起…」:名字以英文开头时中间空一格(验收时写成了「发起accept 报销写回应用」)。"""
|
||||
from app.apps.derive import start_label
|
||||
|
||||
assert start_label("accept 报销写回应用") == "发起 accept 报销写回应用"
|
||||
assert start_label("报销") == "发起报销" and start_label("2026 年报") == "发起 2026 年报"
|
||||
graph = _people_graph()
|
||||
inst = instantiate({**get_template("freeform").example()}, name="accept 报销写回应用", graph=graph)
|
||||
assert inst.app_spec["entries"][0]["label"] == "发起 accept 报销写回应用", inst.app_spec["entries"]
|
||||
approval = instantiate(get_template("approval").example(), name="erp 报销")
|
||||
assert approval.app_spec["entries"][0]["label"] == "发起 erp 报销"
|
||||
|
||||
|
||||
def test_slot_labels_use_the_node_names_and_say_so_when_there_is_none():
|
||||
"""应用里岗位的显示名:图里只写了岗位名(画布上随手写的 id,常是英文 finance),以前显示名就是它,看着像起好的名字。
|
||||
现在按用到它的节点在图上的名字拼(几处用到用顿号连、超时升级的写「…超时升级」);节点没起名字的照实用岗位名。蓝图卡片
|
||||
上起了名字的(审批模板的步骤名)照旧用它。"""
|
||||
inst = instantiate({**get_template("freeform").example()}, name="报销", graph=_people_graph())
|
||||
slots = inst.app_spec["slots"]
|
||||
assert slots["finance"]["label"] == "初审、复审", slots
|
||||
assert slots["boss"]["label"] == "初审超时升级" and slots["ops"]["label"] == "ops", slots
|
||||
approval = instantiate(get_template("approval").example())
|
||||
labels = {k: v["label"] for k, v in approval.app_spec["slots"].items()}
|
||||
assert all(labels[k] != k for k in ("lead", "finance", "finance_up")), labels
|
||||
|
||||
|
||||
async def test_derived_apps_use_the_same_slot_labels_and_entry_label():
|
||||
from app.apps.derive import derive_from_graph
|
||||
from app.db.base import SessionLocal
|
||||
|
||||
async with SessionLocal() as session:
|
||||
spec = await derive_from_graph(session, _people_graph(), name="accept 报销", blueprint_id="bp1")
|
||||
assert {k: v["label"] for k, v in spec["slots"].items()} == {"finance": "初审、复审", "boss": "初审超时升级", "ops": "ops"}
|
||||
assert spec["entries"][0]["label"] == "发起 accept 报销"
|
||||
|
||||
@@ -359,6 +359,27 @@ async def test_table_usage_lists_runs_newest_first(client):
|
||||
assert (await client.get(f"{_base(sid)}/ghost/usage")).status_code == 404
|
||||
|
||||
|
||||
async def test_usage_counts_queries_and_the_runs_list_counts_runs(client):
|
||||
"""两个数口径不同、说明照实:表的「使用次数」按查询计(同一次运行里查了两次算两次),「用过这张表的运行」按运行计。
|
||||
接口说明以前写「取法同使用次数」,看着该是同一个数,验收时 4 对 3 对不上。"""
|
||||
sid = await _source()
|
||||
async with SessionLocal() as session:
|
||||
name = (await session.get(DataSource, sid)).name
|
||||
runs = await _use(sid, name, {"orders": 3})
|
||||
async with SessionLocal() as session:
|
||||
session.add(Artifact(id=uuid.uuid4().hex, run_id=runs[0], kind=catalog.QUERY_SNAPSHOT_KIND,
|
||||
meta={"source_id": sid, "source": name, "tables": ["orders"]}))
|
||||
await session.commit()
|
||||
listed = (await client.get(f"/api/datasources/{sid}/catalog")).json()
|
||||
assert next(t for t in listed["tables"] if t["table_name"] == "orders")["usage"] == 4
|
||||
assert (await client.get(f"{_base(sid)}/orders/usage")).json()["total"] == 3
|
||||
doc = _fastapi_app.openapi()["paths"]["/api/datasources/{source_id}/catalog/{table}/usage"]["get"]["description"]
|
||||
assert "取法同使用次数" not in doc and "按运行" in doc and "按查询" in doc, doc
|
||||
from app.data import catalog_review
|
||||
|
||||
assert "取法同使用次数" not in (catalog_review.table_runs.__doc__ or "")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 进度
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
@@ -48,6 +48,11 @@ def test_next_step_branches():
|
||||
assert turn.next_step(_final(issues=[ERROR], autorun=False)) == "fail", "没过检查比只建不跑优先"
|
||||
assert turn.next_step({"op": "reply", "text": "已经查过了"}) == "reply"
|
||||
assert turn.next_step({"op": "error", "message": "x"}) == "fail"
|
||||
empty = {"op": "error", "message": "助手本轮未完成:模型没有给出任何修改,也没有说明原因",
|
||||
"code": turn.EMPTY_GENERATION, "reason": "模型没有给出任何修改,也没有说明原因"}
|
||||
assert turn.next_step(empty) == "retry", "只写了思考、一个操作都没给:按重来一次的规矩自动重来"
|
||||
assert turn.error_reason(empty) == "模型没有给出任何修改,也没有说明原因"
|
||||
assert turn.error_reason({"op": "error", "message": "助手本轮未完成:接入返回了 502"}) == "接入返回了 502"
|
||||
assert turn.next_step(None) == "asked", "流结束时只有反问"
|
||||
assert set(turn.NEXT_STEPS) == {"run", "ready", "fail"}
|
||||
|
||||
@@ -167,6 +172,74 @@ async def _run(output: dict, graph: dict, code: str) -> str:
|
||||
return run.id
|
||||
|
||||
|
||||
def test_the_frontend_retries_an_error_the_server_says_to_retry():
|
||||
"""问数据页照服务端给的 next 走:error 操作带 next=retry(只写了思考、一个操作都没给)时带着原因重来一次,不再直接报
|
||||
「助手本轮未完成」。读 chat.ts 里 error 那一支(行为另由 scripts/check-chat.mjs 的 emptyretry 一节在浏览器里守)。"""
|
||||
branch = re.search(r"case 'error':(.*?)break\n", _ts(), re.S)
|
||||
assert branch, "chat.ts 里找不到 error 那一支"
|
||||
assert re.search(r"op\.next === 'retry'", branch.group(1)) and "retry(" in branch.group(1), branch.group(1)
|
||||
|
||||
|
||||
async def test_answer_ops_says_to_retry_an_empty_generation(client, monkeypatch):
|
||||
"""生成过程一个操作都没给(deepseek 偶发只出思考):error 操作带机读码 copilot_empty、原因、next=retry。"""
|
||||
import json
|
||||
|
||||
import app.api.copilot as copilot
|
||||
from tests.test_usage_meter import Scripted
|
||||
|
||||
model = Scripted(model="claude-opus-5", replies=[[""]])
|
||||
|
||||
async def _model(*_a, **_k):
|
||||
return model, "claude-opus-5"
|
||||
|
||||
monkeypatch.setattr(copilot, "get_chat_model", _model)
|
||||
events = []
|
||||
async with client.stream("POST", "/api/copilot/generate-stream",
|
||||
json={"instruction": "上周各地区销售额", "intent": "answer"}) as r:
|
||||
async for line in r.aiter_lines():
|
||||
if line.startswith("data: "):
|
||||
events.append(json.loads(line[6:]))
|
||||
[err] = [e for e in events if e["op"] == "error"]
|
||||
assert err["code"] == turn.EMPTY_GENERATION and err["next"] == "retry", err
|
||||
assert err["reason"] == "模型没有给出任何修改,也没有说明原因" and err["message"].startswith("助手本轮未完成")
|
||||
|
||||
|
||||
async def test_the_eval_driver_retries_an_empty_generation_once(monkeypatch):
|
||||
"""评测的作答(app/eval/answer.Driver)走同一个决定:第一次一个操作都没给就带着原因重来一次,原因照实记在 retries;
|
||||
重来还不行才照实报建图失败(助手调用失败),不再试第三次。"""
|
||||
import json
|
||||
|
||||
import app.api.copilot as copilot
|
||||
from app.eval import answer
|
||||
from tests.test_usage_meter import Scripted
|
||||
|
||||
reply = json.dumps({"op": "reply", "text": "上周没有华中区的数据"}, ensure_ascii=False)
|
||||
for replies, path in (([[""], [reply]], "reply"), ([[""]], "build_failed")):
|
||||
model = Scripted(model="claude-opus-5", replies=replies)
|
||||
seen: list[str] = []
|
||||
|
||||
async def _model(*_a, _m=model, **_k):
|
||||
return _m, "claude-opus-5"
|
||||
|
||||
monkeypatch.setattr(copilot, "get_chat_model", _model)
|
||||
real = answer.Driver._generate
|
||||
|
||||
async def spy(self, instruction, base_graph, _seen=seen, _real=real):
|
||||
_seen.append(instruction)
|
||||
return await _real(self, instruction, base_graph)
|
||||
|
||||
monkeypatch.setattr(answer.Driver, "_generate", spy)
|
||||
driver = answer.Driver("上周华中区销售额", datasource_ids=[], ctx=answer.TurnContext(
|
||||
suite_id="s", suite_name="虚构题集", eval_run_id="e", case_id=None, case_seq=1, actor=None))
|
||||
t = await driver.drive()
|
||||
assert t.path == path and t.attempts == 1 and model.calls == 2, (t.path, t.attempts, model.calls, t.reason)
|
||||
assert t.retries == ["模型没有给出任何修改,也没有说明原因"], t.retries
|
||||
assert seen[1] == turn.retry_instruction("上周华中区销售额", t.retries[0], False), seen
|
||||
if path == "build_failed":
|
||||
assert t.copilot_failed and t.reason.startswith("助手本轮未完成"), t.reason
|
||||
assert t.usage["assistant"]["calls"] == 2, "两次生成都是这一题花的"
|
||||
|
||||
|
||||
async def test_review_says_whether_and_how_to_retry(client, monkeypatch):
|
||||
import app.api.copilot as copilot
|
||||
from app.providers.factory import ProviderNotConfigured
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
|
||||
- 几种结局(建图交付、直接回答、只问不改、自查改不好、模型出错、没给操作、输出被截断)的 SSE 输出,和抽出来之前
|
||||
逐条一样(剧本见 SCENARIOS,抽出来之前录下的输出在 fixtures/copilot_answer_ops_baseline.json),末尾只多一条 usage,
|
||||
final 只多一个 next(问数据一轮的决定,chat/turn.py);
|
||||
final 只多一个 next(问数据一轮的决定,chat/turn.py);error 多一个 next 和原因 reason(一个操作都没给的另带机读码 code,
|
||||
next 是 retry:问数据页和评测据此自动重来一次);
|
||||
- 直接迭代 answer_ops 和 SSE 收到的是同一串;
|
||||
- 开流前的拒绝(模型没配好、限定的数据源都不在)照旧是 HTTP 错误,不进流。
|
||||
|
||||
@@ -133,6 +134,8 @@ BODY = {"instruction": "数一数门店", "intent": "answer"}
|
||||
|
||||
#: final 带的 next(chat/turn.next_step):问数据页照它走
|
||||
NEXT = {"graph_ok": "run", "graph_fixed": "run", "check_failed": "fail", "ask_only": "ready"}
|
||||
#: error 带的 (next, code):只有一个操作都没给的重来一次
|
||||
ERROR_NEXT = {"error": ("fail", None), "no_ops": ("retry", "copilot_empty"), "truncated": ("fail", None)}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", list(SCENARIOS))
|
||||
@@ -144,6 +147,11 @@ async def test_the_sse_output_is_unchanged_but_for_a_trailing_usage(client, monk
|
||||
# final 多了 next(问数据一轮的决定下沉到后端),别的一个字不变
|
||||
finals = [op for op in got if op["op"] == "final"]
|
||||
assert [op.pop("next") for op in finals] == ([NEXT[name]] if name in NEXT else [])
|
||||
errors = [op for op in got if op["op"] == "error"]
|
||||
assert [(op.pop("next"), op.pop("code", None)) for op in errors] == \
|
||||
([ERROR_NEXT[name]] if name in ERROR_NEXT else [])
|
||||
for op in errors:
|
||||
assert op["message"] == f"助手本轮未完成:{op.pop('reason')}", op
|
||||
assert got[:-1] == expected
|
||||
|
||||
|
||||
|
||||
@@ -347,6 +347,44 @@ async def test_the_api_starts_estimates_and_reports_progress(client, monkeypatch
|
||||
assert bad.status_code == 422 and bad.json()["code"] == "eval_variant_invalid"
|
||||
|
||||
|
||||
async def test_the_estimate_says_it_cannot_price_instead_of_zero():
|
||||
"""上一次评测的模型都不在价格目录里(价格目录空):估算以前写「约 $0」,像不花钱。现在一分都没估出来的写估不出;估出了
|
||||
一部分的写「至少」、另有几次估不出(老总表里每题平均记成 0、却有估不出的调用的,也按估不出算)。"""
|
||||
from datetime import timedelta
|
||||
|
||||
suite_id = await es.make_suite("none", cases=ask_cases(1))
|
||||
now = utcnow()
|
||||
async with SessionLocal() as session:
|
||||
session.add(EvalRun(suite_id=suite_id, status="succeeded", created_at=now - timedelta(minutes=2),
|
||||
summary={"per_case_cost_usd": 0.0, "unpriced_calls": 6}))
|
||||
await session.commit()
|
||||
got = await runner.estimate(session, suite_id, 3)
|
||||
assert got["total_usd"] is None and got["per_item_usd"] is None and got["unpriced_calls"] == 6, got
|
||||
assert "估不出" in got["note"] and got["basis"], got
|
||||
async with SessionLocal() as session:
|
||||
session.add(EvalRun(suite_id=suite_id, status="succeeded", created_at=now - timedelta(minutes=1),
|
||||
summary={"per_case_cost_usd": 0.02, "unpriced_calls": 2}))
|
||||
await session.commit()
|
||||
got = await runner.estimate(session, suite_id, 3)
|
||||
assert got["total_usd"] == 0.06 and got["at_least"] is True and got["unpriced_calls"] == 2, got
|
||||
assert "至少" in got["note"] and "2 次" in got["note"], got["note"]
|
||||
async with SessionLocal() as session:
|
||||
session.add(EvalRun(suite_id=suite_id, status="succeeded", created_at=now,
|
||||
summary={"per_case_cost_usd": 0.0, "unpriced_calls": 0}))
|
||||
await session.commit()
|
||||
got = await runner.estimate(session, suite_id, 3)
|
||||
assert got["total_usd"] == 0.0 and not got.get("at_least") and not got.get("unpriced_calls"), "目录价是 0 的才是真不花钱"
|
||||
|
||||
|
||||
def test_the_judge_role_keeps_its_unpriced_calls():
|
||||
"""按角色的用量里裁判那一份以前只有金额和次数:裁判模型不在价格目录里时写成 $0。"""
|
||||
usage = runner.judge_usage({"claims": {"cost_usd": 0.0, "calls": 2, "priced": False},
|
||||
"text": {"cost_usd": 0.0, "calls": 1, "usage": {"unpriced_calls": 1}}}, 0.0)
|
||||
assert usage == {"cost_usd": 0.0, "calls": 3, "unpriced_calls": 3}
|
||||
priced = runner.judge_usage({"claims": {"cost_usd": 0.01, "calls": 2, "priced": True}}, 0.01)
|
||||
assert priced == {"cost_usd": 0.01, "calls": 2, "unpriced_calls": 0}
|
||||
|
||||
|
||||
async def test_results_give_the_question_actually_asked_and_the_template_apart(client, monkeypatch):
|
||||
"""逐题结果的 question 是这一题实际问的(变量按这次评测的截至时间渲染好了,和作答时 Driver 收到的一字不差),模板原文
|
||||
另放 question_template(下一轮验收:以前 question 是带 {{ schedule.last_week_start }} 的原始模板)。没有变量的题两个一样。"""
|
||||
|
||||
@@ -116,6 +116,59 @@ def test_tables_sets_none_and_ask():
|
||||
assert score_case(ask, {"status": "ok"}, Answer(run_id="r"), path="ran").verdict == "fail"
|
||||
|
||||
|
||||
def test_a_wrong_answer_shows_the_paired_cell():
|
||||
"""判错时「答案里的数」给评分时配对上的那一格(按列名、指标、维度配),不是答案引用的第一个值;配不上照实写配不上。"""
|
||||
case = _case({**EXP, "column": "refund", "label": "退款金额"})
|
||||
gold = _gold({"e1": _scalar(1234.5, column="refund")})
|
||||
|
||||
def cell(value, column, **dims):
|
||||
return {"value": value, "kind": "cell", "artifact": "a1", "row": 0, "column": column, "dims": dims}
|
||||
|
||||
question = cell("上周退款金额是多少", "问题")
|
||||
both = score_case(case, gold, Answer(run_id="r", values=[question, cell(36, "单数"), cell(1200.0, "refund")]),
|
||||
path="ran")
|
||||
assert both.verdict == "fail" and both.scores[0]["got"] == 1200.0, both.scores
|
||||
assert both.scores[0]["cell"]["column"] == "refund"
|
||||
labelled = score_case(case, gold, Answer(run_id="r", values=[cell(36, "单数"), cell(1200.0, "退款金额")]), path="ran")
|
||||
assert labelled.scores[0]["got"] == 1200.0, "期望的说明(label)和答案的列名一样也算配上"
|
||||
lone = score_case(case, gold, Answer(run_id="r", values=[question, cell(36, "单数")]), path="ran")
|
||||
[s] = lone.scores
|
||||
assert s["outcome"] == "wrong" and "got" not in s, "单数不是退款金额:配不上就不给「答案里的数」"
|
||||
assert "配不上" in s["reason"] and "单数 36" in s["reason"] and "上周退款" not in s["reason"], s["reason"]
|
||||
# 按维度配:华东那一行只有一个数时就是它;同一行有几个数、列名又都配不上时不猜
|
||||
by_dims = _case({**EXP, "column": "refund", "where": {"region": "华东"}})
|
||||
gold = _gold({"e1": _scalar(1234.5, column="refund", where={"region": "华东"}, dims={"region": "华东"})})
|
||||
one = score_case(by_dims, gold, Answer(run_id="r", values=[cell(1200.0, "total", region="华东")]), path="ran")
|
||||
assert one.scores[0]["got"] == 1200.0, one.scores
|
||||
two = score_case(by_dims, gold, Answer(run_id="r", values=[cell(36, "单数", region="华东"),
|
||||
cell(1200.0, "total", region="华东")]), path="ran")
|
||||
assert "got" not in two.scores[0] and "配不上" in two.scores[0]["reason"], two.scores
|
||||
text_only = score_case(case, _gold({"e1": _scalar(1234.5, column="refund")}), Answer(run_id="r", values=[question]),
|
||||
path="ran")
|
||||
assert "got" not in text_only.scores[0] and text_only.scores[0]["reason"] == "答案里没有对得上的数"
|
||||
|
||||
|
||||
def test_none_counts_only_numbers_as_giving_data():
|
||||
"""期望「没有数据」:只有引用了数值格、或叙述里写了数,才算答案给了数。引用的是文字格(地区名)、空值格(NULL)的,
|
||||
答案明说没有数据就按没有数据判对。"""
|
||||
none = _case({"id": "n", "kind": "none", "required": True})
|
||||
gold = _gold({"n": {"kind": "none", "rows": 0}})
|
||||
empty = [{"artifact": "q", "row_count": 0, "sql": "SELECT 1"}]
|
||||
text_cell = {"value": "华中", "kind": "cell", "artifact": "q", "row": 0, "column": "region", "dims": {}}
|
||||
named = score_case(none, gold, Answer(run_id="r", text="华中区没有数据", values=[text_cell], queries=empty), path="ran")
|
||||
assert named.verdict == "pass" and named.scores[0]["outcome"] == "traced", named.scores
|
||||
null_cell = {"value": None, "kind": "cell", "artifact": "q", "row": 0, "column": "gmv", "dims": {}}
|
||||
null = score_case(none, gold, Answer(run_id="r", text="华中区的销售额为空,没有数据", values=[null_cell],
|
||||
queries=empty), path="ran")
|
||||
assert null.verdict == "pass", null.scores
|
||||
number = score_case(none, gold, Answer(run_id="r", text="华中区 12 元", values=[_cell(12.0)], queries=empty),
|
||||
path="ran")
|
||||
assert number.verdict == "fail" and number.scores[0]["got"] == 12.0, number.scores
|
||||
said = score_case(none, gold, Answer(run_id="r", text="华中区 12 元", values=[text_cell], bare=_bare("12 元"),
|
||||
queries=empty), path="ran")
|
||||
assert said.verdict == "fail" and said.scores[0]["raw"] == "12", said.scores
|
||||
|
||||
|
||||
def test_the_path_and_the_gold_decide_before_numbers():
|
||||
case = _case(EXP)
|
||||
gold = _gold({"e1": _scalar(330.0)})
|
||||
@@ -195,6 +248,49 @@ async def test_reading_the_sealed_answer_and_the_per_case_sql_match(engine_on, m
|
||||
assert (await score.sql_match(case, gold, answer))["status"] == "proven"
|
||||
|
||||
|
||||
async def test_the_per_case_sql_match_compares_the_query_behind_the_answer(engine_on, monkeypatch, tmp_path):
|
||||
"""逐题 E1 比的是产出答案的那条查询:答案引用了就比被引用的;没引用、这次运行又查了几条时不硬比第一条(往往是探索
|
||||
查询),照实写没法对上是哪条;没引用、只查了一条时它就是唯一可能的那条,按它比、写明依据。"""
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
from app.engine.runner import run_manager
|
||||
from app.eval import score
|
||||
from app.ledger import match as ledger_match
|
||||
from sqlalchemy import select
|
||||
from tests import ledger_samples as ls
|
||||
|
||||
await ls.fresh_source(tmp_path / "shop.db")
|
||||
async with SessionLocal() as session:
|
||||
sid = (await session.execute(select(DataSource.id).where(DataSource.name == ls.CURRENT["source"]))).scalar()
|
||||
case = {"gold": {"kind": "sql", "datasource_id": sid, "sql": "x"}, "expectations": []}
|
||||
gold = {"status": "ok", "before": {"rendered": {"sql": (
|
||||
"SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' AND paid_at >= '2026-09-01' "
|
||||
"AND paid_at < '2026-09-08'")}, "sets": []}}
|
||||
|
||||
# 先探了一下有哪些地区,再查销售额;答案没引用查询(数写在叙述里)
|
||||
graph = ls.chain(ls.node("start", "input"),
|
||||
ls.node("peek", "tool", tool=ls.tool(), args={"sql": "SELECT DISTINCT region FROM orders"}),
|
||||
ls.node("q", "tool", tool=ls.tool(), args={"sql": ls.WITH_REFUNDS}),
|
||||
ls.node("write", "report", instructions="写答案"),
|
||||
ls.node("out", "output", fields=[{"name": "答案", "value": "{{ nodes.write.text }}"}]))
|
||||
ls.script(monkeypatch, "本周销售额约 200 元。")
|
||||
started = await run_manager.start(graph=graph, input_payload={}, run_class="exploratory", origin_kind="canvas")
|
||||
await ls.wait(started.id)
|
||||
await ledger_match.drain()
|
||||
answer = await score.read_answer(started.id)
|
||||
assert answer.cited == [] and len(answer.queries) == 2, answer.queries
|
||||
match = await score.sql_match(case, gold, answer)
|
||||
assert match["status"] == "skipped" and match["cited"] is False and "没有引用查询" in match["reason"], match
|
||||
assert match.get("query") is None and match.get("aspects") in (None, []), "没法对上是哪条,就不给差在哪"
|
||||
|
||||
# 只查了一条、答案没引用:唯一的那条就是产出答案的,按它比,依据写明
|
||||
run = await ls.explore(monkeypatch, ls.EQUIVALENT, text="本周销售额 170.5 元。")
|
||||
only = await score.read_answer(run.id)
|
||||
assert only.cited == [] and len(only.queries) == 1
|
||||
match = await score.sql_match(case, gold, only)
|
||||
assert match["status"] == "proven" and match["basis"] == "only_query" and not match["cited"], match
|
||||
|
||||
|
||||
async def test_rescoring_reads_the_seal_and_never_calls_a_model(engine_on, monkeypatch, tmp_path):
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models_eval import EvalResult, EvalRun
|
||||
|
||||
@@ -69,6 +69,18 @@ def test_summary_counts_only_gradable_items_and_splits_roles():
|
||||
assert summary["e1"]["passed"] == 1 and summary["e1"]["n"] == 2 and summary["e1"]["aspects"] == {"time": 1}
|
||||
|
||||
|
||||
def test_the_per_case_cost_is_not_zero_when_calls_could_not_be_priced():
|
||||
"""模型不在价格目录里(价格目录空):金额 0 只是「估得出的那部分」,每题平均花费不能写成 0(像不花钱)——一分都没估出来
|
||||
的为 None(估不出),估出了一部分的照给、unpriced_calls 说明它只是下限。"""
|
||||
unpriced = {"assistant": {"by_model": {}, "calls": 4, "cost_usd": 0.0, "input_tokens": 900, "output_tokens": 90,
|
||||
"unpriced_calls": 4}}
|
||||
summary = stats.summarize([_row("c1", "pass", usage=unpriced), _row("c2", "fail", usage=unpriced)])
|
||||
assert summary["unpriced_calls"] == 8 and summary["per_case_cost_usd"] is None, summary["per_case_cost_usd"]
|
||||
partly = {"assistant": {**unpriced["assistant"], "cost_usd": 0.02, "unpriced_calls": 1}}
|
||||
summary = stats.summarize([_row("c1", "pass", usage=partly), _row("c2", "fail", usage=partly)])
|
||||
assert summary["per_case_cost_usd"] == 0.02 and summary["unpriced_calls"] == 2
|
||||
|
||||
|
||||
def test_repeats_give_a_pass_rate_per_case():
|
||||
rows = [_row("c1", "pass", repeat=0), _row("c1", "fail", repeat=1), _row("c1", "pass", repeat=2)]
|
||||
summary = stats.summarize(rows)
|
||||
|
||||
@@ -236,3 +236,20 @@ async def test_out_of_scope_parent_is_not_called_missing(big):
|
||||
checks = catalog_profile.plan_checks("ord_hdr_east", tables["ord_hdr_east"], {"relations": [rel]},
|
||||
tables=tables, masked=set(), outside=outside)
|
||||
assert [c for c in checks if c.kind == "relation"][0].skip[0] == "missing"
|
||||
|
||||
|
||||
def test_out_of_scope_objects_without_a_comment_match_only_their_english_name():
|
||||
"""范围外、又没有注释的对象只有英文名可检索(data/schema_view 模块说明写明的限制):英文名拆出的词(catalog.name_words)
|
||||
没有中文说法——仓库里没有现成的英中业务词对照,不临时编词表,中文需求命不中只叫 refund_orders 的表;有注释的照常命中。
|
||||
以后接了词表,这条要跟着改。"""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from app.data import schema_view
|
||||
|
||||
source = SimpleNamespace(schema_cache={"tables": {}}, options={})
|
||||
outside = [SimpleNamespace(name="refund_orders", comment=None),
|
||||
SimpleNamespace(name="cust_complaint", comment="客户投诉单")]
|
||||
search = schema_view.TableSearch(source, {}, outside)
|
||||
assert [h.name for h in search.search("refund", limit=5)] == ["refund_orders"], "英文名、拆出的词照常命中"
|
||||
assert "refund_orders" not in [h.name for h in search.search("上周的退款", limit=5)], "没有注释、没有词表:中文需求命不中英文名"
|
||||
assert [h.name for h in search.search("投诉", limit=5)] == ["cust_complaint"], "有注释的按注释命中"
|
||||
|
||||
@@ -96,7 +96,8 @@ async def test_metering_counts_by_model_and_prices_from_the_catalog():
|
||||
await Scripted(model=FREE, report=False).ainvoke("你好")
|
||||
got = m.summary()
|
||||
assert got["by_model"][PRICED] == {"input_tokens": 2000, "output_tokens": 400, "calls": 2,
|
||||
"cost_usd": round(estimate_cost(PRICED, 2000, 400), 6), "unpriced_calls": 0}
|
||||
"cost_usd": round(estimate_cost(PRICED, 2000, 400), 6), "unpriced_calls": 0,
|
||||
"unfinished_calls": 0}
|
||||
assert got["by_model"][UNPRICED]["unpriced_calls"] == 1 and got["by_model"][UNPRICED]["cost_usd"] == 0
|
||||
# 目录外的模型名(多了后缀)照样算估不出;目录价是 0 的本地模型没报用量也不算估不出
|
||||
assert got["by_model"][PRICED + "-silent"]["unpriced_calls"] == 1
|
||||
@@ -184,6 +185,64 @@ async def test_the_eval_driver_on_the_demo_model_records_its_name(monkeypatch):
|
||||
assert set(got) == {"mock-fast"} and got["mock-fast"]["calls"] >= 1, got
|
||||
|
||||
|
||||
class Slow(Scripted):
|
||||
"""回得很慢的模型:调用方等不及(挑表超时被 asyncio.wait_for 取消)。partial=True 时先报一段输入用量再卡住。"""
|
||||
|
||||
delay: float = 5.0
|
||||
partial: bool = False
|
||||
|
||||
async def _agenerate(self, messages, stop=None, run_manager=None, **kwargs) -> ChatResult:
|
||||
self.calls += 1
|
||||
await asyncio.sleep(self.delay)
|
||||
return ChatResult(generations=[ChatGeneration(message=AIMessage(content="晚了"))])
|
||||
|
||||
async def _astream(self, messages, stop=None, run_manager=None, **kwargs):
|
||||
self.calls += 1
|
||||
if self.partial:
|
||||
yield ChatGenerationChunk(message=AIMessageChunk(content="{", usage_metadata={
|
||||
"input_tokens": 900, "output_tokens": 0, "total_tokens": 900}))
|
||||
await asyncio.sleep(self.delay)
|
||||
yield ChatGenerationChunk(message=AIMessageChunk(content="晚了"))
|
||||
|
||||
|
||||
class Refused(Scripted):
|
||||
"""请求刚发出去就被拒了(鉴权错、参数不对):模型没干活,不算一次调用。"""
|
||||
|
||||
async def _agenerate(self, messages, stop=None, run_manager=None, **kwargs) -> ChatResult:
|
||||
raise ValueError("401 鉴权失败")
|
||||
|
||||
|
||||
async def test_a_call_cancelled_before_any_result_still_counts_as_an_unfinished_call():
|
||||
"""挑表超时被取消的那次调用以前一点都不计(LangChain 取消时连 on_llm_error 都不一定回调):调用次数少了一次。现在
|
||||
记一次调用、token 不知道(没有部分结果的记 0、算没报全),标「未完成」;有部分用量的照实记已报的;被拒的仍不计。"""
|
||||
from app.look import pick
|
||||
|
||||
m = usage.meter()
|
||||
with usage.metering(m):
|
||||
with pytest.raises(asyncio.TimeoutError):
|
||||
await asyncio.wait_for(Slow(model=PRICED).ainvoke("挑表"), timeout=0.05)
|
||||
with pytest.raises(asyncio.TimeoutError):
|
||||
await asyncio.wait_for(pick._ask(Slow(model=UNPRICED), [("human", "挑表")]), timeout=0.05)
|
||||
|
||||
async def read():
|
||||
async for _ in Slow(model=PRICED + "-partial", partial=True).astream("挑表"):
|
||||
pass
|
||||
|
||||
with pytest.raises(asyncio.TimeoutError):
|
||||
await asyncio.wait_for(read(), timeout=0.05)
|
||||
with pytest.raises(ValueError):
|
||||
await Refused(model=PRICED + "-refused").ainvoke("挑表")
|
||||
got = m.summary()
|
||||
rows = got["by_model"]
|
||||
assert (rows[PRICED]["calls"], rows[PRICED]["unfinished_calls"], rows[PRICED]["input_tokens"]) == (1, 1, 0), rows
|
||||
assert rows[PRICED]["unpriced_calls"] == 1, "token 不知道,金额也就估不出"
|
||||
assert (rows[UNPRICED]["calls"], rows[UNPRICED]["unfinished_calls"]) == (1, 1), "挑表(look/pick._ask)超时同样计"
|
||||
partial = rows[PRICED + "-partial"]
|
||||
assert (partial["calls"], partial["unfinished_calls"], partial["input_tokens"]) == (1, 1, 900), partial
|
||||
assert PRICED + "-refused" not in rows, "被拒的不算一次调用"
|
||||
assert got["calls"] == 3 and got["unfinished_calls"] == 3
|
||||
|
||||
|
||||
async def test_outside_the_meter_nothing_is_counted():
|
||||
m = usage.meter()
|
||||
with usage.metering(m):
|
||||
|
||||
@@ -783,6 +783,12 @@ export const TURN_COST_TEXT = {
|
||||
`其中 ${formatNumber(n)} 次调用的金额无法估算(模型不在价格目录中,或用量没有报全),未计入,实际花费不低于此数`,
|
||||
} as const
|
||||
|
||||
/** 问数据一轮自动重来一次时的进度说明(服务端 chat/turn.next_step 判的 retry) */
|
||||
export const CHAT_RETRY_TEXT = {
|
||||
/** 助手上一次只写了思考、一个操作都没给(error.next = retry) */
|
||||
empty: '助手上一次没有给出任何操作,正在重新生成工作流…',
|
||||
} as const
|
||||
|
||||
export const JUDGE_SETTING_TEXT = {
|
||||
label: '证据裁判',
|
||||
hint: '报告中的结论句(如「增长主要来自新客首单」)由另一个模型按证据逐句判断:正式运行由报告撰写节点当场裁判,探索运行在点击时逐句裁判。判断由模型给出,不是系统核对,界面上一律标注「非确定」。',
|
||||
@@ -2314,7 +2320,7 @@ export const CATALOG_TEXT = {
|
||||
missing: '表结构中已没有这张表',
|
||||
noCatalog: '没有目录',
|
||||
usage: (n: number) => `使用 ${formatNumber(n)} 次`,
|
||||
usageHint: '被运行查询过的次数,同一次运行中结果相同的重复查询只算一次',
|
||||
usageHint: '按查询算:被运行查询过的次数,同一次运行里查了几次算几次(结果相同的重复查询只算一次);「用过这张表的运行」按运行算,两个数可能不一样',
|
||||
relationCount: (n: number) => `${formatNumber(n)} 个关联`,
|
||||
countTitle: (label: string, n: number) => `${label} ${formatNumber(n)} 项`,
|
||||
noMatch: '没有符合条件的表',
|
||||
@@ -6092,6 +6098,8 @@ export const APP_EDITOR_TEXT = {
|
||||
slotAdd: '新增岗位',
|
||||
slotName: '岗位名',
|
||||
slotLabel: '显示名',
|
||||
/** 显示名和岗位名一样:派生时没有节点名可用(服务端 apps/tree.slot_label),照实用的是岗位名,不是起好的名字 */
|
||||
slotLabelRaw: '还没有起显示名,现在用的是图里的岗位名,可以改成一个好认的名字',
|
||||
slotUnbound: '未绑定:发布检查会拦下流程应用',
|
||||
slotRemove: (name: string) => `删除岗位「${name}」`,
|
||||
grants: '授权',
|
||||
|
||||
@@ -356,6 +356,8 @@ export const EVAL_TEXT = {
|
||||
estimateText: (items: number, cases: number, repeats: number, arms: number) =>
|
||||
`${formatNumber(cases)} 道题 × 每题 ${repeats} 次${arms > 1 ? ` × ${arms} 臂` : ''},共 ${formatNumber(items)} 项`,
|
||||
cost: (total: string, per: string) => `约 ${total}(每项约 ${per})`,
|
||||
/** 上一次有估不出金额的调用:估出来的只是下限 */
|
||||
costAtLeast: (total: string, per: string) => `至少约 ${total}(每项至少约 ${per})`,
|
||||
costNone: '估不出花费',
|
||||
budget: '生效的预算',
|
||||
parallel: (n: number) => `同时评 ${n} 题`,
|
||||
@@ -416,6 +418,8 @@ export const EVAL_TEXT = {
|
||||
cost: '花费',
|
||||
costHint: '助手(建图、挑表、复核)+ 写作(答案运行里的模型调用)+ 结论裁判;算期望不调模型',
|
||||
perCase: (v: string) => `每题约 ${v}`,
|
||||
/** 有估不出金额的调用:每题平均只是下限 */
|
||||
perCaseAtLeast: (v: string) => `每题至少约 ${v}`,
|
||||
e1: '逐题的口径命中(E1)',
|
||||
e1Hint: '答案执行的 SQL 和期望的取数被证明等价的比例(口径账本同一个比对函数)',
|
||||
citations: '报告引用率',
|
||||
@@ -441,7 +445,9 @@ export const EVAL_TEXT = {
|
||||
timing: { build_ms: '建图', run_wall_ms: '运行(墙钟)', run_active_ms: '运行(执行)', review_ms: '复核', total_ms: '整题' } as Record<string, string>,
|
||||
usage: '按角色的用量',
|
||||
tokens: (input: string, output: string) => `输入 ${input} · 输出 ${output}`,
|
||||
callsN: (n: number) => `${formatNumber(n)} 次调用`,
|
||||
/** unfinished:没做完的调用(被取消、超时,token 只计了已报的;服务端 providers/usage 的 unfinished_calls) */
|
||||
callsN: (n: number, unfinished = 0) =>
|
||||
`${formatNumber(n)} 次调用${unfinished > 0 ? `(其中 ${formatNumber(unfinished)} 次未完成:被取消或超时,token 只计了已报的部分)` : ''}`,
|
||||
role: { assistant: '助手', writer: '写作', judge: '结论裁判', gold: '算期望' } as Record<string, string>,
|
||||
passRates: '重复评的题的通过率',
|
||||
outcomes: '各期望的结论',
|
||||
@@ -464,6 +470,10 @@ export const EVAL_TEXT = {
|
||||
sqlMatch: {
|
||||
proven: '等价已证明', related: '不等价', none: '答案没有查询', skipped: '没比对',
|
||||
} as Record<string, string>,
|
||||
/** 逐题 SQL 比对比的是哪条查询(服务端 score.sql_match 的 basis)。答案引用了查询的不用多说 */
|
||||
sqlMatchBasis: {
|
||||
only_query: '答案没有引用查询;这次运行只执行了这一条查询(核查用的不算),按它比',
|
||||
} as Record<string, string>,
|
||||
aspect: {
|
||||
time: '时间口径', time_range: '时间范围', filter: '筛选条件', aggregate: '聚合', expression: '计算式', join: '关联',
|
||||
tables: '用到的表', group_by: '分组', limit: '行数限制', other: '其他', parse: '解析不了',
|
||||
@@ -495,6 +505,8 @@ export const EVAL_TEXT = {
|
||||
filter: '只看',
|
||||
repeat: (n: number) => `第 ${n + 1} 次`,
|
||||
attempts: (n: number) => `这一题自动重来了 ${n} 次`,
|
||||
/** 每次自动重来的原因(服务端 eval/answer 的 turn.retries) */
|
||||
retriesWhy: (reasons: string[]) => `:${reasons.join(';')}`,
|
||||
turnReason: '作答的经过',
|
||||
answer: '答案原文',
|
||||
expand: '展开',
|
||||
|
||||
@@ -60,6 +60,7 @@ export function AccessSection() {
|
||||
setBindings((b) => { const next = { ...b.slots }; delete next[name]; return { ...b, slots: next } })
|
||||
}} />
|
||||
</div>
|
||||
{slot.label.trim() === name && <p className="text-2xs text-faint" data-app-slot-raw="">{T.slotLabelRaw}</p>}
|
||||
<PrincipalPicker label={T.slots} value={bound as Picked[]}
|
||||
onChange={(v) => setBindings((b) => ({
|
||||
...b, slots: { ...b.slots, [name]: v.filter((x) => x.kind !== 'everyone') as { kind: 'user' | 'group'; name: string }[] },
|
||||
|
||||
@@ -215,8 +215,7 @@ function Overview({ run }: { run: EvalRun }) {
|
||||
<div className="grid gap-2 sm:grid-cols-2 lg:grid-cols-4">
|
||||
<RateCard title={O.accuracy} hint={O.accuracyHint} rate={s.accuracy} attr="accuracy" />
|
||||
<RateCard title={O.traced} hint={O.tracedHint} rate={s.traced_accuracy} attr="traced" />
|
||||
<StatCard title={O.cost} hint={O.costHint} value={cost} attr="cost"
|
||||
sub={s.per_case_cost_usd != null ? O.perCase(formatCost(s.per_case_cost_usd)) : undefined} />
|
||||
<StatCard title={O.cost} hint={O.costHint} value={cost} attr="cost" sub={perCaseText(s)} />
|
||||
<StatCard title={O.e1} hint={O.e1Hint} value={s.e1?.n ? pct(s.e1.rate) : '—'} sub={rateText(s.e1)} attr="e1" />
|
||||
</div>
|
||||
<Section title={O.verdicts} attr="verdicts">
|
||||
@@ -278,7 +277,7 @@ function Overview({ run }: { run: EvalRun }) {
|
||||
<li key={role} data-eval-usage-role={role}>
|
||||
{O.role[role]}:{usageCost(u)}
|
||||
{u.input_tokens != null && <span className="text-faint"> · {O.tokens(formatTokens(u.input_tokens, { compact: true }), formatTokens(u.output_tokens ?? 0, { compact: true }))}</span>}
|
||||
{u.calls != null && <span className="text-faint"> · {O.callsN(u.calls)}</span>}
|
||||
{u.calls != null && <span className="text-faint"> · {O.callsN(u.calls, u.unfinished_calls ?? 0)}</span>}
|
||||
</li>
|
||||
)
|
||||
})}
|
||||
@@ -296,6 +295,17 @@ function Overview({ run }: { run: EvalRun }) {
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* 每题平均花费:有估不出金额的调用时只是下限(写「至少」);一分都没估出来的不写(老总表记成 0 的也一样,金额那一格已经写了
|
||||
* 「金额无法估算」)
|
||||
*/
|
||||
function perCaseText(s: EvalSummary): string | undefined {
|
||||
const per = s.per_case_cost_usd
|
||||
if (per == null) return undefined
|
||||
if (!s.unpriced_calls) return T.overview.perCase(formatCost(per))
|
||||
return per > 0 ? T.overview.perCaseAtLeast(formatCost(per)) : undefined
|
||||
}
|
||||
|
||||
function Section({ title, children, attr }: { title: string; children: ReactNode; attr?: string }) {
|
||||
return (
|
||||
<section className="space-y-1 rounded-lg border bg-panel p-3 text-xs" aria-label={title} data-eval-section={attr}>
|
||||
@@ -417,6 +427,9 @@ function Results({ id, live, focusCase }: { id: string; live: boolean; focusCase
|
||||
{s.reason && <span> · {s.reason}</span>}
|
||||
</p>
|
||||
))}
|
||||
{r.sql_match?.basis && T.sqlMatchBasis[r.sql_match.basis] && (
|
||||
<p className="text-faint" data-eval-sql-basis={r.sql_match.basis}>{T.sqlMatchBasis[r.sql_match.basis]}</p>
|
||||
)}
|
||||
{r.sql_match?.reason && <p className="text-faint">{r.sql_match.reason}</p>}
|
||||
<ResultDetail evaluationId={id} resultId={r.id} />
|
||||
<div className="flex flex-wrap gap-2">
|
||||
@@ -450,7 +463,11 @@ function ResultDetail({ evaluationId, resultId }: { evaluationId: string; result
|
||||
const R = T.results
|
||||
return (
|
||||
<div className="space-y-1" data-eval-result-details="">
|
||||
{(turn.attempts ?? 0) > 0 && <p className="text-faint">{R.attempts(turn.attempts)}</p>}
|
||||
{(turn.attempts ?? 0) > 0 && (
|
||||
<p className="text-faint" data-eval-retries={turn.retries?.length ?? 0}>
|
||||
{R.attempts(turn.attempts)}{turn.retries?.length ? R.retriesWhy(turn.retries) : ''}
|
||||
</p>
|
||||
)}
|
||||
{turn.reason && <p className="text-dim"><span className="text-faint">{R.turnReason}:</span>{turn.reason}</p>}
|
||||
{answer && (
|
||||
<details>
|
||||
|
||||
@@ -263,9 +263,8 @@ export function StartDialog({ open, onClose, suite, evaluations, onStarted }: {
|
||||
<>
|
||||
<p data-eval-estimate-items={est.estimate.items}>{T.estimateText(est.estimate.items, est.cases, est.repeats, est.arms.length)}</p>
|
||||
<p data-eval-estimate-cost="">
|
||||
{est.estimate.total_usd != null
|
||||
? T.cost(formatCost(est.estimate.total_usd), formatCost(est.estimate.per_item_usd))
|
||||
: T.costNone}
|
||||
{est.estimate.total_usd == null ? T.costNone
|
||||
: (est.estimate.at_least ? T.costAtLeast : T.cost)(formatCost(est.estimate.total_usd), formatCost(est.estimate.per_item_usd))}
|
||||
<span className="text-faint"> · {est.estimate.note}</span>
|
||||
</p>
|
||||
{est.arms.map((a, i) => (
|
||||
|
||||
@@ -2,7 +2,7 @@ import { create } from 'zustand'
|
||||
import { ApiError, api, streamCopilot, streamRun } from '../api/client'
|
||||
import type { CopilotOp, FixKind, RunFinal } from '../run/decode'
|
||||
import { copilotContext, copilotLook, decodeRun, looksOf } from '../run/decode'
|
||||
import { CHAT_BLUEPRINT_TEXT, LEDGER_TEXT, LOOK_TEXT, REJECTED_TEXT } from '../lib/terms'
|
||||
import { CHAT_BLUEPRINT_TEXT, CHAT_RETRY_TEXT, LEDGER_TEXT, LOOK_TEXT, REJECTED_TEXT } from '../lib/terms'
|
||||
import type { LookEvent } from '../typesApp'
|
||||
import { errorMessage, humanizeError, isNetworkError } from '../lib/errors'
|
||||
import { guardReview, hasEvidence } from '../lib/evidence'
|
||||
@@ -174,6 +174,8 @@ export interface UsageTotal {
|
||||
cost_usd?: number | null
|
||||
unpriced_calls?: number
|
||||
calls?: number
|
||||
/** 没做完的调用(被取消、超时):算在 calls 里,token 只计了已报的(服务端 providers/usage 的 unfinished_calls) */
|
||||
unfinished_calls?: number
|
||||
input_tokens?: number
|
||||
output_tokens?: number
|
||||
}
|
||||
@@ -498,7 +500,7 @@ function buildMeta(t: ChatTurn): TurnMeta {
|
||||
}
|
||||
}
|
||||
|
||||
const USAGE_KEYS = ['cost_usd', 'unpriced_calls', 'calls', 'input_tokens', 'output_tokens'] as const
|
||||
const USAGE_KEYS = ['cost_usd', 'unpriced_calls', 'calls', 'unfinished_calls', 'input_tokens', 'output_tokens'] as const
|
||||
|
||||
/** 两份用量相加(缺的当 0)。b 不是对象时原样交回 a */
|
||||
export function addUsage(a: UsageTotal | undefined, b: UsageTotal | null | undefined): UsageTotal | undefined {
|
||||
@@ -1417,14 +1419,14 @@ function plan(flow: Flow, args: PlanArgs) {
|
||||
* 这里会清空这一次的过程和答案——它是自动补救,用户还没拿到过答案。用户主动
|
||||
* 点的重试走 retryTurn,上一次要留档。
|
||||
*/
|
||||
const retry = (reason: string, instruction?: string) => {
|
||||
const retry = (reason: string, instruction?: string, status?: string) => {
|
||||
// 服务端给了那句话(复核的 retry_instruction)就是它已经按次数判过可以重来;老服务端由这里按次数判
|
||||
if ((!instruction && attempt + 1 >= MAX_ATTEMPTS) || !flow.alive()) return false
|
||||
const base = built && (built.nodes?.length ?? 0) > 0 ? built : null
|
||||
patch((t) => ({
|
||||
attempt: attempt + 1,
|
||||
phase: 'planning',
|
||||
status: base ? '上次运行未成功,正在调整工作流…' : '上次运行未成功,正在重新生成工作流…',
|
||||
status: status ?? (base ? '上次运行未成功,正在调整工作流…' : '上次运行未成功,正在重新生成工作流…'),
|
||||
ops: [], events: [], run: null, output: null, final: undefined, runStatus: undefined,
|
||||
review: null, reviewRaw: null, rawOutput: null, lastSeq: 0, failure: null, error: '',
|
||||
// 换下来的那次运行花的钱还是这一轮的
|
||||
@@ -1584,9 +1586,12 @@ function plan(flow: Flow, args: PlanArgs) {
|
||||
void flow.persist()
|
||||
break
|
||||
case 'error':
|
||||
settled = true
|
||||
// 生成过程一个操作都没给(模型只写了思考):服务端说重来一次(error.next = retry,backend/app/chat/turn.py 的
|
||||
// next_step),照它带着原因重来,次数照 MAX_ATTEMPTS;试满了才照实报。这一次建图的用量随后那条 usage 照样记进这一轮
|
||||
if (op.next === 'retry' && retry(String(op.reason ?? op.message ?? ''), undefined, CHAT_RETRY_TEXT.empty)) break
|
||||
// 后端给的是三段:message 人话、hint 怎么办、detail 原始异常。
|
||||
// 原文不再截成一行甩给用户,收进可展开、可复制的技术细节
|
||||
settled = true
|
||||
fail({
|
||||
...failureOf(op.message ?? '生成失败'),
|
||||
...(op.hint ? { hint: String(op.hint) } : {}),
|
||||
|
||||
@@ -365,7 +365,11 @@ export interface EvalEstimate {
|
||||
arms: { label: string | null; variant: EvalVariant; models: EvalResolvedModels; text: string }[]
|
||||
cases: number
|
||||
repeats: number
|
||||
estimate: { items: number; per_item_usd: number | null; total_usd: number | null; basis: string | null; note: string }
|
||||
/** at_least:上一次另有 unpriced_calls 次调用金额估不出,估算只是下限;total_usd 为空、unpriced_calls 有值:一分都没估出来 */
|
||||
estimate: {
|
||||
items: number; per_item_usd: number | null; total_usd: number | null; basis: string | null; note: string
|
||||
at_least?: boolean; unpriced_calls?: number
|
||||
}
|
||||
budget: EvalBudget
|
||||
parallel: number
|
||||
spend: EvalSpend
|
||||
@@ -415,7 +419,10 @@ export interface EvalUsage {
|
||||
calls?: number
|
||||
cost_usd?: number | null
|
||||
unpriced_calls?: number
|
||||
by_model?: Record<string, { input_tokens: number; output_tokens: number; calls: number; cost_usd: number; unpriced_calls: number }>
|
||||
/** 没做完的调用(被取消、超时):算在 calls 里,token 只计了已报的 */
|
||||
unfinished_calls?: number
|
||||
by_model?: Record<string, { input_tokens: number; output_tokens: number; calls: number; cost_usd: number; unpriced_calls: number
|
||||
unfinished_calls?: number }>
|
||||
}
|
||||
|
||||
export interface EvalTiming {
|
||||
@@ -531,6 +538,8 @@ export interface EvalSqlMatch {
|
||||
aspects?: string[]
|
||||
reason?: string | null
|
||||
cited?: boolean
|
||||
/** 比的是哪条查询:cited 答案引用的;only_query 答案没引用、这次运行只查了这一条 */
|
||||
basis?: 'cited' | 'only_query' | string
|
||||
}
|
||||
|
||||
export interface EvalResult {
|
||||
|
||||
+49
-1
@@ -29,7 +29,8 @@
|
||||
// 14. 助手反问(normal-user-apps N11):只问不改的一轮记成「助手反问」,问题作为这一轮的内容、落库进历史,不再一直停在
|
||||
// 「正在理解需求」;输入框提示回答助手的问题;刷新后照样标「助手反问」,不当成没查库的回答;
|
||||
// 15. 回答上的「做成应用」带着这次运行进蓝图(建运行起点的蓝图),不再提取工作流、跳画布的发布弹窗;
|
||||
// 16. 服务端这一轮已经自查修正过(收到 check 的结论)还没过的:不再整轮重新生成一遍,直接说没过、给补救。
|
||||
// 16. 服务端这一轮已经自查修正过(收到 check 的结论)还没过的:不再整轮重新生成一遍,直接说没过、给补救;助手只写了思考、
|
||||
// 一个操作都没给(error.next=retry)的照服务端的决定自动重来一次,还不行再照实报。
|
||||
// 17. 口径账本(next-round 02):交付了的轮次底部写「正在比对口径」,按退避再取到比对出结果、比完不再取;一行摘要
|
||||
// (几个数来自哪份口径)、按口径正式出具和对账按钮照服务端给的动作摆;出具、对账的写请求在浏览器层拦下并核对请求体,
|
||||
// 出具过的正式运行挂在轮次下面(档位、打开运行)。
|
||||
@@ -532,6 +533,26 @@ async function fakeApi(route) {
|
||||
return route.fulfill({ status: 200, contentType: 'text/event-stream',
|
||||
body: ops.map((o) => `data: ${JSON.stringify(o)}\n\n`).join('') })
|
||||
}
|
||||
if (String(b.instruction ?? '').includes('只写了思考')) {
|
||||
// 生成过程一个操作都没给(deepseek 偶发只出思考):服务端的 error 带 next=retry(backend/app/chat/turn.py 的 next_step)。
|
||||
// 重来那一次(指令里缀了「上一次没有成功」)照常直接回答;「两次都」那一问每次都只写思考
|
||||
const again = String(b.instruction).includes('上一次没有成功') && !String(b.instruction).includes('两次都')
|
||||
const spent = { by_model: { m: { input_tokens: 500, output_tokens: 40, calls: 1, cost_usd: 0.001, unpriced_calls: 0 } },
|
||||
input_tokens: 500, output_tokens: 40, calls: 1, cost_usd: 0.001, unpriced_calls: 0 }
|
||||
const ops = again ? [
|
||||
{ op: 'heartbeat', phase: 'planning', elapsed_ms: 200 },
|
||||
{ op: 'reply', text: '已经查过了:一共 3 家门店。' },
|
||||
{ op: 'usage', ...spent },
|
||||
] : [
|
||||
{ op: 'heartbeat', phase: 'planning', elapsed_ms: 200 },
|
||||
{ op: 'thinking', delta: '先想一想要查哪张表……' },
|
||||
{ op: 'error', message: '助手本轮未完成:模型没有给出任何修改,也没有说明原因', hint: '画布未修改。可以重试,或把需求描述得更具体',
|
||||
detail: '', reason: '模型没有给出任何修改,也没有说明原因', code: 'copilot_empty', next: 'retry' },
|
||||
{ op: 'usage', ...spent },
|
||||
]
|
||||
return route.fulfill({ status: 200, contentType: 'text/event-stream',
|
||||
body: ops.map((o) => `data: ${JSON.stringify(o)}\n\n`).join('') })
|
||||
}
|
||||
if (String(b.instruction ?? '').includes('照服务端的决定') || String(b.instruction ?? '').includes(DECIDE_RETRY)) {
|
||||
// 新服务端:final 带 next(backend/app/chat/turn.py),最后一条是这一轮建图的用量(answer_ops 的 usage)。
|
||||
// 「只建不跑」那一问:autorun 没写 false,next 却是 ready——前端得照 next 走
|
||||
@@ -1858,6 +1879,33 @@ for (const theme of THEMES) {
|
||||
await ctx.close()
|
||||
})
|
||||
|
||||
await section('emptyretry', '助手只写了思考、一个操作都没给:照服务端的 next=retry 自动重来一次,还不行再照实报', async () => {
|
||||
const { page, ctx, errors } = await open(theme)
|
||||
await goto(page, 'c0emptg')
|
||||
let gens = log.gens.length
|
||||
await send(page, '只写了思考:有几家门店')
|
||||
const done = await page.waitForFunction(() => window.__chat.getState().byConversation.c0emptg?.at(-1)?.phase === 'done',
|
||||
null, { timeout: 6000 }).then(() => true, () => false)
|
||||
await page.waitForTimeout(300)
|
||||
const turn = (await chatState(page)).byConversation.c0emptg.at(-1)
|
||||
check('第一次没给操作:自动重来一次,第二次照常回答', done && log.gens.length - gens === 2
|
||||
&& String(turn.output?.answer ?? '').includes('3 家'), `${log.gens.length - gens} 次生成 · ${JSON.stringify(turn.output)}`)
|
||||
check('重来时交回去的话带着原因(同服务端 retry_instruction 的从零重建措辞)',
|
||||
log.gens.at(-1)?.instruction === '只写了思考:有几家门店\n\n(上一次没有成功:模型没有给出任何修改,也没有说明原因。请针对这个原因调整,重新给出完整的工作流。)',
|
||||
log.gens.at(-1)?.instruction)
|
||||
check('两次建图的用量都记进这一轮', Math.abs((turn.usage?.copilot?.cost_usd ?? 0) - 0.002) < 1e-9, JSON.stringify(turn.usage))
|
||||
gens = log.gens.length
|
||||
await send(page, '两次都只写了思考:有几家门店')
|
||||
const failed = await page.waitForFunction(() => window.__chat.getState().byConversation.c0emptg?.at(-1)?.phase === 'error',
|
||||
null, { timeout: 6000 }).then(() => true, () => false)
|
||||
await page.waitForTimeout(500)
|
||||
const last = (await chatState(page)).byConversation.c0emptg.at(-1)
|
||||
check('重来还不行:照实报「助手本轮未完成」,不再试第三次', failed && log.gens.length - gens === 2
|
||||
&& JSON.stringify(last.failure ?? {}).includes('助手本轮未完成'), `${log.gens.length - gens} 次生成 · ${JSON.stringify(last.failure)}`)
|
||||
check('没有运行时报错', errors.length === 0, errors[0] ?? '')
|
||||
await ctx.close()
|
||||
})
|
||||
|
||||
await section('decide', '新服务端:照 final.next 和复核的 retry_instruction 走,这一轮的花费 = 助手 + 运行 + 复核', async () => {
|
||||
const { page, ctx, errors } = await open(theme)
|
||||
await goto(page, 'c0emptg')
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
// 数值期望不许人填(请求体里不出现 value 这类键);
|
||||
// - 口径升版待处置:顶部提示、处置三选一(叫法由服务端给,没有缺省);启用开关、导出;
|
||||
// - 发起评测:估算和发起是同一份请求体,评一次 / A/B 对照 / 目录贡献对照(预设)各自的请求体;估算的项数、花费、今天已花照服务端写;
|
||||
// 上一次的调用金额估不出时估算写「估不出花费」、估出一部分的写「至少约」,总览的每题平均不写成 $0;
|
||||
// - 评测详情:进度、每题正在做什么;停止、只重试没做完的、重评、设为基线——摆不摆按钮照服务端的 actions;总览卡片和各项分布;
|
||||
// 逐题表(结论、期望和答案里的数、SQL 差在哪、点开去运行详情)、?case= 定位那道题;和上一次比(指纹差异、转移、归因、换基准);
|
||||
// A/B 配对表(都对、只 A 对、只 B 对、p 值、看不出差别);
|
||||
@@ -695,6 +696,41 @@ await section('加题:选口径、从运行认定、期望反问', async () =>
|
||||
|
||||
// ---------------------------------------------------------------- 发起评测
|
||||
|
||||
await section('发起评测:估不出金额时估算和总览不写成 $0', async () => {
|
||||
// 服务端 eval/runner.estimate:上一次的调用金额都估不出 → total_usd 为空、带次数;估出一部分 → at_least
|
||||
const shaped = (patch) => ({ ...ESTIMATE(['当前目录']), estimate: { ...ESTIMATE(['当前目录']).estimate, ...patch } })
|
||||
const none = shaped({ per_item_usd: null, total_usd: null, unpriced_calls: 6,
|
||||
note: '上一次评测的 6 次模型调用金额都估不出(模型不在价格目录中,或用量没有报全):金额上限只管得住估得出金额的调用' })
|
||||
const some = shaped({ at_least: true, unpriced_calls: 2,
|
||||
note: '按这个题集上一次评测的每题平均花费估算,实际至少这么多:上一次另有 2 次调用金额估不出,没有算进去' })
|
||||
for (const [label, body] of [['都估不出', none], ['估出一部分', some]]) {
|
||||
const state = freshState({ override: (method, path) => (method === 'POST' && path.endsWith('/evaluations/estimate')
|
||||
? { json: body } : null) })
|
||||
const { page, errors } = await open('/evals/suites/suite-a', { state })
|
||||
await page.locator('[data-eval-suite-start]').click()
|
||||
await page.locator('[data-eval-estimate-items]').waitFor()
|
||||
const cost = await text(page, '[data-eval-estimate-cost]')
|
||||
if (label === '都估不出') {
|
||||
check('估算:上一次都估不出时写「估不出花费」和原因,不写 $0', cost.includes('估不出花费') && cost.includes('6 次') && !cost.includes('$0'), cost)
|
||||
} else {
|
||||
check('估算:估出一部分时写「至少约」', cost.includes('至少约 $0.042') && cost.includes('每项至少约 $0.014') && cost.includes('2 次'), cost)
|
||||
}
|
||||
check(`没有页面报错(${label})`, !errors.length, errors.join(' | '))
|
||||
}
|
||||
const state = freshState()
|
||||
state.evaluations['ev-1'].summary = { ...state.evaluations['ev-1'].summary, cost_usd: 0, unpriced_calls: 6, per_case_cost_usd: null }
|
||||
const { page, errors } = await open('/evals/runs/ev-1', { state })
|
||||
await page.locator('[data-eval-overview]').waitFor()
|
||||
const card = await text(page, '[data-eval-card="cost"]')
|
||||
check('总览:都估不出时花费写「金额无法估算」,不写每题 $0', card.includes('金额无法估算') && !card.includes('每题'), card)
|
||||
state.evaluations['ev-1'].summary = { ...state.evaluations['ev-1'].summary, cost_usd: 0.042, unpriced_calls: 2, per_case_cost_usd: 0.014 }
|
||||
await page.reload({ waitUntil: 'networkidle' })
|
||||
await page.locator('[data-eval-overview]').waitFor()
|
||||
const partly = await text(page, '[data-eval-card="cost"]')
|
||||
check('总览:估出一部分时写「≥」和「每题至少约」', partly.includes('≥ $0.042') && partly.includes('每题至少约 $0.014'), partly)
|
||||
check('没有页面报错(总览)', !errors.length, errors.join(' | '))
|
||||
})
|
||||
|
||||
await section('发起评测:估算和发起同一份请求体;评一次、A/B、目录贡献对照', async () => {
|
||||
const { page, state, errors } = await open('/evals/suites/suite-a')
|
||||
await page.locator('[data-eval-suite-start]').click()
|
||||
|
||||
Reference in New Issue
Block a user