From 2b9dcc55e3547e89673f44f582ca611f3888da1a Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 13:49:53 +0800 Subject: [PATCH] =?UTF-8?q?=E8=AF=84=E6=B5=8B=E9=A2=98=E9=9B=86=E4=B8=8E?= =?UTF-8?q?=E6=9C=9F=E6=9C=9B=E7=AD=94=E6=A1=88=EF=BC=88=E7=AC=AC=202=20?= =?UTF-8?q?=E6=B3=A2=20W2-c=EF=BC=89=E7=AC=AC=E4=B8=80=E6=AD=A5=EF=BC=9A?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E3=80=81=E6=B4=BE=E7=94=9F=E5=9B=BE=E3=80=81?= =?UTF-8?q?=E6=9C=9F=E6=9C=9B=E7=9A=84=E4=B8=89=E7=A7=8D=E6=9D=A5=E6=BA=90?= =?UTF-8?q?=E3=80=81=E7=9B=AE=E5=BD=95=E5=BF=AB=E7=85=A7=E4=B8=8E=E5=BD=B1?= =?UTF-8?q?=E5=AD=90=E8=B5=B7=E8=8D=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - db/models_eval.py：EvalSuite、EvalCase、EvalRun、EvalResult 一次定全（第 3 波的执行、评分直接用），另加 EvalCatalogSnapshot 登记目录快照（谁、何时、来源、影子起草的进度和用量）。EvalResult.job_id 是 eval_run_id 的 别名：后台任务运行时的收尾按 task_model.job_id 认逐项。预算缺省按评测-D9（同时 2 题、每题 900 秒、单次 2 美元、 每日 5 美元），这一波只定字段。 - app/ledger/derive.py：全站唯一的「从钉住版本派生只读图」的生成器（总方案 X3）。这一波实现口径探针：拷出口径卡读 的取数节点（id、配置、图级 defaults 原样）+ caliber_from 固定到那一版的口径卡；只认可复用的源头卡，取数节点只许 引用入口字段，别的照实拒收。对账（这次的取数 + 口径 + 对账卡）、召回（新旧两版带前缀、caliber_from.nodes 改名 映射）的扩展点写在模块说明里，assemble / caliber_block 直接可用。 - app/eval/cases.py、suites.py：题目和题集的形状与校验。数值期望不许人填（评测-D4，写了 value 一类的键一律拒收）， 文字要点单列、不计入准确率；只做单轮（评测-D12，history 必须为空）；变量只认定时任务那份 {{ schedule.* }}； 题目指纹去掉确认人、出处运行、升版处置这些记录项。 - app/eval/gold.py：SQL 题、口径题、从运行认定三种期望。每次都以来源 eval、只读 + 遮罩的探索运行重算，期望值取封存的 查询快照（或口径卡的指标集）；只读守卫拒写语句、用到遮罩列的拒收（认法同证据面板的派生遮罩）、取不到值 / 失败 / 超时 / 停在人工确认上各写各的原因。口径升版用和正式运行同一个 unresolved_caliber_upgrades 判、退役按 subgraph_pin_problems 判，待处置就跳过；recompute / dual / incomparable 三选一、没有缺省。作答前后各算一次 （bracket / settle）给第 3 波的执行调。 - app/eval/snapshots.py：目录快照存成 kind=eval_catalog_snapshot 的内容寻址工件（读法即 data/catalog_view.py 定的 那一种）；影子起草是不占数据源队的后台任务，起草写进内存里的目录副本，不调 write_entry、不碰线上目录、不查业务库， 只起草题集用得到的表，开始前估算花费；停止、出错的不出快照。 - data/catalog_draft.py 只加入口 run_shadow_draft（同一个起草引擎，唯一不同是不去业务库数有效行过滤的排除比例）； data/catalog.py 把写库前的并入抽成纯函数 merge_draft、建起草模型抽成 build_draft_model（行为不变，影子起草和写库 用同一套并入规矩、同一个模型建法）。 --- backend/app/data/catalog.py | 48 +- backend/app/data/catalog_draft.py | 26 +- backend/app/db/models_eval.py | 244 ++++++++- backend/app/eval/__init__.py | 10 + backend/app/eval/cases.py | 520 +++++++++++++++++++ backend/app/eval/gold.py | 712 +++++++++++++++++++++++++++ backend/app/eval/snapshots.py | 609 +++++++++++++++++++++++ backend/app/eval/suites.py | 106 ++++ backend/app/ledger/derive.py | 325 ++++++++++++ backend/tests/test_eval_gold.py | 413 ++++++++++++++++ backend/tests/test_eval_snapshots.py | 233 +++++++++ backend/tests/test_ledger_derive.py | 159 ++++++ 12 files changed, 3383 insertions(+), 22 deletions(-) create mode 100644 backend/app/eval/__init__.py create mode 100644 backend/app/eval/cases.py create mode 100644 backend/app/eval/gold.py create mode 100644 backend/app/eval/snapshots.py create mode 100644 backend/app/eval/suites.py create mode 100644 backend/app/ledger/derive.py create mode 100644 backend/tests/test_eval_gold.py create mode 100644 backend/tests/test_eval_snapshots.py create mode 100644 backend/tests/test_ledger_derive.py diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index 25ac4c8c..b8e4f329 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -1827,7 +1827,7 @@ async def resolve_draft_model(session: AsyncSession) -> tuple[Any, str]: 起草出来的目录会被当成真的写进库里。 """ from app.data.draft_model import DraftModelUnavailable, draft_unavailable, resolve_draft_provider - from app.providers.factory import ModelSpec, ProviderNotConfigured, build_chat_model + from app.providers.factory import ProviderNotConfigured try: provider, choice = await resolve_draft_provider( @@ -1835,13 +1835,22 @@ async def resolve_draft_model(session: AsyncSession) -> tuple[Any, str]: mock_reason="当前只有演示用的模型接入，无法用于起草数据目录。请到「设置 → 模型接入」添加真实的模型接入") except DraftModelUnavailable as e: raise CatalogModelUnavailable(str(e)) from e - spec = ModelSpec(provider=choice.provider, model=choice.model, max_tokens=MODEL_MAX_TOKENS, temperature=0, - timeout=MODEL_TIMEOUT_S, max_retries=0) try: - big = build_chat_model(provider, spec) - small = build_chat_model(provider, spec.model_copy(update={"max_tokens": MODEL_FALLBACK_MAX_TOKENS})) + return build_draft_model(provider, choice.provider, choice.model) except ProviderNotConfigured as e: raise CatalogModelUnavailable(draft_unavailable(e, choice.origin)) from e + + +def build_draft_model(provider: Any, provider_name: str | None, model: str | None) -> tuple[DraftModel, str]: + """按起草的规矩建模型对象（温度 0、额度 MODEL_MAX_TOKENS、服务方嫌大时退回 MODEL_FALLBACK_MAX_TOKENS、时限 + MODEL_TIMEOUT_S、SDK 不重试，见 resolve_draft_model）：(模型, 模型 id)。接入由调用方解析好；缺 key 之类抛 + ProviderNotConfigured，说法由调用方定。评测的影子起草指定起草模型时也走这里（app/eval/snapshots.py）。""" + from app.providers.factory import ModelSpec, build_chat_model + + spec = ModelSpec(provider=provider_name, model=model, max_tokens=MODEL_MAX_TOKENS, temperature=0, + timeout=MODEL_TIMEOUT_S, max_retries=0) + big = build_chat_model(provider, spec) + small = build_chat_model(provider, spec.model_copy(update={"max_tokens": MODEL_FALLBACK_MAX_TOKENS})) model_id = spec.model or provider.default_model or "" return DraftModel(big, small, model_id), model_id @@ -2081,6 +2090,23 @@ def missing_table_error(name: str) -> str: WRITE_CONFLICT_ERROR = "这张表的数据目录正被其他人修改，请稍后再起草" +def merge_draft(existing: Mapping[str, Any] | None, source: Any, table: str, modeled: Any = None, *, + at: str) -> tuple[dict[str, Any], MergeStats]: + """一张表的起草并入一份目录（纯函数，不读写库）：数据库注释、外键约束、命名推断照做，modeled（catalog_draft. + ModelOutcome，没用模型时为 None）有可用内容的再并上，规则见 merge_notes（模型各段都写出来或只写出一部分时按完整覆盖 + 删掉说已经不存在的列的那些项）。write_draft 写库之前、评测的影子起草写进内存里的目录副本（app/eval/snapshots.py） + 都用它，并入的规矩只有这一份。""" + structure = draft_structure(source, table) + meta = ((getattr(source, "schema_cache", None) or {}).get("tables") or {}).get(table) or {} + columns = [str(c["name"]) for c in meta.get("columns") or [] if isinstance(c, dict) and c.get("name") is not None] + merged, stats = merge_notes(existing or {}, structure.notes, covered=structure.covered, at=at, columns=columns) + if modeled is not None and modeled.items: + covered = frozenset({"llm"}) if modeled.status in ("ok", "partial") else frozenset() + merged, more = merge_notes(merged, modeled.notes, covered=covered, at=at, columns=columns) + stats += more + return merged, stats + + async def write_draft(session: AsyncSession, source: Any, table: str, modeled: Any = None, *, at: str, actor: str | None) -> TableDraftResult: """一张表的起草并入目录并写入：数据库注释、外键约束、命名推断照做，modeled（catalog_draft.ModelOutcome， @@ -2096,26 +2122,16 @@ async def write_draft(session: AsyncSession, source: Any, table: str, modeled: A - 写入撞上别人刚改过（乐观锁）就重读重并，最多三次。 """ result = TableDraftResult(table=table) - structure = draft_structure(source, table) - meta = ((getattr(source, "schema_cache", None) or {}).get("tables") or {}).get(table) or {} - columns = [str(c["name"]) for c in meta.get("columns") or [] if isinstance(c, dict) and c.get("name") is not None] - model_draft: TableDraft | None = None if modeled is not None: result.parts_total = modeled.parts_total result.model_error = modeled.error if modeled.items: result.model_items = modeled.items result.blank = list(getattr(modeled, "blank", None) or []) - model_draft = TableDraft(notes=modeled.notes, covered=frozenset({"llm"}) - if modeled.status in ("ok", "partial") else frozenset()) for _attempt in range(3): entry = await read_entry(session, source.id, table) version = entry.version if entry else 0 - merged, stats = merge_notes(entry.notes if entry else {}, structure.notes, covered=structure.covered, at=at, - columns=columns) - if model_draft is not None: - merged, more = merge_notes(merged, model_draft.notes, covered=model_draft.covered, at=at, columns=columns) - stats += more + merged, stats = merge_draft(entry.notes if entry else {}, source, table, modeled, at=at) result.added, result.updated, result.removed = stats.added, stats.updated, stats.removed result.version = version if entry is None and not merged: diff --git a/backend/app/data/catalog_draft.py b/backend/app/data/catalog_draft.py index d972c7fb..c0caa5bc 100644 --- a/backend/app/data/catalog_draft.py +++ b/backend/app/data/catalog_draft.py @@ -285,6 +285,22 @@ async def run_model_draft(model: Any, source: Any, tables: Sequence[str], *, return await run.run() +#: 影子起草不核对有效行过滤的排除比例：那一步要经剖析的只读守卫查业务库（FILTER_UNCHECKED 的原因） +SHADOW_NOT_CHECKED = "影子起草只看表结构、不查业务库" + + +async def run_shadow_draft(model: Any, source: Any, tables: Sequence[str], *, + existing: Mapping[str, Mapping[str, Any]] | None = None, + background: Background | None = None, + hooks: Hooks | None = None) -> dict[str, ModelOutcome]: + """影子起草的入口（评测的目录快照，docs/design/next-round/01-效果度量与回归评测.md 做法 6）：和 run_model_draft 同一个 + 起草引擎、同样的分段和失败处理，只有一处不同——开了数据剖析的源，表级那段写完后不经剖析去业务库数有效行过滤排除了 + 多少行（_vet），照收、note 写明没有核对（SHADOW_NOT_CHECKED）。影子起草要的是「只看表结构」：不查业务库、不占数据源 + 的队。写不写目录由调用方定（评测写进内存里的目录副本，不调 write_entry）。""" + run = _Run(model, source, tables, existing or {}, background or Background(), hooks or Hooks(), offline=True) + return await run.run() + + # ========================================================================== # 并发名额 # ========================================================================== @@ -862,8 +878,10 @@ def _halves(pieces: list[_Piece]) -> tuple[list[_Piece], list[_Piece]] | None: class _Run: def __init__(self, model: Any, source: Any, tables: Sequence[str], existing: Mapping[str, Mapping[str, Any]], - background: Background, hooks: Hooks) -> None: + background: Background, hooks: Hooks, *, offline: bool = False) -> None: self.model = model + #: 影子起草（run_shadow_draft）：不查业务库 + self.offline = offline self.hooks = hooks #: structured 结构化输出；text 纯文本、从正文取 JSON。结构化输出用不了时这次起草后面都改走纯文本 self.mode = "structured" @@ -1647,9 +1665,9 @@ class _Run: return from app.data import catalog_profile - if not self.profiling.enabled: + if not self.profiling.enabled or self.offline: found = catalog_profile.FilterCheck(True, catalog_profile.FILTER_UNCHECKED.format( - why=catalog_profile.FILTER_NOT_ENABLED)) + why=SHADOW_NOT_CHECKED if self.offline else catalog_profile.FILTER_NOT_ENABLED)) else: try: found = await catalog_profile.check_valid_filter(self.source, st.name, dict(st.meta), @@ -1828,4 +1846,4 @@ __all__ = ["BUDGET_WORDS", "Background", "CallReport", "FAILURE_LABEL", "Hooks", "REASON_ABANDONED", "REASON_CALL", "REASON_CAPPED", "REASON_EXHAUSTED", "REASON_FORMAT", "REASON_HALTED", "REASON_HOLLOW", "REASON_MISSING", "REASON_OMITTED", "REASON_RETRIED", "REASON_SPLIT", "REASON_SPLIT_PARTS", "REASON_TIMEOUT", "REASON_TRUNCATED", "REASON_UNANSWERED", "SEG", "classify_failure", "model_gate", - "parts_for", "run_model_draft"] + "SHADOW_NOT_CHECKED", "parts_for", "run_model_draft", "run_shadow_draft"] diff --git a/backend/app/db/models_eval.py b/backend/app/db/models_eval.py index e06dd7a9..14924e2a 100644 --- a/backend/app/db/models_eval.py +++ b/backend/app/db/models_eval.py @@ -1,5 +1,245 @@ -"""评测：题集、题目、评测、结果（docs/design/next-round/01-效果度量与回归评测.md 做法 7）。 +"""评测：题集、题目、评测、结果、目录快照（docs/design/next-round/01-效果度量与回归评测.md 做法 6、7）。 -第 2 波 W2-c 填。由 app/db/models.py 末尾整模块导入，create_all 建表。 +由 app/db/models.py 末尾整模块导入，create_all 建表（只加不改：都是新表，总方案 X11）。这里不导入 app.db.models： +models.py 导入本模块时自己还没加载完，外键一律按表名写。 + +**为什么一次定全。** 第 2 波只做题集、题目和期望答案的来源；第 3 波的执行、评分、对比、清理直接用 EvalRun、EvalResult +的这些列。列一次定好，第 3 波不再改表（SQLite 上 ADD COLUMN 式的迁移改不了约束，只加列也要进迁移列表）。 + +**为什么是强类型的几张表，不是「kind + JSON」的通用任务表。** 评测的进度（每题正在做什么、作答路径、结论、分段耗时、 +按角色的用量）要能按列筛、按列汇总；通用任务表丢掉这些列，后台任务运行时的说明里已经否决过（core/job_runtime.py）。 + +- EvalSuite 题集：名称、说明、时区（题目里 {{ schedule.* }} 变量按它算）、默认变体、预算、是否归档、乐观锁 version。 +- EvalCase 题目：kind（ask 问数据题 / workflow 钉版本的工作流题）、问法、数据源、期望（expectations）和期望答案的来源 + （gold：sql / caliber / from_run，见 app/eval/cases.py）、标签、乐观锁 version、case_hash（题目改过 hash 就变，和 + 上一次比时标「题目改过，不可比」）。history 是给多轮追问留的位置（评测-D12：本轮只做单轮，必须为空）。 +- EvalRun 一次评测（第 3 波起用）：状态用后台任务的状态词，变体原样存一份、另存实际解析出的模型 id，指纹、配对（A/B）、 + 基准、预算快照、计数、按角色的用量、停下的原因。 +- EvalResult 每题 × 重复 × 臂一项（第 3 波起用）：状态用逐项状态词，作答路径、结论、评分、复核、出具档位、分段耗时…… + job_id 是 eval_run_id 的别名：后台任务运行时的收尾（close_job、cancel_queued）按 task_model.job_id 认逐项。 +- EvalCatalogSnapshot 目录快照的登记（做法 6）：快照内容是内容寻址工件（kind=eval_catalog_snapshot，读法见 + data/catalog_view.py），这里记谁、何时、来源（当前目录 / 影子起草：起草模型、起草要求、是否先去掉模型起草的项）、影子 + 起草的进度和用量。工件一旦写好就不变，这一行写好 artifact_id 之后也不再改。 """ from __future__ import annotations + +from datetime import datetime +from typing import Any + +from sqlalchemy import ForeignKey, Index, Integer, String, Text, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column, synonym + +from app.db.base import Base, TimestampMixin, UTCDateTime, new_id + +#: 题目的种类：ask 问数据题（走问数据同一条路作答）、workflow 钉版本的工作流题 +CASE_KINDS = ("ask", "workflow") +#: 一次评测怎么发起的：手动、定时、A/B 一次发起两臂 +RUN_TRIGGERS = ("manual", "schedule", "pair") +#: 逐题的作答路径（第 3 波）：直接回答、反问、只建不跑、建图失败、运行失败、跑完了、被驳回（总方案 X9：驳回记成路径， +#: 不算失败） +RESULT_PATHS = ("reply", "asked", "ready", "build_failed", "run_failed", "ran", "rejected") +#: 逐题的结论（第 3 波）：通过、不通过、不能评分（遮罩、期望没算出来、作答期间数据变了）、跳过（口径升版待处置、到了上限） +VERDICTS = ("pass", "fail", "ungradable", "skipped") +#: 目录快照的来源：当前目录、影子起草 +SNAPSHOT_ORIGINS = ("live", "shadow") + +#: 预算的缺省（评测-D9，用户拍板）：同时 2 题、每题 900 秒、单次评测 2 美元、评测每日 5 美元（评测单独一本账，不挤占 +#: 裁判、核查的额度）。题集的 budget 只存改过的项，生效的是缺省加上它（app/eval/suites.effective_budget）；发起评测时 +#: 整份快照进 EvalRun.budget。这一波只定字段，执行、预留与找补在第 3 波。首次真实验收后按实际花费调 +DEFAULT_BUDGET: dict[str, Any] = { + "max_parallel": 2, + "per_case_timeout_s": 900, + "per_run_max_usd": 2.0, + "daily_max_usd": 5.0, +} + + +class EvalSuite(Base, TimestampMixin): + """一个题集。归档的不出现在缺省列表里、不能发起评测（第 3 波），题目和历次评测都留着。""" + + __tablename__ = "eval_suites" + + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) + name: Mapped[str] = mapped_column(String(200)) + description: Mapped[str] = mapped_column(Text, default="") + # 题目、期望 SQL、口径输入里的 {{ schedule.* }} 按评测的截至时间在这个时区算（scheduler/service.template_values） + timezone: Mapped[str] = mapped_column(String(64), default="Asia/Shanghai") + # 发起评测时缺省用的变体（助手模型、写作模型、裁判、目录视图、重复次数……，第 3 波解析） + default_variant: Mapped[dict[str, Any]] = mapped_column(default=dict) + # 只存改过的预算项，缺省见 DEFAULT_BUDGET + budget: Mapped[dict[str, Any]] = mapped_column(default=dict) + archived: Mapped[bool] = mapped_column(default=False) + # 乐观锁：改题集带上读到的 version，对不上回 409 + version: Mapped[int] = mapped_column(Integer, default=1) + created_by: Mapped[str | None] = mapped_column(String(100), default=None) + + +class EvalCase(Base, TimestampMixin): + """题集里的一道题。字段的取值和校验见 app/eval/cases.py（normalize_case）。""" + + __tablename__ = "eval_cases" + __table_args__ = (UniqueConstraint("suite_id", "seq"), Index("ix_eval_cases_suite", "suite_id")) + + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) + suite_id: Mapped[str] = mapped_column(ForeignKey("eval_suites.id", ondelete="CASCADE")) + # 题号：题集里从 1 起，删了的题号不复用（运行名「评测：题集名·第 n 题」按它写） + seq: Mapped[int] = mapped_column(Integer) + kind: Mapped[str] = mapped_column(String(20), default="ask") + # 问法。可以带 {{ schedule.* }} 变量（按评测的截至时间渲染成明确的日期） + question: Mapped[str] = mapped_column(Text, default="") + # 问数据题限定的数据源（空 = 不限）。导出导入按名字对上（app/eval/transfer.py） + datasource_ids: Mapped[list[Any]] = mapped_column(default=list) + # workflow 题：钉住的工作流版本和输入模板（输入模板同定时任务，可以带 {{ schedule.* }}） + workflow_id: Mapped[str | None] = mapped_column(String(32), default=None) + workflow_version: Mapped[int | None] = mapped_column(Integer, default=None) + input_template: Mapped[dict[str, Any]] = mapped_column(default=dict) + # 多轮追问的前几轮（评测-D12：本轮只做单轮，这里必须为空；留位置是为了以后不改表） + history: Mapped[list[Any] | None] = mapped_column(default=None) + # 期望：[{id, kind: scalar|table|set|none|ask|text, required, 取哪一格/列/表……}]。数值期望不许人填（评测-D4） + expectations: Mapped[list[Any]] = mapped_column(default=list) + # 期望答案的来源：{kind: sql|caliber|from_run, …}；只有反问、文字要点的题为空 + gold: Mapped[dict[str, Any] | None] = mapped_column(default=None) + # 标签：口径陷阱、拒答、清单、相对日期……（自由填写，统计时分组用） + tags: Mapped[list[Any]] = mapped_column(default=list) + enabled: Mapped[bool] = mapped_column(default=True) + # 口径升版选了「不可比」：旧题归档（不再评），superseded_by 指向按新版本另建的那道题 + archived: Mapped[bool] = mapped_column(default=False) + superseded_by: Mapped[str | None] = mapped_column(String(32), default=None) + # 乐观锁 + version: Mapped[int] = mapped_column(Integer, default=1) + # 题目内容的指纹（app/eval/cases.case_hash）：问法、数据源、期望、期望来源变了它就变 + case_hash: Mapped[str] = mapped_column(String(64), default="") + # 最近一次试算：{status, code, reason, at, by, as_of, run_ids}。不存期望值：期望每次评测都重算，不冻结数值 + trial: Mapped[dict[str, Any] | None] = mapped_column(default=None) + created_by: Mapped[str | None] = mapped_column(String(100), default=None) + + +class EvalCatalogSnapshot(Base, TimestampMixin): + """一份目录快照的登记。内容在工件里（artifact_id，kind=eval_catalog_snapshot）；变体用工件 id 指定目录 + （data/catalog_view.CatalogView(mode="snapshot", snapshot=工件 id)），同一份快照可以被多次评测引用。 + + 当前目录的快照建出来就是 succeeded；影子起草是后台任务（app/eval/snapshots.py），状态用后台任务的状态词，起草完、 + 工件写好才有 artifact_id。""" + + __tablename__ = "eval_catalog_snapshots" + __table_args__ = (Index("ix_eval_catalog_snapshots_artifact", "artifact_id"),) + + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) + origin: Mapped[str] = mapped_column(String(20), default="live") + # queued | running | succeeded | partial | failed | cancelled | interrupted + status: Mapped[str] = mapped_column(String(20), default="succeeded") + artifact_id: Mapped[str | None] = mapped_column(String(64), default=None) + # 拍了哪些数据源：[{id, name}]（名字是拍的那一刻的） + sources: Mapped[list[Any]] = mapped_column(default=list) + # 影子起草起草了哪些表：{源 id: [表名]}；当前目录的快照为空（整个源的目录照拍） + tables: Mapped[dict[str, Any] | None] = mapped_column(default=None) + # 影子起草的设定：{provider, model, model_id, guidance, note, strip_llm, suite_id} + draft: Mapped[dict[str, Any] | None] = mapped_column(default=None) + # 开始前的估算：{tables, columns, calls, input_tokens, output_tokens, cost_usd, unpriced, note} + estimate: Mapped[dict[str, Any] | None] = mapped_column(default=None) + # 实际用量：{calls, input_tokens, output_tokens, timeouts, failed_calls, cost_usd, unpriced} + usage: Mapped[dict[str, Any]] = mapped_column(default=dict) + # 进度：{tables_total, tables_done, tables_failed, current, notice, failures: {表名: 原因}} + progress: Mapped[dict[str, Any]] = mapped_column(default=dict) + error: Mapped[str | None] = mapped_column(Text, default=None) + created_by: Mapped[str | None] = mapped_column(String(100), default=None) + started_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + + +class EvalRun(Base, TimestampMixin): + """一次评测（第 3 波的 app/eval/runner.py 写）。一次评测是一个后台任务，每题 × 重复次数 × 臂是一项（EvalResult）。 + + 题集有评测时不能删除（只能归档）：每次评测的总表（summary）永久保留，明细可以按保留规则清理（cleaned_at）。""" + + __tablename__ = "eval_runs" + __table_args__ = (Index("ix_eval_runs_suite_created", "suite_id", "created_at"), + Index("ix_eval_runs_pair", "pair_id")) + + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) + suite_id: Mapped[str] = mapped_column(ForeignKey("eval_suites.id", ondelete="CASCADE")) + # queued | running | succeeded | partial | failed | cancelled | interrupted（core/job_runtime 的状态词） + status: Mapped[str] = mapped_column(String(20), default="queued") + trigger: Mapped[str] = mapped_column(String(20), default="manual") + # 定时发起的那一条（ScheduleFire.eval_run_id 反过来指向这里） + schedule_id: Mapped[str | None] = mapped_column(String(32), default=None) + # 变体原样存一份；resolved_models 是实际解析出的模型 id（助手、写作、裁判……） + variant: Mapped[dict[str, Any]] = mapped_column(default=dict) + resolved_models: Mapped[dict[str, Any]] = mapped_column(default=dict) + # 这次用的目录快照（工件 id）：开始时拍一份（缺省「当前目录」也拍），起草、剖析照常进行也不影响可比性 + catalog_snapshot: Mapped[str | None] = mapped_column(String(64), default=None) + # 截至时间：题目里的 {{ schedule.* }} 按它渲染 + as_of: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + repeats: Mapped[int] = mapped_column(Integer, default=1) + # 指纹：提示词指纹、目录快照 id 和哈希、各数据源 schema 哈希、题目哈希、评分规则版本 + fingerprint: Mapped[dict[str, Any]] = mapped_column(default=dict) + # A/B：两臂共用 pair_id，arm 是这一臂的名字 + pair_id: Mapped[str | None] = mapped_column(String(32), default=None) + arm: Mapped[str | None] = mapped_column(String(40), default=None) + # 和哪一次比；pinned 是用户钉住的基线（清理时保留） + baseline_id: Mapped[str | None] = mapped_column(String(32), default=None) + pinned: Mapped[bool] = mapped_column(default=False) + # 发起时的预算快照（DEFAULT_BUDGET 加题集改过的项） + budget: Mapped[dict[str, Any]] = mapped_column(default=dict) + # 计数：{total, done, passed, failed, ungradable, skipped, …}；按角色的用量；总表（永久保留） + counts: Mapped[dict[str, Any]] = mapped_column(default=dict) + usage: Mapped[dict[str, Any]] = mapped_column(default=dict) + summary: Mapped[dict[str, Any]] = mapped_column(default=dict) + # 停下的原因（止损、到了上限、手动停止），整体出错的原因 + stop_reason: Mapped[str | None] = mapped_column(Text, default=None) + error: Mapped[str | None] = mapped_column(Text, default=None) + created_by: Mapped[str | None] = mapped_column(String(100), default=None) + started_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + # 明细（逐题结果、底下的运行）按保留规则清理掉的时间；总表还在 + cleaned_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + + +class EvalResult(Base, TimestampMixin): + """一次评测里的一项：一道题 × 第几次重复 × 哪一臂（第 3 波写）。题目删了结果照留（case_id 不设外键）。""" + + __tablename__ = "eval_results" + __table_args__ = (Index("ix_eval_results_run_case", "eval_run_id", "case_id"),) + + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) + eval_run_id: Mapped[str] = mapped_column(ForeignKey("eval_runs.id", ondelete="CASCADE")) + #: 后台任务运行时按 task_model.job_id 认一个任务的逐项（close_job、cancel_queued） + job_id = synonym("eval_run_id") + # 排序用：同一次评测里从 1 起 + seq: Mapped[int] = mapped_column(Integer, default=0) + case_id: Mapped[str | None] = mapped_column(String(32), default=None) + case_seq: Mapped[int | None] = mapped_column(Integer, default=None) + case_hash: Mapped[str] = mapped_column(String(64), default="") + repeat: Mapped[int] = mapped_column(Integer, default=0) + arm: Mapped[str | None] = mapped_column(String(40), default=None) + # pending | running | ok | partial | failed | skipped | cancelled（core/job_runtime 的逐项状态词） + status: Mapped[str] = mapped_column(String(20), default="pending") + # 作答的那次运行、算期望的那几次运行（作答前后各一次，口径题两版各一次） + run_id: Mapped[str | None] = mapped_column(String(32), default=None) + gold_run_ids: Mapped[list[Any]] = mapped_column(default=list) + # 期望：作答前后两次的结果和是否一致（app/eval/gold.settle 的形状） + gold: Mapped[dict[str, Any] | None] = mapped_column(default=None) + path: Mapped[str | None] = mapped_column(String(20), default=None) + verdict: Mapped[str | None] = mapped_column(String(20), default=None) + # 每个期望的结论（对且可追溯 / 对但没引用 / 错 / 量纲不同 / 不能评分） + scores: Mapped[list[Any]] = mapped_column(default=list) + review: Mapped[dict[str, Any] | None] = mapped_column(default=None) + tier: Mapped[str | None] = mapped_column(String(20), default=None) + judge: Mapped[dict[str, Any] | None] = mapped_column(default=None) + looks: Mapped[dict[str, Any] | None] = mapped_column(default=None) + calls: Mapped[int] = mapped_column(Integer, default=0) + # 分段耗时（建图、运行、复核、整题）、按角色的用量 + timings: Mapped[dict[str, Any]] = mapped_column(default=dict) + usage: Mapped[dict[str, Any]] = mapped_column(default=dict) + # 答错的题：引用那格所在查询的 SQL 和期望 SQL 的比对（逐题的 E1） + sql_match: Mapped[dict[str, Any] | None] = mapped_column(default=None) + details_artifact: Mapped[str | None] = mapped_column(String(64), default=None) + # 正在做什么（正在算期望 / 正在建图 / 运行中 / 正在复核 / 正在评分） + current: Mapped[str | None] = mapped_column(String(200), default=None) + error: Mapped[str | None] = mapped_column(Text, default=None) + started_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + + +__all__ = ["CASE_KINDS", "DEFAULT_BUDGET", "EvalCase", "EvalCatalogSnapshot", "EvalResult", "EvalRun", "EvalSuite", + "RESULT_PATHS", "RUN_TRIGGERS", "SNAPSHOT_ORIGINS", "VERDICTS"] diff --git a/backend/app/eval/__init__.py b/backend/app/eval/__init__.py new file mode 100644 index 00000000..6ff603b2 --- /dev/null +++ b/backend/app/eval/__init__.py @@ -0,0 +1,10 @@ +"""评测（docs/design/next-round/01-效果度量与回归评测.md）：一把尺——题集 + 由 SQL / 口径算出的期望答案。 + +第 2 波（这一包）： +- suites.py、cases.py：题集和题目的形状与校验（数值期望不许人填，评测-D4；只做单轮，评测-D12）、题目指纹； +- gold.py：期望答案的三种来源（SQL 题、口径题、从运行认定）、变量按截至时间渲染、试算、作答前后各算一次、口径升版处置； +- snapshots.py：目录快照（当前目录 / 影子起草）； +- transfer.py：题集导出导入（按数据源名对上，不含期望值、答案、运行）。 + +第 3 波接着写 runner（后台评测）、score、stats、compare、cleanup；第 4 波是界面。接口在 app/api/evals.py（只给搭建者）。 +""" diff --git a/backend/app/eval/cases.py b/backend/app/eval/cases.py new file mode 100644 index 00000000..75e315cd --- /dev/null +++ b/backend/app/eval/cases.py @@ -0,0 +1,520 @@ +"""题目的形状与校验（细案 01 做法 7、8、14；评测-D4、D12）。接口（app/api/evals.py）和导入（app/eval/transfer.py）都经 +normalize_case + resolve_case，规矩只有这一份。 + +**题目**：kind（ask 问数据题 / workflow 钉版本的工作流题）、question（问法）、datasource_ids、workflow 题的 +workflow_id@workflow_version 和 input_template、expectations（期望）、gold（期望答案的来源）、tags、enabled。 +history 是多轮追问的位置：评测-D12 本轮只做单轮，非空一律拒收（多轮要和问数据页的历史拼法完全一致，前提是轮次由后端落库， +不在本轮）。 + +**期望**（expectations，每题至少一个，最多 EXPECTATIONS_MAX 个）。每个期望 {id, kind, required, label, …}： +- scalar 一个数：SQL 题写 column（结果里的列）加 row（第几行，缺省 0）或 where（按维度找那一行，如 {"地区": "华南"}）； + 口径题写 metric（口径卡上的指标 id）。tolerance 是允许的相对误差（不写由评分按显示精度推，第 3 波）。 +- table 多行：keys（按哪几列配对，必填）、columns（比哪几列，缺省全部）、top_n、ordered（要不要按顺序）。 +- set 名单：column、top_n、ordered。 +- none 期望答「没有数据」：期望 SQL 返回 0 行。 +- ask 期望反问（问法含糊）：不能和要答出数的期望写在一起，不要期望答案的来源。 +- text 文字要点（points）：由裁判判，单列「模型判断」，不计入准确率（评测-D4），所以 required 恒为 false。 +**数值期望不许人填**（评测-D4）：期望里写了 value、values、expected、answer、rows 这类键的一律拒收——期望值只能由 SQL、 +口径或从运行认定的 SQL 算出来（app/eval/gold.py），人填的数会过期、会填错，也说不清是哪个口径。 + +**期望答案的来源**（gold；scalar、table、set、none 要它，ask、text 不要）： +- sql：{datasource_id, sql}。SQL 可以带 {{ schedule.* }}，按评测的截至时间渲染；保存前过只读守卫。 +- caliber：{workflow_id, version, node_id, inputs}，钉住 workflow_id@version 的那张口径卡（只认可复用的卡， + app/ledger/derive.load_pinned），inputs 是入口字段的模板（同定时任务的输入模板）。只配 scalar（metric）。 + upgrade 是口径升版的处置（服务端写，见 gold.dispose）：客户端带来的不算，保存时按库里的留着。 +- from_run：{datasource_id, sql, run_id, query, node_id, confirmed_by, confirmed_at}，从某次运行里的一条查询认定 + （只能经 POST …/cases/from-run 建；之后每次评测按这条 SQL 重算，不冻结数值）。改 SQL 就改成 sql 题。 + +**题目指纹**（case_hash）：问法、数据源、工作流和输入、期望、期望来源（去掉确认人、出处运行、升版处置这些记录项）。改过 +就变，和上一次比时标「题目改过，不可比」；口径改钉新版本（recompute）也就断开了历史。 +""" +from __future__ import annotations + +import copy +import re +from collections.abc import Mapping +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models_eval import CASE_KINDS + +#: 题目不合规（422）的机器码，field 是出错的那一项 +CASE_INVALID = "eval_case_invalid" +#: 期望 SQL 没过只读守卫（422） +GOLD_GUARD = "eval_gold_guard" +#: 口径题指的卡不能当口径题（422）：不存在、不是口径卡、不能复用、取数节点引用了入口字段以外的值 +CALIBER_NOT_REUSABLE = "eval_caliber_not_reusable" + +EXPECTATION_KINDS = ("scalar", "table", "set", "none", "ask", "text") +#: 要期望答案来源的期望：数都由 SQL / 口径算 +NEEDS_GOLD = ("scalar", "table", "set", "none") +#: 不计入准确率的（评测-D4：文字要点由裁判判，单列） +UNCOUNTED = ("text",) +GOLD_KINDS = ("sql", "caliber", "from_run") +#: 期望里出现这些键就是在人填数（评测-D4） +HUMAN_VALUE_KEYS = ("value", "values", "expected", "answer", "rows", "number") + +#: 每种期望认的键（另有 id、kind、required、label） +_EXPECTATION_KEYS: dict[str, tuple[str, ...]] = { + "scalar": ("column", "metric", "row", "where", "tolerance"), + "table": ("keys", "columns", "top_n", "ordered", "tolerance"), + "set": ("column", "top_n", "ordered"), + "none": (), + "ask": (), + "text": ("points",), +} +_COMMON_KEYS = ("id", "kind", "required", "label") +#: 期望答案来源的键（客户端能写的）；from_run 只能由服务端建 +_GOLD_KEYS: dict[str, tuple[str, ...]] = { + "sql": ("datasource_id", "sql"), + "caliber": ("workflow_id", "version", "node_id", "inputs"), + "from_run": ("datasource_id", "sql", "run_id", "query", "node_id", "confirmed_by", "confirmed_at"), +} +#: 期望来源里不进指纹的记录项：谁认定的、出自哪次运行、升版怎么处置的（dual 只是多算一版，旧的那一版照旧可比） +_GOLD_RECORDS = ("confirmed_by", "confirmed_at", "run_id", "query", "node_id", "upgrade") +#: 进指纹的题目字段 +HASHED_FIELDS = ("kind", "question", "datasource_ids", "workflow_id", "workflow_version", "input_template", "history", + "expectations", "gold") + +QUESTION_MAX = 2000 +SQL_MAX = 20_000 +LABEL_MAX = 200 +EXPECTATIONS_MAX = 20 +POINTS_MAX = 10 +POINT_MAX = 300 +TAGS_MAX = 20 +TAG_MAX = 40 +DATASOURCES_MAX = 20 +TOP_N_MAX = 1000 +_ID_RE = re.compile(r"^[A-Za-z0-9_-]{1,40}$") + + +class CaseInvalid(ValueError): + """题目不合规。message 给人看，field 是出错的那一项（表单据此落到那一格），code 给程序认。""" + + def __init__(self, message: str, field: str | None = None, code: str = CASE_INVALID) -> None: + super().__init__(message) + self.message = message + self.field = field + self.code = code + + +# ========================================================================== +# 变量：{{ schedule.* }}，和定时任务同一份（scheduler/service.VARIABLES） +# ========================================================================== + + +def check_variables(where: str, text: str, field_name: str) -> None: + """text 里的 {{ }} 只能是定时任务那份变量（可以带过滤器），成对出现。规矩同 scheduler/service.check_template， + 说法换成这里的「哪一项」。""" + from app.engine.expressions import _FILTERS + from app.scheduler import service + + for match in service._TEMPLATE_RE.finditer(text): + expr = match.group(1).strip() + if not expr: + raise CaseInvalid(f"{where}里有空的 {{{{ }}}}，请在中间写变量名，例如 schedule.last_week_start", field_name) + path, *filters = (part.strip() for part in expr.split("|")) + if path not in service._NAMES: + raise CaseInvalid(f"{where}里的「{path}」不是可用的变量，可用的变量：{'、'.join(service._NAMES)}", field_name) + for filt in filters: + if filt not in _FILTERS: + raise CaseInvalid(f"{where}里的「{filt}」不是可用的过滤器，可用的过滤器：{'、'.join(_FILTERS)}", field_name) + rest = service._TEMPLATE_RE.sub("", text) + if "{{" in rest or "}}" in rest: + raise CaseInvalid(f"{where}里 {{{{ 和 }}}} 没有成对出现，请检查是否少写了一半", field_name) + + +def variable_values(as_of: datetime, tz: str, cron_expr: str | None = None) -> dict[str, str]: + """按截至时间 as_of 在时区 tz 算出的变量（键不带 schedule. 前缀）。定时发起的评测给那条定时的 cron（上一次计划触发 + 时间按它往前找）；手动发起的没有 cron，按每天这一刻触发算（同变量清单接口的示例）。""" + from app.scheduler import cron, service + + if not cron_expr: + local = as_of.astimezone(cron.zone(tz)) + cron_expr = f"{local.minute} {local.hour} * * *" + return service.template_values(as_of, tz, cron_expr) + + +def render(text: str, values: Mapping[str, str]) -> str: + """按变量渲染一段文字（问法、期望 SQL、口径输入）。用项目通用的渲染器，变量挂在 schedule 下。""" + from app.engine.expressions import render_template + + return render_template(text or "", {"schedule": dict(values)}) + + +# ========================================================================== +# 逐项校验 +# ========================================================================== + + +def _text(value: Any, where: str, field_name: str, *, limit: int, required: bool = False) -> str: + if value is None: + value = "" + if not isinstance(value, str): + raise CaseInvalid(f"{where}需要是文字", field_name) + text = value.strip() + if required and not text: + raise CaseInvalid(f"请填写{where}", field_name) + if len(text) > limit: + raise CaseInvalid(f"{where}不能超过 {limit} 个字符", field_name) + return text + + +def _bool(value: Any, where: str, field_name: str) -> bool: + if not isinstance(value, bool): + raise CaseInvalid(f"{where}需要是 true 或 false", field_name) + return value + + +def _int(value: Any, where: str, field_name: str, *, low: int, high: int) -> int: + if isinstance(value, bool) or not isinstance(value, int) or not low <= value <= high: + raise CaseInvalid(f"{where}需要是 {low} 到 {high} 的整数", field_name) + return value + + +def _names(value: Any, where: str, field_name: str, *, limit: int, required: bool = False) -> list[str]: + if value is None: + value = [] + if not isinstance(value, list) or not all(isinstance(v, str) and v.strip() for v in value): + raise CaseInvalid(f"{where}需要是一组不为空的文字", field_name) + out = list(dict.fromkeys(v.strip() for v in value)) + if required and not out: + raise CaseInvalid(f"请填写{where}", field_name) + if len(out) > limit: + raise CaseInvalid(f"{where}最多 {limit} 项", field_name) + return out + + +def _expectation(raw: Any, i: int, *, gold_kind: str | None, taken: set[str]) -> dict[str, Any]: + where = f"第 {i + 1} 个期望" + path = f"expectations.{i}" + if not isinstance(raw, Mapping): + raise CaseInvalid(f"{where}需要是一个对象", path) + kind = raw.get("kind") + if kind not in EXPECTATION_KINDS: + raise CaseInvalid(f"{where}的种类只能是 {'、'.join(EXPECTATION_KINDS)}", f"{path}.kind") + human = [k for k in HUMAN_VALUE_KEYS if k in raw] + if human: + raise CaseInvalid(f"{where}写了「{human[0]}」：数值期望不许人填，期望值只能由期望 SQL、口径或从运行认定的 SQL " + "算出来。请写清从期望答案的结果里取哪一格、哪一列或整张表", f"{path}.{human[0]}") + unknown = sorted(set(raw) - set(_COMMON_KEYS) - set(_EXPECTATION_KEYS[kind])) + if unknown: + raise CaseInvalid(f"{where}（{kind}）不认识这些项：{'、'.join(unknown)}", f"{path}.{unknown[0]}") + eid = raw.get("id") + if eid in (None, ""): + n = len(taken) + 1 + while f"e{n}" in taken: + n += 1 + eid = f"e{n}" + if not isinstance(eid, str) or not _ID_RE.match(eid): + raise CaseInvalid(f"{where}的 id 只能是 1 到 40 个字母、数字、下划线或连字符", f"{path}.id") + if eid in taken: + raise CaseInvalid(f"{where}的 id「{eid}」和前面的重复了", f"{path}.id") + out: dict[str, Any] = {"id": eid, "kind": kind} + if (label := _text(raw.get("label"), f"{where}的说明", f"{path}.label", limit=LABEL_MAX)): + out["label"] = label + # 文字要点不计入准确率（评测-D4）：不算「必需」，通过与否不看它 + required = raw.get("required", True) + out["required"] = False if kind in UNCOUNTED else _bool(required, f"{where}的「必需」", f"{path}.required") + + if kind in NEEDS_GOLD and gold_kind is None: + raise CaseInvalid(f"{where}要答出数（或答「没有数据」），需要期望答案的来源：写一条期望 SQL、选一个口径，" + "或从运行里认定一条查询", "gold") + if gold_kind == "caliber" and kind not in ("scalar", "ask", "text"): + raise CaseInvalid(f"{where}（{kind}）不能用口径题算：口径卡给的是一个个指标，只配「一个数」的期望", + f"{path}.kind") + if kind == "scalar": + if gold_kind == "caliber": + if raw.get("column") not in (None, "") or "row" in raw or "where" in raw: + raise CaseInvalid(f"{where}是口径题：写 metric（口径卡上的指标 id），不写列、行", f"{path}.metric") + out["metric"] = _text(raw.get("metric"), f"{where}的指标", f"{path}.metric", limit=LABEL_MAX, + required=True) + else: + if raw.get("metric") not in (None, ""): + raise CaseInvalid(f"{where}的期望答案来自 SQL：写 column（结果里的列），metric 只用于口径题", + f"{path}.metric") + out["column"] = _text(raw.get("column"), f"{where}取哪一列", f"{path}.column", limit=LABEL_MAX, + required=True) + if "where" in raw and raw["where"] not in (None, {}): + cond = raw["where"] + if not isinstance(cond, Mapping) or not cond or not all( + isinstance(k, str) and k.strip() and isinstance(v, (str, int, float)) and not isinstance(v, bool) + for k, v in cond.items()): + raise CaseInvalid(f"{where}的 where 需要是「列名：取值」的对象，例如 {{\"地区\": \"华南\"}}", + f"{path}.where") + if "row" in raw: + raise CaseInvalid(f"{where}按 where 找行时不写 row", f"{path}.row") + out["where"] = {k.strip(): v for k, v in cond.items()} + else: + out["row"] = _int(raw.get("row", 0), f"{where}的行号", f"{path}.row", low=0, high=TOP_N_MAX) + if raw.get("tolerance") is not None: + out["tolerance"] = _tolerance(raw["tolerance"], where, path) + elif kind == "table": + out["keys"] = _names(raw.get("keys"), f"{where}按哪几列配对（keys）", f"{path}.keys", limit=10, required=True) + if raw.get("columns") is not None: + out["columns"] = _names(raw["columns"], f"{where}比哪几列", f"{path}.columns", limit=50, required=True) + if raw.get("top_n") is not None: + out["top_n"] = _int(raw["top_n"], f"{where}的前几名", f"{path}.top_n", low=1, high=TOP_N_MAX) + out["ordered"] = _bool(raw.get("ordered", False), f"{where}要不要按顺序", f"{path}.ordered") + if raw.get("tolerance") is not None: + out["tolerance"] = _tolerance(raw["tolerance"], where, path) + elif kind == "set": + out["column"] = _text(raw.get("column"), f"{where}取哪一列", f"{path}.column", limit=LABEL_MAX, required=True) + if raw.get("top_n") is not None: + out["top_n"] = _int(raw["top_n"], f"{where}的前几名", f"{path}.top_n", low=1, high=TOP_N_MAX) + out["ordered"] = _bool(raw.get("ordered", False), f"{where}要不要按顺序", f"{path}.ordered") + elif kind == "text": + points = raw.get("points") + if not isinstance(points, list) or not points or not all(isinstance(p, str) and p.strip() for p in points): + raise CaseInvalid(f"{where}是文字要点：points 需要是一组不为空的要点", f"{path}.points") + if len(points) > POINTS_MAX or any(len(p.strip()) > POINT_MAX for p in points): + raise CaseInvalid(f"{where}最多 {POINTS_MAX} 条要点，每条不超过 {POINT_MAX} 个字符", f"{path}.points") + out["points"] = [p.strip() for p in points] + return out + + +def _tolerance(value: Any, where: str, path: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)) or not 0 <= float(value) <= 1: + raise CaseInvalid(f"{where}允许的相对误差需要是 0 到 1 之间的数（0.01 是 1%）", f"{path}.tolerance") + return float(value) + + +def normalize_expectations(raw: Any, *, gold_kind: str | None, case_kind: str) -> list[dict[str, Any]]: + if not isinstance(raw, list) or not raw: + raise CaseInvalid("每道题至少要有一个期望", "expectations") + if len(raw) > EXPECTATIONS_MAX: + raise CaseInvalid(f"每道题最多 {EXPECTATIONS_MAX} 个期望", "expectations") + out: list[dict[str, Any]] = [] + taken: set[str] = set() + for i, item in enumerate(raw): + exp = _expectation(item, i, gold_kind=gold_kind, taken=taken) + taken.add(exp["id"]) + out.append(exp) + kinds = {e["kind"] for e in out} + if "ask" in kinds and kinds & set(NEEDS_GOLD): + raise CaseInvalid("期望反问的题不能同时期望答出数：问法含糊时，答案就不该给数", "expectations") + if "ask" in kinds and case_kind == "workflow": + raise CaseInvalid("工作流题不会反问，期望反问只用于问数据题", "expectations") + if gold_kind is not None and not kinds & set(NEEDS_GOLD): + raise CaseInvalid("有期望答案的来源，却没有要它算的期望：请加一个「一个数」「多行」「名单」或「没有数据」的期望，" + "或者去掉期望答案的来源", "gold") + return out + + +def normalize_gold(raw: Any, *, current: Mapping[str, Any] | None = None, trusted: bool = False) -> dict[str, Any] | None: + """期望答案的来源 → 规范化的形状。只看形状和变量，库里的引用（数据源、口径卡）由 resolve_case 核对。 + + from_run 只能由服务端建（trusted），或者和库里的一样原样带回来；口径题的升版处置（upgrade）只认库里的。""" + if raw in (None, {}): + return None + if not isinstance(raw, Mapping): + raise CaseInvalid("期望答案的来源需要是一个对象", "gold") + kind = raw.get("kind") + if kind not in GOLD_KINDS: + raise CaseInvalid(f"期望答案的来源只能是 SQL（sql）、口径（caliber）或从运行认定（from_run）", "gold.kind") + allowed = set(_GOLD_KEYS[kind]) | {"kind", "upgrade"} + unknown = sorted(set(raw) - allowed) + if unknown: + raise CaseInvalid(f"期望答案的来源不认识这些项：{'、'.join(unknown)}", f"gold.{unknown[0]}") + if kind == "from_run" and not trusted: + same = current is not None and current.get("kind") == "from_run" and all( + raw.get(k) == current.get(k) for k in _GOLD_KEYS["from_run"]) + if not same: + raise CaseInvalid("「从运行认定」的期望只能从一次运行里选一条查询认定（POST …/cases/from-run）。" + "要改 SQL，请把期望答案的来源改成 SQL", "gold.kind") + return copy.deepcopy(dict(current)) + out: dict[str, Any] = {"kind": kind} + if kind in ("sql", "from_run"): + out["datasource_id"] = _text(raw.get("datasource_id"), "期望 SQL 查哪个数据源", "gold.datasource_id", + limit=64, required=True) + out["sql"] = _text(raw.get("sql"), "期望 SQL", "gold.sql", limit=SQL_MAX, required=True) + check_variables("期望 SQL ", out["sql"], "gold.sql") + if kind == "from_run": + for key in ("run_id", "query", "node_id", "confirmed_by", "confirmed_at"): + if raw.get(key) not in (None, ""): + out[key] = str(raw[key]) + else: + out["workflow_id"] = _text(raw.get("workflow_id"), "口径所在的工作流", "gold.workflow_id", limit=64, + required=True) + out["version"] = _int(raw.get("version"), "口径固定的版本", "gold.version", low=1, high=10**9) + out["node_id"] = _text(raw.get("node_id"), "口径卡", "gold.node_id", limit=64, required=True) + inputs = raw.get("inputs") or {} + if not isinstance(inputs, Mapping) or not all(isinstance(k, str) and k.strip() and isinstance(v, str) + for k, v in inputs.items()): + raise CaseInvalid("口径的输入需要是「入口字段：模板」的对象，模板可以带 {{ schedule.* }} 变量", "gold.inputs") + out["inputs"] = {k.strip(): v for k, v in inputs.items()} + for name, text in out["inputs"].items(): + check_variables(f"口径输入「{name}」", text, "gold.inputs") + # 升版处置是服务端写的（gold.dispose）：钉的还是同一张卡就照库里的留着，换了卡就没有了 + if current is not None and current.get("kind") == "caliber" and all( + current.get(k) == out[k] for k in ("workflow_id", "version", "node_id")) and current.get("upgrade"): + out["upgrade"] = copy.deepcopy(current["upgrade"]) + return out + + +def normalize_case(body: Mapping[str, Any], *, current: Mapping[str, Any] | None = None, + trusted_gold: bool = False) -> dict[str, Any]: + """请求体（改的时候和库里的合起来：没带的项照旧）→ 题目各列的值。不合规抛 CaseInvalid。""" + if not isinstance(body, Mapping): + raise CaseInvalid("题目需要是一个对象") + cur = dict(current or {}) + + def pick(key: str, default: Any = None) -> Any: + return body[key] if key in body else cur.get(key, default) + + kind = pick("kind", "ask") + if kind not in CASE_KINDS: + raise CaseInvalid("题目的种类只能是问数据题（ask）或工作流题（workflow）", "kind") + history = pick("history") + if history not in (None, []): + raise CaseInvalid("本轮只做单轮题：多轮追问（history）还不支持，请留空", "history") + question = _text(pick("question", ""), "问法", "question", limit=QUESTION_MAX, required=kind == "ask") + check_variables("问法", question, "question") + datasource_ids = _names(pick("datasource_ids", []), "限定的数据源", "datasource_ids", limit=DATASOURCES_MAX) + workflow_id, workflow_version, input_template = None, None, {} + if kind == "workflow": + workflow_id = _text(pick("workflow_id"), "要评的工作流", "workflow_id", limit=64, required=True) + workflow_version = _int(pick("workflow_version"), "工作流固定的版本", "workflow_version", low=1, high=10**9) + from app.scheduler import service + + try: + input_template = service.check_template(pick("input_template", {}) or {}) + except service.TemplateError as e: + raise CaseInvalid(str(e), "input_template") from e + else: + for key in ("workflow_id", "workflow_version"): + if body.get(key) not in (None, ""): + raise CaseInvalid("问数据题不固定工作流：要评某个工作流的某一版，请把题目种类改成工作流题", key) + if body.get("input_template"): + raise CaseInvalid("问数据题没有输入模板，问法就是输入", "input_template") + gold = normalize_gold(pick("gold"), current=cur.get("gold"), trusted=trusted_gold) + expectations = normalize_expectations(pick("expectations"), gold_kind=gold["kind"] if gold else None, + case_kind=kind) + tags = _names(pick("tags", []), "标签", "tags", limit=TAGS_MAX) + if any(len(t) > TAG_MAX for t in tags): + raise CaseInvalid(f"每个标签不超过 {TAG_MAX} 个字符", "tags") + enabled = _bool(pick("enabled", True), "是否启用", "enabled") + return {"kind": kind, "question": question, "datasource_ids": datasource_ids, "workflow_id": workflow_id, + "workflow_version": workflow_version, "input_template": input_template, "history": None, + "expectations": expectations, "gold": gold, "tags": tags, "enabled": enabled} + + +def case_hash(fields: Mapping[str, Any]) -> str: + """题目内容的指纹：HASHED_FIELDS，期望来源去掉记录项（_GOLD_RECORDS）。""" + from app.core.artifact_store import canonical_json, content_hash + + body = {k: fields.get(k) for k in HASHED_FIELDS} + if isinstance(body.get("gold"), Mapping): + body["gold"] = {k: v for k, v in body["gold"].items() if k not in _GOLD_RECORDS} + if not body.get("history"): + body["history"] = None + return content_hash(canonical_json(body)) + + +def case_fields(row: Any) -> dict[str, Any]: + """库里的一道题 → normalize_case 给出的那一组字段（再加 id、seq、suite_id）。gold.py 按这个形状读题目。""" + return {"id": row.id, "suite_id": row.suite_id, "seq": row.seq, "kind": row.kind, "question": row.question, + "datasource_ids": list(row.datasource_ids or []), "workflow_id": row.workflow_id, + "workflow_version": row.workflow_version, "input_template": dict(row.input_template or {}), + "history": row.history, "expectations": copy.deepcopy(list(row.expectations or [])), + "gold": copy.deepcopy(row.gold) if row.gold else None, "tags": list(row.tags or []), + "enabled": bool(row.enabled)} + + +# ========================================================================== +# 核对库里的引用 +# ========================================================================== + + +@dataclass +class CaseRefs: + """resolve_case 核对出来的引用：数据源 {id: 名字}、口径题读好的那一版（Pinned）。""" + + sources: dict[str, str] = field(default_factory=dict) + pinned: Any = None + + +async def resolve_case(session: AsyncSession, fields: Mapping[str, Any], *, tz: str = "Asia/Shanghai", + now: datetime | None = None) -> CaseRefs: + """核对题目引用的数据源、口径卡、工作流版本都在、能用；期望 SQL 按此刻渲染后先过一遍只读守卫（试算时按截至时间 + 再过一遍）。不对抛 CaseInvalid。""" + from app.data.guard import SqlRejected, check + from app.db.base import utcnow + from app.db.models import DataSource + from app.ledger import derive + + refs = CaseRefs() + ids = list(fields.get("datasource_ids") or []) + gold = fields.get("gold") or None + if gold and gold.get("datasource_id"): + ids.append(gold["datasource_id"]) + for sid in dict.fromkeys(ids): + source = await session.get(DataSource, sid) + if source is None: + where = "gold.datasource_id" if gold and sid == gold.get("datasource_id") else "datasource_ids" + raise CaseInvalid(f"数据源 {sid} 不存在，可能已被删除", where) + refs.sources[sid] = source.name + if gold and gold["kind"] in ("sql", "from_run"): + source = await session.get(DataSource, gold["datasource_id"]) + sample = render(gold["sql"], variable_values(now or utcnow(), tz)) + try: + check(sample, readonly=True, source_name=source.name, dialect=(source.kind or "").lower()) + except SqlRejected as e: + raise CaseInvalid(f"期望 SQL 没有通过只读守卫：{e}", "gold.sql", GOLD_GUARD) from e + elif gold and gold["kind"] == "caliber": + try: + pinned = await derive.load_pinned(session, gold["workflow_id"], gold["version"], gold["node_id"]) + except derive.NotDerivable as e: + raise CaseInvalid(f"这张口径卡不能当口径题：{e.message}", "gold", CALIBER_NOT_REUSABLE) from e + refs.pinned = pinned + ids_ = {m["id"] for m in pinned.metrics} + for i, exp in enumerate(fields.get("expectations") or []): + if exp.get("kind") == "scalar" and exp.get("metric") not in ids_: + raise CaseInvalid(f"第 {i + 1} 个期望的指标「{exp.get('metric')}」不在「{pinned.workflow_name}」" + f"v{pinned.version} 的口径卡「{pinned.card_title}」上，这张卡的指标：" + f"{'、'.join(sorted(ids_)) or '（没有）'}", f"expectations.{i}.metric") + _check_inputs(gold.get("inputs") or {}, pinned.entry_fields, pinned.workflow_name) + if fields.get("kind") == "workflow": + await _check_workflow(session, fields) + return refs + + +def _check_inputs(inputs: Mapping[str, str], fields: Any, workflow_name: str) -> None: + from app.scheduler import service + + declared = [{"name": f["name"].strip(), "required": bool(f.get("required")), "default": f.get("default")} + for f in fields or [] if isinstance(f.get("name"), str) and f["name"].strip()] + try: + service.check_fields(dict(inputs), declared, workflow_name) + except service.TemplateError as e: + raise CaseInvalid(str(e), "gold.inputs") from e + + +async def _check_workflow(session: AsyncSession, fields: Mapping[str, Any]) -> None: + from app.apps import versions as app_versions + from app.db.models import Workflow + from app.scheduler import service + + workflow = await session.get(Workflow, str(fields.get("workflow_id") or "")) + if workflow is None: + raise CaseInvalid("要评的工作流不存在，可能已被删除", "workflow_id") + row = await app_versions.version_row(session, workflow.id, int(fields["workflow_version"])) + if row is None: + raise CaseInvalid(f"「{workflow.name}」没有 v{fields['workflow_version']} 这个版本", "workflow_version") + if not app_versions.is_published(row, current=workflow.published_version): + raise CaseInvalid(f"「{workflow.name}」v{row.version} 没有发布过或已退役：工作流题只能固定已发布且未退役的版本", + "workflow_version") + try: + service.check_fields(dict(fields.get("input_template") or {}), service.input_fields(row.graph), workflow.name) + except service.TemplateError as e: + raise CaseInvalid(str(e), "input_template") from e + + +__all__ = ["CALIBER_NOT_REUSABLE", "CASE_INVALID", "CaseInvalid", "CaseRefs", "EXPECTATION_KINDS", "GOLD_GUARD", + "GOLD_KINDS", "HASHED_FIELDS", "HUMAN_VALUE_KEYS", "NEEDS_GOLD", "UNCOUNTED", "case_fields", "case_hash", + "check_variables", "normalize_case", "normalize_expectations", "normalize_gold", "render", "resolve_case", + "variable_values"] diff --git a/backend/app/eval/gold.py b/backend/app/eval/gold.py new file mode 100644 index 00000000..28bb86e5 --- /dev/null +++ b/backend/app/eval/gold.py @@ -0,0 +1,712 @@ +"""期望答案：SQL 探针、口径探针、从运行认定（细案 01 做法 8、14；评测-D4）。 + +**期望不靠人填数，也不冻结数值。** 每次要期望（保存题目时试算一次、评测时作答前后各一次）都在业务库上重算一遍： + +- SQL 题、从运行认定：建一张「入口 → 调用工具（db_query__数据源，args.sql = 按截至时间渲染好的 SQL）→ 出口」的裸图， + 以来源 eval 跑探索运行。期望值取这次运行封存的查询快照里指定的格、列或整张表（cases.py 的 expectations）。从运行认定 + 的题和 SQL 题算法一样，只是 SQL 出自某次运行里的一条查询、记着谁在什么时候认定的。 +- 口径题：app/ledger/derive.caliber_probe 从钉住版本拷出口径卡读的取数节点、接一张 caliber_from 固定到那一版的口径卡， + 跑探索运行，期望值就是口径卡的指标（取整、单位、比率都由口径卡自己定）。不可复用的卡拒收并给原因。 + +为什么不发起钉版本的正式运行：会被 pick_base 和应用视图的「最近一次封存」取到，正式运行也不该为了评测而发；为什么不引用 +最近一次封存的正式运行：数据一直在变，期望会过期；为什么不直接调 run_query：期望没有封存快照，事后说不清当时查的是什么。 + +**守住的几条**（每次都查，不只保存时）： +- 只读：渲染好的 SQL 先过只读守卫（data/guard.check，readonly=True）；运行再带上 read_only_data（可写源上的写语句也由 + 守卫拒掉）和 mask_results 两个选项，同使用者的问数据运行。 +- 遮罩：查询结果在工具边界就按 engine/masking 的派生遮罩遮好（按 SQL 认出改了名字的遮罩列，认不全的整份遮）。快照里有 + 任何一列、一格是遮掉的，期望就拒收——评测页会显示期望值，遮罩值不能从这里露出来。认法和证据面板同一套，认不准就拒。 +- 照实说：取不到值（列不在、行不够、空值、该没有数据的却有数据）、运行失败、超时、停在人工确认上（评测不代批，直接 + 取消），各写各的原因；不混进「答错」。 + +**变量**：问法、期望 SQL、口径输入里的 {{ schedule.* }} 用定时任务那份 VARIABLES / template_values，按评测的「截至时间」 +在题集的时区渲染（cases.variable_values）。 + +**口径升版**（做法 14）：口径题钉 workflow_id@version。每次算期望之前，对探针图调一次 engine/governance. +unresolved_caliber_upgrades（和正式运行用的是同一个判断），再按 apps/versions.subgraph_pin_problems 看钉住的版本退役 +没有。有了新的已发布版本、或者钉住的版本退役了，又没有处置到那一版，这道题就跳过，写「口径已升版到 vN，待处置」。 +题目上三选一（dispose），没有缺省： +- recompute：改钉新版本，历史在这里断开（题目指纹跟着变，和上一次比时标不可比）； +- dual：新旧两版各算一份期望，都报（旧的那一份照旧可比）； +- incomparable：旧题归档，按新版本另建一道新题、重开基线。 + +**作答前后各算一次**（bracket / settle，第 3 波的执行调它）：两次一样才判；不一样就记「作答期间数据变了，不判」。A/B 两臂 +共用同一次算出的期望：在 answer 里同时跑两臂即可。 +""" +from __future__ import annotations + +import asyncio +import copy +import dataclasses +import logging +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, TypeVar + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.eval.cases import CALIBER_NOT_REUSABLE, GOLD_GUARD, NEEDS_GOLD, render, variable_values + +logger = logging.getLogger(__name__) +T = TypeVar("T") + +#: 结论：ok 算出来了；skipped 口径升版待处置（跳过这道题）；refused 规矩不许（守卫、遮罩、卡不能复用）；no_value 运行 +#: 好好的、按期望取不到值；failed 运行没成（SQL 错、数据源用不了、停在人工确认上）；timeout 超时没算完（已停止那次运行） +STATUSES = ("ok", "skipped", "refused", "no_value", "failed", "timeout") +#: 机器码（试算拒收时接口回 422 的 code） +GOLD_MASKED = "eval_gold_masked" +GOLD_NO_VALUE = "eval_gold_no_value" +GOLD_FAILED = "eval_gold_failed" +GOLD_TIMEOUT = "eval_gold_timeout" +UPGRADE_PENDING = "eval_caliber_upgrade_pending" +#: 作答前后两次期望不一样 +GOLD_DRIFTED = "eval_gold_drifted" + +#: 两次算期望的角色（记进运行 origin 的 role）：作答前、作答后 +ROLES = ("gold", "gold_after") +ROLE_LABEL = {"gold": "算期望", "gold_after": "作答后复算期望"} +#: 保存题目时试算一次等多久（秒）。评测里每题的时限另有预算（per_case_timeout_s） +TRIAL_TIMEOUT_S = 120.0 +#: 超时停止那次运行之后，最多再等它收尾多久 +CANCEL_GRACE_S = 15.0 +#: SQL 探针里调用工具节点的 id +QUERY_NODE = "gold_q" + +#: 口径升版的三种处置（评测里的说法；节点上的 upgrade_policy 叫法见 engine/labels） +UPGRADE_LABEL = {"recompute": "改钉新版本（历史在这里断开）", "dual": "新旧两版各算一份期望", + "incomparable": "旧题归档，按新版本另建新题"} + + +# ========================================================================== +# 结果的形状 +# ========================================================================== + + +@dataclass +class GoldSet: + """一次探针算出来的一份期望。version 是口径题这一份按的版本（SQL 题为空）；values 是 {期望 id: 取到的值}。""" + + run_id: str + snapshot: str | None + values: dict[str, dict[str, Any]] + version: int | None = None + + def to_dict(self) -> dict[str, Any]: + return {"run_id": self.run_id, "snapshot": self.snapshot, "version": self.version, + "values": copy.deepcopy(self.values)} + + @classmethod + def from_dict(cls, raw: Mapping[str, Any]) -> GoldSet: + return cls(run_id=str(raw.get("run_id") or ""), snapshot=raw.get("snapshot"), + values=copy.deepcopy(dict(raw.get("values") or {})), version=raw.get("version")) + + +@dataclass +class GoldOutcome: + """一次算期望的结果。sets：一份（SQL 题、口径题）或两份（口径题选了 dual：旧版在前、新版在后）。run_ids 是这次发起的 + 全部运行（失败的也在，证据点得开）。rendered 是按截至时间渲染好的问法、SQL、口径输入。""" + + status: str + as_of: str + reason: str | None = None + code: str | None = None + sets: list[GoldSet] = field(default_factory=list) + run_ids: list[str] = field(default_factory=list) + rendered: dict[str, Any] = field(default_factory=dict) + + @property + def ok(self) -> bool: + return self.status == "ok" + + def to_dict(self) -> dict[str, Any]: + return {"status": self.status, "as_of": self.as_of, "reason": self.reason, "code": self.code, + "sets": [s.to_dict() for s in self.sets], "run_ids": list(self.run_ids), + "rendered": copy.deepcopy(self.rendered)} + + @classmethod + def from_dict(cls, raw: Mapping[str, Any]) -> GoldOutcome: + return cls(status=str(raw.get("status") or "failed"), as_of=str(raw.get("as_of") or ""), + reason=raw.get("reason"), code=raw.get("code"), + sets=[GoldSet.from_dict(s) for s in raw.get("sets") or []], + run_ids=list(raw.get("run_ids") or []), rendered=copy.deepcopy(dict(raw.get("rendered") or {}))) + + def summary(self) -> dict[str, Any]: + """记进题目 trial 的那一份：不带期望值（期望每次重算，不冻结数值）。""" + return {"status": self.status, "code": self.code, "reason": self.reason, "as_of": self.as_of, + "run_ids": list(self.run_ids)} + + +def _outcome(status: str, as_of: datetime, reason: str | None = None, code: str | None = None, **kw: Any) -> GoldOutcome: + return GoldOutcome(status=status, as_of=as_of.isoformat(), reason=reason, code=code, **kw) + + +@dataclass(frozen=True) +class Context: + """算期望时题目以外的那几样：截至时间、时区、定时的 cron（手动发起为空）、发起人、记进运行 origin 的来历。""" + + as_of: datetime + tz: str = "Asia/Shanghai" + cron: str | None = None + actor: str | None = None + suite_id: str | None = None + suite_name: str = "" + eval_run_id: str | None = None + role: str = "gold" + #: 记进运行 origin 的其余字段（臂、第几次重复、试算……） + extra: Mapping[str, Any] = field(default_factory=dict) + timeout_s: float = TRIAL_TIMEOUT_S + + def variables(self) -> dict[str, str]: + return variable_values(self.as_of, self.tz, self.cron) + + +def _run_name(case: Mapping[str, Any], ctx: Context) -> str: + which = f"第 {case['seq']} 题" if case.get("seq") else "新题" + what = "试算" if ctx.extra.get("trial") else ROLE_LABEL.get(ctx.role, "算期望") + return f"评测：{ctx.suite_name or '题集'}·{which}·{what}" + + +def _origin(case: Mapping[str, Any], ctx: Context, **more: Any) -> dict[str, Any]: + data = {"suite_id": ctx.suite_id, "eval_run_id": ctx.eval_run_id, "case_id": case.get("id"), "role": ctx.role, + **dict(ctx.extra), **more} + return {k: v for k, v in data.items() if v is not None} + + +# ========================================================================== +# 发起探针、等它结束 +# ========================================================================== + + +#: 探针运行带的选项：查询一律只读、结果在工具边界就按遮罩遮好（同使用者的问数据运行） +def _probe_options() -> dict[str, Any]: + from app.tools.datasource import MASK_RESULTS_OPTION, READ_ONLY_OPTION + + return {MASK_RESULTS_OPTION: True, READ_ONLY_OPTION: True} + + +@dataclass +class _Ran: + status: str + run_id: str | None + reason: str | None = None + code: str | None = None + + +async def _launch_and_wait(graph: dict[str, Any], payload: dict[str, Any], *, name: str, origin: dict[str, Any], + ctx: Context) -> _Ran: + """发起一次探索运行（来源 eval），等它结束（最多 ctx.timeout_s 秒）。""" + from app.db.base import SessionLocal + from app.db.models import Run + from app.engine.launch import LaunchRefused, launch_run + from app.engine.runner import run_manager + + try: + async with SessionLocal() as session: + run = await launch_run(session, graph=graph, input_payload=payload, actor=ctx.actor, + options=_probe_options(), origin_kind="eval", origin=origin, workflow_name=name) + run_id = run.id + except LaunchRefused as e: + return _Ran("failed", None, f"算期望的运行没有发起：{e.message}", GOLD_FAILED) + loop = asyncio.get_running_loop() + deadline = loop.time() + max(0.0, float(ctx.timeout_s)) + while True: + # 执行任务收完尾（含封存）才回来；任务已经不在了（刚建好还没排上、或者已经结束）就立刻回来，再按库里的状态判 + await run_manager.wait_idle(run_id, timeout=max(0.05, min(5.0, deadline - loop.time()))) + async with SessionLocal() as session: + row = await session.get(Run, run_id) + status, error = (row.status, row.error) if row is not None else ("failed", "运行记录不见了") + if status in ("succeeded", "failed", "cancelled", "interrupted") or loop.time() >= deadline: + break + await asyncio.sleep(min(0.2, max(0.0, deadline - loop.time()))) + if status == "succeeded": + return _Ran("ok", run_id) + if status in ("failed", "cancelled"): + return _Ran("failed", run_id, f"算期望的运行{'失败' if status == 'failed' else '被停止'}：{error or '没有记下原因'}", + GOLD_FAILED) + stopping = status == "interrupted" + await run_manager.cancel(run_id, actor=ctx.actor) + await run_manager.wait_idle(run_id, timeout=CANCEL_GRACE_S) + if stopping: + return _Ran("failed", run_id, "算期望的运行停在人工确认上：评测不代批，已取消这次运行", GOLD_FAILED) + return _Ran("timeout", run_id, f"算期望的运行超过 {ctx.timeout_s:g} 秒还没结束，已停止这次运行", GOLD_TIMEOUT) + + +async def _artifacts(run_id: str, kind: str) -> list[tuple[str | None, tuple[str, Any]]]: + """这次运行落下的某一种工件：[(节点 id, (工件 id, 内容))]，按写入的先后。内容和哈希对不上的内容为 None。""" + from app.core import artifact_store + from app.db.base import SessionLocal + from app.db.models import Artifact + + async with SessionLocal() as session: + rows = (await session.execute(select(Artifact.id, Artifact.node_id).where( + Artifact.run_id == run_id, Artifact.kind == kind).order_by(Artifact.created_at))).all() + out: list[tuple[str | None, tuple[str, Any]]] = [] + for artifact_id, node_id in rows: + try: + out.append((node_id, (artifact_id, artifact_store.load(artifact_id)))) + except ValueError: + out.append((node_id, (artifact_id, None))) + return out + + +def masked_problem(snapshot: Mapping[str, Any]) -> str | None: + """查询快照里有遮掉的列、格（查询时就按遮罩遮好的，engine/masking.mask_query_result）：给人看的原因；没有为 None。""" + if snapshot.get("masked_whole"): + return ("期望 SQL 的写法认不全哪些结果取自遮罩列，整份结果都按遮罩处理：期望答案不能用遮罩列（评测页会显示期望值）。" + "请换一种写法（不要用 *、整行引用），只选不涉及遮罩列的列") + columns = [str(c) for c in snapshot.get("masked_at_query") or []] + if columns: + return (f"期望 SQL 的结果里「{'、'.join(columns[:5])}」取自数据源设了遮罩的列：期望答案不能用遮罩列" + "（评测页会显示期望值）。请去掉这几列") + if snapshot.get("masked_cells"): + return "期望 SQL 的结果里有和遮罩值相同的文字：期望答案不能用遮罩列（评测页会显示期望值）。请换一个不涉及遮罩列的问法" + return None + + +# ========================================================================== +# 从快照、口径卡里取值 +# ========================================================================== + + +class _NoValue(Exception): + pass + + +def _column(columns: Sequence[str], name: str) -> int: + if name in columns: + return list(columns).index(name) + lowered = [i for i, c in enumerate(columns) if c.lower() == name.lower()] + if len(lowered) == 1: + return lowered[0] + raise _NoValue(f"期望 SQL 的结果里没有列「{name}」（结果的列：{'、'.join(columns) or '（没有）'}）") + + +def _same(cell: Any, wanted: Any) -> bool: + if cell == wanted: + return True + if isinstance(cell, (int, float)) and isinstance(wanted, (int, float)) and not isinstance(cell, bool): + return float(cell) == float(wanted) + return cell is not None and str(cell) == str(wanted) + + +def _numeric(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def _dims(columns: Sequence[str], row: Sequence[Any], skip: int) -> dict[str, Any]: + """这一行的维度：非数值的格（评分按它和答案里那一格所在行配对，第 3 波）。""" + return {c: row[i] for i, c in enumerate(columns) if i != skip and i < len(row) and row[i] is not None + and not _numeric(row[i])} + + +def _sql_value(exp: Mapping[str, Any], snapshot: Mapping[str, Any]) -> dict[str, Any]: + columns = [str(c) for c in snapshot.get("columns") or []] + rows = [list(r) if isinstance(r, (list, tuple)) else [r] for r in snapshot.get("rows") or []] + truncated = bool(snapshot.get("truncated")) + kind = exp["kind"] + if kind == "none": + if rows: + raise _NoValue(f"这道题期望「没有数据」，但期望 SQL 返回了 {len(rows)} 行") + return {"kind": "none", "rows": 0} + if kind == "scalar": + ci = _column(columns, exp["column"]) + if exp.get("where"): + keys = {_column(columns, k): v for k, v in exp["where"].items()} + hits = [i for i, r in enumerate(rows) if all(i_ < len(r) and _same(r[i_], v) for i_, v in keys.items())] + cond = "、".join(f"{k}={v}" for k, v in exp["where"].items()) + if not hits: + raise _NoValue(f"期望 SQL 的结果里没有 {cond} 的行" + ("（结果被截断，只取回了前几行）" if truncated else "")) + if len(hits) > 1: + raise _NoValue(f"期望 SQL 的结果里 {cond} 的行有 {len(hits)} 行，取不准是哪一行：请在 where 里多写一列") + ri = hits[0] + else: + ri = int(exp.get("row") or 0) + if ri >= len(rows): + raise _NoValue("期望 SQL 没有返回行，取不到值：期望没有数据的题请把期望写成「没有数据」" if not rows else + f"期望 SQL 只返回了 {len(rows)} 行，取不到第 {ri + 1} 行") + value = rows[ri][ci] if ci < len(rows[ri]) else None + if value is None: + raise _NoValue(f"期望的那一格（第 {ri + 1} 行「{columns[ci]}」）是空值：期望没有数据的题请把期望写成「没有数据」") + return {"kind": "scalar", "value": value, "column": columns[ci], "row": ri, + "dims": _dims(columns, rows[ri], ci), **({"where": dict(exp["where"])} if exp.get("where") else {})} + if truncated: + raise _NoValue("期望 SQL 的结果被截断（只取回了前一部分行），多行、名单的期望算不准：请缩小范围或在 SQL 里汇总") + if kind == "set": + ci = _column(columns, exp["column"]) + values = [r[ci] if ci < len(r) else None for r in rows] + if exp.get("top_n"): + values = values[:exp["top_n"]] + return {"kind": "set", "column": columns[ci], "values": values, "ordered": bool(exp.get("ordered"))} + if kind == "table": + key_idx = [_column(columns, k) for k in exp["keys"]] + wanted = [_column(columns, c) for c in exp.get("columns") or columns] + picked = rows[:exp["top_n"]] if exp.get("top_n") else rows + seen: set[tuple[str, ...]] = set() + out_rows: list[dict[str, Any]] = [] + for r in picked: + key = tuple(str(r[i]) if i < len(r) else "" for i in key_idx) + if key in seen: + raise _NoValue(f"期望 SQL 的结果按「{'、'.join(exp['keys'])}」配对不唯一（{'、'.join(key)} 出现了不止一次）") + seen.add(key) + out_rows.append({columns[i]: (r[i] if i < len(r) else None) for i in dict.fromkeys(key_idx + wanted)}) + return {"kind": "table", "keys": [columns[i] for i in key_idx], "columns": [columns[i] for i in wanted], + "rows": out_rows, "ordered": bool(exp.get("ordered"))} + raise _NoValue(f"「{kind}」的期望不从期望 SQL 取值") + + +def _metric_value(exp: Mapping[str, Any], card: Mapping[str, Any]) -> dict[str, Any]: + metric = next((m for m in card.get("metrics") or [] if m.get("id") == exp["metric"]), None) + if metric is None: + raise _NoValue(f"口径卡这次没有算出指标「{exp['metric']}」") + if metric.get("value") is None: + raise _NoValue(f"口径卡的「{metric.get('name') or metric['id']}」缺输入，没有值（取数没有返回需要的格）") + if metric.get("incomplete"): + raise _NoValue(f"口径卡的「{metric.get('name') or metric['id']}」{metric.get('incomplete_reason') or '基于被截断的查询结果'}" + "，期望算不准") + out = {"kind": "scalar", "value": metric["value"], "metric": metric["id"], "name": metric.get("name") or metric["id"], + "unit": metric.get("unit") or "", "decimals": metric.get("decimals"), "format": metric.get("format"), + "rendered": metric.get("rendered"), "caliber": card.get("caliber"), + "caliber_version": card.get("caliber_version")} + if metric.get("sql_check_failed"): + out["sql_check_failed"] = True + return out + + +def _values(expectations: Sequence[Mapping[str, Any]], pick: Callable[[Mapping[str, Any]], dict[str, Any]] + ) -> dict[str, dict[str, Any]]: + return {e["id"]: pick(e) for e in expectations if e.get("kind") in NEEDS_GOLD} + + +# ========================================================================== +# SQL 探针（SQL 题、从运行认定） +# ========================================================================== + + +def sql_probe_graph(source_name: str, sql: str) -> dict[str, Any]: + """入口 → 调用工具（db_query__数据源，args.sql 是渲染好的字面 SQL）→ 出口。""" + from app.tools.datasource import QUERY_PREFIX + + def node(nid: str, ntype: str, label: str, x: int, **config: Any) -> dict[str, Any]: + return {"id": nid, "type": ntype, "position": {"x": x, "y": 0}, "data": {"label": label, "config": config}} + + nodes = [node("gold_in", "input", "入口", 0), node(QUERY_NODE, "tool", "期望 SQL", 260, + tool=f"{QUERY_PREFIX}{source_name}", args={"sql": sql}), + node("gold_out", "output", "出口", 520)] + return {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +async def _sql_gold(case: Mapping[str, Any], ctx: Context) -> GoldOutcome: + from app.data.guard import SqlRejected, check + from app.db.base import SessionLocal + from app.db.models import DataSource + + gold = case["gold"] + sql = render(gold["sql"], ctx.variables()) + rendered = {"sql": sql} + async with SessionLocal() as session: + source = await session.get(DataSource, gold["datasource_id"]) + name, kind, enabled = (source.name, source.kind, source.enabled) if source is not None else (None, None, False) + if name is None: + return _outcome("failed", ctx.as_of, "期望 SQL 查的数据源不存在，可能已被删除", GOLD_FAILED, rendered=rendered) + if not enabled: + return _outcome("failed", ctx.as_of, f"期望 SQL 查的数据源「{name}」已停用", GOLD_FAILED, rendered=rendered) + try: + check(sql, readonly=True, source_name=name, dialect=(kind or "").lower()) + except SqlRejected as e: + return _outcome("refused", ctx.as_of, f"期望 SQL 没有通过只读守卫：{e}", GOLD_GUARD, rendered=rendered) + ran = await _launch_and_wait(sql_probe_graph(name, sql), {}, name=_run_name(case, ctx), origin=_origin(case, ctx), + ctx=ctx) + runs = [ran.run_id] if ran.run_id else [] + if ran.status != "ok": + return _outcome(ran.status, ctx.as_of, ran.reason, ran.code, run_ids=runs, rendered=rendered) + snapshots = [(nid, item) for nid, item in await _artifacts(ran.run_id, "query_snapshot") if nid == QUERY_NODE] + if not snapshots or snapshots[-1][1][1] is None: + return _outcome("failed", ctx.as_of, "算期望的运行没有留下查询快照，期望无从取值", GOLD_FAILED, run_ids=runs, + rendered=rendered) + artifact_id, snapshot = snapshots[-1][1] + if problem := masked_problem(snapshot): + return _outcome("refused", ctx.as_of, problem, GOLD_MASKED, run_ids=runs, rendered=rendered) + try: + values = _values(case["expectations"], lambda e: _sql_value(e, snapshot)) + except _NoValue as e: + return _outcome("no_value", ctx.as_of, str(e), GOLD_NO_VALUE, run_ids=runs, rendered=rendered) + return _outcome("ok", ctx.as_of, sets=[GoldSet(run_id=ran.run_id, snapshot=artifact_id, values=values)], + run_ids=runs, rendered=rendered) + + +# ========================================================================== +# 口径探针、升版处置 +# ========================================================================== + + +@dataclass(frozen=True) +class UpgradeState: + """口径题钉的那一版现在怎么样。pending：有新的已发布版本（或钉住的退役了）而没处置到那一版，这道题跳过。 + versions：这道题要算的版本（钉住的；dual 时再加新版）。""" + + pinned: int + latest: int | None + retired: bool + pending: bool + message: str | None + policy: str | None + versions: tuple[int, ...] + + def to_dict(self) -> dict[str, Any]: + return {"pinned": self.pinned, "latest": self.latest, "retired": self.retired, "pending": self.pending, + "message": self.message, "policy": self.policy, "versions": list(self.versions)} + + +def _covered(upgrade: Mapping[str, Any] | None, latest: int | None) -> str | None: + """题目上的处置管不管得到最新的那一版：管得到交回处置，否则 None。""" + if not upgrade or upgrade.get("policy") not in UPGRADE_LABEL: + return None + if latest is not None and int(upgrade.get("to_version") or 0) < latest: + return None + return str(upgrade["policy"]) + + +async def upgrade_state(session: AsyncSession, gold: Mapping[str, Any], pinned: Any) -> UpgradeState: + """按正式运行同一个判断（unresolved_caliber_upgrades）、同一个退役认法（subgraph_pin_problems）看这道口径题要不要处置。""" + from app.apps.versions import CALIBER_RETIRED, latest_published, subgraph_pin_problems + from app.engine.governance import unresolved_caliber_upgrades + from app.engine.schema import GraphSpec + from app.ledger import derive + + newest = await latest_published(session, pinned.workflow_id) + latest = newest.version if newest is not None else None + upgrade = gold.get("upgrade") or None + policy = _covered(upgrade, latest) + spec = GraphSpec.model_validate(derive.caliber_probe(pinned, upgrade_policy=policy).graph) + errors, _events = await unresolved_caliber_upgrades(session, spec) + retired = any(p.code == CALIBER_RETIRED for p in await subgraph_pin_problems(session, spec)) + pending = bool(errors) or retired + message = None + if errors: + message = f"口径已升版到 v{latest}，待处置" + elif retired: + message = f"钉住的「{pinned.workflow_name}」v{pinned.version} 已退役，待处置" + versions: tuple[int, ...] = (pinned.version,) + if policy == "dual" and upgrade and int(upgrade.get("to_version") or 0) > pinned.version: + versions = (pinned.version, int(upgrade["to_version"])) + return UpgradeState(pinned=pinned.version, latest=latest, retired=retired, pending=pending, message=message, + policy=policy, versions=versions) + + +async def _caliber_set(case: Mapping[str, Any], pinned: Any, ctx: Context, payload: dict[str, Any], + policy: str | None) -> tuple[GoldSet | None, GoldOutcome | None]: + from app.ledger import derive + + probe = derive.caliber_probe(pinned, upgrade_policy=policy) + ran = await _launch_and_wait(probe.graph, payload, name=_run_name(case, ctx), + origin=_origin(case, ctx, derived=probe.origin), ctx=ctx) + runs = [ran.run_id] if ran.run_id else [] + if ran.status != "ok": + return None, _outcome(ran.status, ctx.as_of, ran.reason, ran.code, run_ids=runs) + for _nid, (_aid, snapshot) in await _artifacts(ran.run_id, "query_snapshot"): + if isinstance(snapshot, dict) and (problem := masked_problem(snapshot)): + return None, _outcome("refused", ctx.as_of, problem.replace("期望 SQL", "口径的取数"), GOLD_MASKED, + run_ids=runs) + cards = [(aid, body) for nid, (aid, body) in await _artifacts(ran.run_id, "metric_set") if nid == probe.card_ids[0]] + if not cards or not isinstance(cards[-1][1], dict): + return None, _outcome("failed", ctx.as_of, "算期望的运行没有留下口径卡的指标集，期望无从取值", GOLD_FAILED, + run_ids=runs) + artifact_id, card = cards[-1] + try: + values = _values(case["expectations"], lambda e: _metric_value(e, card)) + except _NoValue as e: + return None, _outcome("no_value", ctx.as_of, str(e), GOLD_NO_VALUE, run_ids=runs) + return GoldSet(run_id=ran.run_id, snapshot=artifact_id, values=values, version=pinned.version), None + + +async def _caliber_gold(case: Mapping[str, Any], ctx: Context) -> GoldOutcome: + from app.db.base import SessionLocal + from app.ledger import derive + + gold = case["gold"] + variables = ctx.variables() + payload = {k: render(v, variables) for k, v in (gold.get("inputs") or {}).items()} + rendered = {"inputs": payload} + async with SessionLocal() as session: + try: + pinned = await derive.load_pinned(session, gold["workflow_id"], gold["version"], gold["node_id"]) + except derive.NotDerivable as e: + return _outcome("refused", ctx.as_of, f"这张口径卡不能当口径题：{e.message}", CALIBER_NOT_REUSABLE, + rendered=rendered) + state = await upgrade_state(session, gold, pinned) + if state.pending: + return _outcome("skipped", ctx.as_of, state.message, UPGRADE_PENDING, rendered=rendered) + pins = [pinned] + for version in state.versions[1:]: + try: + pins.append(await derive.load_pinned(session, gold["workflow_id"], version, gold["node_id"])) + except derive.NotDerivable as e: + return _outcome("refused", ctx.as_of, f"新版 v{version} 不能当口径题：{e.message}", CALIBER_NOT_REUSABLE, + rendered=rendered) + sets: list[GoldSet] = [] + runs: list[str] = [] + for pin in pins: + found, failed = await _caliber_set(case, pin, ctx, payload, state.policy) + if failed is not None: + failed.run_ids = runs + failed.run_ids + failed.rendered = rendered + if len(pins) > 1: + failed.reason = f"v{pin.version}：{failed.reason}" + return failed + assert found is not None + sets.append(found) + runs.append(found.run_id) + return _outcome("ok", ctx.as_of, sets=sets, run_ids=runs, rendered=rendered) + + +# ========================================================================== +# 对外：算一次、作答前后各算一次 +# ========================================================================== + + +async def compute_gold(case: Mapping[str, Any], ctx: Context) -> GoldOutcome: + """按 case（cases.case_fields 的形状，或者还没保存的 normalize_case 结果）算一次期望。没有期望来源的题（只有反问、 + 文字要点）不用算，交回 ok、没有值。不抛异常：意外也照实记成 failed。""" + gold = case.get("gold") or None + rendered_question = render(case.get("question") or "", ctx.variables()) + try: + if gold is None: + outcome = _outcome("ok", ctx.as_of) + elif gold["kind"] in ("sql", "from_run"): + outcome = await _sql_gold(case, ctx) + else: + outcome = await _caliber_gold(case, ctx) + except asyncio.CancelledError: + raise + except Exception as e: # noqa: BLE001 - 算期望是附加的一步：出了意外照实记下，不让评测整体停下 + from app.core.errors import describe_exception + + logger.exception("评测：算第 %s 题的期望时出错", case.get("seq")) + outcome = _outcome("failed", ctx.as_of, f"算期望时出错：{describe_exception(e)}", GOLD_FAILED) + outcome.rendered = {"question": rendered_question, **outcome.rendered} + return outcome + + +@dataclass +class Settled: + """作答前后两次期望合起来的结论。status：ok 两次一样、可以判；drifted 作答期间数据变了，不判；其余照搬没算出来的 + 那一次（skipped / refused / no_value / failed / timeout）。expected 是能判时用的那一份（作答前的）。""" + + status: str + reason: str | None + code: str | None + before: GoldOutcome + after: GoldOutcome | None + + @property + def ok(self) -> bool: + return self.status == "ok" + + @property + def expected(self) -> list[GoldSet]: + return self.before.sets if self.ok else [] + + def to_dict(self) -> dict[str, Any]: + """EvalResult.gold 的形状。""" + return {"status": self.status, "reason": self.reason, "code": self.code, "before": self.before.to_dict(), + "after": self.after.to_dict() if self.after is not None else None} + + +def _fingerprint(outcome: GoldOutcome) -> str: + from app.core.artifact_store import canonical_json + + return canonical_json([[s.version, s.values] for s in outcome.sets]) + + +def settle(before: GoldOutcome, after: GoldOutcome | None) -> Settled: + """两次期望一样才判。没算出来的那一次的原因照搬（先看作答前的）。""" + if not before.ok: + return Settled(before.status, before.reason, before.code, before, after) + if after is None: + return Settled("failed", "作答后没有复算期望", GOLD_FAILED, before, after) + if not after.ok: + return Settled(after.status, f"作答后复算期望：{after.reason}", after.code, before, after) + if _fingerprint(before) != _fingerprint(after): + return Settled("drifted", "作答期间数据变了（作答前后算出的期望不一样），这道题不判", GOLD_DRIFTED, before, after) + return Settled("ok", None, None, before, after) + + +async def bracket(case: Mapping[str, Any], ctx: Context, answer: Callable[[GoldOutcome], Awaitable[T]] + ) -> tuple[T | None, Settled]: + """作答前算一次期望 → answer(作答前的期望) → 作答后再算一次 → settle。作答前就算不出来的（升版待处置、拒收、失败） + 不作答，省下一次作答的钱，结论照搬原因（第 3 波记「跳过」或「不能评分」，不算答错）。A/B 两臂共用期望：在 answer + 里同时跑两臂。""" + before = await compute_gold(case, ctx) + if not before.ok: + return None, settle(before, None) + result = await answer(before) + after = await compute_gold(case, dataclasses.replace(ctx, role="gold_after")) + return result, settle(before, after) + + +# ========================================================================== +# 口径升版的处置（题目上三选一） +# ========================================================================== + + +class DisposeRefused(Exception): + """处置不了：message 给人看，code 给程序认（接口回 409 / 422）。""" + + def __init__(self, message: str, code: str, status: int = 409) -> None: + super().__init__(message) + self.message = message + self.code = code + self.status = status + + +#: 处置被拒的机器码 +DISPOSE_NOT_CALIBER = "eval_dispose_not_caliber" +DISPOSE_NOTHING = "eval_dispose_nothing_pending" +DISPOSE_INVALID = "eval_dispose_invalid" + + +async def plan_dispose(session: AsyncSession, case: Mapping[str, Any], policy: str) -> tuple[dict[str, Any], int]: + """按 policy 算出处置后的期望来源（还不写库）：(新的 gold, 新版本号)。recompute、incomparable 交回改钉新版本的 gold， + dual 交回带上 upgrade 的旧 gold。新版的卡不能当口径题（不能复用、没有这道题的指标）就拒（DisposeRefused）。""" + from app.db.base import utcnow + from app.ledger import derive + + if policy not in UPGRADE_LABEL: + raise DisposeRefused(f"口径升版的处置只能是 {'、'.join(UPGRADE_LABEL)} 之一，没有缺省", DISPOSE_INVALID, 422) + gold = case.get("gold") or {} + if gold.get("kind") != "caliber": + raise DisposeRefused("只有口径题有升版处置", DISPOSE_NOT_CALIBER, 422) + try: + pinned = await derive.load_pinned(session, gold["workflow_id"], gold["version"], gold["node_id"]) + except derive.NotDerivable as e: + raise DisposeRefused(f"钉住的口径读不出来：{e.message}", CALIBER_NOT_REUSABLE) from e + state = await upgrade_state(session, gold, pinned) + if not state.pending or state.latest is None or state.latest <= pinned.version: + raise DisposeRefused("这道题钉的口径没有待处置的新版本" if not state.pending else + f"{state.message}：已经没有更新的已发布版本可以改钉，只能另建新题或删除这道题", + DISPOSE_NOTHING) + latest = state.latest + if policy == "dual" and state.retired: + raise DisposeRefused(f"钉住的 v{pinned.version} 已退役，不能新旧两版并算：请改钉新版本或另建新题", DISPOSE_INVALID) + try: + newer = await derive.load_pinned(session, gold["workflow_id"], latest, gold["node_id"]) + except derive.NotDerivable as e: + raise DisposeRefused(f"新版 v{latest} 不能当口径题：{e.message}。请另建新题", CALIBER_NOT_REUSABLE) from e + have = {m["id"] for m in newer.metrics} + lost = [e["metric"] for e in case.get("expectations") or [] if e.get("kind") == "scalar" and e["metric"] not in have] + if lost: + raise DisposeRefused(f"新版 v{latest} 的口径卡上没有指标「{'、'.join(lost)}」：请另建新题", CALIBER_NOT_REUSABLE) + record = {"policy": policy, "from_version": pinned.version, "to_version": latest, "at": utcnow().isoformat()} + if policy == "dual": + return {**copy.deepcopy(dict(gold)), "upgrade": record}, latest + return {**{k: copy.deepcopy(v) for k, v in gold.items() if k != "upgrade"}, "version": latest, + **({"upgrade": record} if policy == "recompute" else {})}, latest + + +__all__ = ["CANCEL_GRACE_S", "Context", "DISPOSE_INVALID", "DISPOSE_NOTHING", "DISPOSE_NOT_CALIBER", "DisposeRefused", + "GOLD_DRIFTED", "GOLD_FAILED", "GOLD_MASKED", "GOLD_NO_VALUE", "GOLD_TIMEOUT", "GoldOutcome", "GoldSet", + "QUERY_NODE", "ROLES", "STATUSES", "Settled", "TRIAL_TIMEOUT_S", "UPGRADE_LABEL", "UPGRADE_PENDING", + "UpgradeState", "bracket", "compute_gold", "masked_problem", "plan_dispose", "settle", "sql_probe_graph", + "upgrade_state"] diff --git a/backend/app/eval/snapshots.py b/backend/app/eval/snapshots.py new file mode 100644 index 00000000..48fd82f1 --- /dev/null +++ b/backend/app/eval/snapshots.py @@ -0,0 +1,609 @@ +"""目录快照与影子起草（细案 01 做法 6；评测-D5、D6）。 + +**为什么要快照。** 「目录的贡献」要做对照（当前目录 vs 关、只用已核实的项、起草模型 A 的目录 vs B 的），而评测一跑几十分钟， +期间目录照常被起草、剖析、审阅改动。评测按快照读目录（data/catalog_view.CatalogView(mode="snapshot")），这次用的是哪份 +目录就定下来了，也不用为了可比性占住数据源的队、挡住起草和剖析。 + +**快照的内容**是内容寻址工件（kind=eval_catalog_snapshot），读法由 data/catalog_view.py 定： +{"sources": {源 id: {"tables": {表名: {"version": 版本, "notes": 目录}}}}}；另带 format、taken（谁、何时、来源），读法不看 +它们。目录照原样拍（驳回的项也在，消费方照旧经 visible_notes 去掉），statuses 的筛选在读的时候按视图做。工件一旦写好 +内容就不变（改一个字哈希就变），EvalCatalogSnapshot 这一行写好 artifact_id 之后也不再改：快照不可变，同一份可以被多次 +评测引用。 + +**两种来源**： +- 当前目录（take_live）：选定数据源此刻的目录，当场拍好。 +- 影子起草（后台任务 ShadowDraft）：用指定的起草模型（不指定就按设置里起草模型的取值链，data/draft_model.py）、起草要求， + 把题集用得到的表重新起草进一份**内存里的目录副本**——从当前目录拷一份起步，可选先去掉模型起草、还没人审过的项 + （strip_llm）；起草引擎还是同一个（data/catalog_draft.run_shadow_draft），每一段写出来就按 catalog.merge_draft（和写库 + 同一套并入规矩）并进副本。**不调 write_entry、不碰线上目录；不查业务库**（有效行过滤不去数排除比例，见 run_shadow_draft）； + **不占数据源的队**（lane_of 返回 None），起草、剖析照常进行。全部起草完才写工件：停止、出错、服务重启的不出快照， + 半截的目录拿去对照会误导。有的表没写出来时仍出快照，状态记 partial、逐表原因写在进度里。 +- 起草按表计费，大库很贵：只起草题集用得到的表（从期望 SQL、口径的取数 SQL 推，plan_tables），开始前先估算花费 + （estimate，照实写是按经验估的、不含思考）。 + +进程内的后台任务（core/job_runtime 的 JobManager）：服务重启时没结束的标中断、不自动续跑。startup() 是给 main.py 启动 +整理调的那一步（接进 lifespan 和评测执行一起，第 3 波）；在那之前读接口按 reconcile_stale 当场整理：记着进行中、这个进程 +里却没在跑的，照实标中断。 +""" +from __future__ import annotations + +import copy +import logging +import math +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any + +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core import job_runtime as runtime +from app.data import catalog +from app.data.catalog_view import SNAPSHOT_KIND +from app.db.base import SessionLocal, utcnow +from app.db.models_eval import EvalCatalogSnapshot + +logger = logging.getLogger(__name__) + +FORMAT = 1 +#: 一次最多拍几个数据源、影子起草最多起草几张表（起草按表计费） +MAX_SOURCES = 20 +MAX_SHADOW_TABLES = 200 +GUIDANCE_MAX = 2000 + +#: 机器码 +SNAPSHOT_INVALID = "eval_snapshot_invalid" +SNAPSHOT_MODEL_UNAVAILABLE = "eval_snapshot_model_unavailable" +SNAPSHOT_FINISHED = "eval_snapshot_finished" + +SOURCE_GONE = "数据源不存在，可能已被删除" +INTERRUPTED = "服务重启时影子起草还没结束，没有出快照。请重新发起" +STOPPED = "影子起草已停止，没有出快照" + +#: 估算用的经验值：每列约 50 个输出 token（不算思考，见 data/catalog.py 分段的说明），每张表另加表级的项；输入按提示词 +#: 的字数折算（中文为主的提示词约 1.5 个字一个 token） +EST_OUTPUT_PER_COLUMN = 50 +EST_OUTPUT_PER_TABLE = 120 +EST_CHARS_PER_TOKEN = 1.5 +EST_CHARS_PER_COLUMN = 40 +EST_NOTE = ("按经验估算：每列约 50 个输出 token、输入按提示词字数折算；不含思考（会思考的模型思考也算在输出里，" + "实际可能高几倍）、不含拆小重试") + + +class SnapshotInvalid(ValueError): + """请求不合规（422）或者开始不了（409）。""" + + def __init__(self, message: str, code: str = SNAPSHOT_INVALID, status: int = 422, field: str | None = None) -> None: + super().__init__(message) + self.message = message + self.code = code + self.status = status + self.field = field + + +# ========================================================================== +# 当前目录的快照 +# ========================================================================== + + +def _content(sources: Mapping[str, Mapping[str, Mapping[str, Any]]], *, taken: Mapping[str, Any]) -> dict[str, Any]: + """{源 id: {表名: {version, notes}}} → 快照工件的内容（读法见 data/catalog_view.py）。""" + return {"format": FORMAT, "taken": dict(taken), + "sources": {sid: {"tables": {t: {"version": int(body.get("version") or 0), "notes": body.get("notes") or {}} + for t, body in tables.items()}} + for sid, tables in sources.items()}} + + +async def _sources(session: AsyncSession, source_ids: Sequence[str]) -> list[Any]: + from app.db.models import DataSource + + ids = list(dict.fromkeys(s for s in source_ids if isinstance(s, str) and s)) + if not ids: + raise SnapshotInvalid("请选择要拍快照的数据源", field="source_ids") + if len(ids) > MAX_SOURCES: + raise SnapshotInvalid(f"一次最多拍 {MAX_SOURCES} 个数据源", field="source_ids") + rows = [] + for sid in ids: + row = await session.get(DataSource, sid) + if row is None: + raise SnapshotInvalid(f"数据源 {sid} 不存在，可能已被删除", field="source_ids") + rows.append(row) + return rows + + +async def _live_tables(session: AsyncSession, source_ids: Sequence[str]) -> dict[str, dict[str, dict[str, Any]]]: + """这几个源此刻的目录原样：{源 id: {表名: {version, notes}}}（驳回的项也在）。""" + found = await catalog.read_catalogs(session, list(source_ids)) + return {sid: {t: {"version": e.version, "notes": copy.deepcopy(e.notes)} for t, e in (found.get(sid) or {}).items()} + for sid in source_ids} + + +async def take_live(session: AsyncSession, source_ids: Sequence[str], *, actor: str | None) -> EvalCatalogSnapshot: + """拍选定数据源此刻的目录，当场写工件、登记一行（succeeded）。""" + from app.core.artifact_store import put_json + + rows = await _sources(session, source_ids) + ids = [r.id for r in rows] + at = utcnow() + taken = {"origin": "live", "by": actor, "at": at.isoformat()} + content = _content(await _live_tables(session, ids), taken=taken) + artifact = await put_json(content, kind=SNAPSHOT_KIND, meta={"origin": "live", "sources": ids}) + snap = EvalCatalogSnapshot(origin="live", status="succeeded", artifact_id=artifact, + sources=[{"id": r.id, "name": r.name} for r in rows], created_by=actor, + started_at=at, finished_at=at) + session.add(snap) + await session.commit() + await session.refresh(snap) + return snap + + +# ========================================================================== +# 影子起草：起草哪些表、估算 +# ========================================================================== + + +def _schema_tables(source: Any) -> dict[str, Any]: + return (getattr(source, "schema_cache", None) or {}).get("tables") or {} + + +def _sql_tables(sql: str) -> list[str]: + from app.engine.evidence import sql_tables + + try: + return sql_tables(sql or "") + except Exception: # noqa: BLE001 - 解析不了就当没用到表：只影响起草哪些表 + return [] + + +async def suite_tables(session: AsyncSession, suite_id: str) -> dict[str, list[str]]: + """题集用得到的表：{源 id: [SQL 里写的表名]}。从期望 SQL（SQL 题、从运行认定）和口径题的取数 SQL 推；工作流题、 + 只有反问的题推不出。名字还没对上表结构（plan_shadow 再对）。""" + from app.db.models import DataSource + from app.db.models_eval import EvalCase + from app.eval.cases import render, variable_values + from app.ledger import derive + from app.tools.datasource import QUERY_PREFIX + + values = variable_values(utcnow(), "Asia/Shanghai") + out: dict[str, list[str]] = {} + rows = (await session.execute(select(EvalCase).where(EvalCase.suite_id == suite_id, EvalCase.archived.is_(False)) + .order_by(EvalCase.seq))).scalars().all() + by_name: dict[str, str] = {} + for case in rows: + gold = case.gold or {} + if gold.get("kind") in ("sql", "from_run"): + out.setdefault(gold["datasource_id"], []).extend(_sql_tables(render(gold.get("sql") or "", values))) + elif gold.get("kind") == "caliber": + try: + pinned = await derive.load_pinned(session, gold["workflow_id"], gold["version"], gold["node_id"]) + except derive.NotDerivable: + continue + for node in pinned.fetches: + cfg = (node.get("data") or {}).get("config") or {} + tool = str(cfg.get("tool") or "") + name = tool[len(QUERY_PREFIX):] + if name not in by_name: + sid = (await session.execute(select(DataSource.id).where(DataSource.name == name))).scalar() + by_name[name] = sid or "" + if by_name[name]: + out.setdefault(by_name[name], []).extend(_sql_tables(str((cfg.get("args") or {}).get("sql") or ""))) + return {sid: list(dict.fromkeys(names)) for sid, names in out.items()} + + +def estimate(sources: Sequence[tuple[Any, Sequence[str]]], model_id: str) -> dict[str, Any]: + """估一次影子起草要调几次模型、多少 token、多少钱（照实写是按经验估的）。分段同起草引擎：不超过 MODEL_BATCH_COLUMNS + 列的表按顺序最多 MODEL_BATCH_TABLES 张拼一段，宽表是表级一段加列分段（catalog_draft.parts_for）。""" + from app.data import catalog_draft + from app.providers.catalog import estimate_cost, find_pricing + + width = max(1, int(catalog.MODEL_BATCH_COLUMNS)) + per_call = max(1, int(catalog.MODEL_BATCH_TABLES)) + system = len(catalog.CATALOG_DRAFT_SYSTEM) + calls = tables = columns = 0 + chars = out_tokens = 0 + for source, names in sources: + meta = _schema_tables(source) + batch_tables = batch_columns = 0 + for name in names: + n = len([c for c in (meta.get(name) or {}).get("columns") or [] if isinstance(c, dict)]) + tables += 1 + columns += n + out_tokens += EST_OUTPUT_PER_TABLE + n * EST_OUTPUT_PER_COLUMN + if n > width: + parts = catalog_draft.parts_for(n) + calls += parts + # 表级那段给全部列名，列分段各给自己那几列 + chars += parts * system + n * EST_CHARS_PER_COLUMN * 2 + continue + if batch_tables and (batch_tables >= per_call or batch_columns + n > width): + calls += 1 + chars += system + batch_tables = batch_columns = 0 + batch_tables += 1 + batch_columns += n + chars += n * EST_CHARS_PER_COLUMN + if batch_tables: + calls += 1 + chars += system + in_tokens = int(math.ceil(chars / EST_CHARS_PER_TOKEN)) + priced = bool(model_id) and find_pricing(model_id) is not None + return {"tables": tables, "columns": columns, "calls": calls, "input_tokens": in_tokens, "output_tokens": out_tokens, + "model": model_id or None, "cost_usd": round(estimate_cost(model_id, in_tokens, out_tokens), 4) if priced + else None, "unpriced": not priced, "note": EST_NOTE} + + +@dataclass +class ShadowPlan: + """一次影子起草要做什么：每个源起草哪些表（表结构里的键）、对不上表结构的表名；估算；起草设定。""" + + sources: list[dict[str, Any]] = field(default_factory=list) + estimate: dict[str, Any] = field(default_factory=dict) + draft: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return {"sources": copy.deepcopy(self.sources), "estimate": dict(self.estimate), "draft": dict(self.draft)} + + +async def resolve_model(session: AsyncSession, provider: str | None, model: str | None) -> tuple[Any, str]: + """影子起草用的模型：(模型, 模型 id)。指定了接入或模型就用它（同一个起草的建法，catalog.build_draft_model）；都没指定 + 按设置里起草模型的取值链（catalog.resolve_draft_model）。用不了抛 SnapshotInvalid（409）。不发请求。""" + from app.core.errors import not_configured + from app.providers.factory import ProviderNotConfigured, resolve_provider + + if not provider and not model: + try: + return await catalog.resolve_draft_model(session) + except catalog.CatalogModelUnavailable as e: + raise SnapshotInvalid(str(e), SNAPSHOT_MODEL_UNAVAILABLE, 409) from e + try: + row = await resolve_provider(session, provider, model) + if row is None: + raise SnapshotInvalid("未配置模型接入，无法影子起草。请到「设置 → 模型接入」添加", SNAPSHOT_MODEL_UNAVAILABLE, 409) + if row.kind == "mock": + raise SnapshotInvalid("演示用的模型接入不能用于起草目录：起草出来的目录会被当成真的拿去对照", + SNAPSHOT_MODEL_UNAVAILABLE, 409) + return catalog.build_draft_model(row, provider, model) + except ProviderNotConfigured as e: + raise SnapshotInvalid(f"影子起草指定的模型用不了：{not_configured(e)}", SNAPSHOT_MODEL_UNAVAILABLE, 409) from e + + +def _text(value: Any, where: str, field_name: str) -> str | None: + if value in (None, ""): + return None + if not isinstance(value, str) or len(value.strip()) > GUIDANCE_MAX: + raise SnapshotInvalid(f"{where}需要是不超过 {GUIDANCE_MAX} 个字符的文字", field=field_name) + return value.strip() or None + + +async def plan_shadow(session: AsyncSession, body: Mapping[str, Any], *, model_id: str | None = None) -> ShadowPlan: + """影子起草的请求 → 计划。body：{sources: [{id, tables?}], suite_id?, provider?, model?, guidance?, note?, + strip_llm?}。每个源没给表就按题集推（suite_tables）；推不出、也没给的报错。model_id 给了就按它估价（接口先解析模型）。""" + if not isinstance(body, Mapping): + raise SnapshotInvalid("请求内容需要是 JSON 对象") + raw = body.get("sources") + if not isinstance(raw, list) or not raw or not all(isinstance(s, Mapping) and isinstance(s.get("id"), str) + for s in raw): + raise SnapshotInvalid("请选择要影子起草的数据源：sources 是 [{id, tables}]", field="sources") + suite_id = body.get("suite_id") if isinstance(body.get("suite_id"), str) and body.get("suite_id") else None + from_suite = await suite_tables(session, suite_id) if suite_id else {} + rows = await _sources(session, [s["id"] for s in raw]) + picked: list[dict[str, Any]] = [] + sources_for_estimate: list[tuple[Any, list[str]]] = [] + total = 0 + for row, item in zip(rows, raw): + wanted = item.get("tables") + if wanted is not None and (not isinstance(wanted, list) or not all(isinstance(t, str) for t in wanted)): + raise SnapshotInvalid("tables 需要是一组表名", field="sources") + wanted = list(wanted) if wanted else from_suite.get(row.id, []) + if not wanted: + raise SnapshotInvalid(f"数据源「{row.name}」没有指定要起草的表" + + ("，题集里也推不出用到它的哪些表" if suite_id else "：请列出表名，或给出题集（按题集的期望 SQL 推）"), + field="sources") + keys, unknown = catalog.pick_tables(getattr(row, "schema_cache", None) or {}, wanted) + if not keys: + raise SnapshotInvalid(f"数据源「{row.name}」的表结构里没有这些表：{'、'.join(unknown[:10])}。请先重新探查结构", + field="sources") + total += len(keys) + picked.append({"id": row.id, "name": row.name, "tables": keys, "unknown": unknown}) + sources_for_estimate.append((row, keys)) + if total > MAX_SHADOW_TABLES: + raise SnapshotInvalid(f"一次影子起草最多 {MAX_SHADOW_TABLES} 张表（这次 {total} 张）：起草按表计费，请缩小范围", + field="sources") + strip = body.get("strip_llm", False) + if not isinstance(strip, bool): + raise SnapshotInvalid("strip_llm 需要是 true 或 false", field="strip_llm") + draft = {"provider": body.get("provider") or None, "model": body.get("model") or None, "model_id": model_id, + "guidance": _text(body.get("guidance"), "起草要求", "guidance"), + "note": _text(body.get("note"), "本次说明", "note"), "strip_llm": strip, "suite_id": suite_id} + return ShadowPlan(sources=picked, estimate=estimate(sources_for_estimate, model_id or ""), draft=draft) + + +# ========================================================================== +# 影子起草：后台任务 +# ========================================================================== + + +def strip_model_items(notes: Mapping[str, Any]) -> dict[str, Any]: + """去掉模型起草、还没人审过的项（source=llm、status=proposed）；人工确认、驳回过的模型项留着（审过了就是人的结论）。""" + slots = catalog._slots(notes) + return catalog._assemble({k: v for k, v in slots.items() + if not (v.get("source") == "llm" and v.get("status") == "proposed")}) + + +async def start_shadow(session: AsyncSession, plan: ShadowPlan, *, actor: str | None) -> EvalCatalogSnapshot: + """登记一次影子起草（queued）并在后台开始。""" + snap = EvalCatalogSnapshot(origin="shadow", status="queued", created_by=actor, + sources=[{"id": s["id"], "name": s["name"]} for s in plan.sources], + tables={s["id"]: list(s["tables"]) for s in plan.sources}, draft=dict(plan.draft), + estimate=dict(plan.estimate), + progress={"tables_total": sum(len(s["tables"]) for s in plan.sources), "tables_done": 0, + "tables_failed": 0, "failures": {}}) + session.add(snap) + await session.commit() + await session.refresh(snap) + manager.start(snap.id) + return snap + + +class ShadowDraft(runtime.JobRun): + """一次影子起草在后台的执行。不占数据源的队；全部起草完才写快照工件。""" + + kind = "eval_shadow_draft" + model = EvalCatalogSnapshot + #: 测试注入的起草模型（不给就按登记的设定解析） + model_override: Any = None + + def __init__(self, job_id: str) -> None: + super().__init__(job_id) + self.model_id = "" + self.copy: dict[str, dict[str, dict[str, Any]]] = {} + self.usage = {"calls": 0, "input_tokens": 0, "output_tokens": 0, "timeouts": 0, "failed_calls": 0} + self.done = 0 + self.failed: dict[str, str] = {} + self.notice: str | None = None + + def lane_of(self, job: Any) -> str | None: + return None + + async def _claim(self, session: AsyncSession) -> bool: + return await runtime.claim(session, EvalCatalogSnapshot, self.job_id) + + async def _run(self) -> None: + from app.data import catalog_draft, table_versions + from app.db.models import DataSource + + if not await self._write(self._claim): + return + async with SessionLocal() as session: + snap = await session.get(EvalCatalogSnapshot, self.job_id) + draft = dict(snap.draft or {}) + plan = dict(snap.tables or {}) + ids = [s["id"] for s in snap.sources or []] + sources: list[tuple[Any, list[str]]] = [] + for sid in ids: + row = await session.get(DataSource, sid) + if row is None: + raise runtime.JobFailed(SOURCE_GONE) + try: + sources.append((await table_versions.resolve_source(session, row), list(plan.get(sid) or []))) + except table_versions.SnapshotMissing as e: + raise runtime.JobFailed(str(e)) from e + if self.model_override is not None: + model, model_id = self.model_override, str(getattr(self.model_override, "model_name", "") or "") + else: + try: + model, model_id = await resolve_model(session, draft.get("provider"), draft.get("model")) + except SnapshotInvalid as e: + raise runtime.JobFailed(e.message) from e + live = await _live_tables(session, ids) + self.model_id = model_id + if draft.get("strip_llm"): + live = {sid: {t: {**body, "notes": strip_model_items(body["notes"])} for t, body in tables.items()} + for sid, tables in live.items()} + self.copy = live + background = catalog_draft.Background(guidance=draft.get("guidance"), note=draft.get("note")) + at = catalog.now_iso() + for source, tables in sources: + names = [t for t in tables if isinstance(_schema_tables(source).get(t), dict)] + for gone in [t for t in tables if t not in names]: + self.failed[f"{source.name}.{gone}"] = catalog.missing_table_error(gone) + existing = {t: body["notes"] for t, body in self.copy.get(source.id, {}).items()} + hooks = catalog_draft.Hooks(call_done=self._call_done, part_written=self._part_written(source, at), + table_done=self._table_done(source, at)) + await catalog_draft.run_shadow_draft(model, source, names, existing=existing, background=background, + hooks=hooks) + await self._settle() + await self._finish(at) + + # ---------------------------------------------------------------- 进度回调（起草引擎调） + + def _body(self, source: Any, table: str) -> dict[str, Any]: + # 版本照抄当前目录的：影子起草改的是副本，不是线上目录的新版本（线上那张表没有目录的记 0） + return self.copy.setdefault(source.id, {}).setdefault(table, {"version": 0, "notes": {}}) + + def _merge(self, source: Any, table: str, modeled: Any, at: str) -> None: + """整张表起草完：按写库同一套规矩（catalog.merge_draft）并进副本。""" + body = self._body(source, table) + body["notes"], _ = catalog.merge_draft(body.get("notes") or {}, source, table, modeled, at=at) + if not body["notes"] and not body.get("version"): + self.copy[source.id].pop(table, None) + + async def _call_done(self, report: Any) -> None: + self.usage["calls"] += 1 + self.usage["input_tokens"] += int(report.input_tokens or 0) + self.usage["output_tokens"] += int(report.output_tokens or 0) + self.usage["timeouts"] += 1 if report.timeout else 0 + self.usage["failed_calls"] += 1 if report.failure else 0 + self.notice = report.notice + await self._write(self._save_progress) + + def _part_written(self, source: Any, at: str): + async def hook(table: str, written: Any) -> None: + # 宽表写出来的一段：并进副本，这一次不删任何东西（同 catalog_jobs.write_part）；整张表的收尾在 table_done + body = self._body(source, table) + merged, _ = catalog.merge_notes(body.get("notes") or {}, catalog.draft_structure(source, table).notes, at=at) + body["notes"], _ = catalog.merge_notes(merged, written.notes, at=at) + return hook + + def _table_done(self, source: Any, at: str): + async def hook(table: str, outcome: Any) -> None: + self._merge(source, table, outcome, at) + self.done += 1 + if outcome is not None and outcome.status == "failed": + self.failed[f"{source.name}.{table}"] = outcome.error or catalog.MODEL_EMPTY_REASON + elif outcome is not None and outcome.status == "partial" and outcome.error: + self.failed[f"{source.name}.{table}"] = outcome.error + await self._write(self._save_progress) + return hook + + # ---------------------------------------------------------------- 库操作 + + def _cost(self) -> dict[str, Any]: + from app.providers.catalog import estimate_cost, find_pricing + + model_id = self.model_id or "" + priced = bool(model_id) and find_pricing(model_id) is not None + cost = estimate_cost(model_id, self.usage["input_tokens"], self.usage["output_tokens"]) if priced else None + return {**self.usage, "cost_usd": round(cost, 6) if cost is not None else None, + "unpriced": bool(self.usage["calls"]) and not priced} + + def _progress(self, current: dict[str, Any]) -> dict[str, Any]: + return {**current, "tables_done": self.done, "tables_failed": len(self.failed), "failures": dict(self.failed), + "notice": self.notice} + + async def _save_progress(self, session: AsyncSession) -> None: + snap = await session.get(EvalCatalogSnapshot, self.job_id) + if snap is None: + return + snap.progress = self._progress(dict(snap.progress or {})) + snap.usage = self._cost() + draft = dict(snap.draft or {}) + if self.model_id and draft.get("model_id") != self.model_id: + snap.draft = {**draft, "model_id": self.model_id} + await session.commit() + + async def _finish(self, at: str) -> None: + from app.core.artifact_store import put_json + + async with SessionLocal() as session: + snap = await session.get(EvalCatalogSnapshot, self.job_id) + if snap is None: + return + ids = [s["id"] for s in snap.sources or []] + planned = dict(snap.tables or {}) + taken = {"origin": "shadow", "by": snap.created_by, "at": utcnow().isoformat(), + "draft": {**dict(snap.draft or {}), "model_id": self.model_id or None}, "tables": planned} + content = _content({sid: self.copy.get(sid, {}) for sid in ids}, taken=taken) + total = sum(len(v or []) for v in planned.values()) + ok = total - len(self.failed) + status = "succeeded" if not self.failed else "partial" if ok > 0 else "failed" + artifact = await put_json(content, kind=SNAPSHOT_KIND, meta={"origin": "shadow", "sources": ids}) \ + if status != "failed" else None + + async def write(session: AsyncSession) -> None: + row = await session.get(EvalCatalogSnapshot, self.job_id) + row.progress = self._progress(dict(row.progress or {})) + row.usage = self._cost() + row.status = status + row.artifact_id = artifact + row.finished_at = utcnow() + if status == "failed": + row.error = "每张表都没有起草出来，没有出快照：" + "；".join( + f"{t}：{why}" for t, why in list(self.failed.items())[:5]) + await session.commit() + + await self._write(write) + + async def _close(self, status: str, error: str) -> None: + """停止、出错时收尾：不写工件（半截的目录拿去对照会误导），进度和用量照实留着。""" + async def write(session: AsyncSession) -> None: + row = await session.get(EvalCatalogSnapshot, self.job_id) + if row is None or row.status not in runtime.ACTIVE: + return + row.status, row.error, row.finished_at = status, error, utcnow() + row.usage = self._cost() + row.progress = self._progress(dict(row.progress or {})) + await session.commit() + + await self._settle() + try: + await self._write(write) + except Exception: # noqa: BLE001 + logger.exception("影子起草 %s 收尾时写库出错", self.job_id) + + async def _stopped(self) -> None: + if self.stop_reason == "shutdown": + await self._close("interrupted", INTERRUPTED) + else: + await self._close("cancelled", STOPPED) + + async def _failed(self, reason: str) -> None: + await self._close("failed", reason) + + async def _crashed(self, e: Exception) -> None: + from app.core.errors import describe_exception + + logger.exception("影子起草 %s 出错", self.job_id) + await self._close("failed", f"影子起草出错：{describe_exception(e)}") + + +class ShadowDrafts(runtime.JobManager): + def __init__(self) -> None: + super().__init__(ShadowDraft, label="影子起草任务", name="eval-shadow") + + +manager = ShadowDrafts() + + +async def startup() -> int: + """服务启动时：没结束的影子起草标中断（不自动续跑）。""" + async def close(session: AsyncSession, job_id: str) -> None: + await session.execute(update(EvalCatalogSnapshot).where( + EvalCatalogSnapshot.id == job_id, EvalCatalogSnapshot.status.in_(runtime.ACTIVE)) + .values(status="interrupted", error=INTERRUPTED, finished_at=utcnow()) + .execution_options(synchronize_session=False)) + await session.commit() + + return await runtime.interrupt_on_startup(EvalCatalogSnapshot, close, running=manager.running) + + +async def reconcile_stale(session: AsyncSession, rows: Iterable[EvalCatalogSnapshot]) -> None: + """读接口当场整理：记着排队、进行中，这个进程里却没在跑（服务重启过）的，照实标中断。""" + stale = [r for r in rows if r.status in runtime.ACTIVE and not manager.running(r.id)] + for row in stale: + row.status, row.error, row.finished_at = "interrupted", INTERRUPTED, utcnow() + if stale: + await session.commit() + + +async def stop(job_id: str) -> bool: + """停止一次影子起草。排队中的直接记 cancelled；进行中的停掉、等它收尾。已经结束的返回 False。""" + async with SessionLocal() as session: + row = await session.get(EvalCatalogSnapshot, job_id) + if row is None or row.status not in runtime.ACTIVE: + return False + if await manager.stop(job_id, "user"): + return True + async with SessionLocal() as session: + result = await session.execute(update(EvalCatalogSnapshot).where( + EvalCatalogSnapshot.id == job_id, EvalCatalogSnapshot.status.in_(runtime.ACTIVE)) + .values(status="cancelled", error=STOPPED, finished_at=utcnow()).execution_options(synchronize_session=False)) + await session.commit() + return bool(result.rowcount) + + +def snapshot_out(row: EvalCatalogSnapshot) -> dict[str, Any]: + """接口的形状。""" + return {"id": row.id, "origin": row.origin, "status": row.status, "artifact_id": row.artifact_id, + "sources": list(row.sources or []), "tables": row.tables, "draft": row.draft, "estimate": row.estimate, + "usage": dict(row.usage or {}), "progress": dict(row.progress or {}), "error": row.error, + "created_by": row.created_by, "created_at": row.created_at.isoformat() if row.created_at else None, + "started_at": row.started_at.isoformat() if row.started_at else None, + "finished_at": row.finished_at.isoformat() if row.finished_at else None, + # 变体按工件 id 指定目录：data/catalog_view.CatalogView(mode="snapshot", snapshot=…) + "view": {"mode": "snapshot", "statuses": "all", "snapshot": row.artifact_id} if row.artifact_id else None} + + +__all__ = ["FORMAT", "SNAPSHOT_FINISHED", "SNAPSHOT_INVALID", "SNAPSHOT_MODEL_UNAVAILABLE", "ShadowDraft", "ShadowPlan", + "SnapshotInvalid", "estimate", "manager", "plan_shadow", "reconcile_stale", "resolve_model", "snapshot_out", + "start_shadow", "startup", "stop", "strip_model_items", "suite_tables", "take_live"] diff --git a/backend/app/eval/suites.py b/backend/app/eval/suites.py new file mode 100644 index 00000000..ac545036 --- /dev/null +++ b/backend/app/eval/suites.py @@ -0,0 +1,106 @@ +"""题集的形状与校验（细案 01 做法 7、17；评测-D9）。 + +题集：name（必填）、description、timezone（题目里的 {{ schedule.* }} 按它算）、default_variant（发起评测时缺省用的变体， +第 3 波解析；这里只要求是对象）、budget（只存改过的项）、archived。改题集带上读到的 version（乐观锁，接口回 409）。 + +**预算**（评测-D9，用户拍板）：缺省见 db/models_eval.DEFAULT_BUDGET——同时 2 题、每题 900 秒、单次评测 2 美元、评测每日 +5 美元。daily_max_usd 是评测那本账（engine/spend 的第三本，第 3 波接上）的当日上限：评测单独记账，不挤占裁判、核查的额度。 +这一波只定字段和取值范围，执行（预留、找补、触顶不再开新题）在第 3 波。 +""" +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from app.db.models_eval import DEFAULT_BUDGET +from app.eval.cases import CaseInvalid + +#: 题集不合规（422）的机器码 +SUITE_INVALID = "eval_suite_invalid" + +NAME_MAX = 200 +DESCRIPTION_MAX = 2000 +#: 变体存成 JSON：太大的多半是传错了东西 +VARIANT_MAX = 20_000 + +#: 预算每一项的取值范围：(说法, 整数还是金额, 下限, 上限)。下限都大于 0：0 美元、0 秒的上限只会让评测一题都开不了 +BUDGET_RULES: dict[str, tuple[str, type, float, float]] = { + "max_parallel": ("同时评几道题", int, 1, 8), + "per_case_timeout_s": ("每题时限（秒）", int, 30, 7200), + "per_run_max_usd": ("单次评测的金额上限（美元）", float, 0.01, 1000), + "daily_max_usd": ("评测每日的金额上限（美元）", float, 0.01, 10_000), +} + + +def _invalid(message: str, field: str) -> CaseInvalid: + return CaseInvalid(message, field, SUITE_INVALID) + + +def normalize_budget(raw: Any) -> dict[str, Any]: + """题集里改过的预算项 → 规范化（只留认识的、和缺省不同的）。""" + if raw in (None, {}): + return {} + if not isinstance(raw, Mapping): + raise _invalid("预算需要是一个对象", "budget") + unknown = sorted(set(raw) - set(BUDGET_RULES)) + if unknown: + raise _invalid(f"预算里不认识这些项：{'、'.join(unknown)}（认识的：{'、'.join(BUDGET_RULES)}）", f"budget.{unknown[0]}") + out: dict[str, Any] = {} + for key, value in raw.items(): + label, kind, low, high = BUDGET_RULES[key] + if value is None: + continue + ok = not isinstance(value, bool) and isinstance(value, (int, float)) and (kind is float or isinstance(value, int)) + if not ok or not low <= value <= high: + unit = "整数" if kind is int else "数" + raise _invalid(f"{label}需要是 {low:g} 到 {high:g} 之间的{unit}", f"budget.{key}") + value = kind(value) + if value != DEFAULT_BUDGET[key]: + out[key] = value + if out.get("per_run_max_usd", DEFAULT_BUDGET["per_run_max_usd"]) > \ + out.get("daily_max_usd", DEFAULT_BUDGET["daily_max_usd"]): + raise _invalid("单次评测的金额上限不能高于评测每日的上限", "budget.per_run_max_usd") + return out + + +def effective_budget(stored: Mapping[str, Any] | None) -> dict[str, Any]: + """生效的预算：缺省加上题集改过的项。""" + return {**DEFAULT_BUDGET, **{k: v for k, v in (stored or {}).items() if k in DEFAULT_BUDGET}} + + +def normalize_suite(body: Mapping[str, Any], *, current: Mapping[str, Any] | None = None) -> dict[str, Any]: + """请求体（改的时候和库里的合起来：没带的项照旧）→ 题集各列的值。不合规抛 CaseInvalid（code=eval_suite_invalid）。""" + from app.core.artifact_store import canonical_json + from app.scheduler import cron + + if not isinstance(body, Mapping): + raise _invalid("题集需要是一个对象", "name") + cur = dict(current or {}) + + def pick(key: str, default: Any = None) -> Any: + return body[key] if key in body else cur.get(key, default) + + name = pick("name", "") + name = name.strip() if isinstance(name, str) else "" + if not name: + raise _invalid("请填写题集的名称", "name") + if len(name) > NAME_MAX: + raise _invalid(f"题集的名称不能超过 {NAME_MAX} 个字符", "name") + description = pick("description", "") or "" + if not isinstance(description, str) or len(description.strip()) > DESCRIPTION_MAX: + raise _invalid(f"说明需要是不超过 {DESCRIPTION_MAX} 个字符的文字", "description") + try: + timezone = cron.timezone_name(pick("timezone", "Asia/Shanghai") or "") + except cron.TimezoneError as e: + raise _invalid(str(e), "timezone") from e + variant = pick("default_variant", {}) or {} + if not isinstance(variant, Mapping) or len(canonical_json(variant)) > VARIANT_MAX: + raise _invalid("默认变体需要是一个对象", "default_variant") + archived = pick("archived", False) + if not isinstance(archived, bool): + raise _invalid("是否归档需要是 true 或 false", "archived") + return {"name": name, "description": description.strip(), "timezone": timezone, "default_variant": dict(variant), + "budget": normalize_budget(pick("budget", {})), "archived": archived} + + +__all__ = ["BUDGET_RULES", "SUITE_INVALID", "effective_budget", "normalize_budget", "normalize_suite"] diff --git a/backend/app/ledger/derive.py b/backend/app/ledger/derive.py new file mode 100644 index 00000000..835fabfa --- /dev/null +++ b/backend/app/ledger/derive.py @@ -0,0 +1,325 @@ +"""从钉住的口径版本确定性地派生只读小图：全站唯一的生成器（docs/design/next-round/00-总方案.md X3）。 + +**为什么只有一份。** 评测的口径题要算期望（细案 01 做法 8：在同一份数据上把口径重算一遍），口径账本的对账要在同一份 +数据上把「这次的取数」和口径各算一次（细案 02 做法 6），召回要把新旧两版口径各算一次（做法 7）。三处要的是同一件事： +从一个不可变的已发布版本里，把口径卡读的取数节点原样拷出来，后面接一张 caliber_from 固定到那一版的口径卡，跑一次探索 +运行。各写一份，口径的取法迟早走样——评测量出来的「对」和账本说的「等价」就不是同一个口径。 + +**派生图长什么样**（assemble）：入口（字段照抄钉住版本的入口：类型、必填、缺省值都不变）→ 各段（Block）首尾相接 → +收尾的卡（对账卡、召回卡，这一波没有）→ 出口（每张口径卡一个字段）。一段 = 那一版口径卡读的取数节点（id、配置原样 +照抄；图级 defaults 也照抄，节点上没写的项按同样的规则取值）+ 一张本地口径卡，caliber_from 固定到那一版的那张卡 +（名字、版本、指标式、取整、单位都按那一版，metrics 节点的 _pinned_caliber），「缺输入时」照抄原卡。节点一律首尾相接： +同一次运行里先后查，差值只反映口径（或这次的取数）和数据，不反映图的形状。 + +**能派生的只有可复用的卡**（load_pinned）：口径卡本身是源头（不是转抄别处的 caliber_from）、每个输入都是 +cell(nodes.X, 0, '列')、X 是查库的调用工具节点（look/calibers.card_inputs，和蓝图复用口径同一个认法），而且取数节点的 +模板只引用入口字段（{{ input.字段 }}）——引用 vars.、nodes.、模板函数（file_text……）的，拷出来就拿不到同样的值，照实 +拒收、给原因（NotDerivable）。工作流、版本、节点不在了也照实说。 + +**这一波实现的**：caliber_probe —— 一份口径的取数 + 口径卡（评测口径题的期望）。 + +**扩展点**（第 3 波的账本按这里扩，不另写生成器）： +- 口径对账（02 做法 6）：assemble("reconcile", [ours, caliber_block(pinned)], tail=[对账卡])。ours 是「这次的取数」 + （执行过的 SQL 原文，节点 id ours_q1…，build 一个 Block 即可，不经 caliber_block），对账卡是本地 metrics 节点（差值、 + 相对差，式子引用 cell(nodes.ours_q1, 0, '列') 和口径卡的指标，on_missing=null），出口的契约写 metrics_from。 +- 口径召回（02 做法 7）：assemble("recall", [caliber_block(old, prefix="old_"), caliber_block(new, prefix="new_")], + tail=[召回卡])。同一张图里放两份口径，取数节点 id 会撞：caliber_block 带 prefix 时把拷出来的节点改名，口径卡的 + caliber_from 带上改名映射 nodes={原 id: 新 id}。**运行时认这个映射是 02 做法 4（metrics._pinned_caliber 按 AST 改写 + 式子里的 cell(nodes.X…) 和 nodes.X），归第 3 波的账本包**；在那之前带前缀的段跑起来口径卡取不到输入（缺输入，照实 + 失败，不会算错），这一波只有不带前缀的 caliber_probe 有调用方。 +- 新的派生种类登记进 KINDS，入口字段、出口、首尾相接都走 assemble，不在调用方拼节点。 + +派生图都是探索运行（正式运行只从已发布未退役的版本发起）；来源（origin_kind）由调用方定：评测 eval、对账 reconcile、 +召回 recall，都只能由服务端写（engine/origin.py）。DerivedGraph.origin 是记进运行 origin 的那段派生说明。 +""" +from __future__ import annotations + +import copy +import re +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +#: 派生图的种类。这一波实现 caliber_probe；reconcile、recall 归第 3 波的账本（见模块说明的扩展点） +KINDS = ("caliber_probe", "reconcile", "recall") + +#: 不能派生的原因的机器码 +GONE = "derive.version_gone" +NOT_CARD = "derive.not_caliber_card" +NOT_REUSABLE = "derive.not_reusable" +TEMPLATE = "derive.fetch_template" + +#: 派生图自己的入口、出口节点 id（和拷出来的节点撞了就加后缀） +ENTRY_ID = "derived_in" +EXIT_ID = "derived_out" +#: 节点在画布上的横向间距（派生图也能在运行详情里画出来） +_STEP_X = 260 + +_TEMPLATE_RE = re.compile(r"\{\{(.*?)\}\}", re.DOTALL) +#: 取数节点的模板只许引用入口字段：{{ input.字段 }}，可以带过滤器 +_INPUT_REF = re.compile(r"^input\.([^\s.()\[\]]+)$") + +TEMPLATE_REASON = ("取数节点「{node}」的模板引用了「{ref}」：只能引用入口字段（{{{{ input.字段 }}}}），别的值（vars.、nodes.、" + "模板函数）在派生的图里拿不到同样的值") + + +class NotDerivable(Exception): + """不能从这一版派生只读图。message 给人看（照实说哪一环不行），code 给程序认。""" + + def __init__(self, message: str, code: str) -> None: + super().__init__(message) + self.message = message + self.code = code + + +@dataclass(frozen=True) +class Pinned: + """读好的一版口径：那一版的图、口径卡、它读的取数节点、入口字段。只从不可变的版本快照读。""" + + workflow_id: str + workflow_name: str + version: int + node_id: str + #: 这一版发布时的级别（governed / published），没发布过的草稿为空 + level: str | None + #: 这一版是不是「已发布且未退役」（apps/versions.is_published）。派生不拦：升版、退役怎么处置由调用方按 + #: engine/governance.unresolved_caliber_upgrades 和 apps/versions.subgraph_pin_problems 判（同正式运行） + published: bool + retired: bool + card: dict[str, Any] + #: 口径卡读的取数节点（图里的原始 JSON），按图里的先后 + fetches: tuple[dict[str, Any], ...] + #: card_inputs 认出的 [(取数节点, 工具, 列)] + reads: tuple[tuple[str, str, tuple[str, ...]], ...] + #: 入口节点声明的字段（原始 JSON：name、type、required、default……） + entry_fields: tuple[dict[str, Any], ...] + #: 图级 defaults（节点上没写的配置按它取，照抄才和原图取值一致） + defaults: dict[str, Any] + + @property + def card_title(self) -> str: + data = self.card.get("data") if isinstance(self.card.get("data"), dict) else {} + return str(data.get("label") or self.node_id) + + @property + def card_config(self) -> dict[str, Any]: + data = self.card.get("data") if isinstance(self.card.get("data"), dict) else {} + return data.get("config") if isinstance(data.get("config"), dict) else {} + + @property + def metrics(self) -> list[dict[str, Any]]: + """这张卡的指标定义（id、名称、单位、小数位、式子），只读。""" + return [m for m in self.card_config.get("metrics") or [] if isinstance(m, dict) and m.get("id")] + + @property + def caliber(self) -> str: + cfg = self.card_config + return f"{cfg.get('caliber') or self.card_title} @ {cfg.get('caliber_version') or 'v1'}" + + def ref(self) -> dict[str, Any]: + """caliber_from 的写法（metrics 节点认的形状）。""" + return {"workflow_id": self.workflow_id, "workflow_version": self.version, "node_id": self.node_id} + + def describe(self) -> dict[str, Any]: + """记进运行 origin 的那一份：哪个工作流哪一版的哪张卡。""" + return {**self.ref(), "workflow_name": self.workflow_name, "card": self.card_title} + + +@dataclass +class Block: + """派生图里的一段：照抄（或这次执行过）的取数节点 + 一张口径卡。rename 是拷出来的节点改了名的映射（不带前缀时为空）。""" + + nodes: list[dict[str, Any]] + card_id: str + rename: dict[str, str] = field(default_factory=dict) + #: 这一段出自哪里（记进 DerivedGraph.origin） + source: dict[str, Any] | None = None + + +@dataclass(frozen=True) +class DerivedGraph: + kind: str + graph: dict[str, Any] + #: 各段的口径卡节点 id，按段的先后（出口按它们取指标） + card_ids: tuple[str, ...] + entry_fields: tuple[dict[str, Any], ...] + #: 记进运行 origin 的派生说明：{kind, from: [各段的出处]} + origin: dict[str, Any] + + +# ========================================================================== +# 读一版口径 +# ========================================================================== + + +def _node_of(graph: Mapping[str, Any], node_id: str) -> dict[str, Any] | None: + return next((n for n in graph.get("nodes") or [] if isinstance(n, dict) and n.get("id") == node_id), None) + + +def _config(node: Mapping[str, Any]) -> dict[str, Any]: + data = node.get("data") if isinstance(node.get("data"), dict) else {} + return data.get("config") if isinstance(data.get("config"), dict) else {} + + +def _strings(value: Any) -> Iterable[str]: + if isinstance(value, str): + yield value + elif isinstance(value, Mapping): + for v in value.values(): + yield from _strings(v) + elif isinstance(value, (list, tuple)): + for v in value: + yield from _strings(v) + + +def template_problem(node_id: str, config: Mapping[str, Any]) -> str | None: + """取数节点的配置里引用了入口字段以外的东西：给人看的原因；只引用入口字段（或什么都不引用）为 None。""" + for text in _strings(config): + for match in _TEMPLATE_RE.finditer(text): + expr = match.group(1).strip() + path = expr.split("|")[0].strip() + if not _INPUT_REF.match(path): + return TEMPLATE_REASON.format(node=node_id, ref=path or "{{ }}") + return None + + +def entry_fields(graph: Mapping[str, Any]) -> tuple[dict[str, Any], ...]: + """图里入口节点声明的字段（原始 JSON，照抄）。几个入口节点的字段按先后合在一起，同名的取第一个。""" + out: dict[str, dict[str, Any]] = {} + for node in graph.get("nodes") or []: + if not isinstance(node, dict) or node.get("type") != "input": + continue + for f in _config(node).get("fields") or []: + if isinstance(f, dict) and isinstance(f.get("name"), str) and f["name"].strip(): + out.setdefault(f["name"].strip(), copy.deepcopy(f)) + return tuple(out.values()) + + +async def load_pinned(session: AsyncSession, workflow_id: str, version: int, node_id: str) -> Pinned: + """读 workflow_id@version 里的口径卡 node_id。不能派生的抛 NotDerivable（照实说哪一环不行）。""" + from app.apps import versions as app_versions + from app.db.models import Workflow + from app.look.calibers import card_inputs + + workflow = await session.get(Workflow, str(workflow_id or "")) + if workflow is None: + raise NotDerivable("口径所在的工作流不存在，可能已被删除", GONE) + row = await app_versions.version_row(session, workflow.id, int(version)) + if row is None: + raise NotDerivable(f"「{workflow.name}」没有 v{version} 这个版本", GONE) + graph = row.graph or {} + card = _node_of(graph, node_id) + if card is None: + raise NotDerivable(f"「{workflow.name}」v{version} 里没有节点 {node_id}", GONE) + if card.get("type") != "metrics": + raise NotDerivable(f"「{workflow.name}」v{version} 里的 {node_id} 不是口径卡", NOT_CARD) + inputs, reason = card_inputs(graph, node_id) + if reason is not None: + title = (card.get("data") or {}).get("label") or node_id + raise NotDerivable(f"「{workflow.name}」v{version} 的口径卡「{title}」不能复用：{reason}", NOT_REUSABLE) + wanted = {i["node"] for i in inputs} + fetches = tuple(copy.deepcopy(n) for n in graph.get("nodes") or [] + if isinstance(n, dict) and n.get("id") in wanted) + for node in fetches: + if problem := template_problem(str(node["id"]), _config(node)): + raise NotDerivable(f"「{workflow.name}」v{version} 的{problem}", TEMPLATE) + return Pinned( + workflow_id=workflow.id, workflow_name=workflow.name, version=int(version), node_id=node_id, + level=row.level, published=app_versions.is_published(row, current=workflow.published_version), + retired=row.retired_at is not None, card=copy.deepcopy(card), fetches=fetches, + reads=tuple((i["node"], i["tool"], tuple(i["columns"])) for i in inputs), + entry_fields=entry_fields(graph), defaults=copy.deepcopy(graph.get("defaults") or {})) + + +# ========================================================================== +# 拼图 +# ========================================================================== + + +def _node(nid: str, ntype: str, label: str, **config: Any) -> dict[str, Any]: + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label, "config": config}} + + +def caliber_block(pinned: Pinned, *, prefix: str = "", upgrade_policy: str | None = None) -> Block: + """一份口径：拷出取数节点（id、配置原样；prefix 不为空时改名为 prefix + 原 id）+ 一张 caliber_from 固定到那一版的 + 口径卡（id 同原卡，带前缀时同样改名；「缺输入时」照抄原卡）。 + + upgrade_policy：钉住的版本有了新的已发布版本时这一段怎么处置（recompute / dual / incomparable，没有缺省）。写在 + 本地口径卡上，判断走和正式运行同一个 engine/governance.unresolved_caliber_upgrades；不给就不写（有新版本时那个 + 判断报「待处置」）。prefix 的改名映射要等 metrics 运行时认 caliber_from.nodes（见模块说明）。""" + rename = {str(n["id"]): f"{prefix}{n['id']}" for n in pinned.fetches} if prefix else {} + nodes: list[dict[str, Any]] = [] + for original in pinned.fetches: + node = copy.deepcopy(original) + node["id"] = rename.get(str(node["id"]), str(node["id"])) + node["position"] = {"x": 0, "y": 0} + nodes.append(node) + source = pinned.ref() + if rename: + source = {**source, "nodes": dict(rename)} + config: dict[str, Any] = {"caliber_from": source} + on_missing = pinned.card_config.get("on_missing") + if on_missing not in (None, ""): + config["on_missing"] = on_missing + if upgrade_policy: + config["upgrade_policy"] = upgrade_policy + card_id = f"{prefix}{pinned.node_id}" + nodes.append(_node(card_id, "metrics", pinned.card_title, **config)) + return Block(nodes=nodes, card_id=card_id, rename=rename, source=pinned.describe()) + + +def _free_id(wanted: str, taken: set[str]) -> str: + nid, n = wanted, 2 + while nid in taken: + nid, n = f"{wanted}_{n}", n + 1 + return nid + + +def assemble(kind: str, blocks: Sequence[Block], *, fields: Sequence[Mapping[str, Any]] = (), + defaults: Mapping[str, Any] | None = None, tail: Sequence[dict[str, Any]] = ()) -> DerivedGraph: + """入口 → 各段（首尾相接）→ tail（对账卡、召回卡）→ 出口。出口每张口径卡（含 tail 里的 metrics 节点）一个字段， + 取那张卡的指标清单。几段的节点 id 撞了直接报错：两份口径放进同一张图要带前缀（caliber_block 的 prefix）。""" + if kind not in KINDS: + raise ValueError(f"不认识的派生图种类：{kind}") + body: list[dict[str, Any]] = [n for b in blocks for n in b.nodes] + [copy.deepcopy(n) for n in tail] + ids = [str(n["id"]) for n in body] + dup = sorted({i for i in ids if ids.count(i) > 1}) + if dup: + raise ValueError(f"派生图里节点 id 重复：{'、'.join(dup)}（两份口径放进同一张图要带前缀）") + taken = set(ids) + entry = _free_id(ENTRY_ID, taken) + exit_ = _free_id(EXIT_ID, taken | {entry}) + cards = [b.card_id for b in blocks] + [str(n["id"]) for n in tail if n.get("type") == "metrics"] + titles = {str(n["id"]): str((n.get("data") or {}).get("label") or n["id"]) for n in body} + outputs, seen = [], set() + for cid in cards: + name = _free_id(titles.get(cid, cid), seen) + seen.add(name) + outputs.append({"name": name, "value": f"{{{{ nodes.{cid}.text }}}}"}) + chain = [_node(entry, "input", "入口", fields=[copy.deepcopy(dict(f)) for f in fields]), *body, + _node(exit_, "output", "出口", fields=outputs)] + for i, node in enumerate(chain): + node["position"] = {"x": i * _STEP_X, "y": 0} + graph: dict[str, Any] = { + "nodes": chain, + "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(chain, chain[1:])], + } + if defaults: + graph["defaults"] = copy.deepcopy(dict(defaults)) + origin = {"kind": kind, "from": [b.source for b in blocks if b.source]} + return DerivedGraph(kind=kind, graph=graph, card_ids=tuple(cards), + entry_fields=tuple(copy.deepcopy(dict(f)) for f in fields), origin=origin) + + +def caliber_probe(pinned: Pinned, *, upgrade_policy: str | None = None) -> DerivedGraph: + """口径探针：一份口径的取数 + 口径卡（评测口径题的期望，细案 01 做法 8）。节点 id 和原图一样，入口字段照抄。 + 同样的 pinned 永远得到同一张图（确定性）。""" + return assemble("caliber_probe", [caliber_block(pinned, upgrade_policy=upgrade_policy)], + fields=pinned.entry_fields, defaults=pinned.defaults) + + +__all__ = ["Block", "DerivedGraph", "ENTRY_ID", "EXIT_ID", "GONE", "KINDS", "NOT_CARD", "NOT_REUSABLE", "NotDerivable", + "Pinned", "TEMPLATE", "assemble", "caliber_block", "caliber_probe", "entry_fields", "load_pinned", + "template_problem"] diff --git a/backend/tests/test_eval_gold.py b/backend/tests/test_eval_gold.py new file mode 100644 index 00000000..02a736f7 --- /dev/null +++ b/backend/tests/test_eval_gold.py @@ -0,0 +1,413 @@ +"""评测的期望答案（app/eval/gold.py；细案 01 做法 8、14，评测-D4）。 + +- SQL 探针：{{ schedule.* }} 按截至时间渲染；以来源 eval、只读 + 遮罩的探索运行跑（不进使用统计）；期望值从封存的查询 + 快照里取格、列、整张表；没有数据的题要 0 行。只读守卫拒写语句；用到遮罩列的拒收；取不到值、运行失败、超时照实说。 +- 口径探针：拷出钉住版本的取数节点 + caliber_from 口径卡，算出的值和原工作流一样；不可复用的卡拒收并给原因。 +- 口径升版：有新版本没处置就跳过（「口径已升版到 vN，待处置」），三种处置各自生效；钉住的版本退役同样待处置。 +- 作答前后各算一次：一样才判，数据变了不判。 + +库、表、数字都是虚构的（tests/fixtures/dual_check_shop.py：9/1–9/7 按支付日、不含退款 330）。 +""" +from __future__ import annotations + +import sqlite3 +from datetime import datetime, timezone +from zoneinfo import ZoneInfo + +import pytest +from sqlalchemy import select, update + +from app.core import artifact_store +from app.db.base import SessionLocal, utcnow +from app.db.models import Artifact, DataSource, Run, Workflow, WorkflowVersion +from app.engine.origin import counted_runs +from app.engine.runner import run_manager +from app.eval import cases, gold +from app.eval.gold import Context +from tests.fixtures import dual_check_shop as shop +from tests.fixtures.sources import drop_source + +#: 2026-09-08（周二）早上：上周是 8/31–9/6 +AS_OF = datetime(2026, 9, 8, 9, 0, tzinfo=ZoneInfo("Asia/Shanghai")) +LAST_WEEK = "BETWEEN '{{ schedule.last_week_start }}' AND '{{ schedule.last_week_end }}'" +PAID_GMV = f"SELECT SUM(amount) AS gmv FROM orders WHERE status <> 'refunded' AND paid_date {LAST_WEEK}" + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +@pytest.fixture +async def source(tmp_path): + name, sid = await shop.add_source(shop.build(tmp_path / "shop.db")) + yield {"name": name, "id": sid, "path": tmp_path / "shop.db"} + await drop_source(sid) + + +def sql_case(source_id: str, sql: str, *expectations: dict, seq: int = 1) -> dict: + fields = cases.normalize_case({"question": "上周的销售额是多少？", "gold": {"kind": "sql", "datasource_id": source_id, + "sql": sql}, "expectations": list(expectations)}) + return {**fields, "id": None, "seq": seq} + + +def ctx(**kw) -> Context: + return Context(as_of=AS_OF, tz="Asia/Shanghai", actor="评测测试", suite_id="s1", suite_name="虚构题集", **kw) + + +async def run_of(run_id: str) -> Run: + async with SessionLocal() as session: + return await session.get(Run, run_id) + + +# -------------------------------------------------------------------------- +# SQL 探针 +# -------------------------------------------------------------------------- + + +async def test_the_sql_probe_renders_variables_and_reads_the_sealed_snapshot(source): + case = sql_case(source["id"], PAID_GMV, {"kind": "scalar", "column": "gmv"}) + outcome = await gold.compute_gold(case, ctx()) + assert outcome.ok, outcome.reason + [found] = outcome.sets + assert found.values == {"e1": {"kind": "scalar", "value": 330.0, "column": "gmv", "row": 0, "dims": {}}} + assert "'2026-08-31' AND '2026-09-06'" in outcome.rendered["sql"] + # 期望点得开：快照是那次运行封存的查询快照 + snapshot = artifact_store.load(found.snapshot) + assert snapshot["rows"] == [[330.0]] and "2026-08-31" in snapshot["sql"] + row = await run_of(found.run_id) + assert row.status == "succeeded" and row.manifest_seq is not None + assert row.origin_kind == "eval" and row.origin["role"] == "gold" and row.origin["suite_id"] == "s1" + assert row.workflow_name == "评测：虚构题集·第 1 题·算期望" + # 只读、遮罩：同使用者的问数据运行 + assert row.options["read_only_data"] is True and row.options["mask_results"] is True + # 评测运行是系统派生的，不进使用统计 + async with SessionLocal() as session: + counted = (await session.execute(select(Run.id).where(Run.id == row.id, counted_runs()))).all() + assert counted == [] + + +async def test_cells_columns_tables_and_no_data(source): + by_region = (f"SELECT region, SUM(amount) AS gmv FROM orders WHERE status <> 'refunded' AND paid_date {LAST_WEEK} " + "GROUP BY region ORDER BY gmv DESC") + case = sql_case(source["id"], by_region, + {"kind": "scalar", "column": "gmv", "where": {"region": "华东"}}, + {"kind": "table", "keys": ["region"], "ordered": True}, + {"kind": "set", "column": "region", "top_n": 1}) + outcome = await gold.compute_gold(case, ctx()) + assert outcome.ok, outcome.reason + values = outcome.sets[0].values + assert values["e1"]["value"] == 300.0 and values["e1"]["dims"] == {"region": "华东"} + assert values["e2"]["rows"] == [{"region": "华东", "gmv": 300.0}, {"region": "华北", "gmv": 30.0}] + assert values["e3"]["values"] == ["华东"] + + none = sql_case(source["id"], f"SELECT region, SUM(amount) AS gmv FROM orders WHERE region = '华中' " + f"AND paid_date {LAST_WEEK} GROUP BY region", {"kind": "none"}) + outcome = await gold.compute_gold(none, ctx()) + assert outcome.ok and outcome.sets[0].values == {"e1": {"kind": "none", "rows": 0}} + + +@pytest.mark.parametrize("sql, expectation, words", [ + (PAID_GMV, {"kind": "scalar", "column": "金额"}, "没有列「金额」"), + (PAID_GMV, {"kind": "scalar", "column": "gmv", "row": 3}, "只返回了 1 行"), + (f"SELECT SUM(amount) AS gmv FROM orders WHERE region = '华中'", {"kind": "scalar", "column": "gmv"}, "是空值"), + (f"SELECT region FROM orders WHERE region = '华中'", {"kind": "scalar", "column": "region"}, "没有返回行"), + (PAID_GMV, {"kind": "none"}, "返回了 1 行"), +]) +async def test_values_that_cannot_be_taken_are_said_plainly(source, sql, expectation, words): + outcome = await gold.compute_gold(sql_case(source["id"], sql, expectation), ctx()) + assert outcome.status == "no_value" and outcome.code == gold.GOLD_NO_VALUE + assert words in outcome.reason and outcome.run_ids + + +async def test_the_read_only_guard_refuses_writes_before_anything_runs(source): + async with SessionLocal() as session: + await session.execute(update(DataSource).where(DataSource.id == source["id"]).values(readonly=False)) + await session.commit() + case = sql_case(source["id"], "DELETE FROM orders WHERE id = 1", {"kind": "none"}) + outcome = await gold.compute_gold(case, ctx()) + assert outcome.status == "refused" and outcome.code == cases.GOLD_GUARD and outcome.run_ids == [] + with sqlite3.connect(source["path"]) as db: + assert db.execute("SELECT COUNT(*) FROM orders").fetchone()[0] == len(shop.ROWS) + + +@pytest.mark.parametrize("sql, refused", [ + ("SELECT status, COUNT(*) AS n FROM orders GROUP BY status", True), + ("SELECT status AS 状态 FROM orders", True), # 改了名字也认得出 + ("SELECT * FROM orders", True), + ("SELECT COUNT(*) AS n FROM orders WHERE status = 'paid'", False), # 结果里没有取自遮罩列的格 +]) +async def test_gold_that_uses_a_masked_column_is_refused(tmp_path, sql, refused): + name, sid = await shop.add_source(shop.build(tmp_path / "masked.db"), options={"mask_columns": ["status"]}) + try: + expectation = {"kind": "scalar", "column": "n"} if not refused else {"kind": "table", "keys": ["id"]} \ + if sql.startswith("SELECT *") else {"kind": "set", "column": "状态" if "状态" in sql else "status"} + outcome = await gold.compute_gold(sql_case(sid, sql, expectation), ctx()) + if refused: + assert outcome.status == "refused" and outcome.code == gold.GOLD_MASKED, outcome.reason + assert "遮罩" in outcome.reason + else: + assert outcome.ok and outcome.sets[0].values["e1"]["value"] == 4 + finally: + await drop_source(sid) + + +async def test_failures_and_timeouts_are_said_plainly(tmp_path, source): + outcome = await gold.compute_gold(sql_case(source["id"], "SELECT nope FROM orders", {"kind": "scalar", + "column": "nope"}), ctx()) + assert outcome.status == "failed" and "nope" in outcome.reason and outcome.run_ids + + slow = ("WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM c WHERE x < 500000000) " + "SELECT SUM(x) AS s FROM c") + name, sid = await shop.add_source(shop.build(tmp_path / "slow.db"), options={"query_timeout_s": 2}) + try: + # 等不到：停止那次运行，照实说超时 + outcome = await gold.compute_gold(sql_case(sid, slow, {"kind": "scalar", "column": "s"}), ctx(timeout_s=0.5)) + assert outcome.status == "timeout" and outcome.code == gold.GOLD_TIMEOUT and "0.5 秒" in outcome.reason + [run_id] = outcome.run_ids + assert (await run_of(run_id)).status == "cancelled" + # 数据库按数据源的查询时限停下语句：运行失败，原因照写 + outcome = await gold.compute_gold(sql_case(sid, slow, {"kind": "scalar", "column": "s"}), ctx(timeout_s=60)) + assert outcome.status == "failed" and "超过" in outcome.reason + finally: + await drop_source(sid) + + async with SessionLocal() as session: + await session.execute(update(DataSource).where(DataSource.id == source["id"]).values(enabled=False)) + await session.commit() + outcome = await gold.compute_gold(sql_case(source["id"], PAID_GMV, {"kind": "scalar", "column": "gmv"}), ctx()) + assert outcome.status == "failed" and "已停用" in outcome.reason and outcome.run_ids == [] + + +async def test_cases_without_gold_need_no_run(): + fields = cases.normalize_case({"question": "最近卖得怎么样？", "expectations": [{"kind": "ask"}]}) + outcome = await gold.compute_gold({**fields, "id": None, "seq": 2}, ctx()) + assert outcome.ok and outcome.sets == [] and outcome.run_ids == [] + + +# -------------------------------------------------------------------------- +# 作答前后各算一次 +# -------------------------------------------------------------------------- + + +async def test_bracket_judges_only_when_the_data_did_not_move(source): + case = sql_case(source["id"], PAID_GMV, {"kind": "scalar", "column": "gmv"}) + + async def answer(before): + assert before.ok + return "作答" + + result, settled = await gold.bracket(case, ctx(), answer) + assert result == "作答" and settled.ok and settled.expected[0].values["e1"]["value"] == 330.0 + assert settled.after is not None and (await run_of(settled.after.run_ids[0])).origin["role"] == "gold_after" + + async def answer_while_data_moves(before): + with sqlite3.connect(source["path"]) as db: + db.execute("INSERT INTO orders VALUES (9, '华东', 104, 70.0, 'paid', '2026-09-02', '2026-09-02', " + "'online', 0)") + return "作答" + + _, settled = await gold.bracket(case, ctx(), answer_while_data_moves) + assert settled.status == "drifted" and "数据变了" in settled.reason and settled.expected == [] + + missing = sql_case(source["id"], PAID_GMV, {"kind": "scalar", "column": "金额"}) + called = [] + + async def never(before): + called.append(before) + + result, settled = await gold.bracket(missing, ctx(), never) + assert result is None and called == [] and settled.status == "no_value" + + +# -------------------------------------------------------------------------- +# 口径探针 +# -------------------------------------------------------------------------- + + +def node(nid, ntype, label=None, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}} + + +def chain(*nodes): + return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +CARD_METRICS = [ + {"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "format": "thousands", + "expression": "cell(nodes.fetch, 0, 'gmv')"}, + {"id": "avg", "name": "客单价", "unit": "元", "decimals": 2, "format": "thousands", + "expression": "cell(nodes.fetch, 0, 'gmv') / cell(nodes.fetch, 0, 'n')"}, +] + + +def weekly(source_name: str, *, refunds: bool = False, metrics=CARD_METRICS, fetch_sql: str | None = None) -> dict: + status = "" if refunds else "status <> 'refunded' AND " + sql = fetch_sql or (f"SELECT SUM(amount) AS gmv, COUNT(*) AS n FROM orders WHERE {status}" + "paid_date BETWEEN '{{ input.start }}' AND '{{ input.end }}'") + return chain(node("start", "input", fields=[{"name": "start", "required": True}, + {"name": "end", "required": True}]), + node("fetch", "tool", "取数", tool=f"db_query__{source_name}", args={"sql": sql}), + node("card", "metrics", "周销售口径卡", caliber="周销售", caliber_version="v1", metrics=metrics), + node("out", "output", fields=[{"name": "r", "value": "{{ nodes.card.text }}"}])) + + +async def publish(*graphs, name="周销售") -> str: + async with SessionLocal() as session: + wf = Workflow(name=f"{name}_{len(graphs)}_{id(graphs)}", graph=graphs[-1], status="published", + published_version=len(graphs), version=len(graphs)) + session.add(wf) + await session.flush() + for v, graph in enumerate(graphs, start=1): + session.add(WorkflowVersion(workflow_id=wf.id, version=v, graph=graph, level="published")) + await session.commit() + return wf.id + + +async def add_version(wf_id: str, graph: dict) -> int: + async with SessionLocal() as session: + wf = await session.get(Workflow, wf_id) + wf.published_version = wf.version = (wf.published_version or 0) + 1 + session.add(WorkflowVersion(workflow_id=wf_id, version=wf.published_version, graph=graph, level="published")) + await session.commit() + return wf.published_version + + +def caliber_case(wf_id: str, version: int = 1, *, upgrade=None, metrics=("gmv", "avg")) -> dict: + fields = cases.normalize_case({ + "question": "上周的销售额和客单价？", + "gold": {"kind": "caliber", "workflow_id": wf_id, "version": version, "node_id": "card", + "inputs": {"start": "{{ schedule.last_week_start }}", "end": "{{ schedule.last_week_end }}"}}, + "expectations": [{"kind": "scalar", "metric": m} for m in metrics]}) + if upgrade: + fields["gold"]["upgrade"] = upgrade + return {**fields, "id": None, "seq": 3} + + +async def card_of(run_id: str) -> dict: + async with SessionLocal() as session: + aid = (await session.execute(select(Artifact.id).where( + Artifact.run_id == run_id, Artifact.kind == "metric_set", Artifact.node_id == "card"))).scalar_one() + return artifact_store.load(aid) + + +async def test_the_caliber_probe_computes_what_the_original_workflow_computes(source): + wf_id = await publish(weekly(source["name"])) + outcome = await gold.compute_gold(caliber_case(wf_id), ctx()) + assert outcome.ok, outcome.reason + [found] = outcome.sets + assert found.version == 1 and outcome.rendered["inputs"] == {"start": "2026-08-31", "end": "2026-09-06"} + # 原工作流按同样的输入跑一次（探索运行）：口径卡算出的每个数都一样，取整、单位按口径卡 + original = await run_manager.start(graph=weekly(source["name"]), input_payload=outcome.rendered["inputs"]) + await run_manager.wait_idle(original.id, timeout=30) + card = await card_of(original.id) + by_metric = {v["metric"]: v for v in found.values.values()} + assert {m["id"]: m["value"] for m in card["metrics"]} == {k: v["value"] for k, v in by_metric.items()} + assert by_metric["gmv"]["value"] == 330.0 and by_metric["gmv"]["rendered"] == "330.0元" + assert by_metric["avg"]["value"] == 110.0 and by_metric["avg"]["caliber"] == "周销售" + row = await run_of(found.run_id) + assert row.origin_kind == "eval" and row.origin["derived"]["kind"] == "caliber_probe" + assert row.origin["derived"]["from"][0]["workflow_version"] == 1 + # 探针图里口径卡固定到那一版，取数节点照抄 + nodes = {n["id"]: n for n in row.graph["nodes"]} + assert nodes["card"]["data"]["config"]["caliber_from"] == {"workflow_id": wf_id, "workflow_version": 1, + "node_id": "card"} + assert nodes["fetch"]["data"]["config"] == weekly(source["name"])["nodes"][1]["data"]["config"] + + +@pytest.mark.parametrize("graph_of, words", [ + (lambda s: chain(node("start", "input"), + node("fetch", "transform", mode="expression", expression="{'gmv': 1}", assign_to="kpi"), + node("card", "metrics", caliber="周销售", metrics=[{"id": "gmv", "expression": "vars.kpi.gmv"}]), + node("out", "output")), "输入不全是查库结果"), + (lambda s: weekly(s, fetch_sql="SELECT SUM(amount) AS gmv, COUNT(*) AS n FROM orders WHERE region = " + "'{{ vars.region }}'"), "vars.region"), +]) +async def test_cards_that_cannot_be_reused_are_refused_with_the_reason(source, graph_of, words): + wf_id = await publish(graph_of(source["name"])) + outcome = await gold.compute_gold(caliber_case(wf_id, metrics=("gmv",)), ctx()) + assert outcome.status == "refused" and outcome.code == cases.CALIBER_NOT_REUSABLE and words in outcome.reason + assert outcome.run_ids == [] + + +async def test_a_card_that_copies_another_card_is_not_a_source(source): + upstream = await publish(weekly(source["name"])) + chained = chain(node("start", "input"), node("card", "metrics", caliber_from={ + "workflow_id": upstream, "workflow_version": 1, "node_id": "card"}), node("out", "output")) + wf_id = await publish(chained, name="转抄") + outcome = await gold.compute_gold(caliber_case(wf_id, metrics=("gmv",)), ctx()) + assert outcome.status == "refused" and "转抄" in outcome.reason + + +# -------------------------------------------------------------------------- +# 口径升版的处置 +# -------------------------------------------------------------------------- + + +async def test_an_upgraded_caliber_is_skipped_until_disposed(source): + wf_id = await publish(weekly(source["name"])) + case = caliber_case(wf_id) + assert (await gold.compute_gold(case, ctx())).ok + await add_version(wf_id, weekly(source["name"], refunds=True)) # v2：改成含退款 + outcome = await gold.compute_gold(case, ctx()) + assert outcome.status == "skipped" and outcome.code == gold.UPGRADE_PENDING + assert outcome.reason == "口径已升版到 v2，待处置" and outcome.run_ids == [] + + async with SessionLocal() as session: + with pytest.raises(gold.DisposeRefused): + await gold.plan_dispose(session, case, "keep") # 没有缺省、没有别的选项 + recomputed, latest = await gold.plan_dispose(session, case, "recompute") + dual, _ = await gold.plan_dispose(session, case, "dual") + fresh, _ = await gold.plan_dispose(session, case, "incomparable") + assert latest == 2 + # recompute：改钉新版本，指纹跟着变（历史在这里断开） + assert recomputed["version"] == 2 and recomputed["upgrade"]["policy"] == "recompute" + rebased = {**case, "gold": recomputed} + assert cases.case_hash(rebased) != cases.case_hash(case) + outcome = await gold.compute_gold(rebased, ctx()) + assert outcome.ok and [s.version for s in outcome.sets] == [2] + assert {v["metric"]: v["value"] for v in outcome.sets[0].values.values()}["gmv"] == 380.0 + # dual：新旧两版各算一份，旧的那一份照旧可比（指纹不变） + both = {**case, "gold": dual} + assert cases.case_hash(both) == cases.case_hash(case) + outcome = await gold.compute_gold(both, ctx()) + assert outcome.ok and [s.version for s in outcome.sets] == [1, 2] + assert [{v["metric"]: v["value"] for v in s.values.values()}["gmv"] for s in outcome.sets] == [330.0, 380.0] + # incomparable：按新版本另建新题（这里只给出新题的期望来源，归档旧题在接口里） + assert fresh["version"] == 2 and "upgrade" not in fresh + + # 再来一版：dual 只处置到 v2，v3 又待处置 + await add_version(wf_id, weekly(source["name"])) + outcome = await gold.compute_gold(both, ctx()) + assert outcome.status == "skipped" and outcome.reason == "口径已升版到 v3，待处置" + + +async def test_a_retired_pinned_version_waits_for_disposal(source): + wf_id = await publish(weekly(source["name"]), weekly(source["name"], refunds=True)) + case = caliber_case(wf_id, upgrade={"policy": "dual", "from_version": 1, "to_version": 2}) + assert (await gold.compute_gold(case, ctx())).ok + async with SessionLocal() as session: + await session.execute(update(WorkflowVersion).where(WorkflowVersion.workflow_id == wf_id, + WorkflowVersion.version == 1).values(retired_at=utcnow())) + await session.commit() + outcome = await gold.compute_gold(case, ctx()) + assert outcome.status == "skipped" and "v1 已退役，待处置" in outcome.reason + async with SessionLocal() as session: + with pytest.raises(gold.DisposeRefused, match="已退役"): + await gold.plan_dispose(session, case, "dual") + moved, _ = await gold.plan_dispose(session, case, "recompute") + assert (await gold.compute_gold({**case, "gold": moved}, ctx())).ok + + +def test_settle_reports_the_first_reason(): + now = datetime.now(timezone.utc) + bad = gold.GoldOutcome(status="failed", as_of=now.isoformat(), reason="SQL 错了", code=gold.GOLD_FAILED) + good = gold.GoldOutcome(status="ok", as_of=now.isoformat(), sets=[gold.GoldSet("r1", "a1", {"e1": {"value": 1}})]) + assert gold.settle(bad, good).status == "failed" + assert gold.settle(good, bad).reason == "作答后复算期望：SQL 错了" + assert gold.settle(good, None).status == "failed" + again = gold.GoldOutcome.from_dict(good.to_dict()) + assert gold.settle(good, again).ok + assert gold.GoldOutcome.from_dict(good.to_dict()).to_dict() == good.to_dict() diff --git a/backend/tests/test_eval_snapshots.py b/backend/tests/test_eval_snapshots.py new file mode 100644 index 00000000..00291c3d --- /dev/null +++ b/backend/tests/test_eval_snapshots.py @@ -0,0 +1,233 @@ +"""目录快照与影子起草（app/eval/snapshots.py；细案 01 做法 6）。 + +- 当前目录的快照：按 data/catalog_view.py 的读法读得出、和拍的那一刻一样；之后线上目录改了，快照不变（内容寻址、不可变）。 +- 影子起草：同一个起草引擎、只起草指定（或题集用得到）的表，写进内存里的副本、出快照；线上目录一项不动（版本不变、 + 不调 write_entry）；不查业务库（开了剖析也不去数有效行过滤的排除比例）；不占数据源的队；可以先去掉模型起草、还没人 + 审过的项；停止的不出快照；服务重启后没结束的标中断。开始前给估算。 + +库、表、目录内容都是虚构的。 +""" +from __future__ import annotations + +import asyncio + +import pytest +from sqlalchemy import select + +from app.core import artifact_store, job_runtime +from app.data import catalog, catalog_draft, catalog_profile, catalog_view +from app.data.catalog import make_item +from app.data.catalog_view import CatalogView +from app.db.base import SessionLocal +from app.db.models import CatalogNote +from app.db.models_eval import EvalCatalogSnapshot, EvalSuite +from app.eval import snapshots +from tests.fixtures import dual_check_shop as shop +from tests.fixtures.catalog.fake_model import FakeModel +from tests.fixtures.sources import drop_source + +LIVE_NOTES = { + "label": make_item("门店订单", "human"), # 人工确认 + "grain": make_item("每笔订单一行", "llm"), # 模型起草、没人审过 + "columns": {"amount": {"label": make_item("实收金额", "llm", "confirmed")}}, # 模型起草、人工确认过 +} +REPLY = {"orders": {"label": "影子订单", "grain": "影子：每单一行", "kind": "fact", + "valid_filter": {"condition": "status <> 'refunded'", "reason": "状态列有退款"}, + "columns": [{"name": "status", "label": "订单状态", "meaning": "paid 已支付，refunded 已退款"}]}} + + +@pytest.fixture +async def source(tmp_path): + name, sid = await shop.add_source(shop.build(tmp_path / "shop.db"), + options={catalog_profile.PROFILE_OPTION: {"enabled": True}}) + async with SessionLocal() as session: + await catalog.write_entry(session, sid, "orders", LIVE_NOTES, if_version=0, actor="审阅人") + yield {"name": name, "id": sid} + await drop_source(sid) + + +async def live_rows(source_id: str) -> list[tuple[str, int, dict]]: + async with SessionLocal() as session: + rows = (await session.execute(select(CatalogNote).where(CatalogNote.source_id == source_id))).scalars().all() + return sorted((r.table_name, r.version, r.notes) for r in rows) + + +async def read_view(snapshot_artifact: str, source_id: str) -> dict: + view = CatalogView(mode="snapshot", snapshot=snapshot_artifact) + async with SessionLocal() as session: + found = await catalog_view.read_many(session, view, [source_id]) + return {t: e.notes for t, e in found.get(source_id, {}).items()} + + +async def wait_done(snap_id: str) -> EvalCatalogSnapshot: + await snapshots.manager.wait(snap_id, timeout=30) + async with SessionLocal() as session: + return await session.get(EvalCatalogSnapshot, snap_id) + + +# -------------------------------------------------------------------------- +# 当前目录 +# -------------------------------------------------------------------------- + + +async def test_a_live_snapshot_reads_back_through_the_catalog_view_and_never_changes(source): + async with SessionLocal() as session: + snap = await snapshots.take_live(session, [source["id"]], actor="评测人") + assert snap.status == "succeeded" and snap.origin == "live" and snap.created_by == "评测人" + assert snap.sources == [{"id": source["id"], "name": source["name"]}] + content = artifact_store.load(snap.artifact_id) + assert content["taken"]["origin"] == "live" and content["taken"]["by"] == "评测人" + assert artifact_store.content_hash(artifact_store.canonical_json(content)) == snap.artifact_id + assert (await read_view(snap.artifact_id, source["id"]))["orders"] == LIVE_NOTES + # 之后线上目录改了：快照还是拍的那一份 + async with SessionLocal() as session: + await catalog.write_entry(session, source["id"], "orders", {"label": make_item("改过的", "human")}, + if_version=1, actor="审阅人") + assert (await read_view(snap.artifact_id, source["id"]))["orders"]["label"]["value"] == "门店订单" + # 只用已核实的项：读的时候按视图筛，快照本身照原样 + reviewed = CatalogView(mode="snapshot", statuses="reviewed", snapshot=snap.artifact_id) + async with SessionLocal() as session: + notes = (await catalog_view.read_many(session, reviewed, [source["id"]]))[source["id"]]["orders"].notes + assert "grain" not in notes and notes["label"]["value"] == "门店订单" + + +async def test_a_live_snapshot_needs_existing_sources(): + async with SessionLocal() as session: + with pytest.raises(snapshots.SnapshotInvalid, match="不存在"): + await snapshots.take_live(session, ["nope"], actor=None) + with pytest.raises(snapshots.SnapshotInvalid, match="请选择"): + await snapshots.take_live(session, [], actor=None) + + +# -------------------------------------------------------------------------- +# 影子起草 +# -------------------------------------------------------------------------- + + +async def test_shadow_drafting_writes_only_the_copy(source, monkeypatch): + model = FakeModel(REPLY) + monkeypatch.setattr(snapshots.ShadowDraft, "model_override", model) + + async def no_business_queries(*_a, **_k): + raise AssertionError("影子起草不该查业务库") + + monkeypatch.setattr(catalog_profile, "check_valid_filter", no_business_queries) + before = await live_rows(source["id"]) + async with SessionLocal() as session: + plan = await snapshots.plan_shadow(session, {"sources": [{"id": source["id"], "tables": ["orders"]}], + "guidance": "订单表按支付日统计"}, model_id="mock-fast") + snap = await snapshots.start_shadow(session, plan, actor="评测人") + assert snap.status in ("queued", "running") + # 不占数据源的队：同一个源照样能马上起草、剖析 + await asyncio.sleep(0) + assert job_runtime.lanes.holders(source["id"]) == [] + done = await wait_done(snap.id) + assert done.status == "succeeded", (done.error, done.progress) + # 线上目录一项不动 + assert await live_rows(source["id"]) == before + notes = (await read_view(done.artifact_id, source["id"]))["orders"] + assert notes["label"]["value"] == "门店订单" # 人工确认的不被起草覆盖 + assert notes["grain"]["value"] == "影子：每单一行" # 没人审过的模型项换成这次起草的 + assert notes["columns"]["status"]["label"]["value"] == "订单状态" + # 开了剖析也不去业务库数排除比例：照收，写明没有核对 + assert catalog_draft.SHADOW_NOT_CHECKED in notes["valid_filter"]["note"] + # 起草要求交给了模型；只起草了点名的表 + assert len(model.calls) == 1 and "订单表按支付日统计" in model.calls[0] + content = artifact_store.load(done.artifact_id) + assert content["taken"]["origin"] == "shadow" and content["taken"]["tables"] == {source["id"]: ["orders"]} + assert done.usage["calls"] == 1 and done.progress["tables_done"] == 1 + assert done.estimate["tables"] == 1 and done.estimate["calls"] == 1 + + +async def test_shadow_drafting_can_start_without_unreviewed_model_items(source, monkeypatch): + monkeypatch.setattr(snapshots.ShadowDraft, "model_override", FakeModel({"orders": {"kind": "fact"}})) + async with SessionLocal() as session: + plan = await snapshots.plan_shadow(session, {"sources": [{"id": source["id"], "tables": ["orders"]}], + "strip_llm": True}) + snap = await snapshots.start_shadow(session, plan, actor=None) + done = await wait_done(snap.id) + notes = (await read_view(done.artifact_id, source["id"]))["orders"] + assert "grain" not in notes # 没人审过的模型项去掉了 + assert notes["columns"]["amount"]["label"]["value"] == "实收金额" # 人工确认过的模型项留着 + assert notes["label"]["value"] == "门店订单" and notes["kind"]["value"] == "fact" + + +async def test_tables_come_from_the_suite_when_not_listed(source): + from app.db.models_eval import EvalCase + + async with SessionLocal() as session: + suite = EvalSuite(name="虚构题集") + session.add(suite) + await session.flush() + session.add(EvalCase(suite_id=suite.id, seq=1, question="q", expectations=[], gold={ + "kind": "sql", "datasource_id": source["id"], + "sql": "SELECT SUM(amount) FROM orders WHERE paid_date = '{{ schedule.yesterday }}'"})) + await session.commit() + found = await snapshots.suite_tables(session, suite.id) + assert found == {source["id"]: ["orders"]} + plan = await snapshots.plan_shadow(session, {"sources": [{"id": source["id"]}], "suite_id": suite.id}) + assert plan.sources[0]["tables"] == ["orders"] + with pytest.raises(snapshots.SnapshotInvalid, match="没有指定要起草的表"): + await snapshots.plan_shadow(session, {"sources": [{"id": source["id"]}]}) + with pytest.raises(snapshots.SnapshotInvalid, match="表结构里没有"): + await snapshots.plan_shadow(session, {"sources": [{"id": source["id"], "tables": ["ghost"]}]}) + + +def test_the_estimate_is_said_to_be_an_estimate(): + from types import SimpleNamespace + + cols = lambda n: [{"name": f"c{i}", "type": "TEXT"} for i in range(n)] # noqa: E731 + src = SimpleNamespace(schema_cache={"tables": {"a": {"columns": cols(10)}, "b": {"columns": cols(10)}, + "wide": {"columns": cols(100)}}}) + est = snapshots.estimate([(src, ["a", "b", "wide"])], "no-such-model") + # 窄表拼一段；宽表是表级一段 + 列分段（100 列按 40 列一段是 3 段） + assert est["calls"] == 1 + (1 + 3) and est["tables"] == 3 and est["columns"] == 120 + assert est["unpriced"] is True and est["cost_usd"] is None and "不含思考" in est["note"] + assert est["output_tokens"] == 3 * snapshots.EST_OUTPUT_PER_TABLE + 120 * snapshots.EST_OUTPUT_PER_COLUMN + + +async def test_a_stopped_shadow_draft_leaves_no_snapshot(source, monkeypatch): + monkeypatch.setattr(snapshots.ShadowDraft, "model_override", FakeModel(REPLY, hang=5)) + async with SessionLocal() as session: + plan = await snapshots.plan_shadow(session, {"sources": [{"id": source["id"], "tables": ["orders"]}]}) + snap = await snapshots.start_shadow(session, plan, actor=None) + for _ in range(100): + await asyncio.sleep(0.02) + async with SessionLocal() as session: + if (await session.get(EvalCatalogSnapshot, snap.id)).status == "running": + break + assert await snapshots.stop(snap.id) + done = await wait_done(snap.id) + assert done.status == "cancelled" and done.artifact_id is None and done.error == snapshots.STOPPED + assert not await snapshots.stop(snap.id) + + +async def test_unfinished_shadow_drafts_are_marked_interrupted_after_a_restart(source): + async with SessionLocal() as session: + row = EvalCatalogSnapshot(origin="shadow", status="running", sources=[{"id": source["id"], "name": "x"}]) + session.add(row) + await session.commit() + await snapshots.reconcile_stale(session, [row]) + assert row.status == "interrupted" and row.error == snapshots.INTERRUPTED + other = EvalCatalogSnapshot(origin="shadow", status="queued", sources=[]) + session.add(other) + await session.commit() + other_id = other.id + assert await snapshots.startup() >= 1 + async with SessionLocal() as session: + assert (await session.get(EvalCatalogSnapshot, other_id)).status == "interrupted" + + +async def test_a_mock_provider_is_not_a_drafting_model(): + from app.db.models import Provider + + async with SessionLocal() as session: + session.add(Provider(name="演示接入_快照", kind="mock", default_model="mock-fast", enabled=True)) + await session.commit() + try: + with pytest.raises(snapshots.SnapshotInvalid, match="演示用的模型接入"): + await snapshots.resolve_model(session, "演示接入_快照", None) + finally: + row = (await session.execute(select(Provider).where(Provider.name == "演示接入_快照"))).scalar_one() + await session.delete(row) + await session.commit() diff --git a/backend/tests/test_ledger_derive.py b/backend/tests/test_ledger_derive.py new file mode 100644 index 00000000..bc6792bd --- /dev/null +++ b/backend/tests/test_ledger_derive.py @@ -0,0 +1,159 @@ +"""从钉住版本派生只读图（app/ledger/derive.py；总方案 X3：评测的口径题、账本的对账与召回共用这一个生成器）。 + +- load_pinned：只认可复用的源头口径卡；工作流、版本、节点不在、不是口径卡、转抄别处、输入不是查库的格、取数节点引用了 + 入口字段以外的值，照实拒收（NotDerivable，带机器码）。 +- caliber_probe：入口字段照抄、取数节点 id 和配置原样、口径卡 caliber_from 固定到那一版、「缺输入时」照抄；图能通过 + 校验；同样的输入永远得到同一张图。 +- 扩展点：带前缀的段改名并写出 caliber_from.nodes 映射；两份口径不带前缀放进一张图会撞 id、直接报错。 + +工作流、库、表都是虚构的。 +""" +from __future__ import annotations + +import pytest + +from app.db.base import SessionLocal +from app.db.models import Workflow, WorkflowVersion +from app.engine.schema import GraphSpec, validate_graph +from app.ledger import derive + + +def node(nid, ntype, label=None, **config): + return {"id": nid, "type": ntype, "position": {"x": 7, "y": 9}, "data": {"label": label or nid, "config": config}} + + +def chain(*nodes, defaults=None): + graph = {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + if defaults: + graph["defaults"] = defaults + return graph + + +FIELDS = [{"name": "start", "required": True, "type": "date"}, {"name": "region", "default": "华东"}] +FETCH_SQL = ("SELECT SUM(amount) AS gmv, COUNT(*) AS n FROM orders WHERE paid_date >= '{{ input.start }}' " + "AND region = '{{ input.region | trim }}'") + + +def weekly(fetch_sql: str = FETCH_SQL, **card) -> dict: + return chain(node("start", "input", fields=FIELDS), + node("fetch", "tool", "取数", tool="db_query__shop_x", args={"sql": fetch_sql}, assign_to="rows"), + node("card", "metrics", "周销售口径卡", caliber="周销售", caliber_version="v2", on_missing="null", + metrics=[{"id": "gmv", "name": "销售额", "expression": "cell(nodes.fetch, 0, 'gmv')"}, + {"id": "avg", "expression": "cell(nodes.fetch, 0, 'gmv') / cell(nodes.fetch, 0, 'n')"}], + **card), + node("write", "report", instructions="写周报"), + node("out", "output"), defaults={"approval": "dangerous"}) + + +async def publish(graph: dict, *, level: str | None = "published") -> str: + async with SessionLocal() as session: + wf = Workflow(name=f"派生_{id(graph)}", graph=graph, status="published", published_version=1, version=1) + session.add(wf) + await session.flush() + session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=graph, level=level)) + await session.commit() + return wf.id + + +async def pinned_of(graph: dict, **kw) -> derive.Pinned: + wf_id = await publish(graph, **kw) + async with SessionLocal() as session: + return await derive.load_pinned(session, wf_id, 1, "card") + + +async def test_the_probe_copies_the_fetch_and_pins_the_card(): + pinned = await pinned_of(weekly()) + assert pinned.published and not pinned.retired and pinned.reads == (("fetch", "db_query__shop_x", ("gmv", "n")),) + assert [m["id"] for m in pinned.metrics] == ["gmv", "avg"] and pinned.caliber == "周销售 @ v2" + probe = derive.caliber_probe(pinned) + nodes = {n["id"]: n for n in probe.graph["nodes"]} + assert [n["id"] for n in probe.graph["nodes"]] == [derive.ENTRY_ID, "fetch", "card", derive.EXIT_ID] + # 入口字段照抄（必填、缺省值、类型都不变） + assert nodes[derive.ENTRY_ID]["data"]["config"]["fields"] == FIELDS + # 取数节点 id、配置原样 + original = weekly()["nodes"][1] + assert nodes["fetch"]["data"] == original["data"] and nodes["fetch"]["type"] == "tool" + # 口径卡固定到那一版那张卡；「缺输入时」照抄；本地不写指标式 + card = nodes["card"]["data"]["config"] + assert card == {"caliber_from": {"workflow_id": pinned.workflow_id, "workflow_version": 1, "node_id": "card"}, + "on_missing": "null"} + # 图级 defaults 照抄：节点上没写的项按同样的规则取值 + assert probe.graph["defaults"] == {"approval": "dangerous"} + assert nodes[derive.EXIT_ID]["data"]["config"]["fields"] == [{"name": "周销售口径卡", + "value": "{{ nodes.card.text }}"}] + assert probe.card_ids == ("card",) and probe.origin["kind"] == "caliber_probe" + assert probe.origin["from"][0]["workflow_version"] == 1 + assert validate_graph(GraphSpec.model_validate(probe.graph)).ok + # 确定性：同样的版本永远得到同一张图 + assert derive.caliber_probe(pinned).graph == probe.graph + # 处置写在本地口径卡上，交给和正式运行同一个升版判断 + assert derive.caliber_probe(pinned, upgrade_policy="dual").graph["nodes"][2]["data"]["config"][ + "upgrade_policy"] == "dual" + + +@pytest.mark.parametrize("graph, code, words", [ + (chain(node("start", "input"), node("fetch", "transform", mode="expression", expression="{}", assign_to="k"), + node("card", "metrics", metrics=[{"id": "x", "expression": "vars.k.x"}]), node("out", "output")), + derive.NOT_REUSABLE, "输入不全是查库结果"), + (chain(node("start", "input"), node("fetch", "tool", tool="db_query__shop_x", args={"sql": "SELECT 1 AS v"}), + node("card", "metrics", metrics=[{"id": "x", "expression": "cell(nodes.fetch, 1, 'v')"}]), + node("out", "output")), derive.NOT_REUSABLE, "别的行号"), + (chain(node("start", "input"), node("card", "metrics", caliber_from={"workflow_id": "w", "workflow_version": 1, + "node_id": "c"}), node("out", "output")), + derive.NOT_REUSABLE, "转抄"), + (chain(node("start", "input"), node("card", "report"), node("out", "output")), derive.NOT_CARD, "不是口径卡"), + (weekly("SELECT SUM(amount) AS gmv, COUNT(*) AS n FROM orders WHERE region = '{{ vars.region }}'"), + derive.TEMPLATE, "vars.region"), + (weekly("SELECT SUM(amount) AS gmv, COUNT(*) AS n FROM {{ nodes.pick.table }}"), derive.TEMPLATE, "nodes.pick"), + (weekly("SELECT SUM(amount) AS gmv, COUNT(*) AS n FROM orders -- {{ file_text('x') }}"), derive.TEMPLATE, + "file_text"), +]) +async def test_cards_that_cannot_be_derived_say_why(graph, code, words): + with pytest.raises(derive.NotDerivable) as caught: + await pinned_of(graph) + assert caught.value.code == code and words in caught.value.message + + +async def test_missing_workflows_versions_and_nodes(): + wf_id = await publish(weekly()) + async with SessionLocal() as session: + for args, words in (((("nope", 1, "card")), "不存在"), (((wf_id, 9, "card")), "没有 v9"), + (((wf_id, 1, "ghost")), "没有节点 ghost")): + with pytest.raises(derive.NotDerivable, match=words) as caught: + await derive.load_pinned(session, *args) + assert caught.value.code == derive.GONE + + +async def test_draft_versions_load_but_say_they_are_not_published(): + pinned = await pinned_of(weekly(), level=None) + async with SessionLocal() as session: + wf = await session.get(Workflow, pinned.workflow_id) + wf.published_version = None + await session.commit() + again = await derive.load_pinned(session, pinned.workflow_id, 1, "card") + assert again.published is False + + +async def test_prefixed_blocks_rename_and_map_for_two_calibers_in_one_graph(): + """召回（第 3 波）要把新旧两版放进同一张图：带前缀的段改名，口径卡写出改名映射。""" + pinned = await pinned_of(weekly()) + old = derive.caliber_block(pinned, prefix="old_") + new = derive.caliber_block(pinned, prefix="new_") + assert [n["id"] for n in old.nodes] == ["old_fetch", "old_card"] and old.rename == {"fetch": "old_fetch"} + assert old.nodes[-1]["data"]["config"]["caliber_from"]["nodes"] == {"fetch": "old_fetch"} + graph = derive.assemble("recall", [old, new], fields=pinned.entry_fields) + assert graph.card_ids == ("old_card", "new_card") and len(graph.origin["from"]) == 2 + names = [f["name"] for f in graph.graph["nodes"][-1]["data"]["config"]["fields"]] + assert names == ["周销售口径卡", "周销售口径卡_2"] + # 不带前缀放两份：节点 id 撞了，直接报错，不悄悄拼出一张错图 + with pytest.raises(ValueError, match="重复"): + derive.assemble("recall", [derive.caliber_block(pinned), derive.caliber_block(pinned)]) + with pytest.raises(ValueError, match="不认识"): + derive.assemble("nope", [old]) + + +def test_template_problem_allows_only_entry_fields(): + assert derive.template_problem("q", {"args": {"sql": "SELECT '{{ input.start }}', '{{ input.x | trim }}'"}}) is None + assert derive.template_problem("q", {"args": {"sql": "SELECT 1"}, "limit": 5}) is None + assert "vars.a" in derive.template_problem("q", {"args": {"sql": "{{ vars.a }}"}}) + assert "input" in derive.template_problem("q", {"args": {"sql": "{{ input }}"}})