Files
AgentLab/backend/tests/test_frontend_eval_unions.py
T
ThearchyHelios 9358d9ec48 评测前端认两种分析期望:题目编辑器在预览之后选贡献或判定(分析方法、模式、按哪几列、期次、点名前几段),试算写出服务端算的判定和判定语;结果页逐题写期望和答案的判定,点开写判定语、引用了哪一份结论或正文那一句;总览按期望的种类分开写结论
- typesEval.ts:期望种类加 contribution、verdict,期望带 op、mode、params;EvalAnalysisOp、EvalAnalysisMode、EvalJudgement(同服务端契约和 judgements.JUDGEMENTS);期望值、评分带判定类、点名的段、方向、判定语;总表加 outcomes_by_kind
- termsEval.ts:贡献、判定的叫法;分析方法、模式的叫法和契约的 OP_LABELS、MODE_LABELS 一字不差(test_frontend_eval_unions 逐个对照,连同判定类);判定类、方向、参数的叫法;指纹差异一句补上分析设置
- parts.tsx:期望写成「变化拆解(可加) · 按地区 · 销售额」;判定写成「主要来源:华东(下降)」「超出常规波动(上升)」「差异站得住(西南高于华东)」,算不了的照实写原因;判定类、段、方向都是服务端给的,界面只套叫法
- CaseEditor.tsx:分析期望的表单按分析方法列参数(参数名同分析工具,列从预览结果里选),换方法、模式只留新方法还认的参数;编辑配了几个期望的题时另外的期望列出来、保存时照旧带上(以前只留第一个,第二个期望会丢)
- EvaluationPage.tsx:逐题表、展开的明细、总览按种类
- scripts/check-evals.mjs:夹具补分析期望(加题选算子、试算、保存的请求体不带来源方向判定,编辑保留另外的期望;结果页四种结论、点开的明细、总览按种类),共 130 项
2026-10-21 18:23:57 +08:00

116 lines
5.4 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""评测页的前端约定(frontend/src/typesEval.ts)和文案(lib/termsEval.ts 的 EVAL_TEXT)里的取值,和服务端的常量逐个对上。
评测页上的结论、作答路径、转移、归因、目录视图、状态都是服务端给的取值,前端照着写叫法(界面层不放逻辑,叫法在 termsEval)。
服务端加了一种(比如作答路径「被驳回」),前端的联合类型和叫法表没跟上,界面就写出英文原值;前端多写了一种,就在等一个
永远不会来的值。这里直接读两份 .ts 文件比(同 test_frontend_type_unions.py 的写法),改了一边没改另一边就过不去。
"""
from __future__ import annotations
import re
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
TYPES_EVAL = ROOT / "frontend" / "src" / "typesEval.ts"
TERMS_EVAL = ROOT / "frontend" / "src" / "lib" / "termsEval.ts"
_COMMENT = re.compile(r"/\*.*?\*/|//[^\n]*", re.S)
def _source(path: Path) -> str:
return _COMMENT.sub("", path.read_text(encoding="utf-8"))
def _union(name: str) -> list[str]:
text = _source(TYPES_EVAL)
found = re.search(rf"export type {name} =(.*?)(?=\nexport |\n\n|\Z)", text, re.S)
assert found is not None, f"typesEval.ts 里没有 export type {name}"
return re.findall(r"'([^']+)'", found.group(1))
def _pairs(path: str) -> list[tuple[str, str]]:
"""EVAL_TEXT 里某一张叫法表(a.b 这样的路径)的 (键, 叫法),按写的顺序。只认一行写完、或几行写完的对象字面量。"""
text = _source(TERMS_EVAL)
*parents, leaf = path.split(".")
start = 0
for part in parents:
hit = re.search(rf"\n\s*{part}: \{{", text[start:])
assert hit is not None, f"EVAL_TEXT 里没有 {part}"
start += hit.end()
hit = re.search(rf"\n\s*{leaf}: \{{(.*?)\}}( as Record<[^>]+>)?,", text[start:], re.S)
assert hit is not None, f"EVAL_TEXT 里没有 {path}"
return re.findall(r"(\w+): '([^']*)'", hit.group(1))
def _labels(path: str) -> list[str]:
"""EVAL_TEXT 里某一张叫法表的键,按写的顺序。"""
return [key for key, _ in _pairs(path)]
def test_statuses_triggers_and_kinds():
from app.core.job_runtime import ACTIVE, FINISHED
from app.db.models_eval import CASE_KINDS, RESULT_PATHS, RUN_TRIGGERS, VERDICTS
assert _union("EvalStatus") == [*ACTIVE, *FINISHED]
assert _union("EvalTrigger") == list(RUN_TRIGGERS)
assert _union("EvalCaseKind") == list(CASE_KINDS)
assert _union("EvalVerdict") == list(VERDICTS)
assert _union("EvalPath") == list(RESULT_PATHS)
assert _labels("status") == [*ACTIVE, *FINISHED]
assert _labels("trigger") == list(RUN_TRIGGERS)
assert _labels("verdict") == list(VERDICTS)
assert _labels("path") == list(RESULT_PATHS)
def test_expectations_gold_and_trial():
from app.eval.cases import EXPECTATION_KINDS, GOLD_KINDS
from app.eval.gold import STATUSES, UPGRADE_LABEL
assert _union("EvalExpectationKind") == list(EXPECTATION_KINDS)
assert _labels("expectation.kind") == list(EXPECTATION_KINDS)
assert _union("EvalGoldKind") == list(GOLD_KINDS)
assert _labels("gold.kind") == list(GOLD_KINDS)
assert _union("EvalUpgradePolicy") == list(UPGRADE_LABEL)
# 试算的结论:服务端的几种,加上作答前后算出来不一样时的 drifted(gold.settle)
assert set(STATUSES) | {"drifted"} == set(_union("EvalTrialStatus"))
assert set(_union("EvalTrialStatus")) <= set(_labels("trialStatus"))
def test_scores_compare_and_catalog_modes():
from app.eval.compare import CAUSES, TRANSITIONS
from app.eval.score import OUTCOMES, TEXT_OUTCOMES
from app.eval.variants import CATALOG_MODES
assert _union("EvalOutcome") == list(OUTCOMES)
assert _labels("outcome") == list(OUTCOMES)
assert _union("EvalTextOutcome") == list(TEXT_OUTCOMES)
assert _labels("textOutcome") == list(TEXT_OUTCOMES)
assert _union("EvalTransition") == list(TRANSITIONS)
assert _labels("compare.transition") == list(TRANSITIONS)
assert _union("EvalCause") == list(CAUSES)
assert _labels("compare.cause") == list(CAUSES)
assert _union("EvalCatalogMode") == list(CATALOG_MODES)
assert _labels("start.catalogMode") == list(CATALOG_MODES)
def test_signal_bands_follow_the_server():
from app.api.evals import SIGNAL_CRITERIA
assert _union("EvalE1Band") == [c["band"] for c in SIGNAL_CRITERIA["e1"]]
assert _union("EvalE3Band") == [c["band"] for c in SIGNAL_CRITERIA["e3"]]
assert set(_labels("signals.band")) == {c["band"] for c in [*SIGNAL_CRITERIA["e1"], *SIGNAL_CRITERIA["e3"]]}
def test_analysis_expectations_follow_the_server():
"""分析期望:判定类和服务端 app/eval/judgements 一致;分析方法、模式的叫法和契约(app/analysis/contract 的 OP_LABELS、
MODE_LABELS)一字不差——界面上的「变化拆解(可加)」和服务端拼的、证据面板写的是同一种说法。"""
from app.analysis.contract import MODE_LABELS, MODES, OP_LABELS, OPS
from app.eval.judgements import JUDGEMENTS
assert _union("EvalJudgement") == list(JUDGEMENTS)
assert _labels("analysis.judgement") == list(JUDGEMENTS)
assert _union("EvalAnalysisOp") == list(OPS)
assert _pairs("analysis.op") == list(OP_LABELS.items())
assert _union("EvalAnalysisMode") == list(MODE_LABELS)
assert _pairs("analysis.mode") == list(MODE_LABELS.items())
assert {m for modes in MODES.values() for m in modes} == set(MODE_LABELS)