mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:01:58 +00:00
合入第 2 波复核修复 S4、S9：评测页分析期望的参数格改用分析卡那一份（差异检验的比率、均值模式补上期次三格），参数叫法只留 termsAnalysis 一份；分析测试的「今天」按题集的时区取，不依赖本机时区
This commit is contained in:
ThearchyHelios
@@ -32,8 +32,9 @@ from app.eval.gold import Context
|
||||
from tests import eval_samples as es
|
||||
from tests.fixtures.acceptance import retail_trends
|
||||
|
||||
TZ = "Asia/Shanghai"
|
||||
SUITE = json.loads((Path(__file__).parent / "fixtures" / "eval" / "analysis_suite.json").read_text(encoding="utf-8"))
|
||||
#: 题集的时区：题目里的期次（{{ schedule.* }}）按它算，生成库的「今天」也按它取，不看本机时区
|
||||
TZ = SUITE["suite"]["timezone"]
|
||||
#: 题集里的第 1 题（上周销售额下降主要来自华东）和第 3 题（订单数的常规波动）
|
||||
REGION = next(c for c in SUITE["cases"] if c["seq"] == 1)
|
||||
ORDERS = next(c for c in SUITE["cases"] if c["seq"] == 3)
|
||||
@@ -53,7 +54,7 @@ async def engine_up(monkeypatch):
|
||||
|
||||
@pytest.fixture
|
||||
async def source(tmp_path, scratch_source):
|
||||
path = retail_trends.build(tmp_path / "trends.db", date.today())
|
||||
path = retail_trends.build(tmp_path / "trends.db", now().date())
|
||||
sid = await scratch_source(retail_trends.SOURCE_NAME, path, schema=True)
|
||||
return {"id": sid, "name": retail_trends.SOURCE_NAME, "path": path}
|
||||
|
||||
|
||||
@@ -4,9 +4,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import date
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
@@ -33,9 +34,11 @@ async def engine_up():
|
||||
|
||||
|
||||
async def test_the_analysis_suite_imports_and_every_expectation_computes(tmp_path, scratch_source):
|
||||
path = retail_trends.build(tmp_path / "trends.db", date.today())
|
||||
sid = await scratch_source(retail_trends.SOURCE_NAME, path, schema=True)
|
||||
content = json.loads(SUITE_FILE.read_text(encoding="utf-8"))
|
||||
# 「今天」按题集的时区取（题目里的期次按它算），不看本机时区：本机在别的时区时，跨日那几个钟头两边的今天不是同一天
|
||||
today = datetime.now(ZoneInfo(content["suite"]["timezone"])).date()
|
||||
path = retail_trends.build(tmp_path / "trends.db", today)
|
||||
sid = await scratch_source(retail_trends.SOURCE_NAME, path, schema=True)
|
||||
assert {c["seq"] for c in content["cases"]} == set(WANTED)
|
||||
assert all(c["gold"]["datasource"] == retail_trends.SOURCE_NAME for c in content["cases"])
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
||||
@@ -44,7 +47,7 @@ async def test_the_analysis_suite_imports_and_every_expectation_computes(tmp_pat
|
||||
assert res.json()["imported"] == len(WANTED) and res.json()["skipped"] == []
|
||||
suite = res.json()["suite"]
|
||||
assert all(c["gold"]["datasource_id"] == sid for c in suite["cases"])
|
||||
monday = date.today().weekday() == 0
|
||||
monday = today.weekday() == 0
|
||||
for case in suite["cases"]:
|
||||
assert case["trial"]["status"] == "ok", (case["seq"], case["trial"])
|
||||
trial = (await client.post(f"/api/evals/suites/{suite['id']}/cases/{case['id']}/trial")).json()["trial"]
|
||||
|
||||
@@ -386,7 +386,9 @@ async def test_export_then_import_matches_sources_by_name(client, source):
|
||||
|
||||
async def test_the_retail_sample_suite_trials_on_the_retail_database(client, tmp_path):
|
||||
"""仓库里附的零售样例题集：在虚构零售库上导入，每道题都试算通过，期望和直接查库一样，陷阱按口径算。"""
|
||||
today = datetime.now(ZoneInfo("Asia/Shanghai")).date()
|
||||
# 「今天」按题集的时区取（题目里的期次按它算），不看本机时区：本机在别的时区时，跨日那几个钟头两边的今天不是同一天
|
||||
content = json.loads(SUITE_FILE.read_text(encoding="utf-8"))
|
||||
today = datetime.now(ZoneInfo(content["suite"]["timezone"])).date()
|
||||
path = retail.build(tmp_path / "retail.db", today)
|
||||
await drop_source(name=retail.SOURCE_NAME)
|
||||
async with SessionLocal() as session:
|
||||
@@ -398,7 +400,6 @@ async def test_the_retail_sample_suite_trials_on_the_retail_database(client, tmp
|
||||
await session.commit()
|
||||
sid = row.id
|
||||
try:
|
||||
content = json.loads(SUITE_FILE.read_text(encoding="utf-8"))
|
||||
res = await client.post("/api/evals/suites/import", json={"content": content}, headers=ACTOR)
|
||||
assert res.status_code == 201, res.text
|
||||
assert res.json()["imported"] == 6 and res.json()["skipped"] == []
|
||||
|
||||
@@ -14,6 +14,10 @@ TYPES_EVAL = ROOT / "frontend" / "src" / "typesEval.ts"
|
||||
TERMS_EVAL = ROOT / "frontend" / "src" / "lib" / "termsEval.ts"
|
||||
#: 分析方法、模式的叫法只有这一份（termsEval 引用它，不另抄）
|
||||
TERMS_ANALYSIS = ROOT / "frontend" / "src" / "lib" / "termsAnalysis.ts"
|
||||
#: 分析卡的一项摆哪几个参数格（评测页的分析期望和分析卡用同一份）
|
||||
ANALYSIS_ITEMS = ROOT / "frontend" / "src" / "canvas" / "analysisItems.ts"
|
||||
#: 评测页题目编辑框（分析期望那一块）
|
||||
CASE_EDITOR = ROOT / "frontend" / "src" / "pages" / "evals" / "CaseEditor.tsx"
|
||||
|
||||
_COMMENT = re.compile(r"/\*.*?\*/|//[^\n]*", re.S)
|
||||
|
||||
@@ -138,3 +142,32 @@ def test_analysis_expectations_follow_the_server():
|
||||
terms = _source(TERMS_EVAL)
|
||||
assert re.search(r"\n\s*op: ANALYSIS_OP_LABEL\b", terms) and re.search(r"\n\s*mode: ANALYSIS_MODE_LABEL\b", terms), \
|
||||
"评测页的分析方法、模式叫法要引用 lib/termsAnalysis 的那一份"
|
||||
|
||||
|
||||
def test_analysis_param_grids_follow_the_card_and_the_server():
|
||||
"""分析期望的参数格和分析卡的一项同一份（canvas/analysisItems.paramSpecs）：评测页（CaseEditor）引用它，不另抄。
|
||||
参数名对得上服务端各参数模型（tools/builtin/analysis.param_names）；叫法只有 lib/termsAnalysis 一份
|
||||
（ANALYSIS_TEXT.node.param / basis / grain），评测页（EVAL_TEXT.analysis.param / basis / grain）引用它。"""
|
||||
from app.eval.judgements import VERDICT_OPS, param_names
|
||||
|
||||
items = _source(ANALYSIS_ITEMS)
|
||||
grid = re.search(r"export function paramSpecs\(op: AnalysisOp.*?\n\}\n", items, re.S)
|
||||
assert grid is not None, "analysisItems.ts 里没有 export function paramSpecs"
|
||||
# 期次三格（period、base、target）写在上面的 PERIODS 常量里，一起算
|
||||
periods = re.search(r"const PERIODS: ParamSpec\[\] = \[(.*?)\]\n", items, re.S)
|
||||
assert periods is not None, "analysisItems.ts 里没有 const PERIODS"
|
||||
grid_keys = {a or b for a, b in re.findall(r"(?:col|cols|val)\('(\w+)'\)|key: '(\w+)'",
|
||||
grid.group(0) + periods.group(1))}
|
||||
# 服务端各参数模型里期望能写的那些（去掉输入、总数核对、下钻、两份查询），格子里一个不少、一个不多
|
||||
server = {k for op in ("decompose", *VERDICT_OPS) for k in param_names(op)}
|
||||
assert grid_keys == server, f"格子和服务端的参数模型对不上：少 {server - grid_keys}，多 {grid_keys - server}"
|
||||
editor = _source(CASE_EDITOR)
|
||||
assert "paramSpecs(" in editor and "PARAM_FIELDS" not in editor, "评测页的参数格要引用分析卡的那一份（paramSpecs）"
|
||||
terms = _source(TERMS_EVAL)
|
||||
for table in ("param", "basis", "grain"):
|
||||
assert re.search(rf"\n\s*{table}: ANALYSIS_TEXT\.node\.{table}\b", terms), \
|
||||
f"评测页的 {table} 叫法要引用 lib/termsAnalysis 的那一份"
|
||||
analysis = _source(TERMS_ANALYSIS)
|
||||
labels = re.search(r"\n\s*param: \{(.*?)\} as Record<string, string>", analysis, re.S)
|
||||
assert labels is not None, "termsAnalysis.ts 里没有 ANALYSIS_TEXT.node.param"
|
||||
assert set(grid_keys) <= set(re.findall(r"(\w+): '", labels.group(1))), "参数格里有叫不出名字的参数"
|
||||
|
||||
@@ -10,7 +10,7 @@ import type {
|
||||
EvalTrigger, EvalUpgradePolicy, EvalVerdict,
|
||||
} from '../typesEval'
|
||||
import { formatCost, formatNumber } from './format'
|
||||
import { ANALYSIS_MODE_LABEL, ANALYSIS_OP_LABEL } from './termsAnalysis'
|
||||
import { ANALYSIS_MODE_LABEL, ANALYSIS_OP_LABEL, ANALYSIS_TEXT } from './termsAnalysis'
|
||||
|
||||
/** 比率写成百分数：0.756 → 「75.6%」；没有分母（null）写「—」 */
|
||||
export function pct(rate: number | null | undefined, digits = 1): string {
|
||||
@@ -336,14 +336,10 @@ export const EVAL_TEXT = {
|
||||
ratio: (numerator: string, denominator: string) => `${numerator} ÷ ${denominator}`,
|
||||
/** 答案里同类的分析没有做成 */
|
||||
refused: (reason: string) => `没有做成：${reason}`,
|
||||
param: {
|
||||
by: '维度列', period: '期次列', base: '基期', target: '本期', measure: '度量列', numerator: '分子列', denominator: '分母列',
|
||||
product: '乘积列', factors: '因子列', label: '度量的叫法', basis: '比较方式', grain: '粒度', lag: '和往前第几期比',
|
||||
pairs: '比哪几对', reference: '对照组', mean: '均值列', sd: '标准差列', sum: '合计列', sumsq: '平方和列', n: '样本数列',
|
||||
value: '逐行的值列',
|
||||
} as Record<string, string>,
|
||||
basis: { change: '和比较期比的变化率', level: '值本身' } as Record<string, string>,
|
||||
grain: { day: '天', week: '周', month: '月', quarter: '季度' } as Record<string, string>,
|
||||
/** 参数格的叫法只有 lib/termsAnalysis 一份（ANALYSIS_TEXT.node.param / basis / grain）：分析卡和评测页的分析期望按同一份参数名 */
|
||||
param: ANALYSIS_TEXT.node.param,
|
||||
basis: ANALYSIS_TEXT.node.basis,
|
||||
grain: ANALYSIS_TEXT.node.grain,
|
||||
},
|
||||
upgrade: {
|
||||
title: (seq: number) => `第 ${seq} 题：处置口径升版`,
|
||||
|
||||
@@ -7,6 +7,8 @@ import { Field, Modal, Notice, Spinner, toast, useRadioGroup } from '../../compo
|
||||
import { errorMessage } from '../../lib/errors'
|
||||
import { formatDateTime, formatTime, shortId } from '../../lib/format'
|
||||
import { EVAL_TEXT } from '../../lib/termsEval'
|
||||
import { ANALYSIS_TEXT, isAnalysisOp } from '../../lib/termsAnalysis'
|
||||
import { paramSpecs, paramText, paramValue, type ParamSpec } from '../../canvas/analysisItems'
|
||||
import { useCatalog } from '../../store/catalog'
|
||||
import type { Run } from '../../types'
|
||||
import type {
|
||||
@@ -509,35 +511,19 @@ function SqlPart({ suiteId: _suiteId, sourceId, setSourceId, sql, setSql, previe
|
||||
// 分析期望：贡献、判定
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type ParamKind = 'column' | 'columns' | 'text' | 'int' | 'choice' | 'pairs'
|
||||
interface ParamField { key: string; kind: ParamKind; choices?: Record<string, string> }
|
||||
|
||||
const A = EVAL_TEXT.analysis
|
||||
/** 判定期望能用的分析（贡献恒为变化拆解）、各自的模式（同服务端 contract.MODES） */
|
||||
const VERDICT_OPS: EvalAnalysisOp[] = ['baseline', 'trend', 'differ', 'comparable']
|
||||
const OP_MODES: Partial<Record<EvalAnalysisOp, EvalAnalysisMode[]>> = {
|
||||
decompose: ['additive', 'ratio', 'factor'], differ: ['rate', 'mean', 'rows'],
|
||||
}
|
||||
const col = (key: string): ParamField => ({ key, kind: 'column' })
|
||||
const PERIODS: ParamField[] = [col('period'), { key: 'base', kind: 'text' }, { key: 'target', kind: 'text' }]
|
||||
const LABEL: ParamField = { key: 'label', kind: 'text' }
|
||||
const GRAIN: ParamField = { key: 'grain', kind: 'choice', choices: A.grain }
|
||||
const GROUPS: ParamField[] = [{ key: 'by', kind: 'columns' }, { key: 'pairs', kind: 'pairs' }, { key: 'reference', kind: 'text' }]
|
||||
/** 每种分析（模式）在表单里填哪几个参数：参数名同分析工具（服务端按同一份参数定义校验），叫法在 termsEval */
|
||||
const PARAM_FIELDS: Record<string, ParamField[]> = {
|
||||
'decompose:additive': [{ key: 'by', kind: 'columns' }, ...PERIODS, col('measure'), LABEL],
|
||||
'decompose:ratio': [{ key: 'by', kind: 'columns' }, ...PERIODS, col('numerator'), col('denominator'), LABEL],
|
||||
'decompose:factor': [...PERIODS, col('product'), { key: 'factors', kind: 'columns' }, LABEL],
|
||||
baseline: [col('period'), col('measure'), { key: 'target', kind: 'text' },
|
||||
{ key: 'basis', kind: 'choice', choices: A.basis }, GRAIN, { key: 'lag', kind: 'int' }, LABEL],
|
||||
trend: [col('period'), col('measure'), GRAIN, LABEL],
|
||||
'differ:rate': [...GROUPS, col('numerator'), col('denominator'), LABEL],
|
||||
'differ:mean': [...GROUPS, col('mean'), col('sd'), col('sum'), col('sumsq'), col('n'), LABEL],
|
||||
'differ:rows': [...GROUPS, col('value'), LABEL],
|
||||
comparable: [...PERIODS, col('measure'), GRAIN],
|
||||
}
|
||||
const fieldsOf = (e: EvalExpectation): ParamField[] =>
|
||||
PARAM_FIELDS[e.op === 'decompose' || e.op === 'differ' ? `${e.op}:${e.mode}` : e.op ?? ''] ?? []
|
||||
/**
|
||||
* 每种分析（模式）在表单里填哪几个参数：和分析卡的一项同一份（canvas/analysisItems.paramSpecs），不另抄。
|
||||
* 那张表里本来就没有评测期望不许写的几样——输入（query）、总数核对（headline）、下钻（parent）、两份查询
|
||||
* （base_query、target_query）：期望的输入只有期望 SQL 这一份快照（服务端 judgements._NOT_IN_EXPECTATION）
|
||||
*/
|
||||
const fieldsOf = (e: EvalExpectation): ParamSpec[] =>
|
||||
e.op && isAnalysisOp(e.op) ? paramSpecs(e.op, e.mode) : []
|
||||
|
||||
/** 换了分析方法、模式时，只留下新的那种分析还认的参数 */
|
||||
function keepParams(e: EvalExpectation): EvalExpectation {
|
||||
@@ -607,7 +593,7 @@ function AnalysisPick({ columns, main, setMain }: {
|
||||
</Field>
|
||||
)}
|
||||
{fieldsOf(current).map((f) => (
|
||||
<ParamInput key={`${current.op}:${current.mode}:${f.key}`} field={f} columns={columns}
|
||||
<ParamInput key={`${current.op}:${current.mode}:${f.key}`} spec={f} columns={columns}
|
||||
value={(current.params ?? {})[f.key]} onChange={(v) => setParam(f.key, v)} />
|
||||
))}
|
||||
{fieldsOf(current).some((f) => f.key === 'base') && (
|
||||
@@ -619,15 +605,17 @@ function AnalysisPick({ columns, main, setMain }: {
|
||||
)
|
||||
}
|
||||
|
||||
function ParamInput({ field, columns, value, onChange }: {
|
||||
field: ParamField; columns: string[]; value: unknown; onChange: (v: unknown) => void
|
||||
function ParamInput({ spec, columns, value, onChange }: {
|
||||
spec: ParamSpec; columns: string[]; value: unknown; onChange: (v: unknown) => void
|
||||
}) {
|
||||
const id = `eval-analysis-param-${field.key}`
|
||||
const label = A.param[field.key] ?? field.key
|
||||
if (field.kind === 'columns') {
|
||||
const key = spec.key
|
||||
const id = `eval-analysis-param-${key}`
|
||||
const label = A.param[key] ?? key
|
||||
const text = paramText(spec.kind, value)
|
||||
if (spec.kind === 'columns') {
|
||||
const picked = Array.isArray(value) ? value.map(String) : []
|
||||
return (
|
||||
<div className="sm:col-span-2" data-eval-analysis-param={field.key}>
|
||||
<div className="sm:col-span-2" data-eval-analysis-param={key}>
|
||||
<div className="label">{label}</div>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
{columns.map((c) => (
|
||||
@@ -641,34 +629,46 @@ function ParamInput({ field, columns, value, onChange }: {
|
||||
</div>
|
||||
)
|
||||
}
|
||||
if (field.kind === 'pairs') {
|
||||
const text = Array.isArray(value) ? value.map((p) => (Array.isArray(p) ? p.join('，') : String(p))).join('\n') : ''
|
||||
if (spec.kind === 'pairs') {
|
||||
return (
|
||||
<Field label={label} htmlFor={id} hint={T.analysisPairsHint} className="sm:col-span-2">
|
||||
<textarea id={id} className="field min-h-12 text-xs" defaultValue={text} data-eval-analysis-param={field.key}
|
||||
onChange={(e) => onChange(e.target.value.split('\n').map((line) => line.split(/[,，、]/).map((x) => x.trim())
|
||||
.filter(Boolean)).filter((pair) => pair.length))} />
|
||||
<textarea id={id} className="field min-h-12 text-xs" defaultValue={text} data-eval-analysis-param={key}
|
||||
onChange={(e) => onChange(paramValue('pairs', e.target.value))} />
|
||||
</Field>
|
||||
)
|
||||
}
|
||||
if (field.kind === 'column' || field.kind === 'choice') {
|
||||
const options = field.kind === 'column' ? Object.fromEntries(columns.map((c) => [c, c])) : field.choices ?? {}
|
||||
if (spec.kind === 'basis' || spec.kind === 'grain') {
|
||||
// 和分析卡同两份下拉：空着是缺省（比较方式缺省「和比较期比的变化率」、粒度按期的写法推断），其余每种各有叫法
|
||||
const opts = spec.kind === 'basis' ? A.basis : A.grain
|
||||
return (
|
||||
<Field label={label} htmlFor={id}>
|
||||
<select id={id} className="field" value={value == null ? '' : String(value)} data-eval-analysis-param={field.key}
|
||||
<select id={id} className="field" value={text} data-eval-analysis-param={key}
|
||||
onChange={(e) => onChange(e.target.value)}>
|
||||
<option value="">{field.kind === 'column' ? T.analysisColumn : T.analysisNone}</option>
|
||||
{Object.entries(opts).map(([k, v]) => <option key={k} value={k}>{v}</option>)}
|
||||
</select>
|
||||
</Field>
|
||||
)
|
||||
}
|
||||
if (spec.kind === 'column') {
|
||||
const options = Object.fromEntries(columns.map((c) => [c, c]))
|
||||
return (
|
||||
<Field label={label} htmlFor={id}>
|
||||
<select id={id} className="field" value={text} data-eval-analysis-param={key}
|
||||
onChange={(e) => onChange(e.target.value)}>
|
||||
<option value="">{T.analysisColumn}</option>
|
||||
{Object.entries(options).map(([k, v]) => <option key={k} value={k}>{v}</option>)}
|
||||
</select>
|
||||
</Field>
|
||||
)
|
||||
}
|
||||
// 期次、对照组、叫法可以写 {{ schedule.* }} 这类时间变量；整数的格子写了不是整数的字照原样交给服务端校验
|
||||
return (
|
||||
<Field label={label} htmlFor={id}>
|
||||
<input id={id} className={clsx('field', field.kind === 'int' && 'tnum w-24')} value={value == null ? '' : String(value)}
|
||||
inputMode={field.kind === 'int' ? 'numeric' : undefined} placeholder={field.kind === 'text' ? T.analysisNone : undefined}
|
||||
data-eval-analysis-param={field.key}
|
||||
onChange={(e) => onChange(field.kind === 'int' ? (e.target.value.trim() ? Number(e.target.value) : '') : e.target.value)} />
|
||||
<input id={id} className={clsx('field', spec.kind === 'int' && 'tnum w-24')} value={text}
|
||||
inputMode={spec.kind === 'int' ? 'numeric' : undefined}
|
||||
placeholder={ANALYSIS_TEXT.node.paramPlaceholder[key]}
|
||||
data-eval-analysis-param={key}
|
||||
onChange={(e) => onChange(paramValue(spec.kind, e.target.value))} />
|
||||
</Field>
|
||||
)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user