"""仓库里附的分析题集(tests/fixtures/eval/analysis_suite.json):在虚构库 retail_trends(按今天生成)上导入,每道题都试算 通过,分析期望都算得出来、判定和库里埋的效应一致(导入走接口:按数据源名对上、逐题过校验、试算一次)。今天是周一时本周还 没有数据,「这周和上周比」那一题的期望算不了(照实记,这一项不能评分),导入照样成功。""" from __future__ import annotations import json from datetime import date from pathlib import Path from urllib.parse import quote import pytest from httpx import ASGITransport, AsyncClient from app.engine.runner import run_manager from app.eval import judgements from app.main import app from tests import eval_samples as es from tests.fixtures.acceptance import retail_trends SUITE_FILE = Path(__file__).parent / "fixtures" / "eval" / "analysis_suite.json" ACTOR = {"X-Actor": quote("评测员甲")} #: 每道题的分析期望该算出的判定类(库里埋的效应,见 retail_trends 的说明) WANTED = {1: "source", 2: "mix", 3: "normal", 4: "anomaly", 5: "rising", 6: "source", 7: "no_evidence", 8: "incomparable"} @pytest.fixture(autouse=True) async def engine_up(): await run_manager.setup() yield await es.drop_suites() await run_manager.shutdown() async def test_the_analysis_suite_imports_and_every_expectation_computes(tmp_path, scratch_source): path = retail_trends.build(tmp_path / "trends.db", date.today()) sid = await scratch_source(retail_trends.SOURCE_NAME, path, schema=True) content = json.loads(SUITE_FILE.read_text(encoding="utf-8")) assert {c["seq"] for c in content["cases"]} == set(WANTED) assert all(c["gold"]["datasource"] == retail_trends.SOURCE_NAME for c in content["cases"]) async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: res = await client.post("/api/evals/suites/import", json={"content": content}, headers=ACTOR) assert res.status_code == 201, res.text assert res.json()["imported"] == len(WANTED) and res.json()["skipped"] == [] suite = res.json()["suite"] assert all(c["gold"]["datasource_id"] == sid for c in suite["cases"]) monday = date.today().weekday() == 0 for case in suite["cases"]: assert case["trial"]["status"] == "ok", (case["seq"], case["trial"]) trial = (await client.post(f"/api/evals/suites/{suite['id']}/cases/{case['id']}/trial")).json()["trial"] assert trial["status"] == "ok", (case["seq"], trial["reason"]) [values] = [s["values"] for s in trial["sets"]] found = [v for v in values.values() if v["kind"] in judgements.ANALYSIS_KINDS] assert len(found) == 1, (case["seq"], values) if monday and case["seq"] == 8: assert found[0]["status"] == "refused", found[0] continue assert found[0]["status"] == "ok" and found[0]["judgement"] == WANTED[case["seq"]], (case["seq"], found[0]) # 期望值里没有人填的数:来源、判定都是算子在期望 SQL 上算的,判定语由系统渲染 assert found[0]["phrase"] and found[0]["result"]["inputs"][0]["ref"] == judgements.GOLD_REF for exp in case["expectations"]: assert not set(exp) & set(judgements.JUDGEMENTS) and "subjects" not in exp