mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 06:31:58 +00:00
ThearchyHelios
454e82e211
- test_eval_analysis_runs.py：TZ 从题集读，新增 now()（datetime.now(ZoneInfo(TZ))），生成库用 now().date()，原来写死的 Asia/Shanghai 去掉 - test_eval_analysis_suite.py：从题集内容取时区算今天，判周一也按它 - test_evals_api.py：零售样例题集的「今天」同样从题集取时区
66 lines
3.6 KiB
Python
66 lines
3.6 KiB
Python
"""仓库里附的分析题集（tests/fixtures/eval/analysis_suite.json）：在虚构库 retail_trends（按今天生成）上导入，每道题都试算
|
|
通过，分析期望都算得出来、判定和库里埋的效应一致（导入走接口：按数据源名对上、逐题过校验、试算一次）。今天是周一时本周还
|
|
没有数据，「这周和上周比」那一题的期望算不了（照实记，这一项不能评分），导入照样成功。"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from urllib.parse import quote
|
|
from zoneinfo import ZoneInfo
|
|
|
|
import pytest
|
|
from httpx import ASGITransport, AsyncClient
|
|
|
|
from app.engine.runner import run_manager
|
|
from app.eval import judgements
|
|
from app.main import app
|
|
from tests import eval_samples as es
|
|
from tests.fixtures.acceptance import retail_trends
|
|
|
|
SUITE_FILE = Path(__file__).parent / "fixtures" / "eval" / "analysis_suite.json"
|
|
ACTOR = {"X-Actor": quote("评测员甲")}
|
|
#: 每道题的分析期望该算出的判定类（库里埋的效应，见 retail_trends 的说明）
|
|
WANTED = {1: "source", 2: "mix", 3: "normal", 4: "anomaly", 5: "rising", 6: "source", 7: "no_evidence",
|
|
8: "incomparable"}
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
async def engine_up():
|
|
await run_manager.setup()
|
|
yield
|
|
await es.drop_suites()
|
|
await run_manager.shutdown()
|
|
|
|
|
|
async def test_the_analysis_suite_imports_and_every_expectation_computes(tmp_path, scratch_source):
|
|
content = json.loads(SUITE_FILE.read_text(encoding="utf-8"))
|
|
# 「今天」按题集的时区取（题目里的期次按它算），不看本机时区：本机在别的时区时，跨日那几个钟头两边的今天不是同一天
|
|
today = datetime.now(ZoneInfo(content["suite"]["timezone"])).date()
|
|
path = retail_trends.build(tmp_path / "trends.db", today)
|
|
sid = await scratch_source(retail_trends.SOURCE_NAME, path, schema=True)
|
|
assert {c["seq"] for c in content["cases"]} == set(WANTED)
|
|
assert all(c["gold"]["datasource"] == retail_trends.SOURCE_NAME for c in content["cases"])
|
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
|
res = await client.post("/api/evals/suites/import", json={"content": content}, headers=ACTOR)
|
|
assert res.status_code == 201, res.text
|
|
assert res.json()["imported"] == len(WANTED) and res.json()["skipped"] == []
|
|
suite = res.json()["suite"]
|
|
assert all(c["gold"]["datasource_id"] == sid for c in suite["cases"])
|
|
monday = today.weekday() == 0
|
|
for case in suite["cases"]:
|
|
assert case["trial"]["status"] == "ok", (case["seq"], case["trial"])
|
|
trial = (await client.post(f"/api/evals/suites/{suite['id']}/cases/{case['id']}/trial")).json()["trial"]
|
|
assert trial["status"] == "ok", (case["seq"], trial["reason"])
|
|
[values] = [s["values"] for s in trial["sets"]]
|
|
found = [v for v in values.values() if v["kind"] in judgements.ANALYSIS_KINDS]
|
|
assert len(found) == 1, (case["seq"], values)
|
|
if monday and case["seq"] == 8:
|
|
assert found[0]["status"] == "refused", found[0]
|
|
continue
|
|
assert found[0]["status"] == "ok" and found[0]["judgement"] == WANTED[case["seq"]], (case["seq"], found[0])
|
|
# 期望值里没有人填的数：来源、判定都是算子在期望 SQL 上算的，判定语由系统渲染
|
|
assert found[0]["phrase"] and found[0]["result"]["inputs"][0]["ref"] == judgements.GOLD_REF
|
|
for exp in case["expectations"]:
|
|
assert not set(exp) & set(judgements.JUDGEMENTS) and "subjects" not in exp
|