mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 22:11:58 +00:00
ThearchyHelios
83cd5da72e
- tests/fixtures/acceptance/retail_trends.py（python -m tests.fixtures.acceptance.retail_trends <db> [--today]，数据源名 retail_trends，不动 retail.py）：华东线下客单价降到七成带来的销售额下降（主要来自华东）；三成线下订单转到线上造成的退款率辛普森（各渠道都降、整体升，结构为主）；订单数的常规波动；取消单激增；线上销售额连涨十周以上；线上销售额的变化来自订单数；西南一家小店的退款率看着高但样本太少；本周没过完（和上周不可比，周一时本周还没有数据、那一题期望算不了）。订单数按店按周定好、金额在窄区间随机、退款按渠道比例挑，各效应不靠运气 - tests/fixtures/eval/analysis_suite.json：八道题，问法用使用者的说法（上周销售额为什么下降？退款率为什么上升了？上周订单数下降算不算异常？……），各配一个贡献或判定期望，订单数、取消单两题另配一个不必需的「一个数」 - 测试：同一个今天逐字节相同；四个不同的今天（周三、周日、周六、跨年）上用 app.analysis.run 按题集的期望 SQL 核对八个效应；周一那一题照实算不了；走接口导入题集、逐题试算，分析期望都算得出、判定和埋的效应一致；真运行的算期望（作答前后对得上、阈值按设置快照、参数对不上照实说）、读作答运行里的分析结论和引用、重评老结果、从头到尾评一题（助手剧本搭查库 → 分析卡 → 报告，对且可追溯、逐题 E1 等价已证明）
63 lines
3.4 KiB
Python
63 lines
3.4 KiB
Python
"""仓库里附的分析题集（tests/fixtures/eval/analysis_suite.json）：在虚构库 retail_trends（按今天生成）上导入，每道题都试算
|
|
通过，分析期望都算得出来、判定和库里埋的效应一致（导入走接口：按数据源名对上、逐题过校验、试算一次）。今天是周一时本周还
|
|
没有数据，「这周和上周比」那一题的期望算不了（照实记，这一项不能评分），导入照样成功。"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import date
|
|
from pathlib import Path
|
|
from urllib.parse import quote
|
|
|
|
import pytest
|
|
from httpx import ASGITransport, AsyncClient
|
|
|
|
from app.engine.runner import run_manager
|
|
from app.eval import judgements
|
|
from app.main import app
|
|
from tests import eval_samples as es
|
|
from tests.fixtures.acceptance import retail_trends
|
|
|
|
SUITE_FILE = Path(__file__).parent / "fixtures" / "eval" / "analysis_suite.json"
|
|
ACTOR = {"X-Actor": quote("评测员甲")}
|
|
#: 每道题的分析期望该算出的判定类（库里埋的效应，见 retail_trends 的说明）
|
|
WANTED = {1: "source", 2: "mix", 3: "normal", 4: "anomaly", 5: "rising", 6: "source", 7: "no_evidence",
|
|
8: "incomparable"}
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
async def engine_up():
|
|
await run_manager.setup()
|
|
yield
|
|
await es.drop_suites()
|
|
await run_manager.shutdown()
|
|
|
|
|
|
async def test_the_analysis_suite_imports_and_every_expectation_computes(tmp_path, scratch_source):
|
|
path = retail_trends.build(tmp_path / "trends.db", date.today())
|
|
sid = await scratch_source(retail_trends.SOURCE_NAME, path, schema=True)
|
|
content = json.loads(SUITE_FILE.read_text(encoding="utf-8"))
|
|
assert {c["seq"] for c in content["cases"]} == set(WANTED)
|
|
assert all(c["gold"]["datasource"] == retail_trends.SOURCE_NAME for c in content["cases"])
|
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
|
res = await client.post("/api/evals/suites/import", json={"content": content}, headers=ACTOR)
|
|
assert res.status_code == 201, res.text
|
|
assert res.json()["imported"] == len(WANTED) and res.json()["skipped"] == []
|
|
suite = res.json()["suite"]
|
|
assert all(c["gold"]["datasource_id"] == sid for c in suite["cases"])
|
|
monday = date.today().weekday() == 0
|
|
for case in suite["cases"]:
|
|
assert case["trial"]["status"] == "ok", (case["seq"], case["trial"])
|
|
trial = (await client.post(f"/api/evals/suites/{suite['id']}/cases/{case['id']}/trial")).json()["trial"]
|
|
assert trial["status"] == "ok", (case["seq"], trial["reason"])
|
|
[values] = [s["values"] for s in trial["sets"]]
|
|
found = [v for v in values.values() if v["kind"] in judgements.ANALYSIS_KINDS]
|
|
assert len(found) == 1, (case["seq"], values)
|
|
if monday and case["seq"] == 8:
|
|
assert found[0]["status"] == "refused", found[0]
|
|
continue
|
|
assert found[0]["status"] == "ok" and found[0]["judgement"] == WANTED[case["seq"]], (case["seq"], found[0])
|
|
# 期望值里没有人填的数：来源、判定都是算子在期望 SQL 上算的，判定语由系统渲染
|
|
assert found[0]["phrase"] and found[0]["result"]["inputs"][0]["ref"] == judgements.GOLD_REF
|
|
for exp in case["expectations"]:
|
|
assert not set(exp) & set(judgements.JUDGEMENTS) and "subjects" not in exp
|