mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 02:31:58 +00:00
ThearchyHelios
7f14313416
- cases.resolve_case：口径题钉的版本没发布过或已退役的，保存时照实拒收（422，field=gold.version）；以前要等试算跑到 升版判断才说得出。 - 处置升版（recompute / incomparable）改钉新版本之前，按新版本再过一遍 resolve_case（入口字段、指标），说得清是哪一项不行。 - 接口里的试算要等一次运行跑完（最多两分钟）：先结束这个请求的读事务再等，不一直开着它（导入时同样）。 - 影子起草：只写出一部分的表另记（tables_partial），快照照出、状态 partial；一张都没起草出来才算失败、不出快照。以前 只写出一部分的表算成没起草出来，几张表都只写出一部分时整次记失败、写好的部分也丢了。 - 题集里推起草哪些表时，期望 SQL 按题集的时区渲染。
519 lines
30 KiB
Python
519 lines
30 KiB
Python
"""评测接口（app/api/evals.py）：题集和题目的增删改查（乐观锁）、保存时试算、从运行认定、口径升版处置、导出导入、
|
|
目录快照；零售样例题集（tests/fixtures/eval/retail_suite.json）在虚构零售库上试算通过。
|
|
|
|
库、表、数字、人名都是虚构的。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sqlite3
|
|
import uuid
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
from urllib.parse import quote
|
|
from zoneinfo import ZoneInfo
|
|
|
|
import pytest
|
|
from httpx import ASGITransport, AsyncClient
|
|
from sqlalchemy import delete
|
|
|
|
from app.data import introspect
|
|
from app.db.base import SessionLocal
|
|
from app.db.models import DataSource, Run, Workflow, WorkflowVersion
|
|
from app.db.models_eval import EvalRun, EvalSuite
|
|
from app.engine.runner import run_manager
|
|
from app.eval import snapshots
|
|
from app.main import app
|
|
from tests.fixtures import dual_check_shop as shop
|
|
from tests.fixtures.acceptance import retail
|
|
from tests.fixtures.catalog.fake_model import FakeModel
|
|
from tests.fixtures.sources import drop_source
|
|
|
|
SUITE_FILE = Path(__file__).parent / "fixtures" / "eval" / "retail_suite.json"
|
|
LAST_WEEK = "BETWEEN '{{ schedule.last_week_start }}' AND '{{ schedule.last_week_end }}'"
|
|
#: dual_check_shop 的订单都在 2026-09：「上周」按截至时间算，这里用写死日期的 SQL
|
|
SEPT = "BETWEEN '2026-09-01' AND '2026-09-07'"
|
|
ACTOR = {"X-Actor": quote("评测员甲")}
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
async def engine_up():
|
|
await run_manager.setup()
|
|
yield
|
|
await run_manager.shutdown()
|
|
|
|
|
|
@pytest.fixture
|
|
async def client():
|
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
|
yield c
|
|
|
|
|
|
@pytest.fixture
|
|
async def source(tmp_path):
|
|
name, sid = await shop.add_source(shop.build(tmp_path / "shop.db"))
|
|
yield {"name": name, "id": sid}
|
|
await drop_source(sid)
|
|
|
|
|
|
async def new_suite(client, **body) -> dict:
|
|
res = await client.post("/api/evals/suites", json={"name": f"虚构题集_{uuid.uuid4().hex[:6]}", **body}, headers=ACTOR)
|
|
assert res.status_code == 201, res.text
|
|
return res.json()
|
|
|
|
|
|
def sql_case(source_id: str, sql: str = f"SELECT SUM(amount) AS gmv FROM orders WHERE status <> 'refunded' "
|
|
f"AND paid_date {SEPT}", **kw) -> dict:
|
|
return {"question": "9 月第一周按支付日、不含退款的销售额？", "datasource_ids": [source_id],
|
|
"gold": {"kind": "sql", "datasource_id": source_id, "sql": sql},
|
|
"expectations": [{"kind": "scalar", "column": "gmv"}], **kw}
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 题集
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def test_suites_crud_with_optimistic_lock(client):
|
|
made = await new_suite(client, description="说明", budget={"max_parallel": 3})
|
|
assert made["created_by"] == "评测员甲" and made["version"] == 1 and made["timezone"] == "Asia/Shanghai"
|
|
assert made["budget"] == {"max_parallel": 3}
|
|
assert made["effective_budget"] == {"max_parallel": 3, "per_case_timeout_s": 900, "per_run_max_usd": 2.0,
|
|
"daily_max_usd": 5.0}
|
|
listed = (await client.get("/api/evals/suites")).json()["suites"]
|
|
assert any(s["id"] == made["id"] and s["counts"] == {"cases": 0, "enabled": 0} for s in listed)
|
|
|
|
res = await client.patch(f"/api/evals/suites/{made['id']}", json={"name": "改了名"})
|
|
assert res.status_code == 422 and res.json()["field"] == "version"
|
|
res = await client.patch(f"/api/evals/suites/{made['id']}", json={"name": "改了名", "version": 1})
|
|
assert res.status_code == 200 and res.json()["name"] == "改了名" and res.json()["version"] == 2
|
|
assert res.json()["budget"] == {"max_parallel": 3}, "没带的项照旧"
|
|
res = await client.patch(f"/api/evals/suites/{made['id']}", json={"name": "再改", "version": 1})
|
|
assert res.status_code == 409 and res.json()["code"] == "eval_version_conflict"
|
|
|
|
for body, field in (({"name": ""}, "name"), ({"name": "x", "timezone": "火星"}, "timezone"),
|
|
({"name": "x", "budget": {"per_case_timeout_s": 5}}, "budget.per_case_timeout_s"),
|
|
({"name": "x", "budget": {"per_run_max_usd": 50, "daily_max_usd": 10}}, "budget.per_run_max_usd"),
|
|
({"name": "x", "budget": {"nope": 1}}, "budget.nope")):
|
|
res = await client.post("/api/evals/suites", json=body)
|
|
assert res.status_code == 422 and res.json()["field"] == field, (body, res.text)
|
|
|
|
# 有评测记录的不能删，只能归档
|
|
async with SessionLocal() as session:
|
|
session.add(EvalRun(suite_id=made["id"], status="succeeded"))
|
|
await session.commit()
|
|
res = await client.delete(f"/api/evals/suites/{made['id']}")
|
|
assert res.status_code == 409 and res.json()["code"] == "eval_suite_has_runs"
|
|
async with SessionLocal() as session:
|
|
await session.execute(delete(EvalRun).where(EvalRun.suite_id == made["id"]))
|
|
await session.commit()
|
|
assert (await client.delete(f"/api/evals/suites/{made['id']}")).status_code == 204
|
|
assert (await client.get(f"/api/evals/suites/{made['id']}")).status_code == 404
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 题目：保存时试算
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def test_a_case_is_saved_only_after_a_trial_passes(client, source):
|
|
suite = await new_suite(client)
|
|
url = f"/api/evals/suites/{suite['id']}/cases"
|
|
res = await client.post(url, json=sql_case(source["id"]), headers=ACTOR)
|
|
assert res.status_code == 201, res.text
|
|
body = res.json()
|
|
case, trial = body["case"], body["trial"]
|
|
assert case["seq"] == 1 and case["version"] == 1 and case["created_by"] == "评测员甲" and len(case["case_hash"]) == 64
|
|
assert trial["status"] == "ok" and trial["sets"][0]["values"]["e1"]["value"] == 330.0
|
|
# 题目上只记试算的结论和运行，不存期望值（期望每次重算）
|
|
assert case["trial"]["status"] == "ok" and case["trial"]["by"] == "评测员甲" and "sets" not in case["trial"]
|
|
assert case["gold_source"] == {"id": source["id"], "name": source["name"]}
|
|
async with SessionLocal() as session:
|
|
run = await session.get(Run, trial["run_ids"][0])
|
|
assert run.origin_kind == "eval" and run.origin["trial"] is True and run.origin["suite_id"] == suite["id"]
|
|
|
|
# 只试算、不保存
|
|
res = await client.post(f"{url}/trial", json=sql_case(source["id"], sql="SELECT nope FROM orders"))
|
|
assert res.status_code == 200 and res.json()["trial"]["status"] == "failed"
|
|
|
|
for bad, code, field in (
|
|
(sql_case(source["id"], expectations=[{"kind": "scalar", "column": "gmv", "value": 330}]),
|
|
"eval_case_invalid", "expectations.0.value"), # 评测-D4：数值期望不许人填
|
|
(sql_case(source["id"], history=[{"question": "上一轮"}]), "eval_case_invalid", "history"), # 评测-D12
|
|
(sql_case(source["id"], sql="DELETE FROM orders"), "eval_gold_guard", "gold.sql"),
|
|
(sql_case(source["id"], sql="SELECT SUM(amount) AS gmv FROM orders WHERE {{ schedule.nope }}"),
|
|
"eval_case_invalid", "gold.sql"),
|
|
(sql_case(source["id"], sql="SELECT region FROM orders WHERE region = '华中'"), "eval_gold_no_value", "gold"),
|
|
({"question": "q", "expectations": [{"kind": "scalar", "column": "gmv"}]}, "eval_case_invalid", "gold"),
|
|
({"question": "最近卖得怎么样", "expectations": [{"kind": "ask"}, {"kind": "none"}],
|
|
"gold": {"kind": "sql", "datasource_id": source["id"], "sql": "SELECT 1"}}, "eval_case_invalid", "expectations"),
|
|
(sql_case("nope"), "eval_case_invalid", "gold.datasource_id"),
|
|
):
|
|
res = await client.post(url, json=bad)
|
|
assert res.status_code == 422, (bad, res.text)
|
|
assert (res.json()["code"], res.json().get("field")) == (code, field), res.json()
|
|
detail = (await client.get(f"/api/evals/suites/{suite['id']}")).json()
|
|
assert [c["seq"] for c in detail["cases"]] == [1], "没过的一道都没存"
|
|
|
|
|
|
async def test_masked_columns_are_refused_at_save(client, tmp_path):
|
|
name, sid = await shop.add_source(shop.build(tmp_path / "masked.db"), options={"mask_columns": ["region"]})
|
|
try:
|
|
suite = await new_suite(client)
|
|
body = sql_case(sid, sql="SELECT region AS 地区, SUM(amount) AS gmv FROM orders GROUP BY region",
|
|
expectations=[{"kind": "table", "keys": ["地区"]}])
|
|
res = await client.post(f"/api/evals/suites/{suite['id']}/cases", json=body)
|
|
assert res.status_code == 422 and res.json()["code"] == "eval_gold_masked" and "遮罩" in res.json()["detail"]
|
|
finally:
|
|
await drop_source(sid)
|
|
|
|
|
|
async def test_editing_a_case_retries_only_when_its_content_changes(client, source):
|
|
suite = await new_suite(client)
|
|
made = (await client.post(f"/api/evals/suites/{suite['id']}/cases", json=sql_case(source["id"]))).json()["case"]
|
|
url = f"/api/evals/suites/{suite['id']}/cases/{made['id']}"
|
|
res = await client.patch(url, json={"tags": ["口径陷阱"], "version": 1})
|
|
assert res.status_code == 200 and res.json()["trial"] is None, "只改标签不试算"
|
|
case = res.json()["case"]
|
|
assert case["tags"] == ["口径陷阱"] and case["version"] == 2 and case["case_hash"] == made["case_hash"]
|
|
res = await client.patch(url, json={"gold": {"kind": "sql", "datasource_id": source["id"],
|
|
"sql": f"SELECT SUM(amount) AS gmv FROM orders WHERE paid_date {SEPT}"},
|
|
"version": 2})
|
|
assert res.status_code == 200 and res.json()["trial"]["sets"][0]["values"]["e1"]["value"] == 380.0
|
|
assert res.json()["case"]["case_hash"] != made["case_hash"], "题目改过，指纹跟着变"
|
|
res = await client.patch(url, json={"gold": {"kind": "sql", "datasource_id": source["id"],
|
|
"sql": "DELETE FROM orders"}, "version": 3})
|
|
assert res.status_code == 422 and res.json()["code"] == "eval_gold_guard"
|
|
res = await client.patch(url, json={"tags": [], "version": 2})
|
|
assert res.status_code == 409
|
|
# 已有的题按此刻再试算一次：记进 trial、不改 version
|
|
res = await client.post(f"{url}/trial")
|
|
assert res.status_code == 200 and res.json()["trial"]["status"] == "ok"
|
|
assert (await client.get(f"/api/evals/suites/{suite['id']}")).json()["cases"][0]["version"] == 3
|
|
assert (await client.delete(url)).status_code == 204
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 从运行认定
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def test_a_query_from_a_run_becomes_a_case(client, source):
|
|
from app.engine.launch import launch_run
|
|
|
|
sql = f"SELECT SUM(amount) AS gmv FROM orders WHERE status <> 'refunded' AND paid_date {SEPT}"
|
|
graph = {"nodes": [{"id": "start", "type": "input", "data": {"config": {}}},
|
|
{"id": "q", "type": "tool", "data": {"config": {"tool": f"db_query__{source['name']}",
|
|
"args": {"sql": sql}}}},
|
|
{"id": "out", "type": "output", "data": {"config": {}}}],
|
|
"edges": [{"source": "start", "target": "q"}, {"source": "q", "target": "out"}]}
|
|
async with SessionLocal() as session:
|
|
run = await launch_run(session, graph=graph, input_payload={"question": "9 月第一周卖了多少？"}, actor="提问人",
|
|
origin_kind="chat")
|
|
await run_manager.wait_idle(run.id, timeout=30)
|
|
queries = (await client.get(f"/api/evals/runs/{run.id}/queries")).json()["queries"]
|
|
assert len(queries) == 1 and queries[0]["sql"] == sql and queries[0]["source_id"] == source["id"]
|
|
assert "rows" not in queries[0], "只给 SQL、列名、行数，不给结果的格"
|
|
suite = await new_suite(client)
|
|
res = await client.post(f"/api/evals/suites/{suite['id']}/cases/from-run", headers=ACTOR,
|
|
json={"run_id": run.id, "query": queries[0]["artifact"],
|
|
"expectations": [{"kind": "scalar", "column": "gmv"}]})
|
|
assert res.status_code == 201, res.text
|
|
case = res.json()["case"]
|
|
assert case["question"] == "9 月第一周卖了多少？", "问法没给就取那次运行的问题"
|
|
gold = case["gold"]
|
|
assert gold["kind"] == "from_run" and gold["sql"] == sql and gold["run_id"] == run.id
|
|
assert gold["confirmed_by"] == "评测员甲" and gold["confirmed_at"] and gold["query"] == queries[0]["artifact"]
|
|
assert res.json()["trial"]["sets"][0]["values"]["e1"]["value"] == 330.0
|
|
# 之后按这条 SQL 重算，不冻结数值；要改 SQL 就改成 SQL 题，不能在「从运行认定」上改
|
|
res = await client.patch(f"/api/evals/suites/{suite['id']}/cases/{case['id']}",
|
|
json={"gold": {**gold, "sql": "SELECT 1 AS gmv"}, "version": 1})
|
|
assert res.status_code == 422 and res.json()["field"] == "gold.kind"
|
|
res = await client.patch(f"/api/evals/suites/{suite['id']}/cases/{case['id']}",
|
|
json={"gold": gold, "tags": ["从运行"], "version": 1})
|
|
assert res.status_code == 200 and res.json()["case"]["gold"]["confirmed_by"] == "评测员甲"
|
|
res = await client.post(f"/api/evals/suites/{suite['id']}/cases/from-run",
|
|
json={"run_id": run.id, "query": "nope", "expectations": [{"kind": "none"}]})
|
|
assert res.status_code == 404
|
|
# 客户端不能自己写「从运行认定」
|
|
res = await client.post(f"/api/evals/suites/{suite['id']}/cases", json={**sql_case(source["id"]), "gold": gold})
|
|
assert res.status_code == 422 and res.json()["field"] == "gold.kind"
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 口径题：清单与升版处置
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
def _weekly(source_name: str, *, refunds: bool = False) -> dict:
|
|
status = "" if refunds else "status <> 'refunded' AND "
|
|
|
|
def node(nid, ntype, **config):
|
|
return {"id": nid, "type": ntype, "data": {"label": nid, "config": config}}
|
|
|
|
nodes = [node("start", "input", fields=[{"name": "start", "default": "2026-09-01"},
|
|
{"name": "end", "default": "2026-09-07"}]),
|
|
node("fetch", "tool", tool=f"db_query__{source_name}", args={"sql": (
|
|
f"SELECT SUM(amount) AS gmv FROM orders WHERE {status}"
|
|
"paid_date BETWEEN '{{ input.start }}' AND '{{ input.end }}'")}),
|
|
node("card", "metrics", caliber="周销售", metrics=[{"id": "gmv", "name": "销售额",
|
|
"expression": "cell(nodes.fetch, 0, 'gmv')"}]),
|
|
node("out", "output")]
|
|
return {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
|
|
|
|
|
async def test_caliber_cases_list_and_dispose_upgrades(client, source):
|
|
async with SessionLocal() as session:
|
|
wf = Workflow(name=f"周销售_{uuid.uuid4().hex[:6]}", graph=_weekly(source["name"]), status="published",
|
|
published_version=1, version=1)
|
|
session.add(wf)
|
|
await session.flush()
|
|
session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=_weekly(source["name"]), level="published"))
|
|
await session.commit()
|
|
wf_id = wf.id
|
|
calibers = (await client.get("/api/evals/calibers")).json()["calibers"]
|
|
mine = [c for c in calibers if c["workflow_id"] == wf_id]
|
|
assert mine == [{"workflow_id": wf_id, "workflow_name": mine[0]["workflow_name"], "version": 1,
|
|
"level": "published", "node_id": "card", "card": "card", "caliber": "周销售 @ v1",
|
|
"metric_id": "gmv", "name": "销售额", "unit": "", "reusable": True, "reason": None}]
|
|
suite = await new_suite(client)
|
|
body = {"question": "上周的销售额？", "gold": {"kind": "caliber", "workflow_id": wf_id, "version": 1,
|
|
"node_id": "card", "inputs": {}},
|
|
"expectations": [{"kind": "scalar", "metric": "gmv"}]}
|
|
res = await client.post(f"/api/evals/suites/{suite['id']}/cases", json=body)
|
|
assert res.status_code == 201, res.text
|
|
case = res.json()["case"]
|
|
assert case["upgrade"]["pending"] is False and res.json()["trial"]["sets"][0]["values"]["e1"]["value"] == 330.0
|
|
bad = {**body, "expectations": [{"kind": "scalar", "metric": "nope"}]}
|
|
assert (await client.post(f"/api/evals/suites/{suite['id']}/cases", json=bad)).json()["field"] == \
|
|
"expectations.0.metric"
|
|
|
|
async with SessionLocal() as session:
|
|
wf = await session.get(Workflow, wf_id)
|
|
wf.published_version = wf.version = 2
|
|
session.add(WorkflowVersion(workflow_id=wf_id, version=2, graph=_weekly(source["name"], refunds=True),
|
|
level="published"))
|
|
await session.commit()
|
|
detail = (await client.get(f"/api/evals/suites/{suite['id']}")).json()
|
|
assert detail["pending_upgrades"] == 1 and detail["cases"][0]["upgrade"]["message"] == "口径已升版到 v2，待处置"
|
|
url = f"/api/evals/suites/{suite['id']}/cases/{case['id']}/upgrade"
|
|
info = (await client.get(url)).json()
|
|
assert set(info["choices"]) == {"recompute", "dual", "incomparable"}
|
|
res = await client.post(url, json={"version": 1})
|
|
assert res.status_code == 422 and res.json()["code"] == "eval_dispose_invalid", "没有缺省"
|
|
# 不可比：旧题归档，按新版本另建新题
|
|
res = await client.post(url, json={"policy": "incomparable", "version": 1}, headers=ACTOR)
|
|
assert res.status_code == 200, res.text
|
|
old, new = res.json()["case"], res.json()["new_case"]
|
|
assert old["archived"] and not old["enabled"] and old["superseded_by"] == new["id"]
|
|
assert new["gold"]["version"] == 2 and new["seq"] == 2 and new["upgrade"]["pending"] is False
|
|
assert res.json()["trial"]["sets"][0]["values"]["e1"]["value"] == 380.0
|
|
detail = (await client.get(f"/api/evals/suites/{suite['id']}")).json()
|
|
assert detail["pending_upgrades"] == 0 and detail["counts"]["cases"] == 1
|
|
# 归档的题不再待处置；在新题上再来一版就改钉
|
|
async with SessionLocal() as session:
|
|
wf = await session.get(Workflow, wf_id)
|
|
wf.published_version = wf.version = 3
|
|
session.add(WorkflowVersion(workflow_id=wf_id, version=3, graph=_weekly(source["name"]), level="published"))
|
|
await session.commit()
|
|
res = await client.post(f"/api/evals/suites/{suite['id']}/cases/{new['id']}/upgrade",
|
|
json={"policy": "recompute", "version": 1}, headers=ACTOR)
|
|
assert res.status_code == 200, res.text
|
|
moved = res.json()["case"]
|
|
assert moved["gold"]["version"] == 3 and moved["gold"]["upgrade"]["by"] == "评测员甲"
|
|
assert moved["case_hash"] != new["case_hash"] and moved["version"] == 2
|
|
res = await client.post(f"/api/evals/suites/{suite['id']}/cases/{new['id']}/upgrade",
|
|
json={"policy": "dual", "version": 2})
|
|
assert res.status_code == 409 and res.json()["code"] == "eval_dispose_nothing_pending"
|
|
|
|
|
|
async def test_caliber_cases_pin_only_published_versions(client, source):
|
|
async with SessionLocal() as session:
|
|
wf = Workflow(name=f"草稿口径_{uuid.uuid4().hex[:6]}", graph=_weekly(source["name"]), status="draft", version=1)
|
|
session.add(wf)
|
|
await session.flush()
|
|
session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=_weekly(source["name"])))
|
|
await session.commit()
|
|
wf_id = wf.id
|
|
suite = await new_suite(client)
|
|
res = await client.post(f"/api/evals/suites/{suite['id']}/cases", json={
|
|
"question": "q", "gold": {"kind": "caliber", "workflow_id": wf_id, "version": 1, "node_id": "card"},
|
|
"expectations": [{"kind": "scalar", "metric": "gmv"}]})
|
|
assert res.status_code == 422 and res.json()["field"] == "gold.version" and "没有发布过" in res.json()["detail"]
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 导出导入
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def test_export_then_import_matches_sources_by_name(client, source):
|
|
suite = await new_suite(client, description="导出用")
|
|
await client.post(f"/api/evals/suites/{suite['id']}/cases", json=sql_case(source["id"], tags=["口径陷阱"]))
|
|
await client.post(f"/api/evals/suites/{suite['id']}/cases",
|
|
json={"question": "最近怎么样？", "expectations": [{"kind": "ask"}]})
|
|
res = await client.get(f"/api/evals/suites/{suite['id']}/export")
|
|
assert res.status_code == 200 and "attachment" in res.headers["content-disposition"]
|
|
content = res.json()
|
|
text = json.dumps(content, ensure_ascii=False)
|
|
assert content["format"] == "agentlab-eval-suite" and len(content["cases"]) == 2
|
|
# 按名字、不含期望值、答案、运行
|
|
first = content["cases"][0]
|
|
assert first["datasources"] == [source["name"]] and first["gold"]["datasource"] == source["name"]
|
|
assert "datasource_id" not in first["gold"] and source["id"] not in text
|
|
assert "330" not in text and "run_id" not in text and "trial" not in text
|
|
|
|
preview = (await client.post("/api/evals/suites/import/preview", json={"content": content})).json()
|
|
assert preview["importable"] == 2 and preview["sources"] == {source["name"]: source["id"]}
|
|
res = await client.post("/api/evals/suites/import", json={"content": content, "name": "导进来的"}, headers=ACTOR)
|
|
assert res.status_code == 201, res.text
|
|
imported = res.json()["suite"]
|
|
assert imported["name"] == "导进来的" and res.json()["imported"] == 2 and res.json()["skipped"] == []
|
|
assert [c["question"] for c in imported["cases"]] == [c["question"] for c in content["cases"]]
|
|
assert imported["cases"][0]["gold"]["datasource_id"] == source["id"]
|
|
assert imported["cases"][0]["trial"]["status"] == "ok"
|
|
|
|
# 对不上的数据源：跳过、写原因；一道都导不进来就不建题集
|
|
other = json.loads(json.dumps(content).replace(source["name"], "ghost_source"))
|
|
preview = (await client.post("/api/evals/suites/import/preview", json={"content": other})).json()
|
|
assert preview["skipped"] == 1 and "ghost_source" in preview["cases"][0]["reason"]
|
|
only_bad = {**other, "cases": other["cases"][:1]}
|
|
res = await client.post("/api/evals/suites/import", json={"content": only_bad})
|
|
assert res.status_code == 422 and "ghost_source" in res.json()["detail"]
|
|
res = await client.post("/api/evals/suites/import/preview", json={"content": {"format": "x"}})
|
|
assert res.status_code == 422 and res.json()["code"] == "eval_import_invalid"
|
|
|
|
|
|
async def test_the_retail_sample_suite_trials_on_the_retail_database(client, tmp_path):
|
|
"""仓库里附的零售样例题集：在虚构零售库上导入，每道题都试算通过，期望和直接查库一样，陷阱按口径算。"""
|
|
today = datetime.now(ZoneInfo("Asia/Shanghai")).date()
|
|
path = retail.build(tmp_path / "retail.db", today)
|
|
await drop_source(name=retail.SOURCE_NAME)
|
|
async with SessionLocal() as session:
|
|
row = DataSource(name=retail.SOURCE_NAME, kind="sqlite", database=str(path), readonly=True,
|
|
options={"mask_columns": ["phone"]})
|
|
session.add(row)
|
|
await session.flush()
|
|
row.schema_cache = await introspect.introspect(row)
|
|
await session.commit()
|
|
sid = row.id
|
|
try:
|
|
content = json.loads(SUITE_FILE.read_text(encoding="utf-8"))
|
|
res = await client.post("/api/evals/suites/import", json={"content": content}, headers=ACTOR)
|
|
assert res.status_code == 201, res.text
|
|
assert res.json()["imported"] == 6 and res.json()["skipped"] == []
|
|
suite = res.json()["suite"]
|
|
cases = {c["seq"]: c for c in suite["cases"]}
|
|
assert all(c["trial"]["status"] == "ok" for c in cases.values())
|
|
|
|
monday = today - timedelta(days=today.weekday())
|
|
start, end = (monday - timedelta(days=7)).isoformat(), (monday - timedelta(days=1)).isoformat()
|
|
|
|
def one(sql: str):
|
|
with sqlite3.connect(path) as db:
|
|
return db.execute(sql).fetchone()
|
|
|
|
async def trial(seq: int) -> dict:
|
|
out = (await client.post(f"/api/evals/suites/{suite['id']}/cases/{cases[seq]['id']}/trial")).json()["trial"]
|
|
assert out["status"] == "ok", out
|
|
assert f"'{start}' AND '{end}'" in (out["rendered"].get("sql") or f"'{start}' AND '{end}'")
|
|
return out["sets"][0]["values"] if out["sets"] else {}
|
|
|
|
paid = one(f"SELECT ROUND(SUM(amount), 2) FROM orders WHERE status = 'paid' AND paid_date BETWEEN '{start}' "
|
|
f"AND '{end}'")[0]
|
|
by_order_date = one(f"SELECT ROUND(SUM(amount), 2) FROM orders WHERE status = 'paid' AND order_date BETWEEN "
|
|
f"'{start}' AND '{end}'")[0]
|
|
with_refunds = one(f"SELECT ROUND(SUM(amount), 2) FROM orders WHERE status <> 'cancelled' AND paid_date "
|
|
f"BETWEEN '{start}' AND '{end}'")[0]
|
|
values = await trial(1)
|
|
assert values["sales"]["value"] == paid
|
|
# 陷阱确实埋着：按下单日、含退款算出来的都不一样
|
|
assert paid not in (by_order_date, with_refunds)
|
|
refunds = await trial(2)
|
|
assert (refunds["refund_orders"]["value"], refunds["refund_amount"]["value"]) == one(
|
|
f"SELECT COUNT(*), ROUND(SUM(amount), 2) FROM orders WHERE status = 'refunded' AND paid_date BETWEEN "
|
|
f"'{start}' AND '{end}'")
|
|
assert refunds["refund_orders"]["value"] > 0
|
|
cancelled = await trial(3)
|
|
assert cancelled["cancelled"]["value"] == one(
|
|
f"SELECT COUNT(*) FROM orders WHERE status = 'cancelled' AND order_date BETWEEN '{start}' AND '{end}'")[0]
|
|
regions = (await trial(4))["by_region"]
|
|
assert [r["地区"] for r in regions["rows"]] == ["华南", "华东"] and regions["ordered"] is True
|
|
assert regions["rows"][0]["销售额"] > regions["rows"][1]["销售额"]
|
|
assert (await trial(5)) == {"no_data": {"kind": "none", "rows": 0}}
|
|
ask = (await client.post(f"/api/evals/suites/{suite['id']}/cases/{cases[6]['id']}/trial")).json()["trial"]
|
|
assert ask["status"] == "ok" and ask["run_ids"] == [] and cases[6]["expectations"][0]["kind"] == "ask"
|
|
assert cases[1]["tags"] == ["口径陷阱", "支付日归属", "相对日期"]
|
|
finally:
|
|
async with SessionLocal() as session:
|
|
await session.execute(delete(EvalSuite).where(EvalSuite.name == "零售样例题集"))
|
|
await session.commit()
|
|
await drop_source(sid)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 目录快照
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def test_catalog_snapshots_live_and_shadow(client, source, monkeypatch):
|
|
res = await client.post("/api/evals/catalog-snapshots", json={"source_ids": [source["id"]]}, headers=ACTOR)
|
|
assert res.status_code == 201, res.text
|
|
live = res.json()
|
|
assert live["status"] == "succeeded" and live["view"] == {"mode": "snapshot", "statuses": "all",
|
|
"snapshot": live["artifact_id"]}
|
|
assert (await client.post("/api/evals/catalog-snapshots", json={"source_ids": []})).status_code == 422
|
|
|
|
# 测试环境没有真实接入：起草模型用不了，照实回 409，不建任务
|
|
res = await client.post("/api/evals/catalog-snapshots/shadow/estimate",
|
|
json={"sources": [{"id": source["id"], "tables": ["orders"]}]})
|
|
assert res.status_code == 409 and res.json()["code"] == snapshots.SNAPSHOT_MODEL_UNAVAILABLE
|
|
|
|
async def fake_model(session, provider, model):
|
|
return FakeModel({"orders": {"label": "影子订单"}}), "fake-draft"
|
|
|
|
monkeypatch.setattr(snapshots, "resolve_model", fake_model)
|
|
monkeypatch.setattr(snapshots.ShadowDraft, "model_override", FakeModel({"orders": {"label": "影子订单"}}))
|
|
est = (await client.post("/api/evals/catalog-snapshots/shadow/estimate",
|
|
json={"sources": [{"id": source["id"], "tables": ["orders"]}]})).json()
|
|
assert est["sources"][0]["tables"] == ["orders"] and est["estimate"]["calls"] == 1
|
|
assert est["estimate"]["unpriced"] is True and est["estimate"]["model"] == "fake-draft"
|
|
res = await client.post("/api/evals/catalog-snapshots/shadow", headers=ACTOR,
|
|
json={"sources": [{"id": source["id"], "tables": ["orders"]}], "guidance": "按支付日"})
|
|
assert res.status_code == 202 and res.json()["status"] in ("queued", "running")
|
|
snap_id = res.json()["id"]
|
|
await snapshots.manager.wait(snap_id, timeout=30)
|
|
done = (await client.get(f"/api/evals/catalog-snapshots/{snap_id}")).json()
|
|
assert done["status"] == "succeeded" and done["artifact_id"] and done["created_by"] == "评测员甲"
|
|
assert done["draft"]["guidance"] == "按支付日"
|
|
listed = (await client.get("/api/evals/catalog-snapshots")).json()["snapshots"]
|
|
assert {live["id"], snap_id} <= {s["id"] for s in listed}
|
|
res = await client.post(f"/api/evals/catalog-snapshots/{snap_id}/stop")
|
|
assert res.status_code == 409 and res.json()["code"] == snapshots.SNAPSHOT_FINISHED
|
|
assert (await client.get("/api/evals/catalog-snapshots/nope")).status_code == 404
|
|
|
|
|
|
async def test_evals_are_for_builders_only():
|
|
"""期望值和查询原文只给搭建者及以上：使用者 403，搭建者能用。"""
|
|
from tests.fakes import auth_env
|
|
|
|
try:
|
|
admin = await auth_env.turn_on_local()
|
|
await admin.aclose()
|
|
await auth_env.make_user("eval_user", "user")
|
|
await auth_env.make_user("eval_builder", "builder")
|
|
user = await auth_env.login("eval_user")
|
|
res = await user.get("/api/evals/suites")
|
|
assert res.status_code == 403 and res.json()["code"] == "role_required"
|
|
assert (await user.post("/api/evals/suites", json={"name": "x"})).status_code == 403
|
|
await user.aclose()
|
|
builder = await auth_env.login("eval_builder")
|
|
res = await builder.post("/api/evals/suites", json={"name": "搭建者的题集"})
|
|
assert res.status_code == 201 and res.json()["created_by"] == "eval_builder"
|
|
await builder.aclose()
|
|
finally:
|
|
await auth_env.reset()
|
|
async with SessionLocal() as session:
|
|
await session.execute(delete(EvalSuite).where(EvalSuite.name == "搭建者的题集"))
|
|
await session.commit()
|