Files
AgentLab/backend/tests/test_eval_analysis_runs.py
T
ThearchyHelios 83cd5da72e 虚构库 retail_trends 与分析题集:26 周的零售订单(截止昨天、固定种子),相对上一个整周埋了八个效应;题集 analysis_suite.json 每个效应一题,期望由平台的分析方法在期望 SQL 上算
- tests/fixtures/acceptance/retail_trends.py(python -m tests.fixtures.acceptance.retail_trends <db> [--today],数据源名 retail_trends,不动 retail.py):华东线下客单价降到七成带来的销售额下降(主要来自华东);三成线下订单转到线上造成的退款率辛普森(各渠道都降、整体升,结构为主);订单数的常规波动;取消单激增;线上销售额连涨十周以上;线上销售额的变化来自订单数;西南一家小店的退款率看着高但样本太少;本周没过完(和上周不可比,周一时本周还没有数据、那一题期望算不了)。订单数按店按周定好、金额在窄区间随机、退款按渠道比例挑,各效应不靠运气
- tests/fixtures/eval/analysis_suite.json:八道题,问法用使用者的说法(上周销售额为什么下降?退款率为什么上升了?上周订单数下降算不算异常?……),各配一个贡献或判定期望,订单数、取消单两题另配一个不必需的「一个数」
- 测试:同一个今天逐字节相同;四个不同的今天(周三、周日、周六、跨年)上用 app.analysis.run 按题集的期望 SQL 核对八个效应;周一那一题照实算不了;走接口导入题集、逐题试算,分析期望都算得出、判定和埋的效应一致;真运行的算期望(作答前后对得上、阈值按设置快照、参数对不上照实说)、读作答运行里的分析结论和引用、重评老结果、从头到尾评一题(助手剧本搭查库 → 分析卡 → 报告,对且可追溯、逐题 E1 等价已证明)
2026-10-21 18:12:12 +08:00

316 lines
16 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""分析期望走真运行(app/eval/gold.py、score.py、runner.py;docs/design/analysis-round/00-总方案.md §3.8)。库是虚构的
retail_trends(tests/fixtures/acceptance/retail_trends.py,按今天生成);写作模型、助手都换成剧本,不调任何模型。
- 算期望:期望 SQL 的探针运行封存的那份快照上,用平台的算子(真算子、真现查)算出判定;作答前后两次期望照样对得上(运行当天、
现查时刻、输入快照的工件不比);阈值按交进来的分析设置快照;参数和结果对不上的照实说、整题没有可用的值。
- 读答案:作答运行台账里的分析结论、报告怎么引用了它(判定语、结果表的格、句末依据)、有判断词的句子;逐题 E1 按分析结论的
输入查询比。
- 重评老结果:明细里没有分析结论那一项的照样重评(从封存重新读)。
- 从头到尾:助手剧本搭「查库 → 分析卡 → 报告」,报告引用判定语;评测给这一题「对且可追溯」,逐题 E1 等价已证明,作答运行和
算期望用的是同一份分析设置快照(记进指纹)。
"""
from __future__ import annotations
import json
from datetime import date, datetime, timedelta
from pathlib import Path
from zoneinfo import ZoneInfo
import pytest
from langchain_core.messages import AIMessageChunk
from sqlalchemy import select
from app.core import artifact_store
from app.db.base import SessionLocal
from app.db.models import Run
from app.db.models_eval import EvalCase, EvalResult, EvalRun
from app.engine.runner import run_manager
from app.eval import cases, gold, judgements, runner, score
from app.eval.gold import Context
from tests import eval_samples as es
from tests.fixtures.acceptance import retail_trends
TZ = "Asia/Shanghai"
SUITE = json.loads((Path(__file__).parent / "fixtures" / "eval" / "analysis_suite.json").read_text(encoding="utf-8"))
#: 题集里的第 1 题(上周销售额下降主要来自华东)和第 3 题(订单数的常规波动)
REGION = next(c for c in SUITE["cases"] if c["seq"] == 1)
ORDERS = next(c for c in SUITE["cases"] if c["seq"] == 3)
@pytest.fixture(autouse=True)
async def engine_up(monkeypatch):
from app.eval import snapshots
monkeypatch.setattr(snapshots, "MAX_SOURCES", 10_000)
await run_manager.setup()
yield
await runner.manager.shutdown()
await es.drop_suites()
await run_manager.shutdown()
@pytest.fixture
async def source(tmp_path, scratch_source):
path = retail_trends.build(tmp_path / "trends.db", date.today())
sid = await scratch_source(retail_trends.SOURCE_NAME, path, schema=True)
return {"id": sid, "name": retail_trends.SOURCE_NAME, "path": path}
def now() -> datetime:
return datetime.now(ZoneInfo(TZ))
def ctx(**kw) -> Context:
return Context(as_of=now(), tz=TZ, actor="评测测试", suite_id="s1", suite_name="虚构分析题集", **kw)
def case_of(raw: dict, source_id: str, *, expectations: list[dict] | None = None, sql: str | None = None) -> dict:
fields = cases.normalize_case({"question": raw["question"], "datasource_ids": [source_id],
"gold": {"kind": "sql", "datasource_id": source_id, "sql": sql or raw["gold"]["sql"]},
"expectations": expectations or raw["expectations"]})
return {**fields, "id": None, "seq": raw["seq"]}
def rendered(sql: str) -> str:
return cases.render(sql, cases.variable_values(now(), TZ))
# --------------------------------------------------------------------------
# 算期望
# --------------------------------------------------------------------------
async def test_the_gold_runs_the_platform_analysis_on_the_probe_snapshot(source):
import sqlite3
case = case_of(ORDERS, source["id"])
before = await gold.compute_gold(case, ctx())
assert before.ok, before.reason
band, orders = before.sets[0].values["band"], before.sets[0].values["orders"]
assert (band["kind"], band["status"], band["judgement"]) == ("verdict", "ok", "normal"), band
assert band["phrase"].startswith("订单数本期变化") and "常规波动范围内" in band["phrase"]
result = band["result"]
assert result["inputs"][0]["ref"] == judgements.GOLD_REF and result["inputs"][0]["artifact"] == before.sets[0].snapshot
assert result["params"]["today"] == date.today().isoformat()
assert result["settings"]["k_warn"] == 2.0, "没有设置快照(试算)时现读设置,缺省 2 倍"
# 本期齐不齐按现查的表范围判(SQL 里按日期算出来的界也在库里算一次),都记进结论
assert [(p["table"], p["column"], p["basis"]) for p in result["probes"] if "table" in p] == \
[("orders", "order_date", "live")], result["probes"]
assert all(p.get("at") for p in result["probes"])
with sqlite3.connect(source["path"]) as db:
direct = db.execute(rendered(ORDERS["gold"]["sql"])).fetchall()
assert orders["value"] == direct[24][1] and len(direct) == 25
# 作答后再算一次:快照的工件(记着查询用时)、现查时刻都不同,期望照样对得上
after = await gold.compute_gold(case, ctx(role="gold_after"))
assert after.sets[0].snapshot != before.sets[0].snapshot
assert gold.settle(before, after).status == "ok"
async def test_the_settings_snapshot_decides_the_thresholds(source):
case = case_of(ORDERS, source["id"])
outcome = await gold.compute_gold(case, ctx(analysis={"min_history": 30, "max_history": 52}))
assert outcome.ok, outcome.reason
band = outcome.sets[0].values["band"]
assert band["status"] == "refused" and band["reason_code"] == "history_short", band
assert band["result"]["settings"]["min_history"] == 30
graded = score.score_case(case, gold.settle(outcome, outcome).to_dict(), score.Answer(run_id="r"), path="ran")
assert graded.scores[0]["outcome"] == "ungradable" and graded.scores[0]["reason"].startswith("期望算不了:")
async def test_parameters_that_do_not_fit_the_sql_are_said_plainly(source):
wrong_column = case_of(REGION, source["id"], expectations=[
{"kind": "contribution", "mode": "additive", "params": {"by": ["门店"], "period": "周", "measure": "销售额"}}])
outcome = await gold.compute_gold(wrong_column, ctx())
assert outcome.status == "no_value" and "期望 SQL 里没有「门店」这一列" in outcome.reason, outcome.reason
three_weeks = REGION["gold"]["sql"].replace("'-7 days'", "'-14 days'")
outcome = await gold.compute_gold(case_of(REGION, source["id"], sql=three_weeks), ctx())
assert outcome.status == "no_value" and "有 3 期" in outcome.reason, outcome.reason
# --------------------------------------------------------------------------
# 读答案、重评老结果
# --------------------------------------------------------------------------
REPORT = ("## 上周销售额\n\n上周销售额下降:[[a:A1]]。华东的变化是 [[v:A1.r0.delta]]。\n\n"
"下降主要来自华东,其余地区变化不大。[[see:A1]]")
def _node(nid: str, ntype: str, **config) -> dict:
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
def _card_item() -> dict:
values = cases.variable_values(now(), TZ)
target = values["last_week_start"]
base = (date.fromisoformat(target) - timedelta(days=7)).isoformat()
return {"id": "where", "op": "decompose", "mode": "additive", "from": "q",
"params": {"by": ["地区"], "period": "周", "base": base, "target": target, "measure": "销售额",
"label": "销售额"}}
def answer_graph() -> dict:
nodes = [_node("start", "input"),
_node("q", "tool", tool=f"db_query__{retail_trends.SOURCE_NAME}", args={"sql": rendered(REGION["gold"]["sql"])}),
_node("card", "analysis", items=[_card_item()]),
_node("write", "report", instructions="写上周的周报"),
_node("out", "output", fields=[{"name": "答案", "value": "{{ nodes.write.text }}"}])]
ids = [n["id"] for n in nodes]
return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]}
async def _answer_run(monkeypatch) -> Run:
from tests.test_analysis_card import wait
es.use_report(monkeypatch, REPORT)
started = await run_manager.start(graph=answer_graph(), input_payload={})
row = await wait(started.id)
assert row.status == "succeeded", row.error
return row
async def test_reading_the_analyses_of_the_answer_run(monkeypatch, source):
row = await _answer_run(monkeypatch)
answer = await score.read_answer(row.id)
assert answer.problem is None and answer.report == "write"
[record] = answer.analyses
assert (record["alias"], record["op"], record["mode"], record["judgement"]) == ("A1", "decompose", "additive",
"source")
assert record["subjects"][0] == "华东" and record["direction"] == "down"
assert record["cited"] == ["phrase", "cell", "see"], record
# 判定语不是答案里的数;结果表的那一格是,维度是那一段(华东)
[cell] = answer.values
assert cell["kind"] == "cell" and cell["dims"] == {"地区": "华东"} and cell["artifact"] == record["artifact"]
# 逐题 E1 拿分析结论的输入查询比
[query] = answer.queries
assert answer.cited == [query["artifact"]] and record["inputs"] == [query["artifact"]]
[sentence] = answer.sentences
assert sentence["analyses"] == ["A1"] and "主要来自华东" in sentence["text"]
case = case_of(REGION, source["id"])
before = await gold.compute_gold(case, ctx())
settled = gold.settle(before, before).to_dict()
graded = score.score_case(case, settled, answer, path="ran")
assert graded.verdict == "pass" and graded.traced, graded.scores
assert graded.scores[0]["found"]["alias"] == "A1"
match = await score.sql_match(case, settled, answer)
assert match["status"] == "proven" and match["basis"] == "cited", match
async def test_rescoring_old_results_reads_the_analyses_from_the_seal(monkeypatch, source):
from app.eval import rescore
from app.eval.cases import case_hash
row = await _answer_run(monkeypatch)
case = case_of(REGION, source["id"])
settled = gold.settle(*(2 * [await gold.compute_gold(case, ctx())])).to_dict()
old = score.Answer(run_id=row.id, text="老的答案").to_dict()
old.pop("analyses")
old.pop("sentences")
details = await artifact_store.put_json({"format": 1, "turn": {"reply": None}, "answer": old}, kind="eval_result")
suite_id = await es.make_suite(source["id"], cases=[{k: case[k] for k in ("question", "datasource_ids", "gold",
"expectations")}])
async with SessionLocal() as session:
stored = (await session.execute(select(EvalCase).where(EvalCase.suite_id == suite_id))).scalar_one()
ev = EvalRun(suite_id=suite_id, status="succeeded", fingerprint={"scorer_version": 2})
session.add(ev)
await session.flush()
session.add(EvalResult(eval_run_id=ev.id, case_id=stored.id, case_hash=case_hash(case), status="ok",
path="ran", run_id=row.id, verdict="fail", details_artifact=details, gold=settled,
scores=[{"id": "source", "kind": "contribution", "required": True, "outcome": "wrong"}]))
await session.commit()
ev_id = ev.id
assert await rescore.rescore(ev_id) == 1
async with SessionLocal() as session:
result = (await session.execute(select(EvalResult).where(EvalResult.eval_run_id == ev_id))).scalar_one()
assert result.verdict == "pass" and result.scores[0]["outcome"] == "traced", result.scores
assert (await session.get(EvalRun, ev_id)).fingerprint["scorer_version"] == score.SCORER_VERSION == 3
assert (await session.get(EvalRun, ev_id)).summary["outcomes_by_kind"] == {"contribution": {"traced": 1}}
# --------------------------------------------------------------------------
# 从头到尾
# --------------------------------------------------------------------------
def _line(op: dict) -> AIMessageChunk:
return AIMessageChunk(content=json.dumps(op, ensure_ascii=False) + "\n")
class AnalysisCopilot:
"""助手剧本:直接搭「入口 → 查库 → 分析卡 → 报告 → 出口」并运行;上线前自查直接过。"""
async def astream(self, messages):
if "上线前自查" in messages[1][1]:
yield _line({"op": "done"})
return
yield _line({"op": "plan", "summary": "按地区拆上周销售额的变化"})
graph = answer_graph()
for n in graph["nodes"]:
yield _line({"op": "add_node", "node": {"id": n["id"], "type": n["type"], "label": n["data"]["label"],
"config": n["data"]["config"]}})
for e in graph["edges"]:
yield _line({"op": "add_edge", "edge": e})
yield _line({"op": "done", "explanation": "拆一下", "run": True})
async def test_an_analysis_case_end_to_end(monkeypatch, source):
from app.api.settings import analysis_settings
es.use_copilot(monkeypatch, AnalysisCopilot())
es.use_report(monkeypatch, REPORT)
case = case_of(REGION, source["id"])
suite_id = await es.make_suite(source["id"], cases=[{k: case[k] for k in ("question", "datasource_ids", "gold",
"expectations")}])
from app.db.models_eval import EvalSuite
from app.eval.variants import normalize_variant
async with SessionLocal() as session:
suite = await session.get(EvalSuite, suite_id)
[ev] = await runner.start(session, suite, [normalize_variant({})], actor="评测测试")
expected_settings = await analysis_settings(session)
await es.wait_eval(ev.id)
async with SessionLocal() as session:
ev = await session.get(EvalRun, ev.id)
[item] = (await session.execute(select(EvalResult).where(EvalResult.eval_run_id == ev.id))).scalars()
answer_run = await session.get(Run, item.run_id)
assert ev.status == "succeeded", (ev.error, item.error)
assert item.path == "ran" and item.verdict == "pass", (item.error, item.scores)
[scored] = item.scores
assert scored["outcome"] == "traced" and scored["want"]["subjects"] == ["华东"], scored
assert item.sql_match["status"] == "proven", item.sql_match
# 算期望和作答用同一份分析设置快照,记进指纹
assert ev.fingerprint["analysis"] == expected_settings
assert answer_run.options["analysis"] == expected_settings
assert item.gold["before"]["sets"][0]["values"]["source"]["result"]["settings"]["main_share"] == \
expected_settings["main_share"]
assert ev.summary["outcomes_by_kind"] == {"contribution": {"traced": 1}}
assert ev.fingerprint["scorer_version"] == 3
async def test_suites_without_analysis_expectations_take_no_settings_snapshot(monkeypatch, tmp_path):
from tests.fixtures.sources import drop_source
es.use_copilot(monkeypatch, es.Copilot("none"))
name, sid = await es.make_source(tmp_path)
try:
es.use_copilot(monkeypatch, es.Copilot(name))
es.use_report(monkeypatch)
suite_id = await es.make_suite(sid)
from app.db.models_eval import EvalSuite
from app.eval.variants import normalize_variant
async with SessionLocal() as session:
suite = await session.get(EvalSuite, suite_id)
[ev] = await runner.start(session, suite, [normalize_variant({})], actor="评测测试")
await es.wait_eval(ev.id)
async with SessionLocal() as session:
ev = await session.get(EvalRun, ev.id)
[item] = (await session.execute(select(EvalResult).where(EvalResult.eval_run_id == ev.id))).scalars()
answer_run = await session.get(Run, item.run_id)
assert item.verdict == "pass", item.error
assert "analysis" not in ev.fingerprint and "analysis" not in (answer_run.options or {}), \
"别的题集的指纹、作答运行的选项和以前一样"
finally:
await drop_source(sid)