mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-16 11:11:59 +00:00
83cd5da72e
- tests/fixtures/acceptance/retail_trends.py(python -m tests.fixtures.acceptance.retail_trends <db> [--today],数据源名 retail_trends,不动 retail.py):华东线下客单价降到七成带来的销售额下降(主要来自华东);三成线下订单转到线上造成的退款率辛普森(各渠道都降、整体升,结构为主);订单数的常规波动;取消单激增;线上销售额连涨十周以上;线上销售额的变化来自订单数;西南一家小店的退款率看着高但样本太少;本周没过完(和上周不可比,周一时本周还没有数据、那一题期望算不了)。订单数按店按周定好、金额在窄区间随机、退款按渠道比例挑,各效应不靠运气 - tests/fixtures/eval/analysis_suite.json:八道题,问法用使用者的说法(上周销售额为什么下降?退款率为什么上升了?上周订单数下降算不算异常?……),各配一个贡献或判定期望,订单数、取消单两题另配一个不必需的「一个数」 - 测试:同一个今天逐字节相同;四个不同的今天(周三、周日、周六、跨年)上用 app.analysis.run 按题集的期望 SQL 核对八个效应;周一那一题照实算不了;走接口导入题集、逐题试算,分析期望都算得出、判定和埋的效应一致;真运行的算期望(作答前后对得上、阈值按设置快照、参数对不上照实说)、读作答运行里的分析结论和引用、重评老结果、从头到尾评一题(助手剧本搭查库 → 分析卡 → 报告,对且可追溯、逐题 E1 等价已证明)
316 lines
16 KiB
Python
316 lines
16 KiB
Python
"""分析期望走真运行(app/eval/gold.py、score.py、runner.py;docs/design/analysis-round/00-总方案.md §3.8)。库是虚构的
|
||
retail_trends(tests/fixtures/acceptance/retail_trends.py,按今天生成);写作模型、助手都换成剧本,不调任何模型。
|
||
|
||
- 算期望:期望 SQL 的探针运行封存的那份快照上,用平台的算子(真算子、真现查)算出判定;作答前后两次期望照样对得上(运行当天、
|
||
现查时刻、输入快照的工件不比);阈值按交进来的分析设置快照;参数和结果对不上的照实说、整题没有可用的值。
|
||
- 读答案:作答运行台账里的分析结论、报告怎么引用了它(判定语、结果表的格、句末依据)、有判断词的句子;逐题 E1 按分析结论的
|
||
输入查询比。
|
||
- 重评老结果:明细里没有分析结论那一项的照样重评(从封存重新读)。
|
||
- 从头到尾:助手剧本搭「查库 → 分析卡 → 报告」,报告引用判定语;评测给这一题「对且可追溯」,逐题 E1 等价已证明,作答运行和
|
||
算期望用的是同一份分析设置快照(记进指纹)。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
from datetime import date, datetime, timedelta
|
||
from pathlib import Path
|
||
from zoneinfo import ZoneInfo
|
||
|
||
import pytest
|
||
from langchain_core.messages import AIMessageChunk
|
||
from sqlalchemy import select
|
||
|
||
from app.core import artifact_store
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import Run
|
||
from app.db.models_eval import EvalCase, EvalResult, EvalRun
|
||
from app.engine.runner import run_manager
|
||
from app.eval import cases, gold, judgements, runner, score
|
||
from app.eval.gold import Context
|
||
from tests import eval_samples as es
|
||
from tests.fixtures.acceptance import retail_trends
|
||
|
||
TZ = "Asia/Shanghai"
|
||
SUITE = json.loads((Path(__file__).parent / "fixtures" / "eval" / "analysis_suite.json").read_text(encoding="utf-8"))
|
||
#: 题集里的第 1 题(上周销售额下降主要来自华东)和第 3 题(订单数的常规波动)
|
||
REGION = next(c for c in SUITE["cases"] if c["seq"] == 1)
|
||
ORDERS = next(c for c in SUITE["cases"] if c["seq"] == 3)
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
async def engine_up(monkeypatch):
|
||
from app.eval import snapshots
|
||
|
||
monkeypatch.setattr(snapshots, "MAX_SOURCES", 10_000)
|
||
await run_manager.setup()
|
||
yield
|
||
await runner.manager.shutdown()
|
||
await es.drop_suites()
|
||
await run_manager.shutdown()
|
||
|
||
|
||
@pytest.fixture
|
||
async def source(tmp_path, scratch_source):
|
||
path = retail_trends.build(tmp_path / "trends.db", date.today())
|
||
sid = await scratch_source(retail_trends.SOURCE_NAME, path, schema=True)
|
||
return {"id": sid, "name": retail_trends.SOURCE_NAME, "path": path}
|
||
|
||
|
||
def now() -> datetime:
|
||
return datetime.now(ZoneInfo(TZ))
|
||
|
||
|
||
def ctx(**kw) -> Context:
|
||
return Context(as_of=now(), tz=TZ, actor="评测测试", suite_id="s1", suite_name="虚构分析题集", **kw)
|
||
|
||
|
||
def case_of(raw: dict, source_id: str, *, expectations: list[dict] | None = None, sql: str | None = None) -> dict:
|
||
fields = cases.normalize_case({"question": raw["question"], "datasource_ids": [source_id],
|
||
"gold": {"kind": "sql", "datasource_id": source_id, "sql": sql or raw["gold"]["sql"]},
|
||
"expectations": expectations or raw["expectations"]})
|
||
return {**fields, "id": None, "seq": raw["seq"]}
|
||
|
||
|
||
def rendered(sql: str) -> str:
|
||
return cases.render(sql, cases.variable_values(now(), TZ))
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 算期望
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_the_gold_runs_the_platform_analysis_on_the_probe_snapshot(source):
|
||
import sqlite3
|
||
|
||
case = case_of(ORDERS, source["id"])
|
||
before = await gold.compute_gold(case, ctx())
|
||
assert before.ok, before.reason
|
||
band, orders = before.sets[0].values["band"], before.sets[0].values["orders"]
|
||
assert (band["kind"], band["status"], band["judgement"]) == ("verdict", "ok", "normal"), band
|
||
assert band["phrase"].startswith("订单数本期变化") and "常规波动范围内" in band["phrase"]
|
||
result = band["result"]
|
||
assert result["inputs"][0]["ref"] == judgements.GOLD_REF and result["inputs"][0]["artifact"] == before.sets[0].snapshot
|
||
assert result["params"]["today"] == date.today().isoformat()
|
||
assert result["settings"]["k_warn"] == 2.0, "没有设置快照(试算)时现读设置,缺省 2 倍"
|
||
# 本期齐不齐按现查的表范围判(SQL 里按日期算出来的界也在库里算一次),都记进结论
|
||
assert [(p["table"], p["column"], p["basis"]) for p in result["probes"] if "table" in p] == \
|
||
[("orders", "order_date", "live")], result["probes"]
|
||
assert all(p.get("at") for p in result["probes"])
|
||
with sqlite3.connect(source["path"]) as db:
|
||
direct = db.execute(rendered(ORDERS["gold"]["sql"])).fetchall()
|
||
assert orders["value"] == direct[24][1] and len(direct) == 25
|
||
|
||
# 作答后再算一次:快照的工件(记着查询用时)、现查时刻都不同,期望照样对得上
|
||
after = await gold.compute_gold(case, ctx(role="gold_after"))
|
||
assert after.sets[0].snapshot != before.sets[0].snapshot
|
||
assert gold.settle(before, after).status == "ok"
|
||
|
||
|
||
async def test_the_settings_snapshot_decides_the_thresholds(source):
|
||
case = case_of(ORDERS, source["id"])
|
||
outcome = await gold.compute_gold(case, ctx(analysis={"min_history": 30, "max_history": 52}))
|
||
assert outcome.ok, outcome.reason
|
||
band = outcome.sets[0].values["band"]
|
||
assert band["status"] == "refused" and band["reason_code"] == "history_short", band
|
||
assert band["result"]["settings"]["min_history"] == 30
|
||
graded = score.score_case(case, gold.settle(outcome, outcome).to_dict(), score.Answer(run_id="r"), path="ran")
|
||
assert graded.scores[0]["outcome"] == "ungradable" and graded.scores[0]["reason"].startswith("期望算不了:")
|
||
|
||
|
||
async def test_parameters_that_do_not_fit_the_sql_are_said_plainly(source):
|
||
wrong_column = case_of(REGION, source["id"], expectations=[
|
||
{"kind": "contribution", "mode": "additive", "params": {"by": ["门店"], "period": "周", "measure": "销售额"}}])
|
||
outcome = await gold.compute_gold(wrong_column, ctx())
|
||
assert outcome.status == "no_value" and "期望 SQL 里没有「门店」这一列" in outcome.reason, outcome.reason
|
||
three_weeks = REGION["gold"]["sql"].replace("'-7 days'", "'-14 days'")
|
||
outcome = await gold.compute_gold(case_of(REGION, source["id"], sql=three_weeks), ctx())
|
||
assert outcome.status == "no_value" and "有 3 期" in outcome.reason, outcome.reason
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 读答案、重评老结果
|
||
# --------------------------------------------------------------------------
|
||
|
||
REPORT = ("## 上周销售额\n\n上周销售额下降:[[a:A1]]。华东的变化是 [[v:A1.r0.delta]]。\n\n"
|
||
"下降主要来自华东,其余地区变化不大。[[see:A1]]")
|
||
|
||
|
||
def _node(nid: str, ntype: str, **config) -> dict:
|
||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
||
|
||
|
||
def _card_item() -> dict:
|
||
values = cases.variable_values(now(), TZ)
|
||
target = values["last_week_start"]
|
||
base = (date.fromisoformat(target) - timedelta(days=7)).isoformat()
|
||
return {"id": "where", "op": "decompose", "mode": "additive", "from": "q",
|
||
"params": {"by": ["地区"], "period": "周", "base": base, "target": target, "measure": "销售额",
|
||
"label": "销售额"}}
|
||
|
||
|
||
def answer_graph() -> dict:
|
||
nodes = [_node("start", "input"),
|
||
_node("q", "tool", tool=f"db_query__{retail_trends.SOURCE_NAME}", args={"sql": rendered(REGION["gold"]["sql"])}),
|
||
_node("card", "analysis", items=[_card_item()]),
|
||
_node("write", "report", instructions="写上周的周报"),
|
||
_node("out", "output", fields=[{"name": "答案", "value": "{{ nodes.write.text }}"}])]
|
||
ids = [n["id"] for n in nodes]
|
||
return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]}
|
||
|
||
|
||
async def _answer_run(monkeypatch) -> Run:
|
||
from tests.test_analysis_card import wait
|
||
|
||
es.use_report(monkeypatch, REPORT)
|
||
started = await run_manager.start(graph=answer_graph(), input_payload={})
|
||
row = await wait(started.id)
|
||
assert row.status == "succeeded", row.error
|
||
return row
|
||
|
||
|
||
async def test_reading_the_analyses_of_the_answer_run(monkeypatch, source):
|
||
row = await _answer_run(monkeypatch)
|
||
answer = await score.read_answer(row.id)
|
||
assert answer.problem is None and answer.report == "write"
|
||
[record] = answer.analyses
|
||
assert (record["alias"], record["op"], record["mode"], record["judgement"]) == ("A1", "decompose", "additive",
|
||
"source")
|
||
assert record["subjects"][0] == "华东" and record["direction"] == "down"
|
||
assert record["cited"] == ["phrase", "cell", "see"], record
|
||
# 判定语不是答案里的数;结果表的那一格是,维度是那一段(华东)
|
||
[cell] = answer.values
|
||
assert cell["kind"] == "cell" and cell["dims"] == {"地区": "华东"} and cell["artifact"] == record["artifact"]
|
||
# 逐题 E1 拿分析结论的输入查询比
|
||
[query] = answer.queries
|
||
assert answer.cited == [query["artifact"]] and record["inputs"] == [query["artifact"]]
|
||
[sentence] = answer.sentences
|
||
assert sentence["analyses"] == ["A1"] and "主要来自华东" in sentence["text"]
|
||
|
||
case = case_of(REGION, source["id"])
|
||
before = await gold.compute_gold(case, ctx())
|
||
settled = gold.settle(before, before).to_dict()
|
||
graded = score.score_case(case, settled, answer, path="ran")
|
||
assert graded.verdict == "pass" and graded.traced, graded.scores
|
||
assert graded.scores[0]["found"]["alias"] == "A1"
|
||
match = await score.sql_match(case, settled, answer)
|
||
assert match["status"] == "proven" and match["basis"] == "cited", match
|
||
|
||
|
||
async def test_rescoring_old_results_reads_the_analyses_from_the_seal(monkeypatch, source):
|
||
from app.eval import rescore
|
||
from app.eval.cases import case_hash
|
||
|
||
row = await _answer_run(monkeypatch)
|
||
case = case_of(REGION, source["id"])
|
||
settled = gold.settle(*(2 * [await gold.compute_gold(case, ctx())])).to_dict()
|
||
old = score.Answer(run_id=row.id, text="老的答案").to_dict()
|
||
old.pop("analyses")
|
||
old.pop("sentences")
|
||
details = await artifact_store.put_json({"format": 1, "turn": {"reply": None}, "answer": old}, kind="eval_result")
|
||
suite_id = await es.make_suite(source["id"], cases=[{k: case[k] for k in ("question", "datasource_ids", "gold",
|
||
"expectations")}])
|
||
async with SessionLocal() as session:
|
||
stored = (await session.execute(select(EvalCase).where(EvalCase.suite_id == suite_id))).scalar_one()
|
||
ev = EvalRun(suite_id=suite_id, status="succeeded", fingerprint={"scorer_version": 2})
|
||
session.add(ev)
|
||
await session.flush()
|
||
session.add(EvalResult(eval_run_id=ev.id, case_id=stored.id, case_hash=case_hash(case), status="ok",
|
||
path="ran", run_id=row.id, verdict="fail", details_artifact=details, gold=settled,
|
||
scores=[{"id": "source", "kind": "contribution", "required": True, "outcome": "wrong"}]))
|
||
await session.commit()
|
||
ev_id = ev.id
|
||
assert await rescore.rescore(ev_id) == 1
|
||
async with SessionLocal() as session:
|
||
result = (await session.execute(select(EvalResult).where(EvalResult.eval_run_id == ev_id))).scalar_one()
|
||
assert result.verdict == "pass" and result.scores[0]["outcome"] == "traced", result.scores
|
||
assert (await session.get(EvalRun, ev_id)).fingerprint["scorer_version"] == score.SCORER_VERSION == 3
|
||
assert (await session.get(EvalRun, ev_id)).summary["outcomes_by_kind"] == {"contribution": {"traced": 1}}
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 从头到尾
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def _line(op: dict) -> AIMessageChunk:
|
||
return AIMessageChunk(content=json.dumps(op, ensure_ascii=False) + "\n")
|
||
|
||
|
||
class AnalysisCopilot:
|
||
"""助手剧本:直接搭「入口 → 查库 → 分析卡 → 报告 → 出口」并运行;上线前自查直接过。"""
|
||
|
||
async def astream(self, messages):
|
||
if "上线前自查" in messages[1][1]:
|
||
yield _line({"op": "done"})
|
||
return
|
||
yield _line({"op": "plan", "summary": "按地区拆上周销售额的变化"})
|
||
graph = answer_graph()
|
||
for n in graph["nodes"]:
|
||
yield _line({"op": "add_node", "node": {"id": n["id"], "type": n["type"], "label": n["data"]["label"],
|
||
"config": n["data"]["config"]}})
|
||
for e in graph["edges"]:
|
||
yield _line({"op": "add_edge", "edge": e})
|
||
yield _line({"op": "done", "explanation": "拆一下", "run": True})
|
||
|
||
|
||
async def test_an_analysis_case_end_to_end(monkeypatch, source):
|
||
from app.api.settings import analysis_settings
|
||
|
||
es.use_copilot(monkeypatch, AnalysisCopilot())
|
||
es.use_report(monkeypatch, REPORT)
|
||
case = case_of(REGION, source["id"])
|
||
suite_id = await es.make_suite(source["id"], cases=[{k: case[k] for k in ("question", "datasource_ids", "gold",
|
||
"expectations")}])
|
||
from app.db.models_eval import EvalSuite
|
||
from app.eval.variants import normalize_variant
|
||
|
||
async with SessionLocal() as session:
|
||
suite = await session.get(EvalSuite, suite_id)
|
||
[ev] = await runner.start(session, suite, [normalize_variant({})], actor="评测测试")
|
||
expected_settings = await analysis_settings(session)
|
||
await es.wait_eval(ev.id)
|
||
async with SessionLocal() as session:
|
||
ev = await session.get(EvalRun, ev.id)
|
||
[item] = (await session.execute(select(EvalResult).where(EvalResult.eval_run_id == ev.id))).scalars()
|
||
answer_run = await session.get(Run, item.run_id)
|
||
assert ev.status == "succeeded", (ev.error, item.error)
|
||
assert item.path == "ran" and item.verdict == "pass", (item.error, item.scores)
|
||
[scored] = item.scores
|
||
assert scored["outcome"] == "traced" and scored["want"]["subjects"] == ["华东"], scored
|
||
assert item.sql_match["status"] == "proven", item.sql_match
|
||
# 算期望和作答用同一份分析设置快照,记进指纹
|
||
assert ev.fingerprint["analysis"] == expected_settings
|
||
assert answer_run.options["analysis"] == expected_settings
|
||
assert item.gold["before"]["sets"][0]["values"]["source"]["result"]["settings"]["main_share"] == \
|
||
expected_settings["main_share"]
|
||
assert ev.summary["outcomes_by_kind"] == {"contribution": {"traced": 1}}
|
||
assert ev.fingerprint["scorer_version"] == 3
|
||
|
||
|
||
async def test_suites_without_analysis_expectations_take_no_settings_snapshot(monkeypatch, tmp_path):
|
||
from tests.fixtures.sources import drop_source
|
||
|
||
es.use_copilot(monkeypatch, es.Copilot("none"))
|
||
name, sid = await es.make_source(tmp_path)
|
||
try:
|
||
es.use_copilot(monkeypatch, es.Copilot(name))
|
||
es.use_report(monkeypatch)
|
||
suite_id = await es.make_suite(sid)
|
||
from app.db.models_eval import EvalSuite
|
||
from app.eval.variants import normalize_variant
|
||
|
||
async with SessionLocal() as session:
|
||
suite = await session.get(EvalSuite, suite_id)
|
||
[ev] = await runner.start(session, suite, [normalize_variant({})], actor="评测测试")
|
||
await es.wait_eval(ev.id)
|
||
async with SessionLocal() as session:
|
||
ev = await session.get(EvalRun, ev.id)
|
||
[item] = (await session.execute(select(EvalResult).where(EvalResult.eval_run_id == ev.id))).scalars()
|
||
answer_run = await session.get(Run, item.run_id)
|
||
assert item.verdict == "pass", item.error
|
||
assert "analysis" not in ev.fingerprint and "analysis" not in (answer_run.options or {}), \
|
||
"别的题集的指纹、作答运行的选项和以前一样"
|
||
finally:
|
||
await drop_source(sid)
|