Files
AgentLab/backend/tests/test_schedule_target.py
ThearchyHelios 2371e75f17 评测执行的主干:后台评测、无头作答、评分、统计、对比与 A/B、清理(下一轮第 3 波 W3-c 第一段)
为什么:第 2 波只有题集和期望答案的来源,评测本身跑不起来。这一段把细案 01 做法 9–12、16、17 的后端接上,
量的是用户在问数据页拿到的同一轮,而不是另抄一份编排。

- eval/answer.py:无头作答,串起 answer_ops → chat/turn 的决定 → launch_run(origin=eval,带目录视图、只读、遮罩)
  → 等终态(停在审批的取消、记「评测不代批」;驳回记路径「被驳回」)→ review_of_run → 按同一规则重来一次;
  工作流题按钉住版本的图跑探索运行(不发正式运行,免得被 pick_base 和应用视图取到)。用量按角色记
- eval/runner.py:用 core/job_runtime,一次评测一个任务,题 × 重复 × 臂一项;A/B 几臂共用期望、同一题同时开跑;
  开评拍目录快照、记上传表格版本和表结构哈希的前后;并发不超过 max_parallel 和全局并发的四分之一;每题时限;
  评测单独一本账(开跑前估算、每题预留后找补、触顶不再开新题);止损;能停、只重试没做完的、重启标中断不续跑
- eval/score.py:六种期望、五种结论(量纲不同单列)、维度配对与弱匹配、路径不对直接不通过、口径 dual 两版都报;
  逐题 E1 用口径账本的 compare_queries(总方案 X2,不另写 signals);SCORER_VERSION 可重评(eval/rescore.py,只读封存)
- eval/judging.py:结论裁判走按需裁判同一条路(evidence._judge_now),文字要点由裁判判、单列不计准确率;花费经
  judge.charged_to 记在评测那本账上(engine/judge.py 只加了一个上下文变量和取账本的一处)
- eval/stats.py、eval/compare.py:准确率与可追溯准确率 + Wilson 区间、各项分布、P50/P90、按角色成本、引用率、逐题 E1;
  选基准、逐题转移与归因、指纹差异、A/B 配对表与精确 McNemar;「目录贡献」预设对照(当前 vs 关,可加只用已核实)
- eval/cleanup.py + core/artifact_store.forget:保留最近 N 次 + 钉住的基线 + A/B 另几臂,删结果、明细工件、底下的运行和
  引用行,工件文件一条引用都不剩才删;总表永久保留
- engine/tolerance.py:双盲复算的 _agree / _delta_pct 原样搬来共用(dual_check 改 import,行为不变)
- 设置加 eval 一组(builder 读、admin 写)和 GET /api/settings/eval/spend;题集预算在全站缺省上覆盖,每日上限只能往下收
- api/evals.py:发起(单臂 / A/B / 目录贡献预设)、估算、进度、逐题、停止、重试、重评、钉基线、和上一次比、配对表、清理
- main.py 的 lifespan:影子起草和评测的启动整理、关停(评测先于运行停,停时取消自己发起的运行)
- 测试:test_eval_e2e(dual_check_shop + 剧本模型:按支付日 330 通过且 SQL 等价,按下单日 310 不通过、差在 time;
  目录对照 1:0;和上一次比找到退步、归因做法变);通知事件、定时接口形状的老测试按新事件和新字段更新
2026-10-16 16:38:09 +08:00

137 lines
6.8 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""定时任务的目标(docs/design/next-round/00-总方案.md X4:定时只有一套)。
骨架给 Schedule 加了 target_kind / target_id;workflow_id 是老表上的非空约束,只加不改的迁移放宽不了。规矩只在
scheduler/service.py 的一处封装里:workflow 目标的 target_id 同 workflow_id;不是工作流的目标 workflow_id 存空串。
这里守:
- 存量任务(target_kind 是缺省的 workflow、target_id 为空)照旧按 workflow_id 发起,行为一字不变;
- 建任务、改工作流都经同一处封装写目标,两列对得上;
- fire_once 按目标种类分派:评测题集、数据源维护交给各自的模块(目标不在的照实记「未能发起」并写明原因,不发起运行、
不悄悄当成工作流去跑空串;评测题集的发起见 test_eval_schedule.py);
- 接口返回带 target_kind / target_id / 目标名(测试见 test_schedules_api.py 的 SCHEDULE_KEYS)。
工作流、任务名都是虚构的。
"""
from __future__ import annotations
import uuid
from datetime import datetime, timezone
import pytest
from httpx import ASGITransport, AsyncClient
from sqlalchemy import delete, func, select
from app.db.base import SessionLocal
from app.db.models import Run, Schedule, ScheduleFire, Workflow, WorkflowVersion
from app.engine.runner import run_manager
from app.main import app
from app.scheduler import service
UTC = timezone.utc
NINE = datetime(2026, 10, 5, 1, 0, tzinfo=UTC)
@pytest.fixture(autouse=True)
async def engine_up():
await run_manager.setup()
async with SessionLocal() as session:
await session.execute(delete(Schedule))
await session.commit()
yield
await run_manager.shutdown()
def _node(nid, ntype, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
def _graph() -> dict:
return {"nodes": [_node("start", "input", fields=[{"name": "stat_date", "required": True}]),
_node("out", "output", fields=[{"name": "r", "value": "完成 {{ input.stat_date }}"}])],
"edges": [{"source": "start", "target": "out"}]}
async def _workflow() -> str:
async with SessionLocal() as session:
wf = Workflow(name=f"门店日报-{uuid.uuid4().hex[:6]}", graph=_graph(), status="published", published_version=1)
session.add(wf)
await session.flush()
session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=_graph()))
await session.commit()
return wf.id
def test_target_of_a_legacy_row_reads_the_workflow():
"""加列之前建的任务:target_kind 缺省 workflow、target_id 为空,目标就是 workflow_id。"""
row = Schedule(name="老任务", workflow_id="wf-legacy", cron="0 9 * * *")
assert service.target_of(row) == ("workflow", "wf-legacy")
row.target_kind = "workflow"
row.target_id = None
assert service.target_of(row) == ("workflow", "wf-legacy")
def test_target_columns_keep_the_two_columns_in_step():
assert service.target_columns("workflow", "wf-1") == {"target_kind": "workflow", "target_id": "wf-1",
"workflow_id": "wf-1"}
# 非工作流目标:workflow_id 存空串(老表的非空约束),目标 id 只在 target_id
assert service.target_columns("eval_suite", "suite-1") == {"target_kind": "eval_suite", "target_id": "suite-1",
"workflow_id": ""}
row = Schedule(name="评测", cron="0 9 * * *", **service.target_columns("eval_suite", "suite-1"))
assert service.target_of(row) == ("eval_suite", "suite-1")
with pytest.raises(ValueError):
service.target_columns("nonsense", "x")
with pytest.raises(ValueError):
service.target_columns("workflow", "")
async def test_creating_and_retargeting_through_the_api_write_both_columns():
wf_a, wf_b = await _workflow(), await _workflow()
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
made = await client.post("/api/schedules", json={
"name": "每日早报", "workflow_id": wf_a, "cron": "0 9 * * *", "timezone": "Asia/Shanghai",
"input_template": {"stat_date": "{{ schedule.yesterday }}"}})
assert made.status_code == 201, made.text
body = made.json()
assert (body["target_kind"], body["target_id"]) == ("workflow", wf_a)
assert body["target_name"] == body["workflow_name"] and body["target_label"] == "工作流"
async with SessionLocal() as session:
row = await session.get(Schedule, body["id"])
assert (row.target_kind, row.target_id, row.workflow_id) == ("workflow", wf_a, wf_a)
moved = await client.patch(f"/api/schedules/{body['id']}", json={"workflow_id": wf_b, "version": body["version"]})
assert moved.status_code == 200, moved.text
async with SessionLocal() as session:
row = await session.get(Schedule, body["id"])
assert (row.target_kind, row.target_id, row.workflow_id) == ("workflow", wf_b, wf_b)
async def test_a_legacy_schedule_still_fires_its_workflow():
wf = await _workflow()
async with SessionLocal() as session:
row = Schedule(name="老任务", workflow_id=wf, cron="0 9 * * *", timezone="Asia/Shanghai", run_class="formal",
input_template={"stat_date": "{{ schedule.yesterday }}"}, next_run_at=NINE)
session.add(row)
await session.commit()
schedule_id = row.id
fire = await service.fire_once(schedule_id, NINE, "manual")
assert fire is not None and fire.status == "started" and fire.run_id
async with SessionLocal() as session:
run = await session.get(Run, fire.run_id)
assert run.workflow_id == wf and run.input == {"stat_date": "2026-10-04"}
async def test_a_missing_eval_suite_is_recorded_not_run():
async with SessionLocal() as session:
row = Schedule(name="题集回归", cron="0 9 * * *", timezone="Asia/Shanghai",
**service.target_columns("eval_suite", "suite-x"), next_run_at=NINE)
session.add(row)
await session.commit()
schedule_id = row.id
runs_before = await session.scalar(select(func.count()).select_from(Run))
fire = await service.fire_once(schedule_id, NINE, "manual")
assert fire is not None and fire.status == "failed_to_start" and fire.run_id is None
assert fire.reason == "评测题集不存在,可能已被删除"
async with SessionLocal() as session:
assert await session.scalar(select(func.count()).select_from(Run)) == runs_before, "不能发起任何运行"
assert await session.scalar(select(func.count()).select_from(ScheduleFire)
.where(ScheduleFire.schedule_id == schedule_id)) == 1