mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 22:01:58 +00:00
2371e75f17
为什么:第 2 波只有题集和期望答案的来源,评测本身跑不起来。这一段把细案 01 做法 9–12、16、17 的后端接上, 量的是用户在问数据页拿到的同一轮,而不是另抄一份编排。 - eval/answer.py:无头作答,串起 answer_ops → chat/turn 的决定 → launch_run(origin=eval,带目录视图、只读、遮罩) → 等终态(停在审批的取消、记「评测不代批」;驳回记路径「被驳回」)→ review_of_run → 按同一规则重来一次; 工作流题按钉住版本的图跑探索运行(不发正式运行,免得被 pick_base 和应用视图取到)。用量按角色记 - eval/runner.py:用 core/job_runtime,一次评测一个任务,题 × 重复 × 臂一项;A/B 几臂共用期望、同一题同时开跑; 开评拍目录快照、记上传表格版本和表结构哈希的前后;并发不超过 max_parallel 和全局并发的四分之一;每题时限; 评测单独一本账(开跑前估算、每题预留后找补、触顶不再开新题);止损;能停、只重试没做完的、重启标中断不续跑 - eval/score.py:六种期望、五种结论(量纲不同单列)、维度配对与弱匹配、路径不对直接不通过、口径 dual 两版都报; 逐题 E1 用口径账本的 compare_queries(总方案 X2,不另写 signals);SCORER_VERSION 可重评(eval/rescore.py,只读封存) - eval/judging.py:结论裁判走按需裁判同一条路(evidence._judge_now),文字要点由裁判判、单列不计准确率;花费经 judge.charged_to 记在评测那本账上(engine/judge.py 只加了一个上下文变量和取账本的一处) - eval/stats.py、eval/compare.py:准确率与可追溯准确率 + Wilson 区间、各项分布、P50/P90、按角色成本、引用率、逐题 E1; 选基准、逐题转移与归因、指纹差异、A/B 配对表与精确 McNemar;「目录贡献」预设对照(当前 vs 关,可加只用已核实) - eval/cleanup.py + core/artifact_store.forget:保留最近 N 次 + 钉住的基线 + A/B 另几臂,删结果、明细工件、底下的运行和 引用行,工件文件一条引用都不剩才删;总表永久保留 - engine/tolerance.py:双盲复算的 _agree / _delta_pct 原样搬来共用(dual_check 改 import,行为不变) - 设置加 eval 一组(builder 读、admin 写)和 GET /api/settings/eval/spend;题集预算在全站缺省上覆盖,每日上限只能往下收 - api/evals.py:发起(单臂 / A/B / 目录贡献预设)、估算、进度、逐题、停止、重试、重评、钉基线、和上一次比、配对表、清理 - main.py 的 lifespan:影子起草和评测的启动整理、关停(评测先于运行停,停时取消自己发起的运行) - 测试:test_eval_e2e(dual_check_shop + 剧本模型:按支付日 330 通过且 SQL 等价,按下单日 310 不通过、差在 time; 目录对照 1:0;和上一次比找到退步、归因做法变);通知事件、定时接口形状的老测试按新事件和新字段更新
137 lines
6.8 KiB
Python
137 lines
6.8 KiB
Python
"""定时任务的目标(docs/design/next-round/00-总方案.md X4:定时只有一套)。
|
||
|
||
骨架给 Schedule 加了 target_kind / target_id;workflow_id 是老表上的非空约束,只加不改的迁移放宽不了。规矩只在
|
||
scheduler/service.py 的一处封装里:workflow 目标的 target_id 同 workflow_id;不是工作流的目标 workflow_id 存空串。
|
||
这里守:
|
||
|
||
- 存量任务(target_kind 是缺省的 workflow、target_id 为空)照旧按 workflow_id 发起,行为一字不变;
|
||
- 建任务、改工作流都经同一处封装写目标,两列对得上;
|
||
- fire_once 按目标种类分派:评测题集、数据源维护交给各自的模块(目标不在的照实记「未能发起」并写明原因,不发起运行、
|
||
不悄悄当成工作流去跑空串;评测题集的发起见 test_eval_schedule.py);
|
||
- 接口返回带 target_kind / target_id / 目标名(测试见 test_schedules_api.py 的 SCHEDULE_KEYS)。
|
||
|
||
工作流、任务名都是虚构的。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import uuid
|
||
from datetime import datetime, timezone
|
||
|
||
import pytest
|
||
from httpx import ASGITransport, AsyncClient
|
||
from sqlalchemy import delete, func, select
|
||
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import Run, Schedule, ScheduleFire, Workflow, WorkflowVersion
|
||
from app.engine.runner import run_manager
|
||
from app.main import app
|
||
from app.scheduler import service
|
||
|
||
UTC = timezone.utc
|
||
NINE = datetime(2026, 10, 5, 1, 0, tzinfo=UTC)
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
async def engine_up():
|
||
await run_manager.setup()
|
||
async with SessionLocal() as session:
|
||
await session.execute(delete(Schedule))
|
||
await session.commit()
|
||
yield
|
||
await run_manager.shutdown()
|
||
|
||
|
||
def _node(nid, ntype, **config):
|
||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
||
|
||
|
||
def _graph() -> dict:
|
||
return {"nodes": [_node("start", "input", fields=[{"name": "stat_date", "required": True}]),
|
||
_node("out", "output", fields=[{"name": "r", "value": "完成 {{ input.stat_date }}"}])],
|
||
"edges": [{"source": "start", "target": "out"}]}
|
||
|
||
|
||
async def _workflow() -> str:
|
||
async with SessionLocal() as session:
|
||
wf = Workflow(name=f"门店日报-{uuid.uuid4().hex[:6]}", graph=_graph(), status="published", published_version=1)
|
||
session.add(wf)
|
||
await session.flush()
|
||
session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=_graph()))
|
||
await session.commit()
|
||
return wf.id
|
||
|
||
|
||
def test_target_of_a_legacy_row_reads_the_workflow():
|
||
"""加列之前建的任务:target_kind 缺省 workflow、target_id 为空,目标就是 workflow_id。"""
|
||
row = Schedule(name="老任务", workflow_id="wf-legacy", cron="0 9 * * *")
|
||
assert service.target_of(row) == ("workflow", "wf-legacy")
|
||
row.target_kind = "workflow"
|
||
row.target_id = None
|
||
assert service.target_of(row) == ("workflow", "wf-legacy")
|
||
|
||
|
||
def test_target_columns_keep_the_two_columns_in_step():
|
||
assert service.target_columns("workflow", "wf-1") == {"target_kind": "workflow", "target_id": "wf-1",
|
||
"workflow_id": "wf-1"}
|
||
# 非工作流目标:workflow_id 存空串(老表的非空约束),目标 id 只在 target_id
|
||
assert service.target_columns("eval_suite", "suite-1") == {"target_kind": "eval_suite", "target_id": "suite-1",
|
||
"workflow_id": ""}
|
||
row = Schedule(name="评测", cron="0 9 * * *", **service.target_columns("eval_suite", "suite-1"))
|
||
assert service.target_of(row) == ("eval_suite", "suite-1")
|
||
with pytest.raises(ValueError):
|
||
service.target_columns("nonsense", "x")
|
||
with pytest.raises(ValueError):
|
||
service.target_columns("workflow", "")
|
||
|
||
|
||
async def test_creating_and_retargeting_through_the_api_write_both_columns():
|
||
wf_a, wf_b = await _workflow(), await _workflow()
|
||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
||
made = await client.post("/api/schedules", json={
|
||
"name": "每日早报", "workflow_id": wf_a, "cron": "0 9 * * *", "timezone": "Asia/Shanghai",
|
||
"input_template": {"stat_date": "{{ schedule.yesterday }}"}})
|
||
assert made.status_code == 201, made.text
|
||
body = made.json()
|
||
assert (body["target_kind"], body["target_id"]) == ("workflow", wf_a)
|
||
assert body["target_name"] == body["workflow_name"] and body["target_label"] == "工作流"
|
||
async with SessionLocal() as session:
|
||
row = await session.get(Schedule, body["id"])
|
||
assert (row.target_kind, row.target_id, row.workflow_id) == ("workflow", wf_a, wf_a)
|
||
moved = await client.patch(f"/api/schedules/{body['id']}", json={"workflow_id": wf_b, "version": body["version"]})
|
||
assert moved.status_code == 200, moved.text
|
||
async with SessionLocal() as session:
|
||
row = await session.get(Schedule, body["id"])
|
||
assert (row.target_kind, row.target_id, row.workflow_id) == ("workflow", wf_b, wf_b)
|
||
|
||
|
||
async def test_a_legacy_schedule_still_fires_its_workflow():
|
||
wf = await _workflow()
|
||
async with SessionLocal() as session:
|
||
row = Schedule(name="老任务", workflow_id=wf, cron="0 9 * * *", timezone="Asia/Shanghai", run_class="formal",
|
||
input_template={"stat_date": "{{ schedule.yesterday }}"}, next_run_at=NINE)
|
||
session.add(row)
|
||
await session.commit()
|
||
schedule_id = row.id
|
||
fire = await service.fire_once(schedule_id, NINE, "manual")
|
||
assert fire is not None and fire.status == "started" and fire.run_id
|
||
async with SessionLocal() as session:
|
||
run = await session.get(Run, fire.run_id)
|
||
assert run.workflow_id == wf and run.input == {"stat_date": "2026-10-04"}
|
||
|
||
|
||
async def test_a_missing_eval_suite_is_recorded_not_run():
|
||
async with SessionLocal() as session:
|
||
row = Schedule(name="题集回归", cron="0 9 * * *", timezone="Asia/Shanghai",
|
||
**service.target_columns("eval_suite", "suite-x"), next_run_at=NINE)
|
||
session.add(row)
|
||
await session.commit()
|
||
schedule_id = row.id
|
||
runs_before = await session.scalar(select(func.count()).select_from(Run))
|
||
fire = await service.fire_once(schedule_id, NINE, "manual")
|
||
assert fire is not None and fire.status == "failed_to_start" and fire.run_id is None
|
||
assert fire.reason == "评测题集不存在,可能已被删除"
|
||
async with SessionLocal() as session:
|
||
assert await session.scalar(select(func.count()).select_from(Run)) == runs_before, "不能发起任何运行"
|
||
assert await session.scalar(select(func.count()).select_from(ScheduleFire)
|
||
.where(ScheduleFire.schedule_id == schedule_id)) == 1
|