From df6c3ce10d89f47b72d27d29163962ab04954e2f Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 17:04:05 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=9A=E6=97=B6=E9=A1=B5=E7=85=A7=E5=AE=9E?= =?UTF-8?q?=E6=98=BE=E7=A4=BA=E9=9D=9E=E5=B7=A5=E4=BD=9C=E6=B5=81=E7=9B=AE?= =?UTF-8?q?=E6=A0=87=EF=BC=9B=E8=AF=84=E6=B5=8B=E5=81=9C=E6=AD=A2=E6=97=B6?= =?UTF-8?q?=E5=87=A0=E8=87=82=E4=B8=80=E8=B5=B7=E6=94=B6=E5=B0=BE=EF=BC=9B?= =?UTF-8?q?=E5=B7=A5=E4=BD=9C=E6=B5=81=E9=A2=98=E7=9A=84=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么：这一波后端开始接受评测题集、数据源维护这两种定时目标（总方案 X4），定时页要能把它们照实列出来、点进去 不报错；界面上不能有点了没用的入口。 - 定时页列表：非工作流目标写种类（评测题集 / 数据源维护）和目标名，不标运行类别，「编辑」写成「查看」 - 详情：非工作流目标的设置只读地列出名称、目标、频率、时区、补发宽限，不出工作流那张表单（选工作流、输入模板、 Webhook 对它们不适用）；页头的启用、立即运行、删除和触发历史照常 - 立即运行发起的是一次评测时说「已发起一次评测」（以前会落到「未能发起运行」）；触发历史写评测的编号 - types.ts：Schedule 加 target_kind / target_id / target_label / target_name / target_exists，触发记录加 eval_run_id； 文案进 SCHEDULE_TEXT - check-schedules 加一段「目标不是工作流的定时任务」 - runner：还没准备好就被停下、出错时，按 pair_id 找全几臂一起收尾 - 测试：工作流题按钉住版本跑探索运行（不发正式运行、不建图）；钉住基线的接口 - 后端几句给人看的话按文案规范改（不用「跑」「钉」） --- backend/app/api/evals.py | 2 +- backend/app/eval/answer.py | 2 +- backend/app/eval/runner.py | 8 +-- backend/app/eval/score.py | 6 +-- backend/tests/test_eval_runner.py | 48 +++++++++++++++++ frontend/src/lib/terms.ts | 17 +++++++ frontend/src/pages/schedules/FireHistory.tsx | 3 ++ .../src/pages/schedules/ScheduleDetail.tsx | 41 ++++++++++++++- .../src/pages/schedules/SchedulesPage.tsx | 27 ++++++---- frontend/src/pages/schedules/parts.tsx | 15 +++++- frontend/src/types.ts | 14 +++++ scripts/check-schedules.mjs | 51 +++++++++++++++++++ 12 files changed, 214 insertions(+), 20 deletions(-) diff --git a/backend/app/api/evals.py b/backend/app/api/evals.py index 566ac513..dc5e9419 100644 --- a/backend/app/api/evals.py +++ b/backend/app/api/evals.py @@ -954,7 +954,7 @@ async def pin_evaluation(evaluation_id: str, request: Request, session: AsyncSes if not isinstance(pinned, bool): raise CodedHTTPException(422, "pinned 需要是 true 或 false", case_rules.CASE_INVALID, field="pinned") if pinned and row.status not in ("succeeded", "partial"): - raise CodedHTTPException(409, "只有评完了（或部分评完）的评测能钉成基线", case_rules.CASE_INVALID, field="pinned") + raise CodedHTTPException(409, "只有评完了（或部分评完）的评测能设为基线", case_rules.CASE_INVALID, field="pinned") row.pinned = pinned await session.commit() await session.refresh(row) diff --git a/backend/app/eval/answer.py b/backend/app/eval/answer.py index 46d4c667..9d281947 100644 --- a/backend/app/eval/answer.py +++ b/backend/app/eval/answer.py @@ -333,7 +333,7 @@ class Driver: waiting = await self._pending_approvals(run_id) await stop_run(run_id, self.ctx.actor) t.path = "run_failed" - t.reason = APPROVAL_REASON if waiting else "运行停在断点上（服务可能重启过）：评测不接着跑，已取消这次运行" + t.reason = APPROVAL_REASON if waiting else "运行停在断点上（服务可能重启过）：评测不继续这次运行，已取消" return None if row.status == "cancelled" and row.end_reason == END_REJECTED: t.path, t.reason = "rejected", "运行被驳回（人的决定，不算运行失败）" diff --git a/backend/app/eval/runner.py b/backend/app/eval/runner.py index 1f24e540..c4c89be5 100644 --- a/backend/app/eval/runner.py +++ b/backend/app/eval/runner.py @@ -60,7 +60,7 @@ STOP_COPILOT = 3 STOP_WINDOW = 10 STOP_TIMEOUTS = 7 #: 说法 -INTERRUPTED = "服务重启时评测还没做完，没有自动续跑。可以点「重试」只把没做完的题接着做" +INTERRUPTED = "服务重启时评测还没做完，没有自动继续。可以点「重试」只把没做完的题接着做" STOPPED = "评测已停止" NOT_STARTED = "评测停下时这道题还没开评" LIMIT_RUN = "已到这次评测的金额上限（${limit:g}），没有再开新题" @@ -68,7 +68,7 @@ LIMIT_DAILY = "评测今天的金额已到上限（${limit:g}），没有再开 STOP_COPILOT_TEXT = "连续 {n} 题助手调用失败，评测停下了：{reason}" STOP_TIMEOUT_TEXT = "最近 {window} 题里有 {n} 题超时，评测停下了" CASE_GONE = "这道题已被删除" -TIMED_OUT = "超过每题时限（{s:g} 秒），已停止这一题正在跑的运行" +TIMED_OUT = "超过每题时限（{s:g} 秒），已停止这一题正在执行的运行" #: 结束后多久做清理（在后台、不挡收尾） DETAILS_KIND = "eval_result" DETAILS_FORMAT = 1 @@ -698,7 +698,9 @@ class EvalExecution(runtime.JobRun): await self._settle() async def write(session: AsyncSession) -> None: - ids = [a.run_id for a in self.arms] or [self.job_id] + # 还没准备好就停下、出错的（arms 还是空的）：按 pair_id 找全几臂 + ids = [a.run_id for a in self.arms] or list((await session.execute(select(EvalRun.id).where( + (EvalRun.id == self.job_id) | (EvalRun.pair_id == self.job_id)))).scalars()) item_status = "cancelled" if status in ("cancelled", "interrupted") else "failed" await session.execute(update(EvalResult).where(EvalResult.eval_run_id.in_(ids), EvalResult.status.in_(runtime.OPEN)) diff --git a/backend/app/eval/score.py b/backend/app/eval/score.py index be673a75..f471c750 100644 --- a/backend/app/eval/score.py +++ b/backend/app/eval/score.py @@ -56,8 +56,8 @@ OUTCOME_LABEL = {"traced": "对且可追溯", "untraced": "对但没引用", "wr TEXT_OUTCOMES = ("covered", "partial", "missing", "unjudged") TEXT_LABEL = {"covered": "都答到了", "partial": "答到一部分", "missing": "没答到", "unjudged": "没有判"} VERDICT_LABEL = {"pass": "通过", "fail": "不通过", "ungradable": "不能评分", "skipped": "跳过"} -PATH_LABEL = {"reply": "直接回答", "asked": "反问", "ready": "只建不跑", "build_failed": "建图失败", - "run_failed": "运行失败", "ran": "跑完了", "rejected": "被驳回"} +PATH_LABEL = {"reply": "直接回答", "asked": "反问", "ready": "只建图未运行", "build_failed": "建图失败", + "run_failed": "运行失败", "ran": "运行完成", "rejected": "被驳回"} #: 期望没写容差时，引用的原值按这个相对误差比（同双盲复算的缺省） DEFAULT_TOLERANCE = 1e-6 #: 量纲不同：差这几个倍数（元 / 万元、比率 / 百分数……） @@ -495,7 +495,7 @@ def score_case(case: Mapping[str, Any], gold: Mapping[str, Any] | None, answer: return CaseScore("ungradable", False, path_reason or "运行被驳回（人的决定，不算失败）", []) ask_only = any(e.get("kind") == "ask" for e in expectations) if needs_numbers and path != "ran": - reason = WRONG_PATH.get(path or "", "没有跑出答案") + reason = WRONG_PATH.get(path or "", "没有得到答案") if path_reason: reason = f"{reason}：{path_reason}" return CaseScore("fail", False, reason, [{"id": e["id"], "kind": e.get("kind"), diff --git a/backend/tests/test_eval_runner.py b/backend/tests/test_eval_runner.py index 06a2c233..c98df297 100644 --- a/backend/tests/test_eval_runner.py +++ b/backend/tests/test_eval_runner.py @@ -331,3 +331,51 @@ async def test_eval_settings_group_and_spend_endpoint(client): assert effective_budget({"daily_max_usd": 9.0}, {"daily_max_usd": 3.0})["daily_max_usd"] == 3.0 assert effective_budget({"daily_max_usd": 1.0}, {"daily_max_usd": 3.0})["daily_max_usd"] == 1.0 await client.put("/api/settings", json={"values": {"eval": {}}}) + + +async def test_a_workflow_case_runs_its_pinned_version_as_an_exploratory_run(client, monkeypatch, tmp_path): + """工作流题：钉住那一版的图按输入模板跑一次探索运行（来源 eval），不发正式运行、不建图、不复核。""" + from app.core.artifact_store import graph_hash + from app.db.models import Workflow, WorkflowVersion + from tests.fixtures import dual_check_shop as shop + + name, sid = await es.make_source(tmp_path) + try: + es.use_report(monkeypatch) + + def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + nodes = [node("start", "input", fields=[{"name": "week", "required": True}]), + node("q", "tool", tool=f"db_query__{name}", args={"sql": shop.RIGHT}), + node("write", "report", instructions="写周销售"), + node("out", "output", fields=[{"name": "报告", "value": "{{ nodes.write.text }}"}])] + graph = {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + async with SessionLocal() as session: + wf = Workflow(name="虚构周销售", graph=graph, status="published", published_version=1) + session.add(wf) + await session.flush() + session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=graph, graph_hash=graph_hash(graph), + level="published")) + await session.commit() + wf_id = wf.id + suite_id = await es.make_suite(sid, cases=[{ + "kind": "workflow", "workflow_id": wf_id, "workflow_version": 1, + "input_template": {"week": "{{ schedule.last_week }}"}, + "gold": {"kind": "sql", "datasource_id": sid, "sql": shop.RIGHT}, + "expectations": [{"kind": "scalar", "column": "gmv"}]}]) + [run] = await start(suite_id) + await es.wait_eval(run.id) + [item] = await results(run.id) + assert item.path == "ran" and item.verdict == "pass", (item.error, item.scores) + async with SessionLocal() as session: + answered = await session.get(Run, item.run_id) + assert answered.run_class == "exploratory" and answered.origin_kind == "eval" and answered.workflow_id is None + assert set(answered.input) == {"week"} and answered.input["week"].count("-W") == 1 + # 钉住当基线、取消 + pinned = await client.post(f"/api/evals/evaluations/{run.id}/pin", json={"pinned": True}) + assert pinned.status_code == 200 and pinned.json()["pinned"] is True + assert (await client.post(f"/api/evals/evaluations/{run.id}/pin", json={"pinned": "yes"})).status_code == 422 + assert (await client.get("/api/evals/pairs/nope")).status_code == 404 + finally: + await drop_source(sid) diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index e93f53d7..9f7489df 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -3884,7 +3884,24 @@ export const SCHEDULE_TEXT = { /** 启用开关遇到版本不符（409）：服务端那句是写给表单保存的，开关用这句，原话放进详情 */ toggleConflict: (name: string) => `「${name}」已在别处被修改，已重新载入最新状态，请确认后再切换`, + // ---- 目标（定时只有一套：目标可以是工作流、评测题集或数据源维护） + targetKind: { workflow: '工作流', eval_suite: '评测题集', source_maintenance: '数据源维护' } as Record< + import('../types').ScheduleTargetKind, string>, + /** 目标已经不在了：名字是保存时记下的 */ + targetGone: (name: string) => `${name}（已不存在）`, + view: '查看', + viewLabel: (name: string) => `查看定时任务「${name}」`, + /** 非工作流目标的设置是只读的：启用、暂停、立即运行、触发历史照常可用 */ + targetSettingsTitle: '设置', + targetSettingsNote: (kind: string) => `这个定时任务的目标是${kind}：可以启用、暂停、立即运行和查看触发历史；名称、频率和目标在这里只能查看，不能修改`, + targetField: '目标', + graceField: '补发宽限', + // ---- 立即运行的结果 + /** 目标是评测题集：发起的是一次评测，没有运行可看 */ + evalStarted: '已发起一次评测', + /** 触发历史里评测题集那一次：评测的编号 */ + evalFire: (short: string) => `评测 #${short}`, runStarted: '已发起运行', runSkipped: '上一次运行尚未结束，本次已按重叠策略跳过', runFailed: (reason: string | null) => (reason ? `未能发起运行：${reason}` : '未能发起运行'), diff --git a/frontend/src/pages/schedules/FireHistory.tsx b/frontend/src/pages/schedules/FireHistory.tsx index e69ba212..50f311a0 100644 --- a/frontend/src/pages/schedules/FireHistory.tsx +++ b/frontend/src/pages/schedules/FireHistory.tsx @@ -143,6 +143,9 @@ function FireRow({ fire: f, tz, now }: { fire: ScheduleFire; tz: string; now: nu {T.openRun} + ) : f.eval_run_id ? ( + // 目标是评测题集：这一次发起的是一次评测（评测页在后续版本），这里只写编号 + {T.evalFire(f.eval_run_id.slice(0, 6))} ) : {T.noRun}} {f.reason && ( diff --git a/frontend/src/pages/schedules/ScheduleDetail.tsx b/frontend/src/pages/schedules/ScheduleDetail.tsx index 70d917e4..e42c0f0d 100644 --- a/frontend/src/pages/schedules/ScheduleDetail.tsx +++ b/frontend/src/pages/schedules/ScheduleDetail.tsx @@ -8,7 +8,11 @@ import { SCHEDULE_TEXT as T } from '../../lib/terms' import { useOnReconnect } from '../../store/catalog' import type { Schedule, ScheduleFire } from '../../types' import { FireHistory } from './FireHistory' -import { EnableSwitch, LastFire, NextRun, isUnsupported, rememberScheduleNames, removeSchedule, runNow, setEnabled } from './parts' +import { + EnableSwitch, LastFire, NextRun, isUnsupported, rememberScheduleNames, removeSchedule, runNow, setEnabled, targetKindOf, + targetText, +} from './parts' +import { graceLabel } from './model' import { ScheduleEditor } from './ScheduleEditor' const BACK = ( @@ -234,7 +238,12 @@ export function ScheduleDetailPage({ id }: { id: string }) {
- {settingsSeen && ( + {settingsSeen && targetKindOf(schedule) !== 'workflow' && ( + + )} + {settingsSeen && targetKindOf(schedule) === 'workflow' && (