From 40daf37107cf167bb9cd409b6a599338ee80b223 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 17:15:17 +0800 Subject: [PATCH] =?UTF-8?q?=E8=AF=84=E6=B5=8B=E5=8F=91=E8=B5=B7=E5=90=8E?= =?UTF-8?q?=E7=AB=8B=E5=88=BB=E4=BA=A4=E7=BB=99=E5=90=8E=E5=8F=B0=E5=86=8D?= =?UTF-8?q?=E8=AF=BB=E5=9B=9E=EF=BC=9B=E8=A1=A5=E3=80=8C=E5=A4=8D=E6=A0=B8?= =?UTF-8?q?=E8=A6=81=E9=87=8D=E6=9D=A5=E3=80=8D=E7=9A=84=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么：发起评测时提交之后、交给后台之前还 await 了几次读回，这中间别的请求读到「排队中、却没在这个进程里跑」会把它 当成服务重启留下的、标中断。改成提交之后立刻交给后台，再读回。 - test_eval_runner：复核给了 retry_instruction 时带着这次运行的图、照那句话重来一次，第二次复核（attempt=1）不再要重来， 两次作答运行都记在这一题名下 --- backend/app/eval/runner.py | 4 +++- backend/tests/test_eval_runner.py | 40 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/backend/app/eval/runner.py b/backend/app/eval/runner.py index ca05f81b..5aa86e5b 100644 --- a/backend/app/eval/runner.py +++ b/backend/app/eval/runner.py @@ -216,9 +216,11 @@ async def start(session: AsyncSession, suite: EvalSuite, variants: Sequence[Mapp case_hash=case.case_hash, repeat=repeat, arm=run.arm, status="pending")) runs.append(run) await session.commit() + # 提交之后立刻交给后台（中间不 await）：别的请求读到「排队中」时它已经在这个进程里跑着，不会被当成服务重启留下的 + # 而标中断（reconcile_stale） + manager.start(leader_id) for run in runs: await session.refresh(run) - manager.start(leader_id) return runs diff --git a/backend/tests/test_eval_runner.py b/backend/tests/test_eval_runner.py index 6e6899c6..8e348397 100644 --- a/backend/tests/test_eval_runner.py +++ b/backend/tests/test_eval_runner.py @@ -395,3 +395,43 @@ async def test_a_workflow_case_runs_its_pinned_version_as_an_exploratory_run(cli assert (await client.get("/api/evals/pairs/nope")).status_code == 404 finally: await drop_source(sid) + + +async def test_a_review_asking_for_a_retry_rebuilds_on_the_same_graph(monkeypatch, tmp_path): + """复核给了 retry_instruction（chat/turn 的规则判过可以重来）：带着这次运行的图、照那句话重来一次，两次运行都记在这一题 + 名下、用量都算这一题的；第二次复核不再要重来就收尾。""" + import app.api.copilot as copilot + + name, sid = await es.make_source(tmp_path) + try: + model = es.Copilot(name) + es.use_copilot(monkeypatch, model) + es.use_report(monkeypatch) + reviews: list[int] = [] + real = copilot.review_of_run + + async def review(session, run, *, question, provider=None, model=None, always=False, attempt=0): + reviews.append(attempt) + out = await real(session, run, question=question, provider=provider, model=model, always=always, + attempt=attempt) + if attempt == 0: + out.update(verdict="annotated", severity="broken", retry=True, + retry_instruction="这个工作流上一次运行没有成功：步数用满了。改完继续回答原来的问题") + return out + + monkeypatch.setattr(copilot, "review_of_run", review) + suite_id = await es.make_suite(sid) + [run] = await start(suite_id) + await es.wait_eval(run.id) + [item] = await results(run.id) + assert reviews == [0, 1], "第一次复核要重来，第二次（attempt=1）不再要" + assert item.path == "ran" and item.verdict == "pass", item.error + retry_call = next(c for c in model.calls if "步数用满了" in str(c[-1][1]) or "步数用满了" in str(c[1][1])) + assert "这是当前的工作流" in str(retry_call[1][1]), "带着原图交回去改" + async with SessionLocal() as session: + answers = (await session.execute(select(Run.id).where(Run.origin_kind == "eval"))).scalars().all() + mine = [r for r in answers if (await session.get(Run, r)).origin.get("eval_run_id") == run.id + and (await session.get(Run, r)).origin.get("role") == "answer"] + assert len(mine) == 2 and item.run_id in mine + finally: + await drop_source(sid)