mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 05:51:57 +00:00
ThearchyHelios
3dc5f5add4
真实模型验收里 deepseek 偶发只出思考、不出操作，生成过程交出 error「助手本轮未完成：模型没有给出任何修改，也没有说明 原因」，问数据这一轮照实报了，但不会自动重来——而这种偶发情况重来一次多半就好，「重来一次」的规矩本来就有 （chat/turn 的 MAX_ATTEMPTS = 2，含第一次）。 - chat/turn：error 也由 next_step 判：机读码是 EMPTY_GENERATION（copilot_empty）的判 retry，别的仍判失败（被截断的不重来： 额度不变，重来还是截断）；error_reason 取重来时交给助手的原因（error 带的 reason，没有就去掉「助手本轮未完成：」前缀）； - api/copilot.answer_ops：一个操作都没给时 CopilotIncomplete 带上这个码；error 操作多带 reason 和 next（retry | fail），和 final 的 next 一样由服务端定； - 问数据页（store/chat.ts）照 error.next 走：retry 就带着原因重来一次（同 retry_instruction 的从零重建措辞，次数照 MAX_ATTEMPTS），试满了才照实报；进度写「助手上一次没有给出任何操作，正在重新生成工作流…」（lib/terms 的 CHAT_RETRY_TEXT）； 第一次建图的用量照样记进这一轮； - 评测（eval/answer.Driver.drive）用同一个 next_step / error_reason / retry_instruction：先重来一次，还不行才记建图失败、 助手调用失败；每次自动重来的原因照实记在 turn.retries（没过检查、复核判不可用的重来也记），评测页逐题明细写出来。 测试：test_chat_turn_policy.py 的 test_next_step_branches（error 的两支和 error_reason）、 test_the_frontend_retries_an_error_the_server_says_to_retry、test_answer_ops_says_to_retry_an_empty_generation、 test_the_eval_driver_retries_an_empty_generation_once（重来后回答 / 重来还不行，两次的用量都算）； test_copilot_answer_ops.py 的 SSE 对照多认 error 上新加的 next、reason、code。去掉修复回跑都失败。 浏览器：check-chat 新加 emptyretry 一节（重来一次后照常回答、交回去的话带原因、两次用量都记；两次都不行照实报、不试第三次）， check-chat 840 项全过。
469 lines
25 KiB
Python
469 lines
25 KiB
Python
"""评测里的一轮作答：无头地走一遍问数据页走的那一轮（细案 01 做法 1、9；总方案 2.1）。
|
||
|
||
**量的就是用户拿到的那一轮。** 不另写一份 Python 版编排：生成过程是 api/copilot.answer_ops（问数据页经 SSE 收的同一串
|
||
操作），几个「决定」是 chat/turn 的纯函数（问数据页照服务端给的 next、retry_instruction 走），复核是 api/copilot.review_of_run
|
||
（问数据页经 /copilot/review 调的同一个函数）。这里只把它们串起来：
|
||
|
||
answer_ops → final.next（chat/turn.next_step）→ launch_run(origin=eval) → 等终态 → review_of_run → 要的话带原图重来一次
|
||
|
||
- 生成：问法就是 instruction，intent=answer，限定的数据源同题目；助手模型按变体覆盖（GenerateIn.provider / model，挑表、
|
||
复核跟着换）；读哪份目录按变体（answer_ops 的 catalog_view，查看的表详情、自查的 SQL 检查器都按它）。
|
||
- 结局：reply 直接回答、asked 反问、fail 判失败（建图失败）、ready 只建不跑、retry（老服务端那条路：没自查过的图带着原因
|
||
重来一次）、run 运行。生成过程交出 error（助手本轮未完成）、开流前就拒绝（模型没配好、数据源不在）都是建图失败，另记
|
||
「助手调用失败」——连续几题都这样就止损（runner.py）；只有生成过程一个操作都没给（只写了思考，error.next=retry）时按
|
||
同一个规矩先带着原因重来一次，还不行才记建图失败。每次重来的原因照实记在 retries。
|
||
- 运行：fill_input 填入口字段（单轮，history 为空），来源 eval，名字「评测：题集名·第 n 题」，选项带目录视图（Run.options
|
||
的 catalog_view：运行时的 db_schema、冻结快照、自查都按它）、只读和遮罩（同使用者的问数据运行：评测页会显示答案里的数，
|
||
遮罩值不能从这里露出来；评测不写业务库）。写作模型按变体写进图的 defaults：节点自己点了模型的不覆盖，照实记在
|
||
own_models。
|
||
- 终态：停在审批上（危险工具、人工节点）的直接取消、记「评测不代批」；被驳回的（cancelled + end_reason=rejected，总方案
|
||
X9）记路径「被驳回」，不算运行失败；失败、被停止记运行失败。
|
||
- 复核：总是等完（问数据页超过 25 秒不等，评测里照等，复核用时另记，超过 25 秒的单列，见 stats.py）。复核给了
|
||
retry_instruction 就带着这次运行的图重来一次（chat/turn 的规则，最多试 MAX_ATTEMPTS 次）。
|
||
|
||
用量按角色记（做法 3、11）：助手 = 生成（answer_ops 末尾的 usage）+ 复核（review_of_run 的 usage）；写作 = 答案运行里的
|
||
llm.end（按模型分）。重来的话两次都算——那都是这一题花的钱。
|
||
|
||
时限、停止由调用方管（runner.py 用 asyncio.wait_for 包住 drive）：这里随时把正在跑的那次运行记在 current_run，调用方
|
||
超时、停止时据此取消它（stop_run）。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import copy
|
||
import logging
|
||
import time
|
||
from collections.abc import Awaitable, Callable, Mapping
|
||
from dataclasses import dataclass, field
|
||
from typing import Any
|
||
|
||
from sqlalchemy import func, select
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
#: 问数据页最多等复核这么久（chat.ts 的 REVIEW_TIMEOUT_MS）：评测照等，超过的单列
|
||
REVIEW_PAGE_WAIT_S = 25.0
|
||
#: 取消一次运行之后最多再等它收尾多久
|
||
CANCEL_GRACE_S = 15.0
|
||
#: 运行停在审批上：评测不代批
|
||
APPROVAL_REASON = "运行停在审批上（危险工具、人工节点）：评测不代批，已取消这次运行"
|
||
#: 每一步在做什么（EvalResult.current）
|
||
PHASE_TEXT = {"gold": "正在算期望", "build": "正在建图", "run": "运行中", "review": "正在复核", "score": "正在评分",
|
||
"gold_after": "作答后复算期望", "judge": "正在请裁判判断"}
|
||
|
||
|
||
@dataclass
|
||
class TurnContext:
|
||
"""这一轮作答的来历（记进运行 origin）和要用的东西。"""
|
||
|
||
suite_id: str
|
||
suite_name: str
|
||
eval_run_id: str
|
||
case_id: str | None
|
||
case_seq: int | None
|
||
actor: str | None
|
||
arm: str | None = None
|
||
repeat: int = 0
|
||
#: 变体：助手模型 {provider, model}、写作模型 {provider, model}
|
||
assistant: Mapping[str, Any] = field(default_factory=dict)
|
||
writer: Mapping[str, Any] = field(default_factory=dict)
|
||
#: 目录视图（Run.options['catalog_view'] 的形状）
|
||
view: Mapping[str, Any] | None = None
|
||
|
||
|
||
@dataclass
|
||
class Turn:
|
||
"""一轮作答的结果（也是进行中的状态：current_run、phase 随时更新）。"""
|
||
|
||
path: str | None = None
|
||
run_id: str | None = None
|
||
#: 这一题发起过的全部作答运行（重来的话两次），失败的也在
|
||
runs: list[str] = field(default_factory=list)
|
||
attempts: int = 0
|
||
#: 每次自动重来的原因（和 attempts 一一对应）：没过检查、复核说不可信、生成过程一个操作都没给
|
||
retries: list[str] = field(default_factory=list)
|
||
reason: str | None = None
|
||
#: 助手调用失败（生成过程交出 error、开流前就被拒）：runner 据此止损
|
||
copilot_failed: bool = False
|
||
timed_out: bool = False
|
||
#: 直接回答的原文、反问的问题
|
||
reply: str | None = None
|
||
asked: dict[str, Any] | None = None
|
||
review: dict[str, Any] | None = None
|
||
looks: list[dict[str, Any]] = field(default_factory=list)
|
||
#: 助手这一轮用的模型 id（model 操作）
|
||
assistant_model: str | None = None
|
||
#: {assistant: usage 汇总（生成，含重来）, review: usage 汇总}
|
||
usage: dict[str, Any] = field(default_factory=dict)
|
||
timings: dict[str, int] = field(default_factory=dict)
|
||
#: 写作模型按变体换了，但节点自己点了模型、没被覆盖的节点 id
|
||
own_models: list[str] = field(default_factory=list)
|
||
#: 正在跑的那次运行（调用方超时、停止时取消它）
|
||
current_run: str | None = None
|
||
phase: str | None = None
|
||
|
||
def to_dict(self) -> dict[str, Any]:
|
||
return {"path": self.path, "run_id": self.run_id, "runs": list(self.runs), "attempts": self.attempts,
|
||
"retries": list(self.retries), "reason": self.reason, "copilot_failed": self.copilot_failed, "timed_out": self.timed_out,
|
||
"reply": self.reply, "asked": self.asked, "review": self.review, "looks": list(self.looks),
|
||
"assistant_model": self.assistant_model, "usage": copy.deepcopy(self.usage),
|
||
"timings": dict(self.timings), "own_models": list(self.own_models)}
|
||
|
||
|
||
#: 用量汇总里相加的计数（providers/usage 的 summary）：unfinished_calls 是没做完的调用（被取消、超时，token 只计了已报的）
|
||
_COUNTED = ("input_tokens", "output_tokens", "calls", "unpriced_calls", "unfinished_calls")
|
||
|
||
|
||
def _add_usage(into: dict[str, Any] | None, more: Mapping[str, Any] | None) -> dict[str, Any]:
|
||
"""两份 providers/usage 汇总相加（by_model 逐个模型加）。"""
|
||
out = copy.deepcopy(into) if into else {"by_model": {}, **dict.fromkeys(_COUNTED, 0), "cost_usd": 0.0}
|
||
if not more:
|
||
return out
|
||
for key in _COUNTED:
|
||
out[key] = int(out.get(key) or 0) + int(more.get(key) or 0)
|
||
out["cost_usd"] = round(float(out.get("cost_usd") or 0) + float(more.get("cost_usd") or 0), 6)
|
||
for model, row in (more.get("by_model") or {}).items():
|
||
mine = out["by_model"].setdefault(model, {**dict.fromkeys(_COUNTED, 0), "cost_usd": 0.0})
|
||
for key in _COUNTED:
|
||
mine[key] = int(mine.get(key) or 0) + int(row.get(key) or 0)
|
||
mine["cost_usd"] = round(float(mine.get("cost_usd") or 0) + float(row.get("cost_usd") or 0), 6)
|
||
return out
|
||
|
||
|
||
def add_usage(into: dict[str, Any] | None, more: Mapping[str, Any] | None) -> dict[str, Any]:
|
||
return _add_usage(into, more)
|
||
|
||
|
||
async def writer_usage(run_ids: list[str]) -> dict[str, Any]:
|
||
"""答案运行里的模型调用（llm.end，按模型分）：写作这一角色的用量，形状同 providers/usage 的汇总。封存之后追加的
|
||
（按需裁判、质询）不算：那是事后另记的账（裁判记在评测的账上另算，见 judging.py）。"""
|
||
from app.core.events import EventType
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import RunEvent
|
||
from app.providers.catalog import unpriced_call
|
||
|
||
out = _add_usage(None, None)
|
||
if not run_ids:
|
||
return out
|
||
async with SessionLocal() as session:
|
||
rows = (await session.execute(select(RunEvent.data).where(
|
||
RunEvent.run_id.in_(run_ids), RunEvent.type == EventType.LLM_END))).scalars().all()
|
||
for data in rows:
|
||
data = data if isinstance(data, dict) else {}
|
||
if data.get("post_seal"):
|
||
continue
|
||
model = str(data.get("model") or "unknown")
|
||
piece = {"input_tokens": int(data.get("input_tokens") or 0), "output_tokens": int(data.get("output_tokens") or 0),
|
||
"calls": int(data.get("calls") or 1), "cost_usd": float(data.get("cost_usd") or 0.0),
|
||
"unpriced_calls": 1 if unpriced_call(data) else 0}
|
||
out = _add_usage(out, {**piece, "by_model": {model: piece}})
|
||
return out
|
||
|
||
|
||
def with_writer(graph: dict[str, Any], writer: Mapping[str, Any]) -> tuple[dict[str, Any], list[str]]:
|
||
"""写作模型按变体写进图的 defaults（只写给了的那几项）。返回 (图, 自己点了模型、不会被覆盖的节点 id)。"""
|
||
provider, model = writer.get("provider"), writer.get("model")
|
||
if not (provider or model):
|
||
return graph, []
|
||
out = copy.deepcopy(graph)
|
||
defaults = dict(out.get("defaults") or {})
|
||
if provider:
|
||
defaults["provider"] = provider
|
||
if model:
|
||
defaults["model"] = model
|
||
out["defaults"] = defaults
|
||
own = []
|
||
for node in out.get("nodes") or []:
|
||
config = ((node or {}).get("data") or {}).get("config") or {}
|
||
if isinstance(config, Mapping) and (config.get("provider") or config.get("model")):
|
||
own.append(str(node.get("id")))
|
||
return out, own
|
||
|
||
|
||
def _options(view: Mapping[str, Any] | None) -> dict[str, Any]:
|
||
"""作答运行带的选项：目录视图、只读、遮罩（见模块说明）。"""
|
||
from app.data.catalog_view import OPTION_KEY
|
||
from app.tools.datasource import MASK_RESULTS_OPTION, READ_ONLY_OPTION
|
||
|
||
out: dict[str, Any] = {MASK_RESULTS_OPTION: True, READ_ONLY_OPTION: True}
|
||
if view:
|
||
out[OPTION_KEY] = dict(view)
|
||
return out
|
||
|
||
|
||
def run_name(suite_name: str, case_seq: int | None) -> str:
|
||
return f"评测：{suite_name or '题集'}·第 {case_seq or '?'} 题"
|
||
|
||
|
||
async def stop_run(run_id: str | None, actor: str | None = None) -> None:
|
||
"""取消一次还没结束的运行、等它收尾（超时、停止评测、停在审批上时）。已经结束的什么都不做。"""
|
||
from app.engine.runner import run_manager
|
||
|
||
if not run_id:
|
||
return
|
||
try:
|
||
await run_manager.cancel(run_id, actor=actor)
|
||
await run_manager.wait_idle(run_id, timeout=CANCEL_GRACE_S)
|
||
except Exception: # noqa: BLE001 - 取消是收尾，出错只记日志
|
||
logger.exception("评测：取消运行 %s 时出错", run_id)
|
||
|
||
|
||
class Driver:
|
||
"""一轮作答。drive() 跑完一轮，结果在 turn 上；on_phase(阶段) 在每一步开始时调（runner 据此写「正在做什么」）。"""
|
||
|
||
def __init__(self, question: str, *, datasource_ids: list[str], ctx: TurnContext,
|
||
on_phase: Callable[[str], Awaitable[None]] | None = None) -> None:
|
||
self.question = question
|
||
self.datasource_ids = list(datasource_ids)
|
||
self.ctx = ctx
|
||
self.on_phase = on_phase
|
||
self.turn = Turn()
|
||
|
||
async def _phase(self, phase: str) -> None:
|
||
self.turn.phase = phase
|
||
if self.on_phase is not None:
|
||
try:
|
||
await self.on_phase(phase)
|
||
except Exception: # noqa: BLE001 - 写进度失败不挡作答
|
||
logger.exception("评测：记进度出错")
|
||
|
||
# ------------------------------------------------------------------ 生成
|
||
|
||
async def _generate(self, instruction: str, base_graph: dict[str, Any] | None) -> tuple[list[dict[str, Any]], str | None]:
|
||
"""跑一次生成过程，收下全部操作。开流前就被拒（HTTPException）交回原因。"""
|
||
from fastapi import HTTPException
|
||
|
||
from app.api.copilot import GenerateIn, answer_ops
|
||
|
||
payload = GenerateIn(instruction=instruction, base_graph=base_graph, intent="answer",
|
||
provider=self.ctx.assistant.get("provider"), model=self.ctx.assistant.get("model"),
|
||
datasource_ids=self.datasource_ids or None)
|
||
ops: list[dict[str, Any]] = []
|
||
try:
|
||
async for op in answer_ops(payload, catalog_view=self.ctx.view):
|
||
if op.get("op") in ("thinking", "heartbeat"):
|
||
continue
|
||
ops.append(op)
|
||
except HTTPException as e:
|
||
return ops, str(e.detail)
|
||
return ops, None
|
||
|
||
def _take(self, ops: list[dict[str, Any]]) -> dict[str, Any] | None:
|
||
"""收下这一次生成的用量、查看记录、模型；交回结局（final / reply / error），只有反问的为 None。"""
|
||
terminal = None
|
||
for op in ops:
|
||
kind = op.get("op")
|
||
if kind == "usage":
|
||
self.turn.usage["assistant"] = _add_usage(self.turn.usage.get("assistant"),
|
||
{k: v for k, v in op.items() if k != "op"})
|
||
elif kind == "look":
|
||
self.turn.looks.append({k: op.get(k) for k in ("id", "by", "need", "status") if k in op})
|
||
elif kind == "model":
|
||
self.turn.assistant_model = str(op.get("model") or "") or None
|
||
elif kind in ("final", "reply", "error"):
|
||
terminal = op
|
||
return terminal
|
||
|
||
# ------------------------------------------------------------------ 运行
|
||
|
||
async def _launch(self, graph: dict[str, Any], payload: dict[str, Any] | None = None) -> str | None:
|
||
from app.chat import turn as policy
|
||
from app.db.base import SessionLocal
|
||
from app.engine.launch import LaunchRefused, launch_run
|
||
|
||
graph, own = with_writer(graph, self.ctx.writer)
|
||
self.turn.own_models = sorted(set(self.turn.own_models) | set(own))
|
||
origin = {k: v for k, v in {
|
||
"suite_id": self.ctx.suite_id, "eval_run_id": self.ctx.eval_run_id, "case_id": self.ctx.case_id,
|
||
"role": "answer", "arm": self.ctx.arm, "repeat": self.ctx.repeat, "attempt": self.turn.attempts}.items()
|
||
if v is not None}
|
||
try:
|
||
async with SessionLocal() as session:
|
||
inputs = payload if payload is not None else policy.fill_input(graph, self.question, "")
|
||
run = await launch_run(session, graph=graph, input_payload=inputs,
|
||
actor=self.ctx.actor, options=_options(self.ctx.view), origin_kind="eval",
|
||
origin=origin, workflow_name=run_name(self.ctx.suite_name, self.ctx.case_seq))
|
||
except LaunchRefused as e:
|
||
self.turn.path, self.turn.reason = "run_failed", f"运行没有发起：{e.message}"
|
||
return None
|
||
self.turn.runs.append(run.id)
|
||
self.turn.run_id = self.turn.current_run = run.id
|
||
return run.id
|
||
|
||
async def _wait(self, run_id: str) -> Any:
|
||
"""等运行到终态（或停在审批上）。交回库里的运行。"""
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import Run
|
||
from app.engine.runner import run_manager
|
||
|
||
while True:
|
||
await run_manager.wait_idle(run_id, timeout=5.0)
|
||
async with SessionLocal() as session:
|
||
row = await session.get(Run, run_id)
|
||
if row is None or row.status in ("succeeded", "failed", "cancelled", "interrupted"):
|
||
# 封存在终态之后：等执行任务收完尾（含封存），评分只读封存范围
|
||
await run_manager.wait_idle(run_id, timeout=CANCEL_GRACE_S)
|
||
async with SessionLocal() as session:
|
||
return await session.get(Run, run_id)
|
||
await asyncio.sleep(0.2)
|
||
|
||
async def _pending_approvals(self, run_id: str) -> int:
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import Approval
|
||
|
||
async with SessionLocal() as session:
|
||
return int(await session.scalar(select(func.count()).select_from(Approval).where(
|
||
Approval.run_id == run_id, Approval.status.in_(("pending", "answered")))) or 0)
|
||
|
||
# ------------------------------------------------------------------ 一轮
|
||
|
||
async def _run(self, graph: dict[str, Any], payload: dict[str, Any] | None = None) -> Any | None:
|
||
"""发起、等到终态。成功结束的交回运行；别的结局记好路径和原因，交回 None。"""
|
||
from app.engine.runner import END_REJECTED
|
||
|
||
t = self.turn
|
||
run_id = await self._launch(graph, payload)
|
||
if run_id is None:
|
||
return None
|
||
row = await self._wait(run_id)
|
||
t.current_run = None
|
||
usage = dict((row.usage or {}) if row is not None else {})
|
||
t.timings["run_wall_ms"] = t.timings.get("run_wall_ms", 0) + int(usage.get("wall_ms") or 0)
|
||
t.timings["run_active_ms"] = t.timings.get("run_active_ms", 0) + int(
|
||
usage.get("active_ms") or usage.get("duration_ms") or 0)
|
||
if row is None:
|
||
t.path, t.reason = "run_failed", "运行记录不见了"
|
||
return None
|
||
if row.status == "interrupted":
|
||
waiting = await self._pending_approvals(run_id)
|
||
await stop_run(run_id, self.ctx.actor)
|
||
t.path = "run_failed"
|
||
t.reason = APPROVAL_REASON if waiting else "运行停在断点上（服务可能重启过）：评测不继续这次运行，已取消"
|
||
return None
|
||
if row.status == "cancelled" and row.end_reason == END_REJECTED:
|
||
t.path, t.reason = "rejected", "运行被驳回（人的决定，不算运行失败）"
|
||
return None
|
||
if row.status != "succeeded":
|
||
t.path = "run_failed"
|
||
t.reason = f"运行{'失败' if row.status == 'failed' else '被停止'}：{row.error or '没有记下原因'}"
|
||
return None
|
||
return row
|
||
|
||
async def drive_workflow(self, graph: dict[str, Any], inputs: dict[str, Any]) -> Turn:
|
||
"""工作流题：钉住那一版的图按输入跑一次探索运行（来源 eval）。不建图、不复核：考的是这一版本身。不发正式运行——
|
||
正式运行会被 pick_base 和应用视图的「最近一次封存」取到（同期望探针不发正式运行的理由）。"""
|
||
t = self.turn
|
||
started = time.monotonic()
|
||
try:
|
||
await self._phase("run")
|
||
if await self._run(graph, inputs) is not None:
|
||
t.path = "ran"
|
||
return t
|
||
finally:
|
||
t.timings["total_ms"] = int((time.monotonic() - started) * 1000)
|
||
|
||
async def drive(self) -> Turn:
|
||
from app.chat import turn as policy
|
||
|
||
t = self.turn
|
||
started = time.monotonic()
|
||
instruction, base_graph = self.question, None
|
||
try:
|
||
while True:
|
||
await self._phase("build")
|
||
building = time.monotonic()
|
||
ops, refused = await self._generate(instruction, base_graph)
|
||
t.timings["build_ms"] = t.timings.get("build_ms", 0) + int((time.monotonic() - building) * 1000)
|
||
terminal = self._take(ops)
|
||
if refused is not None:
|
||
t.path, t.reason, t.copilot_failed = "build_failed", f"助手没有开始：{refused}", True
|
||
return t
|
||
if terminal is None:
|
||
asked = next((o for o in reversed(ops) if o.get("op") == "ask"), None)
|
||
t.path, t.asked = "asked", {k: asked.get(k) for k in ("question", "options")} if asked else None
|
||
return t
|
||
kind = terminal.get("op")
|
||
if kind == "error":
|
||
# 生成过程一个操作都没给（只写了思考）：按重来一次的规矩带着原因重来（同问数据页，chat/turn.next_step）
|
||
step = terminal.get("next") or policy.next_step(terminal)
|
||
if step == policy.NEXT_RETRY and t.attempts + 1 < policy.MAX_ATTEMPTS:
|
||
reason = policy.error_reason(terminal)
|
||
t.attempts += 1
|
||
t.retries.append(reason)
|
||
instruction = policy.retry_instruction(self.question, reason, policy.has_nodes(base_graph))
|
||
continue
|
||
t.path, t.reason, t.copilot_failed = "build_failed", str(terminal.get("message") or "助手本轮未完成"), True
|
||
return t
|
||
if kind == "reply":
|
||
t.path, t.reply = "reply", str(terminal.get("text") or "")
|
||
return t
|
||
if terminal.get("asked"):
|
||
t.path, t.asked = "asked", dict(terminal["asked"])
|
||
return t
|
||
self_checked = any(o.get("op") == "check" for o in ops)
|
||
step = terminal.get("next") or policy.next_step(terminal, self_checked=self_checked)
|
||
graph = terminal.get("graph") or {}
|
||
if step == policy.NEXT_RETRY:
|
||
if t.attempts + 1 < policy.MAX_ATTEMPTS:
|
||
t.attempts += 1
|
||
t.retries.append(policy.blocked_reason(terminal))
|
||
instruction = policy.retry_instruction(self.question, policy.blocked_reason(terminal),
|
||
policy.has_nodes(graph))
|
||
base_graph = graph if policy.has_nodes(graph) else None
|
||
continue
|
||
t.path, t.reason = "build_failed", policy.blocked_reason(terminal)
|
||
return t
|
||
if step == policy.NEXT_FAIL:
|
||
errors = [str(i.get("message") or "") for i in terminal.get("issues") or []
|
||
if isinstance(i, Mapping) and i.get("level") == "error"]
|
||
t.path, t.reason = "build_failed", "生成的工作流没有通过校验：" + "；".join(errors)
|
||
return t
|
||
if step == policy.NEXT_READY:
|
||
t.path, t.reason = "ready", "助手只建了图、没有运行（它判断用户要的是流程本身）"
|
||
return t
|
||
|
||
await self._phase("run")
|
||
row = await self._run(graph)
|
||
if row is None:
|
||
return t
|
||
|
||
await self._phase("review")
|
||
reviewing = time.monotonic()
|
||
review = await self._review(row)
|
||
spent = int((time.monotonic() - reviewing) * 1000)
|
||
t.timings["review_ms"] = t.timings.get("review_ms", 0) + spent
|
||
t.review = {k: v for k, v in review.items() if k != "usage"}
|
||
t.usage["review"] = _add_usage(t.usage.get("review"), review.get("usage"))
|
||
if review.get("retry_instruction"):
|
||
from app.chat.turn import review_reason
|
||
|
||
t.attempts += 1
|
||
t.retries.append(review_reason(review) or "复核判答案不可用")
|
||
instruction = str(review["retry_instruction"])
|
||
base_graph = row.graph if policy.has_nodes(row.graph) else None
|
||
continue
|
||
t.path = "ran"
|
||
return t
|
||
finally:
|
||
t.timings["total_ms"] = int((time.monotonic() - started) * 1000)
|
||
|
||
async def _review(self, run: Any) -> dict[str, Any]:
|
||
"""复核（review_of_run，问数据页同一个函数）。复核自己出错不挡评分：照实记一条失败的复核。"""
|
||
from app.api.copilot import review_of_run
|
||
from app.core.errors import describe_exception
|
||
from app.db.base import SessionLocal
|
||
|
||
try:
|
||
async with SessionLocal() as session:
|
||
return await review_of_run(session, run, question=self.question,
|
||
provider=self.ctx.assistant.get("provider"),
|
||
model=self.ctx.assistant.get("model"), attempt=self.turn.attempts)
|
||
except Exception as e: # noqa: BLE001
|
||
logger.exception("评测：复核运行 %s 出错", run.id)
|
||
return {"verdict": "failed", "note": f"复核出错：{describe_exception(e)}", "signals": [],
|
||
"retry_instruction": None}
|
||
|
||
|
||
__all__ = ["APPROVAL_REASON", "Driver", "PHASE_TEXT", "REVIEW_PAGE_WAIT_S", "Turn", "TurnContext", "add_usage",
|
||
"run_name", "stop_run", "with_writer", "writer_usage"]
|