From d18a496ef9427d93eca1701985f35485151a3681 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 18 Oct 2026 16:05:53 +0800 Subject: [PATCH 1/7] =?UTF-8?q?=E9=97=AE=E6=95=B0=E6=8D=AE=E6=95=B4?= =?UTF-8?q?=E8=BD=AE=E6=94=BE=E5=88=B0=E6=9C=8D=E5=8A=A1=E7=AB=AF=E8=B7=91?= =?UTF-8?q?=EF=BC=88=E5=90=8E=E7=AB=AF=EF=BC=89=EF=BC=9A=E4=B8=80=E8=BD=AE?= =?UTF-8?q?=E6=98=AF=E4=B8=80=E4=B8=AA=E5=90=8E=E5=8F=B0=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=EF=BC=8C=E5=BB=BA=E5=9B=BE=E3=80=81=E8=BF=90=E8=A1=8C=E3=80=81?= =?UTF-8?q?=E5=A4=8D=E6=A0=B8=E3=80=81=E6=8C=89=E5=A4=8D=E6=A0=B8=E9=87=8D?= =?UTF-8?q?=E6=9D=A5=E3=80=81=E8=90=BD=E5=BA=93=E9=83=BD=E5=9C=A8=E6=9C=8D?= =?UTF-8?q?=E5=8A=A1=E7=AB=AF=EF=BC=8C=E9=A1=B5=E9=9D=A2=E6=96=AD=E4=BA=86?= =?UTF-8?q?=E4=B9=9F=E4=BC=9A=E8=B7=91=E5=AE=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么：一轮问数据以前是浏览器在推（store/chat.ts 的 plan / launch / settle）：网络一抖建图那条流就断、建了一半的图丢掉； 发起运行、复核、重来、落库都要页面在，页面一关这一轮就停在 running，15 分钟后才被判成断了（总方案 §0 的调查结论）。 做了什么： - 流程和评测共用一份：eval/answer.Driver 里的流程抽到 chat/drive.TurnDriver，差异写成子类给的几项（生成的请求、目录视图、 改图、入口、发起参数、停在审批上怎么办、复核等多久）。评测的 Driver 成了它的子类，行为和测试一字不变（test_eval_* 照跑）。 - chat/job.py：问数据的一轮是 core/job_runtime 上的后台任务（不占数据源的队，并发归「交互」类，第 2 波再给上限和优先级）。 停在审批上照旧等人（评测是取消）；复核最多等 25 秒，晚到的那次用量照记（review.late）；按复核自动重来一次；用户点的 重试本轮、重新运行、放宽步数、跑一下、继续运行、接着收尾都是同一轮里的一次尝试，上一次的结局归档进 meta.attempts。 - 事件日志（新表 chat_turn_events，models_chat.py，不进运行的封存范围）：每一步按 seq 落库再叫醒订阅方，订阅方按 seq 回库里读——回放和实时是同一段代码；心跳一拍也回库里看一次，第 2 波多进程时别的进程写的最迟一拍送到。 - 接口 api/chat_turns.py：POST /api/chat/turns（202 带 turn_id）、GET /{id}、GET /{id}/stream?after=（SSE：先回放、再推 实时、带心跳、结束补 stream.end）、POST /{id}/cancel。挂 chat_scope，每个接口按这一轮所属的对话核对归属；访问密钥调不了。 - conversation_turns 那一行由服务端写（chat/record.py：状态、运行 id、答案、复核、meta 各项和页面以前写的一一对应）；失败 只记原因（meta.cause），说法留在界面层按 lib/explain 讲。TurnOut 带这一轮的任务（job），列表判「还在跑」先看任务。 - 使用者的问数据运行规则（白名单、自己的记忆范围、遮罩、只读）抽到 chat/launch.user_chat_run，POST /api/runs 和服务端的 一轮共用，说法一字不变。 - 服务重启：还在建图的标中断（可重试本轮）；已发起运行的接着等那次运行、照常复核收尾；和调度循环同一个开关，拷来的数据 目录启动时不替人收尾，页面打开对话时再请服务端接着收尾（mode=resume）。老轮次停在建图中的同样标中断。 - 删对话时先停掉它正在跑的几轮（取消各自的运行），任务和事件日志一并删。 测试：tests/test_chat_turns.py——一轮跑完不依赖页面、直接回答与反问、开流前被拒、实时流回放后接实时与按 seq 重连（补齐、 不重复）、思考合并、建图中与运行中停止、重启时建图中标中断与已发起运行的接着收尾、开关关着时等人来接、等审批后收尾、 按复核重来、用户重试归档、一段对话一次一轮、继续运行被拒照 /runs 的说法、删对话停掉这一轮、使用者的归属与遮罩只读、 访问密钥调不了、驳回与失败与发起被拒的原因、晚到的复核、跑一下与重新运行不重建。去掉几处关键实现回跑，相应的测试都会失败。 --- backend/app/api/chat_turns.py | 116 ++++ backend/app/api/conversations.py | 87 ++- backend/app/api/deps.py | 4 +- backend/app/api/runs.py | 14 +- backend/app/chat/drive.py | 414 ++++++++++++ backend/app/chat/job.py | 1013 ++++++++++++++++++++++++++++++ backend/app/chat/launch.py | 30 + backend/app/chat/log.py | 138 ++++ backend/app/chat/record.py | 257 ++++++++ backend/app/chat/turn.py | 21 +- backend/app/db/models.py | 2 +- backend/app/db/models_chat.py | 73 +++ backend/app/eval/answer.py | 333 +--------- backend/app/main.py | 14 + backend/tests/test_chat_turns.py | 836 ++++++++++++++++++++++++ 15 files changed, 3020 insertions(+), 332 deletions(-) create mode 100644 backend/app/api/chat_turns.py create mode 100644 backend/app/chat/drive.py create mode 100644 backend/app/chat/job.py create mode 100644 backend/app/chat/launch.py create mode 100644 backend/app/chat/log.py create mode 100644 backend/app/chat/record.py create mode 100644 backend/app/db/models_chat.py create mode 100644 backend/tests/test_chat_turns.py diff --git a/backend/app/api/chat_turns.py b/backend/app/api/chat_turns.py new file mode 100644 index 00000000..22b33176 --- /dev/null +++ b/backend/app/api/chat_turns.py @@ -0,0 +1,116 @@ +"""问数据的一轮：发起、看、停（docs/design/orchestration-round/00-总方案.md §1；做法在 app/chat/job.py）。 + +- POST /api/chat/turns：发起一轮，202 {turn_id, turn}。新问带 conversation_id、question（可选 datasource_ids 限定数据源）；对已有 + 的一轮重试、重新运行、放宽步数、跑一下、继续运行、接着收尾带 retry_of 和 mode（MODES），重试时 datasource_ids 给了就换范围 + （空数组是不限），不给沿用这一轮的。同一段对话里有一轮正在跑（停在审批上的不算）时 409 chat_turn_busy；继续运行被拒照 + POST /api/runs/{id}/continue 的说法（409，带那边的机器码）。 +- GET /api/chat/turns/{id}：这一轮的任务现在怎样（status、stage、run_id、last_seq、attempt_seq……），连同那一行（turn）。老轮次 + 没有任务时 job 为 null。 +- GET /api/chat/turns/{id}/stream?after=：SSE。先回放 seq > after 的，再推实时；隔一拍没有新事件发一行注释心跳；这一轮不在 + 进行中、也没有新的了，补一条 stream.end 再关（chat/log.py）。断了按收到的最后一个 seq 重连，补齐、不重复。 +- POST /api/chat/turns/{id}/cancel：停止（取消生成、取消运行，照现有的取消规则）。已经结束的 409 chat_turn_finished。 + +权限照问数据的规矩：路由挂 chat_scope（使用者要管理员开了问数据），每个接口再按这一轮所属的对话核对归属（对话按人隔离， +conversations.turn_access / ensure_conversation_access）；访问密钥调不了（不在 auth/keys.KEY_ROUTES 里）。 +""" +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Depends, Header, HTTPException, Query +from fastapi.responses import StreamingResponse +from pydantic import BaseModel, Field +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.coded import CodedHTTPException +from app.api.deps import actor_of +from app.db.base import get_session + +router = APIRouter(prefix="/api/chat/turns", tags=["chat"]) + + +class TurnStartIn(BaseModel): + conversation_id: str = Field(min_length=1) + question: str = "" + #: 这一问只查这几个数据源（id 或名字）。不给：新问不限，重试沿用这一轮的；空数组：不限 + datasource_ids: list[str] | None = None + #: 对已有的哪一轮（重试、重新运行、继续运行……） + retry_of: str | None = None + #: ask | retry | rerun | max_steps | run | continue | resume（app/chat/job.MODES）。不给：新问是 ask，带 retry_of 是 retry + mode: str | None = None + #: 放宽到几步（mode=max_steps） + max_steps: int | None = Field(default=None, ge=1, le=10_000) + + +def _refused(e: Any) -> HTTPException: + if e.code: + return CodedHTTPException(e.status_code, e.message, e.code) + return HTTPException(e.status_code, e.message) + + +@router.post("", status_code=202) +async def start_turn(payload: TurnStartIn, session: AsyncSession = Depends(get_session), + x_actor: str | None = Header(default=None)) -> dict[str, Any]: + from app.api.conversations import _get, turn_access, turn_out + from app.chat import job as chat_job + + retry_of = None + if payload.retry_of: + conversation, retry_of = await turn_access(session, payload.retry_of) + if conversation.id != payload.conversation_id: + raise HTTPException(404, "该轮对话不属于这段对话") + else: + conversation = await _get(session, payload.conversation_id) + try: + turn_id = await chat_job.start(session, conversation=conversation, question=payload.question, + datasource_ids=payload.datasource_ids, actor=actor_of(x_actor), mode=payload.mode, + retry_of=retry_of, max_steps=payload.max_steps) + except chat_job.TurnRefused as e: + raise _refused(e) from e + from app.db.base import SessionLocal + from app.db.models import ConversationTurn + + async with SessionLocal() as fresh: + row = await fresh.get(ConversationTurn, turn_id) + turn = (await turn_out(fresh, row)).model_dump(mode="json") if row is not None else None + return {"turn_id": turn_id, "turn": turn} + + +@router.get("/{turn_id}") +async def get_turn(turn_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + from app.api.conversations import turn_access, turn_out + from app.chat import job as chat_job + + _, row = await turn_access(session, turn_id) + return {"id": row.id, "conversation_id": row.conversation_id, "job": await chat_job.state(session, row.id), + "turn": (await turn_out(session, row)).model_dump(mode="json")} + + +@router.get("/{turn_id}/stream") +async def stream_turn(turn_id: str, after: int = Query(default=0, ge=0), + session: AsyncSession = Depends(get_session)) -> StreamingResponse: + from app.api.conversations import turn_access + from app.chat import job as chat_job + from app.chat import log as turn_log + + await turn_access(session, turn_id) + await session.close() # 流可能开很久：核对完就把这次请求的会话还回去 + + async def active() -> str | None: + return await chat_job.stream_status(turn_id) + + return StreamingResponse(turn_log.stream(turn_id, after, active=active), media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}) + + +@router.post("/{turn_id}/cancel") +async def cancel_turn(turn_id: str, session: AsyncSession = Depends(get_session), + x_actor: str | None = Header(default=None)) -> dict[str, Any]: + from app.api.conversations import turn_access + from app.chat import job as chat_job + + await turn_access(session, turn_id) + await session.close() + if not await chat_job.cancel(turn_id, actor_of(x_actor)): + raise CodedHTTPException(409, "这一轮已经结束，无需停止", chat_job.TURN_FINISHED) + return {"ok": True} diff --git a/backend/app/api/conversations.py b/backend/app/api/conversations.py index b54d7410..b3d54cf0 100644 --- a/backend/app/api/conversations.py +++ b/backend/app/api/conversations.py @@ -5,10 +5,11 @@ 同时服务两件事：给用户看的历史列表，和给模型看的上下文（`_history_text` 被 copilot 拿去拼进 prompt）。 -轮次的写入由前端驱动，分两步：开轮（只有问题）→ 回填（图、run、答案）。 -让 copilot 或 runner 自己去写会更"自动"，但建图失败、运行失败、用户中途 -取消这三种半途而废的情况它们各自只知道一半，最后还是要在三个地方各接一次。 -前端本来就完整地知道这一轮走到哪了。 +问数据的轮次由服务端写（docs/design/orchestration-round §1）：一轮是一个后台任务（app/chat/job.py），建图、运行、复核、 +按复核重来、落库都在服务端，这一行的状态、运行 id、答案、复核、meta 由它写（app/chat/record.py 说明了每一项）；页面只发起 +（POST /api/chat/turns）、只看。以前这一行是页面写的（开轮 → 回填），页面一关就没人收尾。开轮、回填这两个接口留着：画布助手的 +对话和老页面还在用，回填也用来改标题这类事，但问数据的一轮不再依赖它们。每一轮带着它的任务现在怎样（TurnOut.job），列表判 +「还在跑」先看任务，不再只靠 15 分钟没动静来猜。 对话按人隔离（D-d）：新建时记下建立人（created_by = actor_of）。开登录后 user 角色只列得出、读得到、改得动、删得掉 自己建的对话；created_by 为空的老对话（关登录时建的）只有 builder 以上看得到。应用编辑器的助手对话（kind=app）另要 @@ -99,6 +100,8 @@ class TurnOut(BaseModel): created_at: Any = None #: 最后一次落库。列表判「建图断了」按它算（BUILD_STALE），前端用同一只钟 updated_at: Any = None + #: 这一轮在服务端的任务现在怎样（app/chat/job.job_view：status、stage、last_seq、attempt_seq……）。老轮次没有任务，为 None + job: dict[str, Any] | None = None model_config = {"from_attributes": True} @@ -153,17 +156,35 @@ def turn_run_id(turn: Any) -> str | None: return turn.run_id or None -def _turn_out(turn: ConversationTurn) -> TurnOut: +def _turn_out(turn: ConversationTurn, job: Any = None) -> TurnOut: + from app.chat.job import job_view + out = TurnOut.model_validate(turn) out.meta = turn_meta(turn) + out.job = job_view(job) if job is not None else None return out -def _detail(row: Conversation, turns: list[ConversationTurn]) -> ConversationDetail: +async def _jobs(session: AsyncSession, turn_ids: list[str]) -> dict[str, Any]: + """这几轮在服务端的任务（一次查完）。""" + from app.db.models_chat import ChatTurnJob + + if not turn_ids: + return {} + rows = (await session.execute(select(ChatTurnJob).where(ChatTurnJob.id.in_(turn_ids)))).scalars() + return {j.id: j for j in rows} + + +async def turn_out(session: AsyncSession, turn: ConversationTurn) -> TurnOut: + """一轮（带着它的任务），同 GET /api/conversations/{id} 里的一轮：服务端跑的一轮收尾时把它放进 end 事件。""" + return _turn_out(turn, (await _jobs(session, [turn.id])).get(turn.id)) + + +def _detail(row: Conversation, turns: list[ConversationTurn], jobs: dict[str, Any] | None = None) -> ConversationDetail: base = ConversationOut.model_validate(row) return ConversationDetail( **base.model_dump(), - turns=[_turn_out(t) for t in turns], + turns=[_turn_out(t, (jobs or {}).get(t.id)) for t in turns], ) @@ -174,10 +195,14 @@ BUILD_STALE = timedelta(minutes=15) def last_status( turn: Any, run_status: str | None, pending: bool, now: datetime | None = None, end_reason: str | None = None, + job_stage: str | None = None, ) -> str: """一轮停在哪。轮次状态只有 running / done / error 三种，细分要看运行和元数据。 - 轮次状态是前端写的：页面关掉、服务重启时它停在 running，这时要以运行的状态为准； + job_stage：这一轮在服务端的任务还在跑时它在哪一步（app/chat/job.py；没有任务、任务不在跑时为 None）。任务在跑就是在跑 + （停在审批上的是 waiting），不按 15 分钟没动静去猜——建图慢的一轮，那一行要到建完才改。 + + 没有任务的老轮次是页面写的：页面关掉、服务重启时它停在 running，这时要以运行的状态为准； 运行也没有的，是断在了建图阶段。已经收尾的轮次以轮次自己为准——它写下的就是 用户看到的结局（取消、服务重启挂起在 error 里分开记）。 @@ -186,6 +211,8 @@ def last_status( """ meta = turn_meta(turn) or {} outcome = meta.get("outcome") + if job_stage is not None and turn.status == "running": + return "waiting" if job_stage == "waiting" else "running" if turn.status == "done": return "done" if turn.status == "error": @@ -258,12 +285,17 @@ async def _statuses( select(Approval.run_id).where( Approval.run_id.in_(stopped), Approval.status == "pending") )).scalars()) + from app.core.job_runtime import ACTIVE + + jobs = await _jobs(session, [t.id for t in turns.values()]) now = utcnow() out: dict[str, tuple[str, str | None]] = {} for cid, turn in turns.items(): run_id = run_of[cid] if run_of[cid] in runs else None + job = jobs.get(turn.id) + stage = (job.stage or "build") if job is not None and job.status in ACTIVE else None out[cid] = (last_status(turn, runs.get(run_id or ""), run_id in waiting, now, - end_reason=ended.get(run_id or "")), + end_reason=ended.get(run_id or ""), job_stage=stage), run_id) return out @@ -437,7 +469,7 @@ async def conversation_detail(session: AsyncSession, row: Conversation) -> Conve .where(ConversationTurn.conversation_id == conversation_id) .order_by(ConversationTurn.seq) )).scalars()) - out = _detail(row, turns) + out = _detail(row, turns, await _jobs(session, [t.id for t in turns])) out.turn_count = len(turns) out.last_question = turns[-1].question if turns else "" if turns: @@ -465,9 +497,14 @@ async def patch_conversation( async def delete_conversation( conversation_id: str, session: AsyncSession = Depends(get_session) ) -> None: + from app.chat import job as chat_job + row = await _get(session, conversation_id) + # 正在服务端跑的几轮先停下（取消各自的运行）：不然删掉之后它还在跑、还在计费，收尾时又找不到那一行 + await chat_job.stop_conversation(conversation_id) # 显式删 turns 而不是只靠 ondelete=CASCADE：SQLite 的外键级联要 PRAGMA - # 打开才生效（base.py 里是打开的），但这条路径不该依赖一个连接级的开关 + # 打开才生效（base.py 里是打开的），但这条路径不该依赖一个连接级的开关。任务和事件日志同理 + await chat_job.forget(session, conversation_id) await session.execute( delete(ConversationTurn).where(ConversationTurn.conversation_id == conversation_id) ) @@ -487,25 +524,39 @@ async def create_turn( session: AsyncSession = Depends(get_session), ) -> TurnOut: conversation = await _get(session, conversation_id) + turn = await new_turn(session, conversation, payload.question) + await session.commit() + await session.refresh(turn) + return _turn_out(turn) + + +async def new_turn(session: AsyncSession, conversation: Conversation, question: str) -> ConversationTurn: + """开一轮（status=running，只有问题），不提交：开轮接口和服务端发起的一轮（app/chat/job.start）共用。""" next_seq = int((await session.execute( select(func.coalesce(func.max(ConversationTurn.seq), -1)) - .where(ConversationTurn.conversation_id == conversation_id) + .where(ConversationTurn.conversation_id == conversation.id) )).scalar_one()) + 1 turn = ConversationTurn( - conversation_id=conversation_id, seq=next_seq, - question=payload.question, status="running", + conversation_id=conversation.id, seq=next_seq, + question=question, status="running", ) session.add(turn) conversation.last_active_at = utcnow() # 第一轮的问题就是这次对话的标题。用户改过名就不要再覆盖回去 if next_seq == 0 and conversation.title in ("", "新对话"): - conversation.title = title_from(payload.question) + conversation.title = title_from(question) + await session.flush() + return turn - await session.commit() - await session.refresh(turn) - return _turn_out(turn) + +async def turn_access(session: AsyncSession, turn_id: str) -> tuple[Conversation, ConversationTurn]: + """这一轮这个人能不能用（按它所属的对话判，规则同 _get：对话按人隔离）。不存在 404，不能用 403。""" + turn = await session.get(ConversationTurn, turn_id) + if turn is None: + raise HTTPException(404, "该轮对话不存在，可能已被删除") + return await _get(session, turn.conversation_id), turn @router.patch("/{conversation_id}/turns/{turn_id}", response_model=TurnOut) diff --git a/backend/app/api/deps.py b/backend/app/api/deps.py index 21b4fbda..782e2ec8 100644 --- a/backend/app/api/deps.py +++ b/backend/app/api/deps.py @@ -349,12 +349,14 @@ async def _chat_scope(conn: HTTPConnection) -> None: ensure_role("user") path = conn.url.path.rstrip("/") method = conn.scope.get("method", "GET") - if user_can_chat() and (path.startswith("/api/conversations") or (method, path) in _CHAT_ROUTES): + if user_can_chat() and (path.startswith(_CHAT_PREFIXES) or (method, path) in _CHAT_ROUTES): return raise CodedHTTPException(403, "没有权限：问数据和助手需要搭建者及以上的角色，或由管理员为使用者开启问数据", CHAT_FORBIDDEN) +#: user 角色开了问数据时能用的整组接口：对话，和服务端跑的问数据一轮（发起、看、停，app/chat/job.py）。归属在接口里按对话判 +_CHAT_PREFIXES = ("/api/conversations", "/api/chat/turns") #: user 角色开了问数据时能用的其余接口：问数据的生成流、读当前助手模型、问数据页选数据源用的数据源名单 #: （GET /api/datasources/names，只有名字、说明和种类，不含连接信息） _CHAT_ROUTES = frozenset({("POST", "/api/copilot/generate-stream"), ("GET", "/api/copilot/model"), diff --git a/backend/app/api/runs.py b/backend/app/api/runs.py index 606f82ad..bd87fc14 100644 --- a/backend/app/api/runs.py +++ b/backend/app/api/runs.py @@ -112,7 +112,6 @@ async def start_run( """发起规则全在 engine/launch.py（定时任务到点发起也用它），这里只把拒绝换回 HTTP 响应。""" from app.api.deps import restricted from app.engine.origin import CLIENT_ORIGINS - from app.tools.datasource import MASK_RESULTS_OPTION, READ_ONLY_OPTION origin = payload.origin or "api" if origin not in CLIENT_ORIGINS: @@ -133,15 +132,16 @@ async def start_run( # 数据源查询一律只读，写语句不交给审批（Agent 的 approval 是它自己写在图里的） if origin != "chat" or payload.run_class != "exploratory" or payload.workflow_id or payload.version is not None: raise HTTPException(403, "没有权限：使用者只能发起问数据的运行，工作流的运行需要搭建者及以上的角色") - from app.auth.chat_policy import SCOPE_REFUSED, chat_graph_problem, chat_memory_scope + from app.auth.chat_policy import SCOPE_REFUSED + from app.chat.launch import ChatRunRefused, user_chat_run - if problem := chat_graph_problem(payload.graph): - raise HTTPException(403, problem) + # 白名单、记忆范围、遮罩与只读这几条和服务端跑的一轮问数据（chat/job.py）共用 chat/launch.user_chat_run + try: + memory_scope, options = user_chat_run(payload.graph, actor) + except ChatRunRefused as e: + raise HTTPException(403, str(e)) from e if payload.memory_scope or payload.collection: raise HTTPException(403, SCOPE_REFUSED) - # 记忆固定为这位使用者自己的范围、知识库取设置里的默认值：记忆、知识库的内容不经遮罩（见 chat_policy 开头） - memory_scope = chat_memory_scope(actor) - options = {MASK_RESULTS_OPTION: True, READ_ONLY_OPTION: True} try: return await launch_run( session, diff --git a/backend/app/chat/drive.py b/backend/app/chat/drive.py new file mode 100644 index 00000000..f9f1f899 --- /dev/null +++ b/backend/app/chat/drive.py @@ -0,0 +1,414 @@ +"""一轮问数据的流程：问数据（服务端跑整轮，app/chat/job.py）和评测（app/eval/answer.py）共用这一份。 + +**为什么只有一份。** 评测量的必须是用户拿到的那一轮（细案 01 做法 1），以前问数据的一轮是浏览器在推（frontend/src/store/chat.ts +的 plan / launch / settle），评测照着在服务端另串了一遍（eval/answer.Driver）。现在问数据也整轮放到服务端（总方案 §1），两边各写 +一份流程迟早走样，所以把流程抽到这里，差异写成子类给的几项（总方案上一轮 X「一种说法」）： + + answer_ops → final.next（chat/turn.next_step）→ launch_run → 等终态 → review_of_run → 要的话带原图重来一次 + +- 生成：generate_in 给这一次生成的请求（问数据带对话、限定的数据源；评测带变体的助手模型），catalog_view 给读哪一份数据目录 + （评测按变体，问数据是当前目录）。每一条操作都先交给 on_op（问数据写进这一轮的事件日志，评测不管），思考和心跳不进结局判断。 +- 结局：reply 直接回答、asked 反问、ready 只建不跑、build_failed 建图失败、retry 带着原因重来（次数照 MAX_ATTEMPTS）、run 运行。 + 每次重来前调 on_retry（问数据记一条「重来」，把换下来的运行的用量并进这一轮）。 +- 运行：prepare_graph 改图（评测按变体写进写作模型）、run_input 填入口（问数据带前几轮的问答，评测单轮）、launch_args 给来源、 + 选项、发起人（遮罩、只读、目录视图照各自现有的规则），发起之后调 on_launched。 +- 终态：停在审批上交给 on_interrupted——评测取消（不代批），问数据照旧等人（这是两边唯一的流程差别）；被驳回的记 rejected， + 失败、被停止记 run_failed。 +- 复核：_review（评测一直等、复核自己出错照实记一条失败的复核；问数据最多等 25 秒，见 chat/job.py）。给了 + retry_instruction 且 retry 打开时带着这次运行的图重来一次。 + +drive 的几个参数只有问数据用：graph（不建图、直接运行这张图：重新运行本轮、放宽步数、跑一下）、run_id（不发起、接着等这次 +运行：继续运行、服务重启后接着收尾）、retry（复核之后要不要自动重来：用户点的「重新运行」照以前不自动重来）、instruction / +base_graph（用户点「重试本轮」时带着原因和上一次的图重建）。不给就是评测那一轮，和抽出来之前逐条一样。 + +时限、停止由调用方管：这里随时把正在跑的那次运行记在 turn.current_run，调用方超时、停止时据此取消它。 +""" +from __future__ import annotations + +import asyncio +import copy +import logging +import time +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass, field +from typing import Any + +from sqlalchemy import func, select + +logger = logging.getLogger(__name__) + +#: 取消一次运行之后最多再等它收尾多久 +CANCEL_GRACE_S = 15.0 + + +@dataclass +class Turn: + """一轮作答的结果（也是进行中的状态：current_run、phase 随时更新）。""" + + path: str | None = None + run_id: str | None = None + #: 这一轮发起过的全部运行（重来的话两次），失败的也在 + runs: list[str] = field(default_factory=list) + attempts: int = 0 + #: 每次自动重来的原因（和 attempts 一一对应）：没过检查、复核说不可信、生成过程一个操作都没给 + retries: list[str] = field(default_factory=list) + reason: str | None = None + #: 助手调用失败（生成过程交出 error、开流前就被拒）：评测据此止损 + copilot_failed: bool = False + timed_out: bool = False + #: 直接回答的原文、反问的问题 + reply: str | None = None + asked: dict[str, Any] | None = None + review: dict[str, Any] | None = None + looks: list[dict[str, Any]] = field(default_factory=list) + #: 助手这一轮用的模型 id（model 操作） + assistant_model: str | None = None + #: {assistant: usage 汇总（生成，含重来）, review: usage 汇总} + usage: dict[str, Any] = field(default_factory=dict) + timings: dict[str, int] = field(default_factory=dict) + #: 写作模型按变体换了，但节点自己点了模型、没被覆盖的节点 id（评测） + own_models: list[str] = field(default_factory=list) + #: 正在跑的那次运行（调用方超时、停止时取消它） + current_run: str | None = None + phase: str | None = None + + def to_dict(self) -> dict[str, Any]: + return {"path": self.path, "run_id": self.run_id, "runs": list(self.runs), "attempts": self.attempts, + "retries": list(self.retries), "reason": self.reason, "copilot_failed": self.copilot_failed, "timed_out": self.timed_out, + "reply": self.reply, "asked": self.asked, "review": self.review, "looks": list(self.looks), + "assistant_model": self.assistant_model, "usage": copy.deepcopy(self.usage), + "timings": dict(self.timings), "own_models": list(self.own_models)} + + +#: 用量汇总里相加的计数（providers/usage 的 summary）：unfinished_calls 是没做完的调用（被取消、超时，token 只计了已报的） +_COUNTED = ("input_tokens", "output_tokens", "calls", "unpriced_calls", "unfinished_calls") + + +def add_usage(into: dict[str, Any] | None, more: Mapping[str, Any] | None) -> dict[str, Any]: + """两份 providers/usage 汇总相加（by_model 逐个模型加）。""" + out = copy.deepcopy(into) if into else {"by_model": {}, **dict.fromkeys(_COUNTED, 0), "cost_usd": 0.0} + if not more: + return out + for key in _COUNTED: + out[key] = int(out.get(key) or 0) + int(more.get(key) or 0) + out["cost_usd"] = round(float(out.get("cost_usd") or 0) + float(more.get("cost_usd") or 0), 6) + for model, row in (more.get("by_model") or {}).items(): + mine = out["by_model"].setdefault(model, {**dict.fromkeys(_COUNTED, 0), "cost_usd": 0.0}) + for key in _COUNTED: + mine[key] = int(mine.get(key) or 0) + int(row.get(key) or 0) + mine["cost_usd"] = round(float(mine.get("cost_usd") or 0) + float(row.get("cost_usd") or 0), 6) + return out + + +class TurnDriver: + """一轮问数据。drive() 跑完一轮，结果在 turn 上；on_phase(阶段) 在每一步开始时调。子类给差异（见模块说明）。 + + 跑完之后这几项给调用方收尾用（问数据据此写 conversation_turns）：terminal 最后一次生成的结局操作；refusal 开流前的拒绝 + （HTTPException，带 detail / code）；launch_refused 发起被拒（LaunchRefused）；graph 最后一次建出的图；self_checked 服务端 + 这一次自查过；last_run 最后一次运行的库里记录；last_review 最后一次复核的完整结果。""" + + def __init__(self, question: str, *, datasource_ids: list[str], + on_phase: Callable[[str], Awaitable[None]] | None = None, + provider: str | None = None, model: str | None = None) -> None: + self.question = question + self.datasource_ids = list(datasource_ids) + self.on_phase = on_phase + #: 助手模型（生成、复核都用它）：评测按变体覆盖，问数据用设置里的 + self.provider = provider + self.model = model + self.turn = Turn() + self.terminal: dict[str, Any] | None = None + self.refusal: Exception | None = None + self.launch_refused: Exception | None = None + self.graph: dict[str, Any] | None = None + self.self_checked = False + self.last_run: Any = None + self.last_review: dict[str, Any] | None = None + + async def _phase(self, phase: str) -> None: + self.turn.phase = phase + if self.on_phase is not None: + try: + await self.on_phase(phase) + except Exception: # noqa: BLE001 - 写进度失败不挡作答 + logger.exception("一轮作答：记进度出错") + + # ------------------------------------------------------------------ 子类给的差异 + + def generate_in(self, instruction: str, base_graph: dict[str, Any] | None) -> Any: + """这一次生成的请求（api/copilot.GenerateIn）。""" + from app.api.copilot import GenerateIn + + return GenerateIn(instruction=instruction, base_graph=base_graph, intent="answer", provider=self.provider, + model=self.model, datasource_ids=self.datasource_ids or None) + + def catalog_view(self) -> Any: + """这一轮读哪一份数据目录（data/catalog_view.py），None 是当前目录。""" + return None + + async def on_op(self, op: dict[str, Any]) -> None: + """生成过程的每一条操作（含思考、心跳），收下之前先交给这里。""" + + async def on_retry(self, reason: str, *, kind: str, base_graph: dict[str, Any] | None) -> None: + """要自动重来一次了（kind：empty 生成过程一个操作都没给、blocked 没过检查、review 复核判不可用）。""" + + def prepare_graph(self, graph: dict[str, Any]) -> dict[str, Any]: + """发起之前改一下图（评测按变体写进写作模型）。""" + return graph + + async def run_input(self, session: Any, graph: dict[str, Any]) -> dict[str, Any]: + """入口字段怎么填（chat/turn.fill_input）。缺省单轮：前几轮的问答为空。""" + from app.chat import turn as policy + + return policy.fill_input(graph, self.question, "") + + async def launch_args(self, session: Any, graph: dict[str, Any]) -> dict[str, Any]: + """launch_run 的其余参数：来源、选项、发起人……可以抛 LaunchRefused（照发起被拒记）。""" + raise NotImplementedError + + async def on_launched(self, run: Any) -> None: + """运行发起了。""" + + async def on_interrupted(self, run_id: str, row: Any) -> Any | None: + """运行停在断点上（审批、服务重启挂起）：交回接着等到的运行记录，或者记好路径、交回 None。""" + raise NotImplementedError + + # ------------------------------------------------------------------ 生成 + + async def _generate(self, instruction: str, base_graph: dict[str, Any] | None) -> tuple[list[dict[str, Any]], str | None]: + """跑一次生成过程，收下全部操作。开流前就被拒（HTTPException）交回原因。""" + from fastapi import HTTPException + + from app.api.copilot import answer_ops + + payload = self.generate_in(instruction, base_graph) + ops: list[dict[str, Any]] = [] + try: + async for op in answer_ops(payload, catalog_view=self.catalog_view()): + await self.on_op(op) + if op.get("op") in ("thinking", "heartbeat"): + continue + ops.append(op) + except HTTPException as e: + self.refusal = e + return ops, str(e.detail) + return ops, None + + def _take(self, ops: list[dict[str, Any]]) -> dict[str, Any] | None: + """收下这一次生成的用量、查看记录、模型；交回结局（final / reply / error），只有反问的为 None。""" + terminal = None + for op in ops: + kind = op.get("op") + if kind == "usage": + self.turn.usage["assistant"] = add_usage(self.turn.usage.get("assistant"), + {k: v for k, v in op.items() if k != "op"}) + elif kind == "look": + self.turn.looks.append({k: op.get(k) for k in ("id", "by", "need", "status") if k in op}) + elif kind == "model": + self.turn.assistant_model = str(op.get("model") or "") or None + elif kind in ("final", "reply", "error"): + terminal = op + return terminal + + # ------------------------------------------------------------------ 运行 + + async def _launch(self, graph: dict[str, Any], payload: dict[str, Any] | None = None) -> str | None: + from app.db.base import SessionLocal + from app.engine.launch import LaunchRefused, launch_run + + graph = self.prepare_graph(graph) + try: + async with SessionLocal() as session: + inputs = payload if payload is not None else await self.run_input(session, graph) + run = await launch_run(session, graph=graph, input_payload=inputs, + **(await self.launch_args(session, graph))) + except LaunchRefused as e: + self.launch_refused = e + self.turn.path, self.turn.reason = "run_failed", f"运行没有发起：{e.message}" + return None + self.turn.runs.append(run.id) + self.turn.run_id = self.turn.current_run = run.id + await self.on_launched(run) + return run.id + + async def _wait(self, run_id: str) -> Any: + """等运行到终态（或停在审批上）。交回库里的运行。""" + from app.db.base import SessionLocal + from app.db.models import Run + from app.engine.runner import run_manager + + while True: + await run_manager.wait_idle(run_id, timeout=5.0) + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row is None or row.status in ("succeeded", "failed", "cancelled", "interrupted"): + # 封存在终态之后：等执行任务收完尾（含封存），评分只读封存范围 + await run_manager.wait_idle(run_id, timeout=CANCEL_GRACE_S) + async with SessionLocal() as session: + return await session.get(Run, run_id) + await asyncio.sleep(0.2) + + async def _pending_approvals(self, run_id: str) -> int: + from app.db.base import SessionLocal + from app.db.models import Approval + + async with SessionLocal() as session: + return int(await session.scalar(select(func.count()).select_from(Approval).where( + Approval.run_id == run_id, Approval.status.in_(("pending", "answered")))) or 0) + + async def _follow(self, run_id: str) -> Any | None: + """等这次运行到终态。成功结束的交回运行；别的结局记好路径和原因，交回 None。""" + from app.engine.runner import END_REJECTED + + t = self.turn + row = await self._wait(run_id) + t.current_run = None + self.last_run = row + usage = dict((row.usage or {}) if row is not None else {}) + t.timings["run_wall_ms"] = t.timings.get("run_wall_ms", 0) + int(usage.get("wall_ms") or 0) + t.timings["run_active_ms"] = t.timings.get("run_active_ms", 0) + int( + usage.get("active_ms") or usage.get("duration_ms") or 0) + if row is None: + t.path, t.reason = "run_failed", "运行记录不见了" + return None + if row.status == "interrupted": + return await self.on_interrupted(run_id, row) + if row.status == "cancelled" and row.end_reason == END_REJECTED: + t.path, t.reason = "rejected", "运行被驳回（人的决定，不算运行失败）" + return None + if row.status != "succeeded": + t.path = "run_failed" + t.reason = f"运行{'失败' if row.status == 'failed' else '被停止'}：{row.error or '没有记下原因'}" + return None + return row + + async def _run(self, graph: dict[str, Any], payload: dict[str, Any] | None = None) -> Any | None: + """发起、等到终态。成功结束的交回运行；别的结局记好路径和原因，交回 None。""" + run_id = await self._launch(graph, payload) + if run_id is None: + return None + return await self._follow(run_id) + + # ------------------------------------------------------------------ 一轮 + + async def drive(self, *, graph: dict[str, Any] | None = None, run_id: str | None = None, retry: bool = True, + instruction: str | None = None, base_graph: dict[str, Any] | None = None) -> Turn: + from app.chat import turn as policy + + t = self.turn + started = time.monotonic() + instruction = instruction or self.question + # 不建图、直接运行这张图（graph），或者不发起、接着等这次运行（run_id）：只有第一圈这样，重来还是照常建图 + direct, follow = graph, run_id + try: + while True: + if direct is None and follow is None: + await self._phase("build") + building = time.monotonic() + ops, refused = await self._generate(instruction, base_graph) + t.timings["build_ms"] = t.timings.get("build_ms", 0) + int((time.monotonic() - building) * 1000) + terminal = self._take(ops) + self.terminal = terminal + if refused is not None: + t.path, t.reason, t.copilot_failed = "build_failed", f"助手没有开始：{refused}", True + return t + if terminal is None: + asked = next((o for o in reversed(ops) if o.get("op") == "ask"), None) + t.path, t.asked = "asked", {k: asked.get(k) for k in ("question", "options")} if asked else None + return t + kind = terminal.get("op") + if kind == "error": + # 生成过程一个操作都没给（只写了思考）：按重来一次的规矩带着原因重来（chat/turn.next_step） + step = terminal.get("next") or policy.next_step(terminal) + if step == policy.NEXT_RETRY and t.attempts + 1 < policy.MAX_ATTEMPTS: + reason = policy.error_reason(terminal) + t.attempts += 1 + t.retries.append(reason) + instruction = policy.retry_instruction(self.question, reason, policy.has_nodes(base_graph)) + await self.on_retry(reason, kind="empty", base_graph=base_graph) + continue + t.path, t.reason, t.copilot_failed = "build_failed", str(terminal.get("message") or "助手本轮未完成"), True + return t + if kind == "reply": + t.path, t.reply = "reply", str(terminal.get("text") or "") + return t + if terminal.get("asked"): + self.graph = terminal.get("graph") or None + t.path, t.asked = "asked", dict(terminal["asked"]) + return t + self.self_checked = any(o.get("op") == "check" for o in ops) + step = terminal.get("next") or policy.next_step(terminal, self_checked=self.self_checked) + built = terminal.get("graph") or {} + self.graph = built + if step == policy.NEXT_RETRY: + if t.attempts + 1 < policy.MAX_ATTEMPTS: + t.attempts += 1 + t.retries.append(policy.blocked_reason(terminal)) + instruction = policy.retry_instruction(self.question, policy.blocked_reason(terminal), + policy.has_nodes(built)) + base_graph = built if policy.has_nodes(built) else None + await self.on_retry(t.retries[-1], kind="blocked", base_graph=base_graph) + continue + t.path, t.reason = "build_failed", policy.blocked_reason(terminal) + return t + if step == policy.NEXT_FAIL: + errors = [str(i.get("message") or "") for i in terminal.get("issues") or [] + if isinstance(i, Mapping) and i.get("level") == "error"] + t.path, t.reason = "build_failed", "生成的工作流没有通过校验：" + "；".join(errors) + return t + if step == policy.NEXT_READY: + t.path, t.reason = "ready", "助手只建了图、没有运行（它判断用户要的是流程本身）" + return t + direct = built + + await self._phase("run") + if follow is not None: + t.runs.append(follow) + t.run_id = t.current_run = follow + row = await self._follow(follow) + else: + row = await self._run(direct) + direct = follow = None + if row is None: + return t + + await self._phase("review") + reviewing = time.monotonic() + self.last_review = None + review = await self._review(row) + spent = int((time.monotonic() - reviewing) * 1000) + t.timings["review_ms"] = t.timings.get("review_ms", 0) + spent + if review is not None: + self.last_review = review + t.review = {k: v for k, v in review.items() if k != "usage"} + t.usage["review"] = add_usage(t.usage.get("review"), review.get("usage")) + if retry and review.get("retry_instruction"): + t.attempts += 1 + t.retries.append(policy.review_reason(review) or "复核判答案不可用") + instruction = str(review["retry_instruction"]) + base_graph = row.graph if policy.has_nodes(row.graph) else None + await self.on_retry(t.retries[-1], kind="review", base_graph=base_graph) + continue + t.path = "ran" + return t + finally: + t.timings["total_ms"] = int((time.monotonic() - started) * 1000) + + async def _review(self, run: Any) -> dict[str, Any] | None: + """复核（review_of_run，问数据和评测同一个函数）。复核自己出错不挡收尾：照实记一条失败的复核。交回 None 表示没有 + 复核结论（问数据等不及的那种，见 chat/job.py）。""" + from app.api.copilot import review_of_run + from app.core.errors import describe_exception + from app.db.base import SessionLocal + + try: + async with SessionLocal() as session: + return await review_of_run(session, run, question=self.question, provider=self.provider, + model=self.model, attempt=self.turn.attempts) + except Exception as e: # noqa: BLE001 + logger.exception("一轮作答：复核运行 %s 出错", run.id) + return {"verdict": "failed", "note": f"复核出错：{describe_exception(e)}", "signals": [], + "retry_instruction": None} + + +__all__ = ["CANCEL_GRACE_S", "Turn", "TurnDriver", "add_usage"] diff --git a/backend/app/chat/job.py b/backend/app/chat/job.py new file mode 100644 index 00000000..32bc6f3b --- /dev/null +++ b/backend/app/chat/job.py @@ -0,0 +1,1013 @@ +"""问数据的一轮在服务端跑（docs/design/orchestration-round/00-总方案.md §1）。 + +**为什么。** 一轮问数据（建图 → 发起运行 → 复核 → 按复核重来、落库）以前是浏览器在推（frontend/src/store/chat.ts）：网络一抖， +建图那条流断了、建了一半的图丢掉；发起运行、复核、重来、落库都要页面在，页面一关这一轮就停在 running，15 分钟后才被判成断了。 +现在一轮是一个后台任务：页面只发起、只看（GET /api/chat/turns/{id}/stream 先回放、再接实时，断了按 seq 补齐），这一轮在不在 +页面上都会跑完、复核完、落好库。 + +**流程和评测是同一份**（chat/drive.TurnDriver）。问数据这边的差异写在 ChatDriver： +- 生成带着对话（历史、上一轮的图）、限定的数据源；每一条操作写进事件日志（思考按 THINKING_FLUSH_S 合并成一条，几百条增量 + 不逐条落库）； +- 发起：来源 chat（origin 记对话和这一轮），入口填问题和前几轮的问答（conversations.history_text）；开登录后 user 角色照 + chat/launch.user_chat_run（白名单、自己的记忆范围、遮罩、只读）——和以前浏览器发 POST /api/runs 时一字不差； +- 停在审批上：照旧等人（评测是取消），审批过了接着等这次运行；没有待审批的断点（服务重启挂起）这一轮收成「服务重启，本轮已 + 中断」，可以「继续运行」； +- 复核最多等 REVIEW_WAIT_S（同以前页面上的 25 秒）：等不到就先交原答案，复核那一次晚到了照样把用量记进这一轮（review.late）。 + +**事件日志**（chat/log.py；每条 {seq, type, data, ts}）： + turn.attempt {mode, attempts, scope, max_steps?} 这一次尝试开始了（发起、重试本轮、重新运行、跑一下、继续运行、接着收尾） + op {...} 助手的一条操作原样（含心跳、查看、final、usage） + stage {stage} 到了哪一步：build 建图、run 运行、waiting 停在审批上等人、review 复核 + run {run_id, status, run_class} 发起了一次运行（页面据此接上运行自己的实时流看过程） + review {result} 复核结论原样（含 usage、retry_instruction） + turn.retry {kind, reason, base, attempt} 自动重来一次（kind：empty 一个操作都没给、blocked 没过检查、review 复核判不可用） + end {outcome, turn, output, run} 这一轮到此为止：outcome 是 done | reply | asked | ready | error | cancelled | suspended， + turn 是落库之后的那一行（同 GET /api/conversations/{id} 里的一轮），output 是完整成果 + review.late {usage} 等不及的那次复核晚到了：它的用量（已经并进那一行的 meta.usage.review） + +**重启。** 还在建图的标中断（照实说「服务重启，本轮在生成工作流时中断」，可以重试本轮）；已经发起了运行的，重启后接着等那次 +运行、照常复核收尾（运行本身的中断规则不变：服务重启时在跑的运行挂起，这一轮就收成「服务重启，本轮已中断」）。接着收尾可能要 +调模型（复核、按复核重来），所以和调度循环同一个开关（AGENTLAB_SCHEDULER）：拷来的数据目录（检查脚本的 --stacks）启动时不替人 +做，这几轮停着，等人打开这段对话时页面再请服务端接着收尾（mode=resume）。 + +**并发。** 归「交互」类（总方案 §5 W2-b）：这一波还是进程内的 core/job_runtime（一个 asyncio 任务一轮，不占数据源的队）， +和以前浏览器各推各的一样不设上限；第 2 波按库里的计数认领、按类别给并发上限和优先级（问数据优先于评测），租约到期别的进程接手。 +""" +from __future__ import annotations + +import asyncio +import logging +import time +from collections.abc import Mapping +from typing import Any + +from sqlalchemy import delete, select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.chat import log as turn_log +from app.chat import record +from app.chat.drive import TurnDriver +from app.core import job_runtime as runtime +from app.db.base import SessionLocal, utcnow + +logger = logging.getLogger(__name__) + +#: 发起一轮的几种方式：ask 新问；retry 重试本轮（带着上一次的原因重建）；rerun 原图重新运行；max_steps 放宽步数重新运行； +#: run 只建了图、用户点「运行」；continue 从断点接着跑（失败的、服务重启挂起的）；resume 接着等这一轮的那次运行、复核收尾 +MODES = ("ask", "retry", "rerun", "max_steps", "run", "continue", "resume") +#: 复核之后会自动重来的方式（用户点的「重新运行」「跑一下」「继续运行」照以前不自动重来） +AUTO_RETRY_MODES = ("ask", "retry") +#: 并发类别（第 2 波按它给上限和优先级） +CONCURRENCY_CLASS = "interactive" +#: 最多等复核多久（秒）：等不到先交原答案（同以前页面上的 REVIEW_TIMEOUT_MS） +REVIEW_WAIT_S = 25.0 +#: 思考增量攒多久写一条 +THINKING_FLUSH_S = 1.0 +#: 停在审批上时多久回库里看一次（审批过了运行事件总线也会叫醒它） +APPROVAL_POLL_S = 5.0 +#: 停止时最多等后台任务收尾多久 +STOP_GRACE_S = 10.0 + + +# -------------------------------------------------------------------------- +# 发起人：服务端接着做时按这个人的身份（遮罩、只读、对话归属都看它） +# -------------------------------------------------------------------------- + + +def principal_snapshot() -> dict[str, Any]: + """这次请求的身份快照（api/deps.Principal 里和问数据有关的几项）。访问密钥调不到这些接口，不记密钥的那几项。""" + from app.api.deps import current_principal + + p = current_principal() + return {"username": p.username, "role": p.role, "display_name": p.display_name, "user_id": p.user_id, + "groups": list(p.groups), "verified": p.verified, "mode": p.mode, "via": p.via} + + +def principal_of(snapshot: Mapping[str, Any] | None) -> Any: + from app.api.deps import Principal + + s = dict(snapshot or {}) + if not s: + return Principal(username=None, role="admin", verified=False, mode="off", via="off") + return Principal(username=s.get("username"), role=str(s.get("role") or "user"), display_name=s.get("display_name"), + user_id=s.get("user_id"), groups=tuple(s.get("groups") or ()), verified=bool(s.get("verified")), + mode=str(s.get("mode") or "off"), via=str(s.get("via") or "session")) + + +# -------------------------------------------------------------------------- +# 驱动：问数据和评测不一样的那几项 +# -------------------------------------------------------------------------- + + +class ChatDriver(TurnDriver): + """问数据的一轮（流程在 TurnDriver，见模块说明）。""" + + def __init__(self, execution: ChatTurnExecution, *, question: str, datasource_ids: list[str]) -> None: + super().__init__(question, datasource_ids=datasource_ids, on_phase=execution.on_phase) + self.execution = execution + self._thinking = "" + self._thinking_at = time.monotonic() + #: 这一轮的查看记录（存进 meta.looks） + self.looks: list[dict[str, Any]] = [] + #: 等不及的那次复核（还在跑）：收尾之后由它把用量记进这一轮 + self.late_review: asyncio.Task[Any] | None = None + + def generate_in(self, instruction: str, base_graph: dict[str, Any] | None) -> Any: + from app.api.copilot import GenerateIn + + return GenerateIn(instruction=instruction, base_graph=base_graph, intent="answer", + conversation_id=self.execution.conversation_id, datasource_ids=self.datasource_ids or None) + + async def on_op(self, op: dict[str, Any]) -> None: + kind = op.get("op") + if kind == "thinking": + self._thinking += str(op.get("delta") or "") + if time.monotonic() - self._thinking_at >= THINKING_FLUSH_S: + await self._flush_thinking() + return + await self._flush_thinking() + await self.execution.emit("op", op) + if kind == "look" and (look := record.look_event(op)) is not None: + self.looks.append(look) + elif kind == "usage": + await self.execution.add_usage("copilot", op) + elif kind == "final" and not op.get("asked"): + await self.execution.save(graph=op.get("graph"), explanation=str(op.get("explanation") or "")) + + async def _flush_thinking(self) -> None: + if self._thinking: + text, self._thinking = self._thinking, "" + await self.execution.emit("op", {"op": "thinking", "delta": text}) + self._thinking_at = time.monotonic() + + async def _generate(self, instruction: str, base_graph: dict[str, Any] | None) -> tuple[list[dict[str, Any]], str | None]: + try: + return await super()._generate(instruction, base_graph) + finally: + await self._flush_thinking() + + async def on_retry(self, reason: str, *, kind: str, base_graph: dict[str, Any] | None) -> None: + from app.chat import turn as policy + + await self.execution.retrying(reason, kind=kind, base=policy.has_nodes(base_graph), attempt=self.turn.attempts, + run=self.last_run if kind == "review" else None) + + async def run_input(self, session: AsyncSession, graph: dict[str, Any]) -> dict[str, Any]: + from app.api.conversations import history_text + from app.chat import turn as policy + + return policy.fill_input(graph, self.question, await history_text(session, self.execution.conversation_id)) + + async def launch_args(self, session: AsyncSession, graph: dict[str, Any]) -> dict[str, Any]: + from app.api.deps import restricted + from app.chat.launch import ChatRunRefused, user_chat_run + from app.engine.launch import LaunchRefused + + args: dict[str, Any] = {"actor": self.execution.actor, "origin_kind": "chat", + "origin": {"conversation_id": self.execution.conversation_id, + "turn_id": self.execution.turn_id}} + if restricted(): + try: + args["memory_scope"], args["options"] = user_chat_run(graph, self.execution.actor) + except ChatRunRefused as e: + raise LaunchRefused(403, str(e)) from e + return args + + async def on_launched(self, run: Any) -> None: + await self.execution.launched(run) + + async def on_interrupted(self, run_id: str, row: Any) -> Any | None: + """停在审批上：照旧等人，审批过了接着等这次运行。没有待审批（服务重启挂起的）：这一轮收成中断，可以继续运行。""" + t = self.turn + if not await self._pending_approvals(run_id): + t.path, t.reason = "suspended", record.SUSPENDED + return None + await self._phase("waiting") + await self._await_resume(run_id) + await self._phase("run") + t.current_run = run_id + return await self._follow(run_id) + + async def _await_resume(self, run_id: str) -> None: + """等到这次运行不再停在审批上（审批过了、被驳回、被放弃）。运行的事件总线叫醒，另外隔一拍回库里看一次。""" + from app.core.bus import bus + from app.db.models import Run + + with bus.attach(run_id) as sub: + while True: + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row is None or row.status != "interrupted" or not await self._pending_approvals(run_id): + return + try: + await asyncio.wait_for(anext(aiter(sub.stream())), timeout=APPROVAL_POLL_S) + except (asyncio.TimeoutError, StopAsyncIteration): + pass + + async def _review(self, run: Any) -> dict[str, Any] | None: + """复核最多等 REVIEW_WAIT_S：等到了交回结论；等不到交回 None（先交原答案），那一次接着在后台跑完、记用量。复核 + 自己出错也交回 None：复核是补一层说明，它失败了不能把已经到手的结果一起带走（同以前页面上的做法）。""" + from app.api.copilot import review_of_run + + async def review() -> dict[str, Any] | None: + try: + async with SessionLocal() as session: + return await review_of_run(session, run, question=self.question, attempt=self.turn.attempts) + except Exception: # noqa: BLE001 + logger.exception("问数据：复核运行 %s 出错", run.id) + return None + + task = asyncio.ensure_future(review()) + try: + done, _ = await asyncio.wait({task}, timeout=REVIEW_WAIT_S) + except asyncio.CancelledError: + task.cancel() + raise + if task in done: + result = task.result() + if result is not None: + await self.execution.emit("review", {"result": result}) + await self.execution.add_usage("review", result.get("usage")) + return result + self.late_review = task + return None + + +# -------------------------------------------------------------------------- +# 后台任务 +# -------------------------------------------------------------------------- + + +class ChatTurnExecution(runtime.JobRun): + """一轮问数据在后台的一次执行。""" + + kind = "chat_turn" + + def __init__(self, job_id: str) -> None: + from app.db.models_chat import ChatTurnJob + + super().__init__(job_id) + self.model = ChatTurnJob + self.turn_id = job_id + self.conversation_id = "" + self.actor: str | None = None + self.mode = "ask" + self.request: dict[str, Any] = {} + #: 这一轮的用量 {copilot, review, earlier, run}（UsageTotal），从库里那一行接着记 + self.usage: dict[str, Any] = {} + self.run_id: str | None = None + self.run_class: str | None = None + self.driver: ChatDriver | None = None + #: end 已经写了：之后的停止只改任务状态，不再改那一行 + self.ended = False + + def lane_of(self, job: Any) -> str | None: + return None # 不占数据源的队：问数据的一轮不挡起草、剖析，起草也不挡问数据（同以前浏览器发起时） + + # ---------------- 写库 ---------------- + + async def emit(self, type_: str, data: dict[str, Any] | None = None) -> int: + """追加一条事件（和这一轮的库写入排在同一条队里，停止时不会写到一半）。""" + return await self._write(turn_log.append, self.turn_id, type_, data or {}) + + async def save(self, *, meta: Mapping[str, Any] | None = None, **fields: Any) -> None: + """改 conversation_turns 那一行：fields 照写，meta 并进去（值为 None 的键删掉），顺手记 meta.at。""" + await self._write(_save_turn, self.turn_id, fields, dict(meta or {})) + + async def _job(self, **values: Any) -> None: + async def go(session: AsyncSession) -> None: + from app.db.models_chat import ChatTurnJob + + await session.execute(update(ChatTurnJob).where(ChatTurnJob.id == self.turn_id).values(**values) + .execution_options(synchronize_session=False)) + await session.commit() + await self._write(go) + turn_log.notify(self.turn_id) + + async def add_usage(self, part: str, usage: Mapping[str, Any] | None) -> None: + if not isinstance(usage, Mapping): + return + self.usage[part] = record.add_total(self.usage.get(part), usage) + await self.save(meta={"usage": dict(self.usage)}) + + async def on_phase(self, phase: str) -> None: + await self._job(stage=phase) + await self.emit("stage", {"stage": phase}) + + async def launched(self, run: Any) -> None: + self.run_id, self.run_class = run.id, run.run_class + self.usage.pop("run", None) + await self._job(run_id=run.id) + await self.save(run_id=run.id, status="running", + meta={"runId": run.id, "runClass": run.run_class, "runStatus": run.status}) + await self.emit("run", {"run_id": run.id, "status": run.status, "run_class": run.run_class}) + + async def retrying(self, reason: str, *, kind: str, base: bool, attempt: int, run: Any = None) -> None: + """自动重来一次：换下来的那次运行的用量并进 earlier；这一行回到「还没有运行」。""" + if run is not None: + self.usage["run"] = record.usage_total(run.usage) or {} + self.usage = record.retire_run(self.usage) + self.run_id = None + await self._job(attempt=attempt, run_id=None) + await self.save(run_id=None, meta={"runId": None, "runStatus": None, "usage": dict(self.usage)}) + await self.emit("turn.retry", {"kind": kind, "reason": reason, "base": base, "attempt": attempt}) + + # ---------------- 执行 ---------------- + + async def _run(self) -> None: + from app.api.deps import set_principal + from app.chat import turn as policy + from app.db.models import ConversationTurn + from app.db.models_chat import ChatTurnJob + + async with SessionLocal() as session: + if not await runtime.claim(session, ChatTurnJob, self.turn_id): + return # 开始之前就被停止了 + job = await session.get(ChatTurnJob, self.turn_id) + row = await session.get(ConversationTurn, self.turn_id) + if job is None or row is None: + return + self.conversation_id, self.actor, self.mode = job.conversation_id, job.actor, job.mode + self.request = dict(job.request or {}) + meta = dict(row.meta or {}) if isinstance(row.meta, dict) else {} + self.usage = dict(meta.get("usage") or {}) + self.run_id = job.run_id + # 这一轮按发起的人的身份跑：answer_ops 里的 restricted()、对话归属，发起运行时的遮罩、只读都看它 + set_principal(principal_of(job.principal)) + scope = [s.get("id") for s in self.request.get("scope") or [] if isinstance(s, Mapping) and s.get("id")] + driver = self.driver = ChatDriver(self, question=row.question, datasource_ids=scope) + driver.turn.attempts = int(job.attempt or 0) + retry = self.mode in AUTO_RETRY_MODES + graph = row.graph if policy.has_nodes(row.graph) else None + if self.request.get("follow") and job.run_id: + # 接着等这次运行（继续运行、接着收尾、服务重启后接手）：自动重来的规矩照这一轮原来的方式 + await driver.drive(run_id=job.run_id, retry=retry) + elif self.mode in ("rerun", "max_steps", "run") and graph is not None: + if self.mode == "max_steps": + graph = record.with_max_steps(graph, int(self.request.get("max_steps") or 0)) + await self.save(graph=graph) + await driver.drive(graph=graph, retry=False) + elif self.mode == "retry": + await driver.drive(instruction=str(self.request.get("instruction") or row.question), + base_graph=graph if self.request.get("base") else None) + else: + await driver.drive() + await self._conclude() + + async def _conclude(self) -> None: + """按这一轮的结局写那一行、发 end、任务记成功。""" + driver = self.driver + assert driver is not None + fields, meta, outcome, output = await _settled(driver, self) + await self.save(meta=meta, **fields) + await self._end(outcome, output) + # 晚到的复核先登记、再改任务状态：实时流看到任务结束时，要知道还得等这一次复核回来 + if driver.late_review is not None: + _track_late(self.turn_id, driver.late_review) + await self._job(status="succeeded", stage=None, finished_at=utcnow()) + + async def _end(self, outcome: str, output: Any = None) -> None: + turn, run = await self._write(_snapshot, self.turn_id, self.run_id) + await self.emit("end", {"outcome": outcome, "turn": turn, "output": output, "run": run}) + self.ended = True + + async def _stopped(self) -> None: + reason = self.stop_reason + if reason == "shutdown": + # 服务关停：不改这一轮，启动时照「重启」的规矩整理（见模块说明） + return + driver = self.driver + run_id = self.run_id or (driver.turn.current_run if driver else None) + if run_id: + from app.engine.runner import run_manager + + try: + await run_manager.cancel(run_id, actor=self.actor) + except Exception: # noqa: BLE001 - 取消运行是收尾，出错只记日志 + logger.exception("问数据：停止这一轮时取消运行 %s 出错", run_id) + if reason == "deleted": + return # 对话删了：那一行已经不在 + if driver is not None and driver.late_review is not None: + driver.late_review.cancel() + await self._settle() + if not self.ended: + await self.save(status="error", error=record.CANCELLED, + meta={"outcome": "cancelled", "runStatus": "cancelled" if run_id else None, + "usage": dict(self.usage), **_timing(self.request)}) + await self._end("cancelled") + await self._job(status="cancelled", stage=None, finished_at=utcnow()) + + async def _failed(self, reason: str) -> None: + await self._crash_out(reason) + + async def _crashed(self, e: Exception) -> None: + from app.core.errors import describe_exception + + logger.exception("问数据：这一轮的后台任务出错", exc_info=e) + await self._crash_out(describe_exception(e)) + + async def _crash_out(self, message: str) -> None: + await self._settle() + if not self.ended: + await self.save(status="error", error=message, + meta={"cause": {"kind": "crash", "message": message}, "usage": dict(self.usage), + **_timing(self.request)}) + await self._end("error") + await self._job(status="failed", stage=None, error=message, finished_at=utcnow()) + + +class ChatTurnManager(runtime.JobManager): + def __init__(self) -> None: + super().__init__(ChatTurnExecution, label="问数据的一轮", name="chat-turn") + + +manager = ChatTurnManager() + +#: 等不及的复核还在后台跑的几轮：{turn_id: task}。它们的实时流在复核回来（记上用量）之前不关 +_late: dict[str, asyncio.Task[Any]] = {} + + +def _track_late(turn_id: str, task: asyncio.Task[Any]) -> None: + async def finish() -> None: + try: + result = await task + except asyncio.CancelledError: + return + except Exception: # noqa: BLE001 + logger.exception("问数据：晚到的复核出错") + return + finally: + if _late.get(turn_id) is watcher: + _late.pop(turn_id, None) + turn_log.notify(turn_id) + usage = (result or {}).get("usage") if isinstance(result, Mapping) else None + if not usage: + return + async with SessionLocal() as session: + from app.db.models import ConversationTurn + + row = await session.get(ConversationTurn, turn_id) + if row is None: + return + meta = dict(row.meta or {}) if isinstance(row.meta, dict) else {} + spent = dict(meta.get("usage") or {}) + spent["review"] = record.add_total(spent.get("review"), usage) + row.meta = {**meta, "usage": spent} + await session.commit() + await turn_log.append(session, turn_id, "review.late", {"usage": usage}) + + watcher = asyncio.ensure_future(finish()) + _late[turn_id] = watcher + + +def late_pending(turn_id: str) -> bool: + task = _late.get(turn_id) + return task is not None and not task.done() + + +# -------------------------------------------------------------------------- +# 写那一行 +# -------------------------------------------------------------------------- + + +async def _save_turn(session: AsyncSession, turn_id: str, fields: Mapping[str, Any], meta: Mapping[str, Any]) -> None: + from app.db.models import ConversationTurn + + row = await session.get(ConversationTurn, turn_id) + if row is None: + return + for key, value in fields.items(): + setattr(row, key, value) + merged = dict(row.meta or {}) if isinstance(row.meta, dict) else {} + for key, value in meta.items(): + if value is None: + merged.pop(key, None) + else: + merged[key] = value + merged["v"] = 1 + merged["at"] = int(time.time() * 1000) + row.meta = merged + await session.commit() + + +async def _snapshot(session: AsyncSession, turn_id: str, run_id: str | None) -> tuple[dict[str, Any] | None, + dict[str, Any] | None]: + """end 里带的那一行（同 GET /api/conversations/{id} 里的一轮）和这次运行的摘要。""" + from app.api.conversations import turn_out + from app.db.models import ConversationTurn, Run + + row = await session.get(ConversationTurn, turn_id) + turn = (await turn_out(session, row)).model_dump(mode="json") if row is not None else None + run = await session.get(Run, run_id) if run_id else None + summary = None if run is None else { + "id": run.id, "status": run.status, "run_class": run.run_class, "usage": run.usage or {}, + "error": run.error, "end_reason": run.end_reason, + "started_at": run.started_at.isoformat() if run.started_at else None, + "finished_at": run.finished_at.isoformat() if run.finished_at else None, + } + return turn, summary + + +def _timing(request: Mapping[str, Any]) -> dict[str, Any]: + """这一次尝试的墙钟毫秒（meta.ms）：从发起这一次算（继续运行接着上次的算），到现在为止；补交付的到运行结束为止。""" + started, ended = request.get("started_ms"), request.get("ended_ms") + if not isinstance(started, (int, float)): + return {} + end = int(ended) if isinstance(ended, (int, float)) else int(time.time() * 1000) + return {"ms": max(0, end - int(started))} + + +async def _queries(run_id: str | None) -> int | None: + """这次运行查了几次库（db_query 的每一次调用，同页面的 countQueries）。""" + from app.core.events import EventType + from app.db.models import RunEvent + + if not run_id: + return None + async with SessionLocal() as session: + rows = (await session.execute(select(RunEvent.data).where( + RunEvent.run_id == run_id, RunEvent.type == EventType.TOOL_START))).scalars().all() + return sum(1 for d in rows if isinstance(d, dict) and str(d.get("tool") or "").startswith("db_query")) + + +async def _rejected_by(run_id: str) -> dict[str, Any]: + """被驳回结束的运行：run.cancelled 带的谁、哪一步、备注、怎么驳回的（页面据此写「张工 驳回了「查数」：…」）。""" + from app.core.events import EventType + from app.db.models import RunEvent + + async with SessionLocal() as session: + data = (await session.execute(select(RunEvent.data).where( + RunEvent.run_id == run_id, RunEvent.type == EventType.RUN_CANCELLED).order_by(RunEvent.seq.desc()) + .limit(1))).scalar_one_or_none() + data = data if isinstance(data, dict) else {} + return {k: data[k] for k in ("actor", "label", "note", "decision", "node_id", "kind") if data.get(k) is not None} + + +async def _run_failed_detail(run_id: str) -> str | None: + from app.core.events import EventType + from app.db.models import RunEvent + + async with SessionLocal() as session: + data = (await session.execute(select(RunEvent.data).where( + RunEvent.run_id == run_id, RunEvent.type == EventType.RUN_FAILED).order_by(RunEvent.seq.desc()) + .limit(1))).scalar_one_or_none() + return str(data.get("detail")) if isinstance(data, dict) and data.get("detail") else None + + +async def _settled(driver: ChatDriver, execution: ChatTurnExecution + ) -> tuple[dict[str, Any], dict[str, Any], str, Any]: + """这一轮的结局 → (那一行要写的列, 并进 meta 的, outcome, 完整成果)。和以前页面在各个结局处落库写的一一对应。""" + from app.engine.runner import END_REJECTED + + t = driver.turn + term = driver.terminal or {} + run = driver.last_run + meta: dict[str, Any] = {"looks": driver.looks or None, **_timing(execution.request)} + fields: dict[str, Any] = {} + output: Any = None + path = t.path + if path == "reply": + text = t.reply or "" + fields.update(status="done", answer=text, review={}) + meta.update(noQuery=True, runId=None) + return fields, {**meta, "usage": dict(execution.usage)}, "reply", {"answer": text} + if path == "asked": + asked = t.asked or {} + question = str(asked.get("question") or "").strip() + fields.update(status="done", answer=question, review={}) + meta.update(asked=dict(asked) or True, runId=None) + outcome = "asked" + if term.get("op") == "final": + # 改了图又问了用户、或者只问不改（图原样交回）：图摆着等「运行」，问题作为这一轮的内容 + fields.update(graph=driver.graph or term.get("graph"), explanation=str(term.get("explanation") or "")) + outcome = "ready" + return fields, {**meta, "usage": dict(execution.usage)}, outcome, {"answer": question} if question else None + if path == "ready": + fields.update(status="done", graph=driver.graph, explanation=str(term.get("explanation") or ""), review={}) + meta["runId"] = None + return fields, {**meta, "usage": dict(execution.usage)}, "ready", None + if path == "build_failed": + if driver.refusal is not None: + message = str(getattr(driver.refusal, "detail", "") or driver.refusal) + cause = {"kind": "refused", "message": message, "code": getattr(driver.refusal, "code", None)} + error = message + elif term.get("op") == "error": + error = str(term.get("message") or "助手本轮未完成") + cause = {"kind": "copilot", "message": error, "hint": term.get("hint"), "detail": term.get("detail")} + else: + issues = [str(i.get("message") or "") for i in term.get("issues") or [] + if isinstance(i, Mapping) and i.get("level") == "error"] + cause = {"kind": "blocked", "issues": issues, "self_checked": driver.self_checked} + error = t.reason or "生成的工作流没有通过校验" + if driver.graph: + fields.update(graph=driver.graph, explanation=str(term.get("explanation") or "")) + fields.update(status="error", error=error, review={}) + meta.update(cause={k: v for k, v in cause.items() if v is not None}, runId=None) + return fields, {**meta, "usage": dict(execution.usage)}, "error", None + + # 发起过运行的结局 + if run is not None: + execution.usage["run"] = record.usage_total(run.usage) or {} + usage = dict(execution.usage) + run_id = run.id if run is not None else execution.run_id + meta.update(runId=run_id, runClass=getattr(run, "run_class", None) or execution.run_class, + runStatus=getattr(run, "status", None), queries=await _queries(run_id), usage=usage) + if path == "run_failed": + refused = driver.launch_refused + if refused is not None: + message = getattr(refused, "message", str(refused)) + fields.update(status="error", error=message, review={}) + meta.update(cause={"kind": "start", "message": message, "code": getattr(refused, "code", None), + "status": getattr(refused, "status_code", None)}, runId=None, runStatus=None, queries=None) + return fields, meta, "error", None + if run is None: + fields.update(status="error", error=record.GONE, review={}) + meta.update(cause={"kind": "gone"}, runStatus="gone") + return fields, meta, "error", None + if run.status == "cancelled" and run.end_reason != END_REJECTED: + fields.update(status="error", error=record.CANCELLED, review={}) + meta["outcome"] = "cancelled" + return fields, meta, "cancelled", None + fields.update(status="error", error=run.error or "运行失败", review={}) + meta["cause"] = {k: v for k, v in {"kind": "run", "error": run.error or "", + "detail": await _run_failed_detail(run.id)}.items() if v is not None} + return fields, meta, "error", None + if path == "rejected": + fields.update(status="error", error=record.CANCELLED, review={}) + meta.update(outcome="cancelled", rejected=await _rejected_by(run.id) or {"decision": "reject"}) + return fields, meta, "cancelled", None + if path == "suspended": + fields.update(status="error", error=record.SUSPENDED, review={}) + meta.update(outcome="suspended", runStatus="interrupted") + return fields, meta, "suspended", None + + # 跑完了：成果以库里那份为准（事件里那份为了推送体积被切过） + output = run.output if isinstance(run.output, dict) else {} + review = driver.last_review if driver.late_review is None else None + rewritten = (review or {}).get("answer") if (review or {}).get("verdict") != "ok" else None + fields.update(status="done", answer=rewritten or record.answer_text(output), review=review or {}) + meta.update(issuance=record.slim_issuance(output.get("_issuance")), + evidence=True if record.has_evidence(output) else None) + return fields, meta, "done", output + + +# -------------------------------------------------------------------------- +# 发起、停止、查询（接口层 api/chat_turns.py 调） +# -------------------------------------------------------------------------- + + +class TurnRefused(Exception): + """不发起：status_code、message 就是 HTTP 的状态码和 detail，code 是机器码。""" + + def __init__(self, status_code: int, message: str, code: str | None = None) -> None: + super().__init__(message) + self.status_code = status_code + self.message = message + self.code = code + + +#: 机器码 +TURN_BUSY = "chat_turn_busy" +TURN_INVALID = "chat_turn_invalid" +TURN_FINISHED = "chat_turn_finished" +BUSY = "这段对话正有一轮在进行：等它结束或先停止，再提问" + + +async def _busy(session: AsyncSession, conversation_id: str, *, besides: str | None = None) -> bool: + """这段对话里有没有正在跑的一轮（停在审批上等人的不算：以前页面上也照样能接着问）。""" + from app.db.models_chat import ChatTurnJob + + rows = (await session.execute(select(ChatTurnJob.id, ChatTurnJob.stage).where( + ChatTurnJob.conversation_id == conversation_id, ChatTurnJob.status.in_(runtime.ACTIVE)))).all() + return any(stage != "waiting" for job_id, stage in rows if job_id != besides) + + +async def _scope(session: AsyncSession, ids: list[str] | None) -> list[dict[str, str]]: + """限定的数据源 → [{id, name}]（名字跟着存：库删了之后，这一轮查的是谁还得说得出来）。认 id 和名字。""" + from app.db.models import DataSource + + wanted = [s.strip() for s in ids or [] if isinstance(s, str) and s.strip()] + if not wanted: + return [] + rows = {r.id: r for r in (await session.execute(select(DataSource).where( + DataSource.id.in_(wanted) | DataSource.name.in_(wanted)))).scalars()} + by_name = {r.name: r for r in rows.values()} + out = [] + for item in wanted: + hit = rows.get(item) or by_name.get(item) + out.append({"id": hit.id, "name": hit.name} if hit else {"id": item, "name": item}) + return out + + +async def start(session: AsyncSession, *, conversation: Any, question: str, datasource_ids: list[str] | None, + actor: str | None, mode: str | None = None, retry_of: Any = None, + max_steps: int | None = None) -> str: + """发起一轮（新问，或者对已有的一轮重试、重新运行、继续运行、接着收尾）。交回这一轮的 id（conversation_turns.id）。 + + 新问：建 conversation_turns 那一行（同 POST /api/conversations/{id}/turns）和任务。已有的一轮：上一次的结局归档进 + meta.attempts（重试、重新运行、放宽步数），那一行回到 running，任务重新排队。不发起抛 TurnRefused。""" + from app.api.conversations import new_turn, turn_run_id + from app.chat import turn as policy + from app.db.models import Approval, Run + from app.db.models_chat import ChatTurnJob + + mode = mode or ("retry" if retry_of is not None else "ask") + if mode not in MODES or (mode == "ask") != (retry_of is None): + raise TurnRefused(422, "发起方式不对：新问不带 retry_of，重试、重新运行、继续运行要带上是哪一轮", TURN_INVALID) + besides = retry_of.id if retry_of is not None else None + if await _busy(session, conversation.id, besides=besides): + raise TurnRefused(409, BUSY, TURN_BUSY) + now_ms = int(time.time() * 1000) + request: dict[str, Any] = {"started_ms": now_ms} + + if retry_of is None: + if not question.strip(): + raise TurnRefused(422, "问题不能为空", TURN_INVALID) + scope = await _scope(session, datasource_ids) + row = await new_turn(session, conversation, question) + row.meta = {"v": 1, **({"scope": scope} if scope else {}), "at": now_ms} + request["scope"] = scope + session.add(ChatTurnJob(id=row.id, conversation_id=conversation.id, status="queued", mode=mode, request=request, + principal=principal_snapshot(), actor=actor, started_at=None)) + await session.flush() + turn_id = row.id + attempts: list[Any] = [] + else: + row = retry_of + job = await session.get(ChatTurnJob, row.id) + if job is not None and job.status in runtime.ACTIVE: + raise TurnRefused(409, BUSY, TURN_BUSY) + meta = dict(row.meta or {}) if isinstance(row.meta, dict) else {} + scope = await _scope(session, datasource_ids) if datasource_ids is not None else list(meta.get("scope") or []) + request["scope"] = scope + run_id = turn_run_id(row) + run = await session.get(Run, run_id) if run_id else None + attempts = list(meta.get("attempts") or []) + graphful = policy.has_nodes(row.graph) + if mode in ("rerun", "max_steps", "run") and not graphful: + raise TurnRefused(409, "这一轮没有可运行的工作流，请重试本轮", TURN_INVALID) + if mode == "max_steps": + if not max_steps or max_steps < 1: + raise TurnRefused(422, "放宽到几步要写清楚", TURN_INVALID) + request["max_steps"] = int(max_steps) + if mode in ("continue", "resume"): + if run is None: + raise TurnRefused(409, "这一轮没有可接着的运行，请重试本轮", TURN_INVALID) + if mode == "continue": + await _continue_run(run.id, actor) + request["follow"] = True + # 计时：收过尾的（失败、中断之后点「继续运行」）接着上次停下的地方走——隔了一夜再点，头部不该写「已运行 14 小时」； + # 停在半路的（页面以前没来得及收尾的）从运行开始算；运行早就结束了的（补交付）按它真正结束的时刻算，不按现在 + spent = meta.get("ms") + if row.status != "running" and isinstance(spent, (int, float)): + request["started_ms"] = now_ms - int(spent) + elif run.started_at is not None: + request["started_ms"] = int(run.started_at.timestamp() * 1000) + if mode == "resume" and run.status in ("succeeded", "failed", "cancelled") and run.finished_at is not None: + request["ended_ms"] = int(run.finished_at.timestamp() * 1000) + else: + # 重试、重新运行、放宽步数、跑一下：服务重启挂起的旧运行没人会再去续它，顺手取消（停在审批上的不算：重试不会从那里发起） + if run is not None and run.status == "interrupted" and mode != "run": + pending = (await session.execute(select(Approval.id).where( + Approval.run_id == run.id, Approval.status.in_(("pending", "answered"))).limit(1))).first() + if pending is None: + from app.engine.runner import run_manager + + try: + await run_manager.cancel(run.id, actor=actor) + except Exception: # noqa: BLE001 + logger.exception("问数据：重试时取消挂起的旧运行 %s 出错", run.id) + if mode != "run": + archived = record.attempt_record(row, meta, n=len(attempts) + 1, + at_ms=int(row.updated_at.timestamp() * 1000) if row.updated_at else now_ms) + attempts.append(archived) + if mode == "retry": + reason = record.attempt_reason(archived) + request["base"] = graphful + request["instruction"] = policy.user_retry_instruction(row.question, reason, graphful) + usage = record.retire_run(meta.get("usage")) + new_meta = {"v": 1, "attempts": attempts or None, "scope": scope or None, "usage": usage or None, + "runId": None, "at": now_ms} + if mode == "run": + # 「跑一下」不是重来：这一轮还没有答过，别的照旧 + new_meta = {**meta, "runId": None, "at": now_ms, "asked": meta.get("asked")} + row.meta = {k: v for k, v in new_meta.items() if v is not None} + row.status, row.error, row.answer, row.review = "running", "", "", {} + row.run_id = None + if mode in ("continue", "resume"): + row.status, row.error = "running", "" + row.meta = {**meta, "at": now_ms} + if job is None: + job = ChatTurnJob(id=row.id, conversation_id=conversation.id) + session.add(job) + job.status, job.stage, job.mode, job.request = "queued", None, mode, request + job.run_id = run_id if mode in ("continue", "resume") else None + job.attempt = 0 + job.principal, job.actor, job.error, job.started_at, job.finished_at = ( + principal_snapshot(), actor, None, None, None) + conversation.last_active_at = utcnow() + await session.flush() + turn_id = row.id + await session.commit() + + # 这一次尝试从这一条开始：刷新、重开页面时从这里回放 + async with SessionLocal() as s: + seq = await turn_log.append(s, turn_id, "turn.attempt", { + "mode": mode, "attempts": attempts, "scope": request.get("scope") or [], + **({"max_steps": request["max_steps"]} if request.get("max_steps") else {})}) + await s.execute(update(ChatTurnJob).where(ChatTurnJob.id == turn_id).values(attempt_seq=seq) + .execution_options(synchronize_session=False)) + await s.commit() + manager.start(turn_id) + return turn_id + + +async def _continue_run(run_id: str, actor: str | None) -> None: + """继续运行：规则同 POST /api/runs/{id}/continue（失败的、服务重启挂起的才续得上），被拒照那边的说法。""" + from app.engine.runner import ContinueRefused, run_manager + + try: + await run_manager.continue_failed(run_id, graph=None, actor=actor) + except KeyError as e: + raise TurnRefused(404, str(e).strip("'")) from e + except ContinueRefused as e: + raise TurnRefused(409, str(e), e.code) from e + except ValueError as e: + raise TurnRefused(409, str(e)) from e + + +async def cancel(turn_id: str, actor: str | None) -> bool: + """停止这一轮（取消生成、取消运行，照现有的取消规则）。这一轮已经结束了交回 False。""" + from app.db.models_chat import ChatTurnJob + + if await manager.stop(turn_id, "user", grace=STOP_GRACE_S): + return True + async with SessionLocal() as session: + job = await session.get(ChatTurnJob, turn_id) + if job is None or job.status not in (*runtime.ACTIVE, "interrupted"): + return False + # 不在这个进程里跑（排队还没开始、服务重启后停着等人来接着收尾）：这里收成已取消 + claimed = await session.execute(update(ChatTurnJob).where( + ChatTurnJob.id == turn_id, ChatTurnJob.status == job.status).values( + status="cancelled", stage=None, finished_at=utcnow()).execution_options(synchronize_session=False)) + await session.commit() + if claimed.rowcount != 1: + return False + run_id = job.run_id + execution = ChatTurnExecution(turn_id) + execution.actor, execution.run_id, execution.request = actor, run_id, dict(job.request or {}) + if run_id: + from app.engine.runner import run_manager + + try: + await run_manager.cancel(run_id, actor=actor) + except Exception: # noqa: BLE001 + logger.exception("问数据：停止这一轮时取消运行 %s 出错", run_id) + await execution.save(status="error", error=record.CANCELLED, + meta={"outcome": "cancelled", "runStatus": "cancelled" if run_id else None}) + await execution._end("cancelled") + await execution._settle() + turn_log.notify(turn_id) + return True + + +async def state(session: AsyncSession, turn_id: str) -> dict[str, Any] | None: + """这一轮的任务现在怎样（GET /api/chat/turns/{id}）。没有任务（老轮次）交回 None。""" + from app.db.models_chat import ChatTurnJob + + job = await session.get(ChatTurnJob, turn_id) + return None if job is None else job_view(job) + + +def job_view(job: Any) -> dict[str, Any]: + return {"status": job.status, "stage": job.stage, "mode": job.mode, "run_id": job.run_id, "attempt": job.attempt, + "last_seq": job.last_seq, "attempt_seq": job.attempt_seq, "error": job.error, + "late_review": late_pending(job.id)} + + +async def stream_status(turn_id: str) -> str | None: + """实时流还要不要等：这一轮还在进行中（或晚到的复核还没回来）交回 None，否则交回任务的状态。""" + from app.db.models_chat import ChatTurnJob + + async with SessionLocal() as session: + job = await session.get(ChatTurnJob, turn_id) + if job is None: + return "gone" + if job.status in runtime.ACTIVE or late_pending(turn_id): + return None + return job.status + + +async def stop_conversation(conversation_id: str) -> None: + """对话要删了：停掉它正在跑的几轮（取消各自的运行），不再写那几行。""" + from app.db.models_chat import ChatTurnJob + + async with SessionLocal() as session: + ids = list((await session.execute(select(ChatTurnJob.id).where( + ChatTurnJob.conversation_id == conversation_id, ChatTurnJob.status.in_(runtime.ACTIVE)))).scalars()) + for turn_id in ids: + await manager.stop(turn_id, "deleted", grace=STOP_GRACE_S) + + +async def forget(session: AsyncSession, conversation_id: str) -> None: + """删掉一段对话的任务和事件日志（外键是级联的，这里照样显式删：理由同 conversations.delete_conversation）。""" + from app.db.models import ConversationTurn + from app.db.models_chat import ChatTurnEvent, ChatTurnJob + + turns = select(ConversationTurn.id).where(ConversationTurn.conversation_id == conversation_id) + await session.execute(delete(ChatTurnEvent).where(ChatTurnEvent.turn_id.in_(turns))) + await session.execute(delete(ChatTurnJob).where(ChatTurnJob.conversation_id == conversation_id)) + + +# -------------------------------------------------------------------------- +# 启动、关停 +# -------------------------------------------------------------------------- + + +async def startup(*, resume: bool | None = None) -> dict[str, int]: + """服务启动时（run_manager.setup 之后：服务重启时在跑的运行这时已经标成挂起）： + + - 还在建图（没有运行）的这一轮标中断：那一行记 error 和原因（build_interrupted），可以重试本轮； + - 已经发起了运行的：resume 为真（缺省看 AGENTLAB_SCHEDULER）就接着等那次运行、照常复核收尾；为假就停着（任务记 + interrupted、那一行不动），等人打开这段对话时页面再请服务端接着收尾； + - 老轮次（这一波之前浏览器推的，没有任务）停在 running、又没有运行的：同样标中断——它的建图那条流随上次的进程一起没了。 + + 交回整理了几轮：{interrupted, resumed, parked, legacy}。""" + from app.api.conversations import turn_run_id + from app.core.config import settings + from app.db.models import Conversation, ConversationTurn + from app.db.models_chat import ChatTurnJob + + resume = bool(settings.scheduler) if resume is None else resume + counts = {"interrupted": 0, "resumed": 0, "parked": 0, "legacy": 0} + async with SessionLocal() as session: + jobs = [j for j in (await session.execute(select(ChatTurnJob).where( + ChatTurnJob.status.in_(runtime.ACTIVE)))).scalars() if not manager.running(j.id)] + legacy = list((await session.execute( + select(ConversationTurn).join(Conversation, Conversation.id == ConversationTurn.conversation_id) + .outerjoin(ChatTurnJob, ChatTurnJob.id == ConversationTurn.id) + .where(ConversationTurn.status == "running", Conversation.kind == "chat", ChatTurnJob.id.is_(None)) + )).scalars()) + for job in jobs: + if not job.run_id: + await _interrupt_build(job.id) + counts["interrupted"] += 1 + elif resume: + async with SessionLocal() as session: + await session.execute(update(ChatTurnJob).where(ChatTurnJob.id == job.id).values( + status="queued", request={**dict(job.request or {}), "follow": True}) + .execution_options(synchronize_session=False)) + await session.commit() + manager.start(job.id) + counts["resumed"] += 1 + else: + async with SessionLocal() as session: + await session.execute(update(ChatTurnJob).where(ChatTurnJob.id == job.id).values( + status="interrupted", finished_at=utcnow()).execution_options(synchronize_session=False)) + await session.commit() + turn_log.notify(job.id) + counts["parked"] += 1 + for row in legacy: + if turn_run_id(row): + continue # 有运行的老轮次：页面打开时请服务端接着收尾 + async with SessionLocal() as session: + fresh = await session.get(ConversationTurn, row.id) + if fresh is None or fresh.status != "running": + continue + meta = dict(fresh.meta or {}) if isinstance(fresh.meta, dict) else {} + fresh.status, fresh.error = "error", record.BUILD_INTERRUPTED + fresh.meta = {**meta, "v": 1, "cause": {"kind": "build_interrupted"}, "at": int(time.time() * 1000)} + await session.commit() + counts["legacy"] += 1 + if any(counts.values()): + logger.info("问数据：启动时整理了上次没跑完的几轮 %s", counts) + return counts + + +async def _interrupt_build(turn_id: str) -> None: + execution = ChatTurnExecution(turn_id) + await execution.save(status="error", error=record.BUILD_INTERRUPTED, + meta={"cause": {"kind": "build_interrupted"}, "runId": None}) + await execution._end("error") + await execution._settle() + async with SessionLocal() as session: + from app.db.models_chat import ChatTurnJob + + await session.execute(update(ChatTurnJob).where(ChatTurnJob.id == turn_id).values( + status="interrupted", stage=None, finished_at=utcnow()).execution_options(synchronize_session=False)) + await session.commit() + turn_log.notify(turn_id) + + +async def shutdown() -> None: + """关停：停掉在跑的几轮（不改那几行，下次启动照「重启」的规矩整理）；晚到的复核不等。""" + await manager.shutdown() + for task in list(_late.values()): + task.cancel() + _late.clear() + + +__all__ = ["APPROVAL_POLL_S", "AUTO_RETRY_MODES", "BUSY", "CONCURRENCY_CLASS", "ChatDriver", "ChatTurnExecution", + "MODES", "REVIEW_WAIT_S", "TURN_BUSY", "TURN_FINISHED", "TURN_INVALID", "TurnRefused", "cancel", "forget", + "job_view", "late_pending", "manager", "principal_of", "principal_snapshot", "shutdown", "start", "startup", + "state", "stop_conversation", "stream_status"] diff --git a/backend/app/chat/launch.py b/backend/app/chat/launch.py new file mode 100644 index 00000000..dd796a82 --- /dev/null +++ b/backend/app/chat/launch.py @@ -0,0 +1,30 @@ +"""问数据的运行怎么发起：开登录后 user 角色的那几条规则，HTTP 发起（api/runs.start_run）和服务端跑的一轮（chat/job.py）共用。 + +以前只有 start_run 一处（浏览器发 POST /api/runs）。一轮问数据搬到服务端之后，发起运行的是这一轮的后台任务，规则不能另抄一份—— +使用者的问数据运行只许白名单里的节点和工具、记忆固定为自己的范围、查询结果在工具边界遮、数据源一律只读，这几条一处说清。 +请求体能不能带记忆范围、知识库，能不能发正式运行，这些是 HTTP 请求本身的事，仍在 start_run 里判。 +""" +from __future__ import annotations + +from typing import Any + + +class ChatRunRefused(Exception): + """这张图使用者在问数据里不能运行：str(e) 就是给人看的原因（HTTP 发起时是 403 的 detail）。""" + + +def user_chat_run(graph: Any, actor: str | None) -> tuple[str, dict[str, Any]]: + """开登录后 user 角色发起问数据运行：交回 (记忆范围, 运行选项)。图里有白名单之外的节点、工具抛 ChatRunRefused。 + + 记忆固定为这位使用者自己的范围、知识库取设置里的默认值（记忆、知识库的内容不经遮罩，见 auth/chat_policy 开头）；查询结果 + 在工具边界就遮（事件、回答里只有遮过的值——它读得到自己这次运行的事件，api/deps.py 的 _run_scope）；数据源查询一律只读， + 写语句不交给审批。""" + from app.auth.chat_policy import chat_graph_problem, chat_memory_scope + from app.tools.datasource import MASK_RESULTS_OPTION, READ_ONLY_OPTION + + if problem := chat_graph_problem(graph): + raise ChatRunRefused(problem) + return chat_memory_scope(actor), {MASK_RESULTS_OPTION: True, READ_ONLY_OPTION: True} + + +__all__ = ["ChatRunRefused", "user_chat_run"] diff --git a/backend/app/chat/log.py b/backend/app/chat/log.py new file mode 100644 index 00000000..4b120a25 --- /dev/null +++ b/backend/app/chat/log.py @@ -0,0 +1,138 @@ +"""问数据一轮的事件日志（chat_turn_events，见 db/models_chat.py）：追加、回放、实时推送。 + +**为什么「先落库、再通知、订阅方回库里读」。** 页面要的是：先回放、再接实时，断了按 seq 补齐，刷新、关掉再打开都能接着看。 +事件一律先写库，再叫醒等着的订阅方，订阅方按自己收到的最后一个 seq 回库里读——实时和回放是同一段代码，补齐不会漏、不会重复， +也不用像运行的事件总线那样处理慢消费者的积压。订阅方在读库**之前**先登记等待（watch），读完没东西才睡：读和写之间新写进来的 +那一条会把它叫醒，不会漏掉。 + +叫醒只在本进程里（进程内的事件）；隔一拍心跳（HEARTBEAT_S）订阅方也会回库里看一次，所以换成多进程（第 2 波）时，别的进程写的 +事件最迟一拍心跳就送到——那时再换成 PostgreSQL 的 LISTEN/NOTIFY，这里的接口不用变。 + +线上格式（SSE，每条一行 data: ）：{seq, type, data, ts}；回放完、这一轮也不在进行中了，补一条不带 seq 的 +{type: stream.end, status}（任务的状态）再关；隔一拍没有新事件发一行注释心跳「: ping」（页面据此知道连接还活着）。 +事件种类见 app/chat/job.py 的模块说明。 +""" +from __future__ import annotations + +import asyncio +import contextlib +import json +import time +from collections import defaultdict +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator +from typing import Any + +from sqlalchemy import func, select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.base import SessionLocal + +#: 隔多久没有新事件就发一拍心跳（秒），也是订阅方回库里看一次的间隔 +HEARTBEAT_S = 15.0 +#: 一次回放最多读多少条 +READ_BATCH = 500 +#: 回放、实时推送完、这一轮不在进行中时的结束标记（不落库、不带 seq） +STREAM_END = "stream.end" + + +class _Waiters: + """本进程里等某一轮新事件的订阅方：一个订阅方一个 asyncio.Event。""" + + def __init__(self) -> None: + self._by_turn: dict[str, set[asyncio.Event]] = defaultdict(set) + + @contextlib.contextmanager + def watch(self, turn_id: str) -> Iterator[asyncio.Event]: + event = asyncio.Event() + self._by_turn[turn_id].add(event) + try: + yield event + finally: + waiters = self._by_turn.get(turn_id) + if waiters is not None: + waiters.discard(event) + if not waiters: + self._by_turn.pop(turn_id, None) + + def notify(self, turn_id: str) -> None: + for event in list(self._by_turn.get(turn_id, ())): + event.set() + + +waiters = _Waiters() + + +def notify(turn_id: str) -> None: + """这一轮有了变化（新事件、任务状态变了）：叫醒本进程里等着的订阅方。""" + waiters.notify(turn_id) + + +async def append(session: AsyncSession, turn_id: str, type_: str, data: dict[str, Any] | None = None) -> int: + """追加一条，提交，叫醒订阅方。交回它的 seq。 + + seq 按这一轮库里已有的最大值往下接：同一轮同一时刻只有一个写的人（它的后台任务；任务不在跑时才由停止、启动整理来写）， + 唯一约束 (turn_id, seq) 兜底。任务行的 last_seq 顺手跟上（GET /api/chat/turns/{id} 照它说现在到哪了）。""" + from app.db.models_chat import ChatTurnEvent, ChatTurnJob + + last = int(await session.scalar(select(func.coalesce(func.max(ChatTurnEvent.seq), 0)) + .where(ChatTurnEvent.turn_id == turn_id)) or 0) + seq = last + 1 + session.add(ChatTurnEvent(turn_id=turn_id, seq=seq, type=type_, ts=time.time(), data=dict(data or {}))) + await session.execute(update(ChatTurnJob).where(ChatTurnJob.id == turn_id).values(last_seq=seq) + .execution_options(synchronize_session=False)) + await session.commit() + notify(turn_id) + return seq + + +def wire(row: Any) -> dict[str, Any]: + """库里的一条 → 线上的一条。""" + return {"seq": row.seq, "type": row.type, "data": row.data or {}, "ts": row.ts} + + +async def read(turn_id: str, after: int = 0, limit: int = READ_BATCH) -> list[dict[str, Any]]: + """seq > after 的事件（按 seq），最多 limit 条。""" + from app.db.models_chat import ChatTurnEvent + + async with SessionLocal() as session: + rows = (await session.execute( + select(ChatTurnEvent).where(ChatTurnEvent.turn_id == turn_id, ChatTurnEvent.seq > after) + .order_by(ChatTurnEvent.seq).limit(limit))).scalars().all() + return [wire(r) for r in rows] + + +def frame(event: dict[str, Any]) -> str: + return f"data: {json.dumps(event, ensure_ascii=False, default=str)}\n\n" + + +#: 心跳（SSE 注释行）：页面的解析器只认 data: 开头的帧，这一行只是告诉它连接还活着 +PING = ": ping\n\n" + + +async def stream(turn_id: str, after: int, *, active: Callable[[], Awaitable[str | None]], + heartbeat: float | None = None) -> AsyncIterator[str]: + """SSE 的每一帧：先回放 seq > after 的，再推实时的；这一轮不在进行中、也没有新的了，补一条 stream.end 再结束。 + + active() 交回这一轮还在进行中时为 None，不在进行中时交回任务的状态（stream.end 里带的就是它）。""" + beat = HEARTBEAT_S if heartbeat is None else heartbeat + last = after + while True: + with waiters.watch(turn_id) as woke: + rows = await read(turn_id, last) + for row in rows: + last = max(last, int(row["seq"])) + yield frame(row) + if rows: + continue + status = await active() + if status is not None: + yield frame({"type": STREAM_END, "status": status, "data": {"status": status}}) + return + try: + await asyncio.wait_for(woke.wait(), timeout=beat) + except asyncio.TimeoutError: + yield PING + + +__all__ = ["HEARTBEAT_S", "PING", "READ_BATCH", "STREAM_END", "append", "frame", "notify", "read", "stream", "waiters", + "wire"] diff --git a/backend/app/chat/record.py b/backend/app/chat/record.py new file mode 100644 index 00000000..07254265 --- /dev/null +++ b/backend/app/chat/record.py @@ -0,0 +1,257 @@ +"""问数据一轮落进 conversation_turns 的那一行：服务端写（状态、运行 id、答案、复核、meta），页面只读。 + +以前这一行是页面写的（frontend/src/store/chat.ts 的 persist / buildMeta），一轮的编排搬到服务端之后由这里写，字段和页面以前 +写的一一对应，读的一方（问数据页的 restoreTurn、助手拼历史的 copilot._history_section、列表的 conversations.last_status）不用变： + +- status：running | done | error；error 列是一句原话（运行的报错、助手的说法、「已取消」「服务重启，本轮已中断」），助手拼 + 历史时照它写「这一轮失败了：…」； +- answer：答案正文（成果各字段拼起来，同页面的 answerText，上限 ANSWER_CAP）；复核改写了的存改写后的； +- review：复核原样（含 verdict=ok、usage、retry_instruction）；没复核过是 {}； +- meta（v=1）：runId、runClass、runStatus、issuance（瘦过的出具信息）、noQuery、outcome（cancelled | suspended）、rejected + （驳回的人、哪一步、备注：run.cancelled 的原样）、cause（失败的原因，见下）、attempts（用户点重试之前的各次）、ms、queries、 + scope、asked、looks、evidence、usage（{copilot, review, earlier, run}）、at。 + +**失败只记原因（cause），不记说法。** 页面以前记的是翻成人话之后的 failure（标题、原因、怎么办：lib/explain 那一套），说法在 +界面层；服务端只记发生了什么，页面恢复时照同一套 lib/explain 讲（frontend/src/store/chat.ts 的 failureFromCause）： + {kind: refused, message, code} 开流之前就被拒（模型没配好、限定的数据源都不在） + {kind: copilot, message, hint, detail} 生成过程交出 error（助手本轮未完成） + {kind: blocked, issues, self_checked} 生成的图没过检查 + {kind: start, message, code, status} 发起运行被拒（绑定的工具不在、使用者用不了的节点） + {kind: run, error, detail} 运行失败 + {kind: gone} 运行记录不见了 + {kind: build_interrupted} 建图时服务重启，没有可恢复的断点 + {kind: crash, message} 这一轮的后台任务自己出错 +老轮次的 meta.failure 照旧认（页面优先读它）。 +""" +from __future__ import annotations + +import json +import time +from collections.abc import Mapping +from typing import Any + +#: 落库答案的上限（同页面的 ANSWER_CAP）：只是防跑飞，不是显示上限 +ANSWER_CAP = 20_000 +ANSWER_CLIPPED = "\n\n…（内容过长，已截断，完整内容请查看运行记录）" +#: 归档的旧答案存多少（同页面的 ATTEMPT_ANSWER_CAP） +ATTEMPT_ANSWER_CAP = 4000 +#: 停止、服务重启挂起时 error 列里的那一句（同页面落库时写的） +CANCELLED = "已取消" +SUSPENDED = "服务重启，本轮已中断" +#: 建图时服务重启 +BUILD_INTERRUPTED = "服务重启，本轮在生成工作流时中断" +GONE = "运行记录不见了" +#: 用量里记的几项（同页面的 USAGE_KEYS） +USAGE_KEYS = ("cost_usd", "unpriced_calls", "calls", "unfinished_calls", "input_tokens", "output_tokens") +#: 出具信息里无法回指的数字最多存几个（同页面的 slimIssuance） +ISSUANCE_UNMATCHED_CAP = 20 + + +def answer_text(output: Mapping[str, Any] | None) -> str: + """成果压成一句话（同页面的 answerText）：下划线开头的、空的不要；字符串原样，别的按 JSON；超过上限截断并说出来。""" + if not output: + return "" + parts = [value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, separators=(",", ":")) + for key, value in output.items() if not str(key).startswith("_") and value not in (None, "")] + text = "\n".join(parts) + return text[:ANSWER_CAP] + ANSWER_CLIPPED if len(text) > ANSWER_CAP else text + + +def slim_issuance(issuance: Any) -> dict[str, Any] | None: + """出具信息存进 meta 前瘦一下身（同页面的 slimIssuance）：无法回指的数字只留前 20 个。""" + if not isinstance(issuance, Mapping) or not issuance.get("tier"): + return None + out = dict(issuance) + unmatched = issuance.get("unmatched_numbers") + if isinstance(unmatched, list): + out["unmatched_numbers"] = unmatched[:ISSUANCE_UNMATCHED_CAP] + return out + + +def has_evidence(output: Mapping[str, Any] | None) -> bool: + """成果带不带逐段证据（同页面 lib/evidence 的 hasEvidence）：_evidence 或它的 others 里有文档、有字段。""" + ev = (output or {}).get("_evidence") if isinstance(output, Mapping) else None + if not isinstance(ev, Mapping): + return False + refs = [ev, *[r for r in ev.get("others") or [] if isinstance(r, Mapping)]] + return any(isinstance(r.get("doc_artifact"), str) and r.get("doc_artifact") and isinstance(r.get("fields"), list) + and r.get("fields") for r in refs) + + +def _names(value: Any) -> list[str]: + return [str(v) for v in value if v not in (None, "")] if isinstance(value, list) else [] + + +def _num(value: Any) -> float | None: + return value if isinstance(value, (int, float)) and not isinstance(value, bool) else None + + +def look_event(op: Mapping[str, Any]) -> dict[str, Any] | None: + """助手的 look 操作 → 存进 meta.looks 的那一份（同页面 run/decode 的 lookEventOf）：下一轮服务端据此写「上一轮查看过」。""" + if op.get("op") != "look": + return None + listed = op.get("objects") if isinstance(op.get("objects"), list) else op.get("sources") + if not isinstance(listed, list): + return None + objects = [] + for x in listed: + if not isinstance(x, Mapping): + continue + kind = x.get("kind") if x.get("kind") in ("capability", "workflow_version", "caliber") else "table" + total, explored = _num(x.get("total")), _num(x.get("explored")) + obj: dict[str, Any] = {"kind": kind} + if isinstance(x.get("source"), str) and x.get("source"): + obj["source"] = x["source"] + if isinstance(x.get("source_id"), str) and x.get("source_id"): + obj["source_id"] = x["source_id"] + obj.update(named=_names(x.get("named")), picked=_names(x.get("picked")), joined=_names(x.get("joined")), + missing=_names(x.get("missing"))) + if total is not None: + obj["total"] = total + if explored is not None and explored < total: + obj["explored"] = explored + if _num(x.get("catalog_tables")) is not None: + obj["catalog_tables"] = x["catalog_tables"] + if _names(x.get("added_to_scope")): + obj["added_to_scope"] = _names(x.get("added_to_scope")) + if _num(x.get("prefiltered")) is not None: + obj["prefiltered"] = x["prefiltered"] + objects.append(obj) + out: dict[str, Any] = {"op": "look", "id": str(op.get("id") or ""), "by": "system" if op.get("by") == "system" else "model", + "need": str(op.get("need") or ""), + "status": op.get("status") if op.get("status") in ("index", "failed") else "ok"} + if isinstance(op.get("reason"), str) and op.get("reason"): + out["reason"] = op["reason"] + out["elapsed_ms"] = _num(op.get("elapsed_ms")) or 0 + out["objects"] = objects + return out + + +# -------------------------------------------------------------------------- +# 用量：{copilot, review, earlier, run}，每份是 UsageTotal（同页面的 TurnUsage） +# -------------------------------------------------------------------------- + + +def usage_total(usage: Mapping[str, Any] | None) -> dict[str, Any] | None: + """一份用量（providers/usage 的汇总、运行的 usage）只留要记的几项。不是对象交回 None。""" + return add_total({}, usage) if isinstance(usage, Mapping) else None + + +def add_total(a: Mapping[str, Any] | None, b: Mapping[str, Any] | None) -> dict[str, Any]: + """两份用量相加（缺的当 0，同页面的 addUsage）：只记不为 0 的项，金额保留 6 位。""" + out = {k: v for k, v in (a or {}).items() if k in USAGE_KEYS} + for key in USAGE_KEYS: + try: + value = float((b or {}).get(key) or 0) + except (TypeError, ValueError): + continue + if value: + out[key] = (out.get(key) or 0) + value + for key in USAGE_KEYS: + if key in out and key != "cost_usd": + out[key] = int(out[key]) + if "cost_usd" in out: + out["cost_usd"] = round(float(out["cost_usd"]), 6) + return out + + +def retire_run(usage: Mapping[str, Any] | None) -> dict[str, Any]: + """当前这次运行换下来了（自动重来、用户点重试）：它的用量并进 earlier，这一轮的花费不丢这一笔（同页面的 retireRun）。""" + out = dict(usage or {}) + run = out.pop("run", None) + if run: + out["earlier"] = add_total(out.get("earlier"), run) + return out + + +# -------------------------------------------------------------------------- +# 用户点重试：上一次的结局归档 +# -------------------------------------------------------------------------- + +#: 归档那一次的一句话（同页面以前 attemptOf 的写法）。也是重试时交给助手的原因 +ATTEMPT_TEXT = { + "cancelled": "已手动停止", + "suspended": "服务重启，运行中断", + "rejected": "被驳回", + "unusable": "结论不可用：{note}", + "partial": "存在缺口：{note}", + "done": "已给出答案", +} + + +def _rejected_line(rejected: Mapping[str, Any]) -> str: + """被驳回的那一句（同页面 REJECTED_TEXT.sub）：谁在哪一步驳回、备注；页面记过 sub 的照用。""" + if isinstance(rejected.get("sub"), str) and rejected["sub"]: + return rejected["sub"] + who = str(rejected.get("actor") or "").strip() + label = str(rejected.get("label") or "").strip() + where = f"「{label}」" if label else "这一步" + note = str(rejected.get("note") or "").strip() + if rejected.get("decision") == "expired_reject": + return f"{where}超过时限，按设置驳回" + return (f"{who} 驳回了{where}" if who else f"{where}被驳回") + (f"：{note}" if note else "") + + +def attempt_record(row: Any, meta: Mapping[str, Any], *, n: int, at_ms: int | None = None) -> dict[str, Any]: + """这一轮现在的样子折成一条留档（同页面以前的 attemptOf）：{n, outcome, summary, detail?, runId, answer?, at, cause?}。 + + 用户主动重试不能把上一次抹掉：那一次的答案可能只是有缺口、并非全错，新的一次也未必更好，两份要能对照。""" + from app.api.conversations import turn_run_id + + review = row.review if isinstance(row.review, Mapping) else {} + answer = row.answer or "" + if len(answer) > ATTEMPT_ANSWER_CAP: + answer = answer[:ATTEMPT_ANSWER_CAP] + "…" + base: dict[str, Any] = {"n": n, "at": at_ms if at_ms is not None else int(time.time() * 1000), + "runId": turn_run_id(row)} + if answer: + base["answer"] = answer + signals = "\n".join(str(s.get("detail") or "") for s in review.get("signals") or [] if isinstance(s, Mapping)) or None + outcome = meta.get("outcome") + if outcome == "cancelled" and isinstance(meta.get("rejected"), Mapping): + sub = _rejected_line(meta["rejected"]) + return {**base, "outcome": "rejected", "summary": ATTEMPT_TEXT["rejected"] + (f"：{sub}" if sub else "")} + if outcome == "cancelled" or (row.status == "error" and (row.error or "").strip() == CANCELLED): + return {**base, "outcome": "cancelled", "summary": ATTEMPT_TEXT["cancelled"]} + if outcome == "suspended": + return {**base, "outcome": "suspended", "summary": ATTEMPT_TEXT["suspended"]} + if row.status == "error": + failure = meta.get("failure") if isinstance(meta.get("failure"), Mapping) else None + line = (f"{failure.get('title')}：{failure.get('reason')}" if failure and failure.get("reason") + else str(failure.get("title")) if failure and failure.get("title") else (row.error or "发生错误")) + out = {**base, "outcome": "failed", "summary": line} + if failure and failure.get("detail"): + out["detail"] = str(failure["detail"]) + if isinstance(meta.get("cause"), Mapping): + out["cause"] = dict(meta["cause"]) + return out + note = str(review.get("note") or "") + if review.get("severity") == "broken": + return {**base, "outcome": "unusable", "summary": ATTEMPT_TEXT["unusable"].format(note=note), "detail": signals} + if review.get("severity") == "degraded": + return {**base, "outcome": "partial", "summary": ATTEMPT_TEXT["partial"].format(note=note), "detail": signals} + return {**base, "outcome": "done", "summary": ATTEMPT_TEXT["done"]} + + +def attempt_reason(record: Mapping[str, Any]) -> str: + """重试时交给助手的原因（同页面以前的写法）：那一次的一句话，有细节的括号里带前 300 字。""" + summary = str(record.get("summary") or "") + detail = record.get("detail") + return f"{summary}（{str(detail)[:300]}）" if detail else summary + + +def with_max_steps(graph: Mapping[str, Any], steps: int) -> dict[str, Any]: + """图里每个 agent 节点的步数上限调到 steps（「放宽步数重跑」，同页面的 withMaxSteps）。""" + nodes = [] + for node in graph.get("nodes") or []: + if isinstance(node, Mapping) and node.get("type") == "agent": + data = dict(node.get("data") or {}) + data["config"] = {**dict(data.get("config") or {}), "max_steps": steps} + node = {**node, "data": data} + nodes.append(node) + return {**dict(graph), "nodes": nodes} + + +__all__ = ["ANSWER_CAP", "ATTEMPT_ANSWER_CAP", "ATTEMPT_TEXT", "BUILD_INTERRUPTED", "CANCELLED", "GONE", "SUSPENDED", + "USAGE_KEYS", "add_total", "answer_text", "attempt_reason", "attempt_record", "has_evidence", "look_event", + "retire_run", "slim_issuance", "usage_total", "with_max_steps"] diff --git a/backend/app/chat/turn.py b/backend/app/chat/turn.py index f416d95a..ac361b52 100644 --- a/backend/app/chat/turn.py +++ b/backend/app/chat/turn.py @@ -14,7 +14,8 @@ /copilot/review 的结果带 retry_instruction（要重来时是那句话，不重来是 null）。问数据页照后端给的走；老后端没有这两个字段时退回它自己那份同样的 规则（tests/test_chat_turn_policy.py 逐条对照两边的常量和措辞）。评测（第 3 波）用同一组函数把一轮串起来。 -只搬「决定」：重连、断点续看、取消、复核等多久这些仍在问数据页（把整轮编排搬到后端要重做这些，单独立项）。 +整轮编排后来也搬到了服务端（docs/design/orchestration-round §1）：流程在 chat/drive.py（问数据和评测共用），问数据的后台任务在 +chat/job.py；页面只发起、只看，不再自己判这些。老服务端那条退路（页面自己那份规则）随之去掉。 """ from __future__ import annotations @@ -50,6 +51,12 @@ RETRY_WITH_BASE = ("这个工作流上一次运行没有成功：{reason}。\n\n "改完继续回答原来的问题：{question}") #: 没有图可交回（上一次连图都没建出来）时：原问题后面缀上原因，从零再建 RETRY_FROM_SCRATCH = "{question}\n\n（上一次没有成功：{reason}。请针对这个原因调整，重新给出完整的工作流。）" +#: 用户点「重试本轮」时交给助手的话（以前在问数据页 retryTurn 里拼，一轮搬到服务端之后在这里）：有图时说的是「没有给出可用的 +#: 结果」——上一次未必是运行失败（停止了、被驳回、结论不可用都算），原因是那一次的归档说法（chat/record.attempt_reason）； +#: 没有图的同 RETRY_FROM_SCRATCH +RETRY_USER_WITH_BASE = ("这个工作流上一次没有给出可用的结果：{reason}。\n\n" + "请针对这个原因改它（比如调大最大步数、改正工具参数、换个查询方式），" + "改完继续回答原来的问题：{question}") def _blockers(final: Mapping[str, Any]) -> list[str]: @@ -142,12 +149,18 @@ def retry_instruction(question: str, reason: str, has_base: bool) -> str: return template.format(question=question, reason=reason) +def user_retry_instruction(question: str, reason: str, has_base: bool) -> str: + """用户点「重试本轮」时交给助手的话。has_base：有上一次建出的图可以交回去改。""" + template = RETRY_USER_WITH_BASE if has_base else RETRY_FROM_SCRATCH + return template.format(question=question, reason=reason) + + def has_nodes(graph: Any) -> bool: """一张图有没有节点（有才交回去改）。""" return isinstance(graph, Mapping) and bool(graph.get("nodes")) __all__ = ["EMPTY_GENERATION", "MAX_ATTEMPTS", "NEXT_ASKED", "NEXT_FAIL", "NEXT_READY", "NEXT_REPLY", "NEXT_RETRY", - "NEXT_RUN", "NEXT_STEPS", "QUESTION_FIELD", "RETRY_ERRORS", "RETRY_FROM_SCRATCH", "RETRY_WITH_BASE", - "blocked_reason", "error_reason", "fill_input", "has_nodes", "next_step", "retry_after_review", - "retry_instruction", "review_reason"] + "NEXT_RUN", "NEXT_STEPS", "QUESTION_FIELD", "RETRY_ERRORS", "RETRY_FROM_SCRATCH", "RETRY_USER_WITH_BASE", + "RETRY_WITH_BASE", "blocked_reason", "error_reason", "fill_input", "has_nodes", "next_step", "retry_after_review", + "retry_instruction", "review_reason", "user_retry_instruction"] diff --git a/backend/app/db/models.py b/backend/app/db/models.py index 9565cecd..a400acb3 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -1093,4 +1093,4 @@ from app.db.models_blueprints import ( # noqa: E402,F401 ) # 下一轮第 2 波（docs/design/next-round）：数据源对象索引与后台探查（models_sources）、评测（models_eval）。整模块导入， # 各自的模型类由那两个文件自己登记，这里不逐个列名 -from app.db import models_eval, models_sources # noqa: E402,F401 +from app.db import models_chat, models_eval, models_sources # noqa: E402,F401 diff --git a/backend/app/db/models_chat.py b/backend/app/db/models_chat.py new file mode 100644 index 00000000..f13542f7 --- /dev/null +++ b/backend/app/db/models_chat.py @@ -0,0 +1,73 @@ +"""问数据的一轮在服务端跑（docs/design/orchestration-round/00-总方案.md §1）：这一轮的后台任务和它的事件日志。 + +由 app/db/models.py 末尾整模块导入，create_all 建表（都是新表，不改已有表）。这里不导入 app.db.models：models.py 导入本模块时 +自己还没加载完，外键一律按表名写。 + +**为什么一轮一行任务、id 就是 conversation_turns 的 id。** 一轮问数据（建图 → 运行 → 复核 → 按复核重来、落库）以前是浏览器在推， +页面一断这一轮就没人管了。现在它是服务端的一个后台任务：发起、重试本轮、重新运行本轮、继续运行都还是「这一轮」，用户看到的 +是同一轮里的几次尝试，所以任务按轮记、一轮一行，重试时把这一行重新排队（上一次的结局归档进 conversation_turns.meta.attempts）。 + +- ChatTurnJob：状态用后台任务的状态词（core/job_runtime：queued → running → succeeded / failed / cancelled / interrupted）， + stage 是这一轮现在在哪一步（build 建图、run 运行、waiting 停在审批上等人、review 复核），run_id 是当前这次运行。request 是 + 这次要做什么（问题、限定的数据源、方式、重试的原因、放宽到几步……），principal 是发起的人（身份快照：服务重启后接着做时 + 按这个人的身份——遮罩、只读、对话归属都看它，见 app/chat/job.py）。attempt_seq 是这一次尝试在事件日志里从哪一条开始： + 刷新、重开页面时从这里回放，不必把之前几次尝试的过程整条重放。 +- ChatTurnEvent：这一轮的事件日志。每一步（助手的操作、心跳、查看、final、用量、运行 id、复核结论、重来、终局）按 seq + 一条，页面先回放、再接实时，断了按 seq 补齐（GET /api/chat/turns/{id}/stream?after=）。不进运行的封存范围：运行自己的 + 事件仍在 run_events，这里只记「这一轮」的事。 +""" +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from sqlalchemy import Float, ForeignKey, Integer, String, Text, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base, TimestampMixin, UTCDateTime, new_id + +#: 这一轮现在在哪一步 +STAGES = ("build", "run", "waiting", "review") + + +class ChatTurnJob(Base, TimestampMixin): + """问数据一轮的后台任务（见模块说明）。""" + + __tablename__ = "chat_turn_jobs" + + #: 就是 conversation_turns.id + id: Mapped[str] = mapped_column(ForeignKey("conversation_turns.id", ondelete="CASCADE"), primary_key=True) + conversation_id: Mapped[str] = mapped_column(ForeignKey("conversations.id", ondelete="CASCADE"), index=True) + #: queued | running | succeeded | failed | cancelled | interrupted（core/job_runtime 的状态词） + status: Mapped[str] = mapped_column(String(20), default="queued", index=True) + stage: Mapped[str | None] = mapped_column(String(20), default=None) + #: ask | retry | rerun | max_steps | run | continue | resume（app/chat/job.py 的 MODES） + mode: Mapped[str] = mapped_column(String(20), default="ask") + run_id: Mapped[str | None] = mapped_column(String(32), default=None) + #: 这一次尝试里自动重来了几次（从 0 数，同 chat/turn.MAX_ATTEMPTS 的口径） + attempt: Mapped[int] = mapped_column(Integer, default=0) + request: Mapped[dict[str, Any]] = mapped_column(default=dict) + principal: Mapped[dict[str, Any]] = mapped_column(default=dict) + actor: Mapped[str | None] = mapped_column(String(100), default=None) + last_seq: Mapped[int] = mapped_column(Integer, default=0) + attempt_seq: Mapped[int] = mapped_column(Integer, default=0) + error: Mapped[str | None] = mapped_column(Text, default=None) + started_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + + +class ChatTurnEvent(Base): + """问数据一轮的事件日志里的一条（见模块说明）。""" + + __tablename__ = "chat_turn_events" + __table_args__ = (UniqueConstraint("turn_id", "seq", name="uq_chat_turn_events_turn_seq"),) + + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) + turn_id: Mapped[str] = mapped_column(ForeignKey("conversation_turns.id", ondelete="CASCADE"), index=True) + seq: Mapped[int] = mapped_column(Integer) + type: Mapped[str] = mapped_column(String(40)) + ts: Mapped[float] = mapped_column(Float) + data: Mapped[dict[str, Any]] = mapped_column(default=dict) + + +__all__ = ["STAGES", "ChatTurnEvent", "ChatTurnJob"] diff --git a/backend/app/eval/answer.py b/backend/app/eval/answer.py index d9e66982..6acc25cf 100644 --- a/backend/app/eval/answer.py +++ b/backend/app/eval/answer.py @@ -1,8 +1,8 @@ """评测里的一轮作答：无头地走一遍问数据页走的那一轮（细案 01 做法 1、9；总方案 2.1）。 -**量的就是用户拿到的那一轮。** 不另写一份 Python 版编排：生成过程是 api/copilot.answer_ops（问数据页经 SSE 收的同一串 -操作），几个「决定」是 chat/turn 的纯函数（问数据页照服务端给的 next、retry_instruction 走），复核是 api/copilot.review_of_run -（问数据页经 /copilot/review 调的同一个函数）。这里只把它们串起来： +**量的就是用户拿到的那一轮。** 不另写一份编排：一轮的流程是 app/chat/drive.TurnDriver（问数据的一轮在服务端跑也是它， +app/chat/job.py），生成过程是 api/copilot.answer_ops，几个「决定」是 chat/turn 的纯函数，复核是 api/copilot.review_of_run。这里 +只写评测和问数据不一样的那几项（Driver 是 TurnDriver 的子类）： answer_ops → final.next（chat/turn.next_step）→ launch_run(origin=eval) → 等终态 → review_of_run → 要的话带原图重来一次 @@ -29,22 +29,21 @@ llm.end（按模型分）。重来的话两次都算——那都是这一题花 """ from __future__ import annotations -import asyncio import copy import logging -import time from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass, field from typing import Any -from sqlalchemy import func, select +from sqlalchemy import select + +# 一轮的流程和结果抽到了 chat/drive.py（问数据、评测共用），这里留着老名字：runner、stats 和测试照旧从这里取 +from app.chat.drive import CANCEL_GRACE_S, Turn, TurnDriver, add_usage # noqa: F401 logger = logging.getLogger(__name__) -#: 问数据页最多等复核这么久（chat.ts 的 REVIEW_TIMEOUT_MS）：评测照等，超过的单列 +#: 问数据页最多等复核这么久（chat/job.REVIEW_WAIT_S，和以前页面上的 REVIEW_TIMEOUT_MS 同一个数）：评测照等，超过的单列 REVIEW_PAGE_WAIT_S = 25.0 -#: 取消一次运行之后最多再等它收尾多久 -CANCEL_GRACE_S = 15.0 #: 运行停在审批上：评测不代批 APPROVAL_REASON = "运行停在审批上（危险工具、人工节点）：评测不代批，已取消这次运行" #: 每一步在做什么（EvalResult.current） @@ -71,69 +70,6 @@ class TurnContext: view: Mapping[str, Any] | None = None -@dataclass -class Turn: - """一轮作答的结果（也是进行中的状态：current_run、phase 随时更新）。""" - - path: str | None = None - run_id: str | None = None - #: 这一题发起过的全部作答运行（重来的话两次），失败的也在 - runs: list[str] = field(default_factory=list) - attempts: int = 0 - #: 每次自动重来的原因（和 attempts 一一对应）：没过检查、复核说不可信、生成过程一个操作都没给 - retries: list[str] = field(default_factory=list) - reason: str | None = None - #: 助手调用失败（生成过程交出 error、开流前就被拒）：runner 据此止损 - copilot_failed: bool = False - timed_out: bool = False - #: 直接回答的原文、反问的问题 - reply: str | None = None - asked: dict[str, Any] | None = None - review: dict[str, Any] | None = None - looks: list[dict[str, Any]] = field(default_factory=list) - #: 助手这一轮用的模型 id（model 操作） - assistant_model: str | None = None - #: {assistant: usage 汇总（生成，含重来）, review: usage 汇总} - usage: dict[str, Any] = field(default_factory=dict) - timings: dict[str, int] = field(default_factory=dict) - #: 写作模型按变体换了，但节点自己点了模型、没被覆盖的节点 id - own_models: list[str] = field(default_factory=list) - #: 正在跑的那次运行（调用方超时、停止时取消它） - current_run: str | None = None - phase: str | None = None - - def to_dict(self) -> dict[str, Any]: - return {"path": self.path, "run_id": self.run_id, "runs": list(self.runs), "attempts": self.attempts, - "retries": list(self.retries), "reason": self.reason, "copilot_failed": self.copilot_failed, "timed_out": self.timed_out, - "reply": self.reply, "asked": self.asked, "review": self.review, "looks": list(self.looks), - "assistant_model": self.assistant_model, "usage": copy.deepcopy(self.usage), - "timings": dict(self.timings), "own_models": list(self.own_models)} - - -#: 用量汇总里相加的计数（providers/usage 的 summary）：unfinished_calls 是没做完的调用（被取消、超时，token 只计了已报的） -_COUNTED = ("input_tokens", "output_tokens", "calls", "unpriced_calls", "unfinished_calls") - - -def _add_usage(into: dict[str, Any] | None, more: Mapping[str, Any] | None) -> dict[str, Any]: - """两份 providers/usage 汇总相加（by_model 逐个模型加）。""" - out = copy.deepcopy(into) if into else {"by_model": {}, **dict.fromkeys(_COUNTED, 0), "cost_usd": 0.0} - if not more: - return out - for key in _COUNTED: - out[key] = int(out.get(key) or 0) + int(more.get(key) or 0) - out["cost_usd"] = round(float(out.get("cost_usd") or 0) + float(more.get("cost_usd") or 0), 6) - for model, row in (more.get("by_model") or {}).items(): - mine = out["by_model"].setdefault(model, {**dict.fromkeys(_COUNTED, 0), "cost_usd": 0.0}) - for key in _COUNTED: - mine[key] = int(mine.get(key) or 0) + int(row.get(key) or 0) - mine["cost_usd"] = round(float(mine.get("cost_usd") or 0) + float(row.get("cost_usd") or 0), 6) - return out - - -def add_usage(into: dict[str, Any] | None, more: Mapping[str, Any] | None) -> dict[str, Any]: - return _add_usage(into, more) - - async def writer_usage(run_ids: list[str]) -> dict[str, Any]: """答案运行里的模型调用（llm.end，按模型分）：写作这一角色的用量，形状同 providers/usage 的汇总。封存之后追加的 （按需裁判、质询）不算：那是事后另记的账（裁判记在评测的账上另算，见 judging.py）。""" @@ -142,7 +78,7 @@ async def writer_usage(run_ids: list[str]) -> dict[str, Any]: from app.db.models import RunEvent from app.providers.catalog import unpriced_call - out = _add_usage(None, None) + out = add_usage(None, None) if not run_ids: return out async with SessionLocal() as session: @@ -156,7 +92,7 @@ async def writer_usage(run_ids: list[str]) -> dict[str, Any]: piece = {"input_tokens": int(data.get("input_tokens") or 0), "output_tokens": int(data.get("output_tokens") or 0), "calls": int(data.get("calls") or 1), "cost_usd": float(data.get("cost_usd") or 0.0), "unpriced_calls": 1 if unpriced_call(data) else 0} - out = _add_usage(out, {**piece, "by_model": {model: piece}}) + out = add_usage(out, {**piece, "by_model": {model: piece}}) return out @@ -208,150 +144,46 @@ async def stop_run(run_id: str | None, actor: str | None = None) -> None: logger.exception("评测：取消运行 %s 时出错", run_id) -class Driver: - """一轮作答。drive() 跑完一轮，结果在 turn 上；on_phase(阶段) 在每一步开始时调（runner 据此写「正在做什么」）。""" +class Driver(TurnDriver): + """一轮作答（流程在 TurnDriver）。drive() 跑完一轮，结果在 turn 上；on_phase(阶段) 在每一步开始时调（runner 据此写 + 「正在做什么」）。和问数据不一样的：助手模型、目录视图按变体；写作模型按变体写进图；来源 eval、只读和遮罩；停在审批上 + 直接取消（评测不代批）；复核一直等。""" def __init__(self, question: str, *, datasource_ids: list[str], ctx: TurnContext, on_phase: Callable[[str], Awaitable[None]] | None = None) -> None: - self.question = question - self.datasource_ids = list(datasource_ids) + super().__init__(question, datasource_ids=datasource_ids, on_phase=on_phase, + provider=ctx.assistant.get("provider"), model=ctx.assistant.get("model")) self.ctx = ctx - self.on_phase = on_phase - self.turn = Turn() - async def _phase(self, phase: str) -> None: - self.turn.phase = phase - if self.on_phase is not None: - try: - await self.on_phase(phase) - except Exception: # noqa: BLE001 - 写进度失败不挡作答 - logger.exception("评测：记进度出错") - - # ------------------------------------------------------------------ 生成 - - async def _generate(self, instruction: str, base_graph: dict[str, Any] | None) -> tuple[list[dict[str, Any]], str | None]: - """跑一次生成过程，收下全部操作。开流前就被拒（HTTPException）交回原因。""" - from fastapi import HTTPException - - from app.api.copilot import GenerateIn, answer_ops - - payload = GenerateIn(instruction=instruction, base_graph=base_graph, intent="answer", - provider=self.ctx.assistant.get("provider"), model=self.ctx.assistant.get("model"), - datasource_ids=self.datasource_ids or None) - ops: list[dict[str, Any]] = [] - try: - async for op in answer_ops(payload, catalog_view=self.ctx.view): - if op.get("op") in ("thinking", "heartbeat"): - continue - ops.append(op) - except HTTPException as e: - return ops, str(e.detail) - return ops, None - - def _take(self, ops: list[dict[str, Any]]) -> dict[str, Any] | None: - """收下这一次生成的用量、查看记录、模型；交回结局（final / reply / error），只有反问的为 None。""" - terminal = None - for op in ops: - kind = op.get("op") - if kind == "usage": - self.turn.usage["assistant"] = _add_usage(self.turn.usage.get("assistant"), - {k: v for k, v in op.items() if k != "op"}) - elif kind == "look": - self.turn.looks.append({k: op.get(k) for k in ("id", "by", "need", "status") if k in op}) - elif kind == "model": - self.turn.assistant_model = str(op.get("model") or "") or None - elif kind in ("final", "reply", "error"): - terminal = op - return terminal - - # ------------------------------------------------------------------ 运行 - - async def _launch(self, graph: dict[str, Any], payload: dict[str, Any] | None = None) -> str | None: - from app.chat import turn as policy - from app.db.base import SessionLocal - from app.engine.launch import LaunchRefused, launch_run + def catalog_view(self) -> Any: + return self.ctx.view + def prepare_graph(self, graph: dict[str, Any]) -> dict[str, Any]: graph, own = with_writer(graph, self.ctx.writer) self.turn.own_models = sorted(set(self.turn.own_models) | set(own)) + return graph + + async def launch_args(self, session: Any, graph: dict[str, Any]) -> dict[str, Any]: origin = {k: v for k, v in { "suite_id": self.ctx.suite_id, "eval_run_id": self.ctx.eval_run_id, "case_id": self.ctx.case_id, "role": "answer", "arm": self.ctx.arm, "repeat": self.ctx.repeat, "attempt": self.turn.attempts}.items() if v is not None} - try: - async with SessionLocal() as session: - inputs = payload if payload is not None else policy.fill_input(graph, self.question, "") - run = await launch_run(session, graph=graph, input_payload=inputs, - actor=self.ctx.actor, options=_options(self.ctx.view), origin_kind="eval", - origin=origin, workflow_name=run_name(self.ctx.suite_name, self.ctx.case_seq)) - except LaunchRefused as e: - self.turn.path, self.turn.reason = "run_failed", f"运行没有发起：{e.message}" - return None - self.turn.runs.append(run.id) - self.turn.run_id = self.turn.current_run = run.id - return run.id - - async def _wait(self, run_id: str) -> Any: - """等运行到终态（或停在审批上）。交回库里的运行。""" - from app.db.base import SessionLocal - from app.db.models import Run - from app.engine.runner import run_manager - - while True: - await run_manager.wait_idle(run_id, timeout=5.0) - async with SessionLocal() as session: - row = await session.get(Run, run_id) - if row is None or row.status in ("succeeded", "failed", "cancelled", "interrupted"): - # 封存在终态之后：等执行任务收完尾（含封存），评分只读封存范围 - await run_manager.wait_idle(run_id, timeout=CANCEL_GRACE_S) - async with SessionLocal() as session: - return await session.get(Run, run_id) - await asyncio.sleep(0.2) - - async def _pending_approvals(self, run_id: str) -> int: - from app.db.base import SessionLocal - from app.db.models import Approval - - async with SessionLocal() as session: - return int(await session.scalar(select(func.count()).select_from(Approval).where( - Approval.run_id == run_id, Approval.status.in_(("pending", "answered")))) or 0) - - # ------------------------------------------------------------------ 一轮 - - async def _run(self, graph: dict[str, Any], payload: dict[str, Any] | None = None) -> Any | None: - """发起、等到终态。成功结束的交回运行；别的结局记好路径和原因，交回 None。""" - from app.engine.runner import END_REJECTED + return {"actor": self.ctx.actor, "options": _options(self.ctx.view), "origin_kind": "eval", "origin": origin, + "workflow_name": run_name(self.ctx.suite_name, self.ctx.case_seq)} + async def on_interrupted(self, run_id: str, row: Any) -> Any | None: t = self.turn - run_id = await self._launch(graph, payload) - if run_id is None: - return None - row = await self._wait(run_id) - t.current_run = None - usage = dict((row.usage or {}) if row is not None else {}) - t.timings["run_wall_ms"] = t.timings.get("run_wall_ms", 0) + int(usage.get("wall_ms") or 0) - t.timings["run_active_ms"] = t.timings.get("run_active_ms", 0) + int( - usage.get("active_ms") or usage.get("duration_ms") or 0) - if row is None: - t.path, t.reason = "run_failed", "运行记录不见了" - return None - if row.status == "interrupted": - waiting = await self._pending_approvals(run_id) - await stop_run(run_id, self.ctx.actor) - t.path = "run_failed" - t.reason = APPROVAL_REASON if waiting else "运行停在断点上（服务可能重启过）：评测不继续这次运行，已取消" - return None - if row.status == "cancelled" and row.end_reason == END_REJECTED: - t.path, t.reason = "rejected", "运行被驳回（人的决定，不算运行失败）" - return None - if row.status != "succeeded": - t.path = "run_failed" - t.reason = f"运行{'失败' if row.status == 'failed' else '被停止'}：{row.error or '没有记下原因'}" - return None - return row + waiting = await self._pending_approvals(run_id) + await stop_run(run_id, self.ctx.actor) + t.path = "run_failed" + t.reason = APPROVAL_REASON if waiting else "运行停在断点上（服务可能重启过）：评测不继续这次运行，已取消" + return None async def drive_workflow(self, graph: dict[str, Any], inputs: dict[str, Any]) -> Turn: """工作流题：钉住那一版的图按输入跑一次探索运行（来源 eval）。不建图、不复核：考的是这一版本身。不发正式运行—— 正式运行会被 pick_base 和应用视图的「最近一次封存」取到（同期望探针不发正式运行的理由）。""" + import time + t = self.turn started = time.monotonic() try: @@ -362,107 +194,6 @@ class Driver: finally: t.timings["total_ms"] = int((time.monotonic() - started) * 1000) - async def drive(self) -> Turn: - from app.chat import turn as policy - - t = self.turn - started = time.monotonic() - instruction, base_graph = self.question, None - try: - while True: - await self._phase("build") - building = time.monotonic() - ops, refused = await self._generate(instruction, base_graph) - t.timings["build_ms"] = t.timings.get("build_ms", 0) + int((time.monotonic() - building) * 1000) - terminal = self._take(ops) - if refused is not None: - t.path, t.reason, t.copilot_failed = "build_failed", f"助手没有开始：{refused}", True - return t - if terminal is None: - asked = next((o for o in reversed(ops) if o.get("op") == "ask"), None) - t.path, t.asked = "asked", {k: asked.get(k) for k in ("question", "options")} if asked else None - return t - kind = terminal.get("op") - if kind == "error": - # 生成过程一个操作都没给（只写了思考）：按重来一次的规矩带着原因重来（同问数据页，chat/turn.next_step） - step = terminal.get("next") or policy.next_step(terminal) - if step == policy.NEXT_RETRY and t.attempts + 1 < policy.MAX_ATTEMPTS: - reason = policy.error_reason(terminal) - t.attempts += 1 - t.retries.append(reason) - instruction = policy.retry_instruction(self.question, reason, policy.has_nodes(base_graph)) - continue - t.path, t.reason, t.copilot_failed = "build_failed", str(terminal.get("message") or "助手本轮未完成"), True - return t - if kind == "reply": - t.path, t.reply = "reply", str(terminal.get("text") or "") - return t - if terminal.get("asked"): - t.path, t.asked = "asked", dict(terminal["asked"]) - return t - self_checked = any(o.get("op") == "check" for o in ops) - step = terminal.get("next") or policy.next_step(terminal, self_checked=self_checked) - graph = terminal.get("graph") or {} - if step == policy.NEXT_RETRY: - if t.attempts + 1 < policy.MAX_ATTEMPTS: - t.attempts += 1 - t.retries.append(policy.blocked_reason(terminal)) - instruction = policy.retry_instruction(self.question, policy.blocked_reason(terminal), - policy.has_nodes(graph)) - base_graph = graph if policy.has_nodes(graph) else None - continue - t.path, t.reason = "build_failed", policy.blocked_reason(terminal) - return t - if step == policy.NEXT_FAIL: - errors = [str(i.get("message") or "") for i in terminal.get("issues") or [] - if isinstance(i, Mapping) and i.get("level") == "error"] - t.path, t.reason = "build_failed", "生成的工作流没有通过校验：" + "；".join(errors) - return t - if step == policy.NEXT_READY: - t.path, t.reason = "ready", "助手只建了图、没有运行（它判断用户要的是流程本身）" - return t - - await self._phase("run") - row = await self._run(graph) - if row is None: - return t - - await self._phase("review") - reviewing = time.monotonic() - review = await self._review(row) - spent = int((time.monotonic() - reviewing) * 1000) - t.timings["review_ms"] = t.timings.get("review_ms", 0) + spent - t.review = {k: v for k, v in review.items() if k != "usage"} - t.usage["review"] = _add_usage(t.usage.get("review"), review.get("usage")) - if review.get("retry_instruction"): - from app.chat.turn import review_reason - - t.attempts += 1 - t.retries.append(review_reason(review) or "复核判答案不可用") - instruction = str(review["retry_instruction"]) - base_graph = row.graph if policy.has_nodes(row.graph) else None - continue - t.path = "ran" - return t - finally: - t.timings["total_ms"] = int((time.monotonic() - started) * 1000) - - async def _review(self, run: Any) -> dict[str, Any]: - """复核（review_of_run，问数据页同一个函数）。复核自己出错不挡评分：照实记一条失败的复核。""" - from app.api.copilot import review_of_run - from app.core.errors import describe_exception - from app.db.base import SessionLocal - - try: - async with SessionLocal() as session: - return await review_of_run(session, run, question=self.question, - provider=self.ctx.assistant.get("provider"), - model=self.ctx.assistant.get("model"), attempt=self.turn.attempts) - except Exception as e: # noqa: BLE001 - logger.exception("评测：复核运行 %s 出错", run.id) - return {"verdict": "failed", "note": f"复核出错：{describe_exception(e)}", "signals": [], - "retry_instruction": None} - __all__ = ["APPROVAL_REASON", "Driver", "PHASE_TEXT", "REVIEW_PAGE_WAIT_S", "Turn", "TurnContext", "add_usage", "run_name", "stop_run", "with_writer", "writer_usage"] diff --git a/backend/app/main.py b/backend/app/main.py index 7343ab82..c96663a3 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -27,6 +27,8 @@ from app.api import ( blueprints as blueprints_api, copilot_blueprint, files as files_api, personal as personal_api, templates as templates_api, ) +# 编排完善第 1 波（docs/design/orchestration-round §1）：问数据的一轮在服务端跑 +from app.api import chat_turns as chat_turns_api from app.api.coded import CodedHTTPException, coded_handler from app.api.deps import chat_scope, evidence_scope, private_run_scope, require_role, role_by_method, run_scope from app.auth.keys import key_gate @@ -132,6 +134,14 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: logger.exception("评测任务的启动整理失败，服务照常启动") await run_manager.setup() + # 问数据的一轮（app/chat/job.py）：在 run_manager 之后整理——服务重启时在跑的运行这时已经标成挂起。还在建图的标中断； + # 已经发起了运行的接着等它、照常复核收尾（和调度循环同一个开关：拷来的数据目录不替人调模型，等人打开对话时再接着收尾） + from app.chat import job as chat_job + + try: + await chat_job.startup() + except Exception: # noqa: BLE001 + logger.exception("问数据的启动整理失败，服务照常启动") # 流程：存量的待审批补建待办（派给任何人，幂等）；上次进程没了时已发出、没拿到结果的写入收成结果未知、安排核实 from app.process import actions as _actions, tasks as _tasks @@ -163,6 +173,8 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: await catalog_jobs.manager.shutdown() await catalog_profile_jobs.manager.shutdown() await introspect_jobs.manager.shutdown() + # 问数据的一轮停下不改那几行（下次启动照重启的规矩整理），也要在运行之前停 + await chat_job.shutdown() # 评测要在运行之前停：它停下时会取消自己发起的作答、算期望的运行，再标中断 await eval_runner.shutdown() await eval_snapshots.manager.shutdown() @@ -262,6 +274,8 @@ for router, guard in ( (sandbox.router, _BUILDER), (copilot.router, [chat_scope]), (conversations.router, [chat_scope]), + # 问数据的一轮（发起、看、停）：同问数据的门，接口里再按这一轮所属的对话核对归属 + (chat_turns_api.router, [chat_scope]), # 工件：user 角色只能取能读的运行引用的、遮罩后的（api/artifacts.py） (artifacts.router, _USER), (governance.router, _BUILDER), diff --git a/backend/tests/test_chat_turns.py b/backend/tests/test_chat_turns.py new file mode 100644 index 00000000..59e6a226 --- /dev/null +++ b/backend/tests/test_chat_turns.py @@ -0,0 +1,836 @@ +"""问数据的一轮在服务端跑（docs/design/orchestration-round/00-总方案.md §1；app/chat/job.py、app/api/chat_turns.py）。 + +守的是： +- 一轮跑完（建图 → 运行 → 复核 → 落库）不依赖任何前端请求：发起之后只等后台任务，conversation_turns 那一行由服务端写好； +- 事件日志：每一步按 seq 记下，实时流先回放、再推实时，中途断开、按 seq 重连，补齐、不重复；思考增量合并成一条； +- 停止：建图中、运行中都能停，运行跟着取消，那一行收成「已取消」； +- 服务重启：建图中的标中断（照实说、可重试本轮）；已发起运行的接着等那次运行、照常复核收尾；开关关着（拷来的数据目录）时停着， + 页面请服务端接着收尾（mode=resume）才做； +- 按复核重来：复核给了 retry_instruction，带着这次运行的图重建一次，换下来的运行的用量并进这一轮； +- 用户点的重试本轮：上一次的结局归档进 meta.attempts，交给助手的话照以前页面上的措辞； +- 同一段对话一次只有一轮在跑（停在审批上等人的不算）； +- 使用者（user 角色）：只碰得到自己的对话的轮次；发起的运行照旧遮罩、只读、记忆是自己的范围；访问密钥调不了这些接口。 + +库、问题、名字都是虚构的；助手和报告撰写都是剧本，不调真接口。 +""" +from __future__ import annotations + +import asyncio +import json +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import delete, select + +from app.chat import job as chat_job +from app.chat import log as turn_log +from app.db.base import SessionLocal +from app.db.models import Approval, Conversation, ConversationTurn, Run +from app.db.models_chat import ChatTurnEvent, ChatTurnJob +from app.engine.runner import run_manager +from app.main import app +from tests import eval_samples as es +from tests.fixtures.sources import drop_source + +QUESTION = es.QUESTION + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await chat_job.shutdown() + await run_manager.shutdown() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +@pytest.fixture +async def source(tmp_path): + name, sid = await es.make_source(tmp_path) + yield {"name": name, "id": sid} + await drop_source(sid) + + +async def _conversation(client, title: str = "虚构的问数据会话") -> str: + r = await client.post("/api/conversations", json={"kind": "chat", "title": title}) + assert r.status_code == 201, r.text + return r.json()["id"] + + +async def _ask(client, conversation_id: str, question: str = QUESTION, **extra: Any) -> str: + r = await client.post("/api/chat/turns", json={"conversation_id": conversation_id, "question": question, **extra}) + assert r.status_code == 202, r.text + body = r.json() + assert body["turn"]["status"] == "running" and body["turn"]["job"]["status"] in ("queued", "running") + return body["turn_id"] + + +async def _done(turn_id: str, timeout: float = 60.0) -> ChatTurnJob: + await chat_job.manager.wait(turn_id, timeout=timeout) + async with SessionLocal() as session: + job = await session.get(ChatTurnJob, turn_id) + assert job is not None and job.status not in ("queued", "running"), job and (job.status, job.stage) + return job + + +async def _row(turn_id: str) -> ConversationTurn: + async with SessionLocal() as session: + return await session.get(ConversationTurn, turn_id) + + +async def _events(turn_id: str) -> list[dict[str, Any]]: + return await turn_log.read(turn_id, 0, limit=10_000) + + +async def _until(cond, timeout: float = 20.0) -> None: + for _ in range(int(timeout / 0.05)): + if await cond(): + return + await asyncio.sleep(0.05) + raise AssertionError("等不到条件成立") + + +async def _stage(turn_id: str, stage: str) -> None: + async def hit() -> bool: + async with SessionLocal() as session: + job = await session.get(ChatTurnJob, turn_id) + return job is not None and job.stage == stage + await _until(hit) + + +def _sse(text: str) -> list[dict[str, Any]]: + return [json.loads(line[6:]) for line in text.splitlines() if line.startswith("data: ")] + + +# -------------------------------------------------------------------------- +# 一轮跑完不依赖页面 +# -------------------------------------------------------------------------- + + +async def test_a_turn_runs_to_the_end_without_the_page(client, monkeypatch, source): + """发起之后页面一个请求都不再发：建图、运行、复核、落库都在服务端做完。""" + es.use_copilot(monkeypatch, es.Copilot(source["name"])) + es.use_report(monkeypatch) + conversation_id = await _conversation(client) + turn_id = await _ask(client, conversation_id, datasource_ids=[source["id"]]) + job = await _done(turn_id) + assert job.status == "succeeded" and job.stage is None and job.run_id, (job.status, job.error) + + row = await _row(turn_id) + assert row.status == "done", (row.status, row.error) + assert "330" in row.answer, row.answer + assert row.run_id == job.run_id and row.graph and row.graph.get("nodes") + meta = row.meta + assert meta["runId"] == job.run_id and meta["runClass"] == "exploratory" and meta["runStatus"] == "succeeded" + assert meta["queries"] == 1 and isinstance(meta["ms"], int) + assert meta["scope"] == [{"id": source["id"], "name": source["name"]}] + assert meta["looks"] and meta["looks"][0]["op"] == "look", "查看记录照页面以前的形状存进 meta.looks" + assert set(meta["usage"]) >= {"copilot", "run"}, "这一轮的花费 = 助手建图 + 运行（+ 复核）" + assert isinstance(row.review, dict) and row.review.get("verdict") == "ok", "复核原样落库（含 verdict=ok）" + + async with SessionLocal() as session: + run = await session.get(Run, job.run_id) + assert run.origin_kind == "chat" and run.origin.get("conversation_id") == conversation_id + assert run.origin.get("turn_id") == turn_id and run.input.get("question") == QUESTION + + events = await _events(turn_id) + seqs = [e["seq"] for e in events] + assert seqs == list(range(1, len(seqs) + 1)), "seq 从 1 起连续" + kinds = [e["type"] for e in events] + assert kinds[0] == "turn.attempt" and kinds[-1] == "end" + assert [e["data"]["stage"] for e in events if e["type"] == "stage"] == ["build", "run", "review"] + ops = [e["data"]["op"] for e in events if e["type"] == "op"] + assert {"model", "look", "add_node", "final", "usage"} <= set(ops), ops + [launched] = [e for e in events if e["type"] == "run"] + assert launched["data"]["run_id"] == job.run_id + end = events[-1]["data"] + assert end["outcome"] == "done" and end["turn"]["status"] == "done" and end["turn"]["answer"] == row.answer + assert end["run"]["id"] == job.run_id and end["output"], "end 带完整成果，页面不必再回源" + + # 对话详情里这一轮带着它的任务；列表状态是 done + detail = (await client.get(f"/api/conversations/{conversation_id}")).json() + assert detail["turns"][-1]["job"]["status"] == "succeeded" and detail["last_status"] == "done" + state = (await client.get(f"/api/chat/turns/{turn_id}")).json() + assert state["job"]["status"] == "succeeded" and state["job"]["last_seq"] == seqs[-1] + assert state["job"]["attempt_seq"] == 1 and state["turn"]["id"] == turn_id + + +async def test_reply_and_ask_settle_like_the_page_did(client, monkeypatch, source): + """直接回答：done、没查库（noQuery）；助手反问：done，答案列是那句问题，meta.asked 记着问题和候选。""" + es.use_copilot(monkeypatch, es.Copilot(source["name"], mode="reply")) + conversation_id = await _conversation(client) + turn_id = await _ask(client, conversation_id, "刚才那个数是多少") + await _done(turn_id) + row = await _row(turn_id) + assert row.status == "done" and row.answer == "销售额是 330。" and row.meta["noQuery"] is True + assert (await _events(turn_id))[-1]["data"]["outcome"] == "reply" + + es.use_copilot(monkeypatch, es.Copilot(source["name"], mode="ask")) + turn_id = await _ask(client, conversation_id, "上周销售额") + await _done(turn_id) + row = await _row(turn_id) + assert row.status == "done" and row.answer == "要按下单日还是支付日？" + assert row.meta["asked"]["options"] == ["下单日", "支付日"] and not row.meta.get("noQuery") + + +async def test_a_refusal_before_the_stream_is_recorded_as_its_cause(client, monkeypatch, source): + """限定的数据源都不在了：开流之前就被拒，这一轮记 error，原因带机器码（页面据此给「不限数据源重试」）。""" + es.use_copilot(monkeypatch, es.Copilot(source["name"])) + conversation_id = await _conversation(client) + turn_id = await _ask(client, conversation_id, datasource_ids=["not-a-source"]) + await _done(turn_id) + row = await _row(turn_id) + assert row.status == "error" and row.meta["cause"]["kind"] == "refused" + assert row.meta["cause"]["code"] == "datasource_scope_empty" and row.error.startswith("限定的数据源") + + +# -------------------------------------------------------------------------- +# 事件日志、实时流 +# -------------------------------------------------------------------------- + + +class Gated(es.Copilot): + """第一次起草前停在闸门上，放行之后照剧本走（测实时流中途接入、断开重连）。思考先吐几段增量。""" + + def __init__(self, source: str, gate: asyncio.Event, **kw: Any) -> None: + super().__init__(source, **kw) + self.gate = gate + + async def astream(self, messages): + from langchain_core.messages import AIMessageChunk + + if len(self.calls) == 0: + for piece in ("先想一想", "要查哪张表", "……"): + yield AIMessageChunk(content="", additional_kwargs={"reasoning_content": piece}) + await self.gate.wait() + async for chunk in super().astream(messages): + yield chunk + + +async def test_the_stream_replays_then_follows_and_reconnects_by_seq(client, monkeypatch, source): + """订阅方中途接入：先回放已有的、再推实时的；断开之后按收到的最后一个 seq 重连，补齐、不重复，最后是 stream.end。""" + gate = asyncio.Event() + es.use_copilot(monkeypatch, Gated(source["name"], gate)) + es.use_report(monkeypatch) + conversation_id = await _conversation(client) + turn_id = await _ask(client, conversation_id, datasource_ids=[source["id"]]) + await _stage(turn_id, "build") + + async def active() -> str | None: + return await chat_job.stream_status(turn_id) + + first: list[dict[str, Any]] = [] + stream = turn_log.stream(turn_id, 0, active=active, heartbeat=0.2) + async for frame in stream: + if frame.startswith("data: "): + first.append(json.loads(frame[6:])) + if any(e.get("type") == "stage" for e in first): + break + await stream.aclose() # 断开 + assert first[0]["type"] == "turn.attempt" and first[0]["seq"] == 1 + + gate.set() + rest: list[dict[str, Any]] = [] + pings = 0 + async for frame in turn_log.stream(turn_id, first[-1]["seq"], active=active, heartbeat=0.2): + if frame == turn_log.PING: + pings += 1 + continue + rest.append(json.loads(frame[6:])) + assert rest[-1]["type"] == "stream.end" and rest[-1]["status"] == "succeeded" + got = [e["seq"] for e in first + rest[:-1]] + assert got == list(range(1, len(got) + 1)), "补齐、不重复" + assert got[-1] == (await _events(turn_id))[-1]["seq"] + + # 跑完之后按 seq 重连（HTTP）：只给 seq 之后的，再给 stream.end + after = got[-3] + text = (await client.get(f"/api/chat/turns/{turn_id}/stream", params={"after": after})).text + frames = _sse(text) + assert [f["seq"] for f in frames[:-1]] == got[-2:] and frames[-1]["type"] == "stream.end" + assert [f["seq"] for f in _sse((await client.get(f"/api/chat/turns/{turn_id}/stream")).text)[:-1]] == got + + # 思考增量合并成一条，不逐条落库 + thinking = [e for e in await _events(turn_id) if e["type"] == "op" and e["data"]["op"] == "thinking"] + assert len(thinking) == 1 and thinking[0]["data"]["delta"] == "先想一想要查哪张表……", thinking + + +# -------------------------------------------------------------------------- +# 停止 +# -------------------------------------------------------------------------- + + +async def test_stop_while_building(client, monkeypatch, source): + gate = asyncio.Event() + es.use_copilot(monkeypatch, Gated(source["name"], gate)) + conversation_id = await _conversation(client) + turn_id = await _ask(client, conversation_id) + await _stage(turn_id, "build") + r = await client.post(f"/api/chat/turns/{turn_id}/cancel") + assert r.status_code == 200, r.text + job = await _done(turn_id) + assert job.status == "cancelled" + row = await _row(turn_id) + assert row.status == "error" and row.error == "已取消" and row.meta["outcome"] == "cancelled" + assert (await _events(turn_id))[-1]["data"]["outcome"] == "cancelled" + r = await client.post(f"/api/chat/turns/{turn_id}/cancel") + assert r.status_code == 409 and r.json()["code"] == "chat_turn_finished" + + +async def test_stop_while_running_cancels_the_run(client, monkeypatch, source): + """运行中停止：运行跟着取消（不只是断开订阅——以前点了停止，模型和 SQL 其实还在跑、还在计费）。""" + from app.providers import mock_model + + es.use_copilot(monkeypatch, es.Copilot(source["name"])) + hold = asyncio.Event() + + async def slow(self, messages, stop=None, run_manager=None, **kwargs): + await hold.wait() + raise AssertionError("不该放行") + + async def slow_stream(self, messages, stop=None, run_manager=None, **kwargs): + await hold.wait() + raise AssertionError("不该放行") + yield # noqa: unreachable - 让它成为异步生成器 + + # 报告撰写卡在模型调用上（流式、非流式都卡住）：运行一直在跑，等着被停止 + monkeypatch.setattr(mock_model.MockChatModel, "_agenerate", slow) + monkeypatch.setattr(mock_model.MockChatModel, "_astream", slow_stream) + conversation_id = await _conversation(client) + turn_id = await _ask(client, conversation_id, datasource_ids=[source["id"]]) + await _stage(turn_id, "run") + async with SessionLocal() as session: + run_id = (await session.get(ChatTurnJob, turn_id)).run_id + assert (await client.post(f"/api/chat/turns/{turn_id}/cancel")).status_code == 200 + job = await _done(turn_id) + assert job.status == "cancelled" + await run_manager.wait_idle(run_id) + async with SessionLocal() as session: + assert (await session.get(Run, run_id)).status == "cancelled" + row = await _row(turn_id) + assert row.status == "error" and row.error == "已取消" and row.meta["runStatus"] == "cancelled" + + +# -------------------------------------------------------------------------- +# 服务重启 +# -------------------------------------------------------------------------- + + +async def _stranded(conversation_id: str, *, run: Run | None, stage: str, mode: str = "ask") -> str: + """造一轮「上次进程没了时还在跑」的：那一行 running、任务 running（不在这个进程里）。""" + async with SessionLocal() as session: + if run is not None: + session.add(run) + await session.flush() + turn = ConversationTurn(conversation_id=conversation_id, seq=0, question="上周各区域销售额", status="running", + run_id=run.id if run else None, meta={"v": 1, "runId": run.id if run else None}) + session.add(turn) + await session.flush() + session.add(ChatTurnJob(id=turn.id, conversation_id=conversation_id, status="running", stage=stage, mode=mode, + run_id=run.id if run else None, request={"started_ms": 0}, principal={})) + await session.commit() + return turn.id + + +async def test_restart_interrupts_builds_and_finishes_launched_runs(client, monkeypatch): + conversation_id = await _conversation(client) + building = await _stranded(conversation_id, run=None, stage="build") + finished = await _stranded(conversation_id, run=Run(status="succeeded", output={"答案": "华东 300。"}, + graph={"nodes": [], "edges": []}, run_class="exploratory", + origin_kind="chat"), stage="review") + suspended = await _stranded(conversation_id, run=Run(status="interrupted", output={}, graph={"nodes": [], "edges": []}, + run_class="exploratory", origin_kind="chat"), stage="run") + counts = await chat_job.startup(resume=True) + assert counts["interrupted"] >= 1 and counts["resumed"] >= 2 + for turn_id in (finished, suspended): + await chat_job.manager.wait(turn_id, timeout=30) + + row = await _row(building) + assert row.status == "error" and row.meta["cause"] == {"kind": "build_interrupted"}, "建图中的照实标中断" + async with SessionLocal() as session: + assert (await session.get(ChatTurnJob, building)).status == "interrupted" + assert (await _events(building))[-1]["data"]["outcome"] == "error" + + row = await _row(finished) + assert row.status == "done" and row.answer == "华东 300。", "已发起运行的：接着等那次运行、照常复核收尾" + assert row.review.get("verdict") == "ok" + + row = await _row(suspended) + assert row.status == "error" and row.meta["outcome"] == "suspended" and row.error == "服务重启，本轮已中断" + + +async def test_with_the_switch_off_launched_runs_wait_for_someone_to_open_the_conversation(client): + """拷来的数据目录（AGENTLAB_SCHEDULER=0）：启动时不替人收尾，那一行不动；页面请服务端接着收尾（resume）才做。""" + conversation_id = await _conversation(client) + turn_id = await _stranded(conversation_id, run=Run(status="succeeded", output={"答案": "华北 30。"}, + graph={"nodes": [], "edges": []}, run_class="exploratory", + origin_kind="chat"), stage="review") + counts = await chat_job.startup(resume=False) + assert counts["parked"] >= 1 + assert (await _row(turn_id)).status == "running" + async with SessionLocal() as session: + assert (await session.get(ChatTurnJob, turn_id)).status == "interrupted" + r = await client.post("/api/chat/turns", json={"conversation_id": conversation_id, "retry_of": turn_id, + "mode": "resume"}) + assert r.status_code == 202, r.text + await _done(turn_id) + row = await _row(turn_id) + assert row.status == "done" and row.answer == "华北 30。" + assert not row.meta.get("attempts"), "接着收尾不是重试：不归档" + + +async def test_a_legacy_turn_stuck_building_is_marked_interrupted_at_startup(client): + """这一波之前页面推的老轮次（没有任务）停在 running、又没有运行的：它的建图那条流随上次的进程一起没了。""" + conversation_id = await _conversation(client) + async with SessionLocal() as session: + turn = ConversationTurn(conversation_id=conversation_id, seq=0, question="老轮次", status="running") + session.add(turn) + await session.commit() + legacy = turn.id + await chat_job.startup(resume=False) + row = await _row(legacy) + assert row.status == "error" and row.meta["cause"] == {"kind": "build_interrupted"} + + +async def test_a_turn_waits_for_the_approver_and_finishes_after(client, monkeypatch): + """停在审批上：照旧等人（评测是取消）。审批过了运行接着跑，这一轮照常复核收尾。""" + conversation_id = await _conversation(client) + run = Run(status="interrupted", output={}, graph={"nodes": [], "edges": []}, run_class="exploratory", origin_kind="chat") + turn_id = await _stranded(conversation_id, run=run, stage="run") + async with SessionLocal() as session: + session.add(Approval(run_id=run.id, node_id="ask", mode="approve", title="确认一下", payload={}, status="pending")) + await session.commit() + monkeypatch.setattr(chat_job, "APPROVAL_POLL_S", 0.1) + await chat_job.startup(resume=True) + await _stage(turn_id, "waiting") + detail = (await client.get(f"/api/conversations/{conversation_id}")).json() + assert detail["last_status"] == "waiting" + # 别的轮次照样能问（停在审批上的不算占着这段对话） + async with SessionLocal() as session: + busy = await chat_job._busy(session, conversation_id) + assert busy is False + # 审批人处理了、运行跑完了 + async with SessionLocal() as session: + row = await session.get(Run, run.id) + row.status, row.output = "succeeded", {"答案": "审批之后的答案"} + approval = (await session.execute(select(Approval).where(Approval.run_id == run.id))).scalar_one() + approval.status = "approved" + await session.commit() + await _done(turn_id) + row = await _row(turn_id) + assert row.status == "done" and row.answer == "审批之后的答案" + stages = [e["data"]["stage"] for e in await _events(turn_id) if e["type"] == "stage"] + assert stages[-3:] == ["waiting", "run", "review"], stages + + +# -------------------------------------------------------------------------- +# 重来：按复核自动重来一次、用户点的重试本轮 +# -------------------------------------------------------------------------- + + +async def test_the_review_retries_once_with_the_run_graph(client, monkeypatch, source): + import app.api.copilot as copilot + from app.chat import turn as policy + + model = es.Copilot(source["name"]) + es.use_copilot(monkeypatch, model) + es.use_report(monkeypatch) + real = copilot.review_of_run + attempts: list[int] = [] + + async def review(session, run, *, question, attempt=0, **kw): + attempts.append(attempt) + out = await real(session, run, question=question, attempt=attempt, **kw) + if attempt == 0: + out = {**out, "verdict": "annotated", "severity": "broken", "note": "查到一半就停了", "retry": True, + "signals": [{"kind": "step_limit", "detail": "用满了步数", "severity": "broken"}], + "retry_instruction": policy.retry_instruction(question, "用满了步数", True), + "usage": {"calls": 1, "cost_usd": 0.003}} + return out + + monkeypatch.setattr(copilot, "review_of_run", review) + conversation_id = await _conversation(client) + turn_id = await _ask(client, conversation_id, datasource_ids=[source["id"]]) + await _done(turn_id) + assert attempts == [0, 1], "第二次复核按第二次算，不再给重来" + events = await _events(turn_id) + [retry] = [e["data"] for e in events if e["type"] == "turn.retry"] + assert retry["kind"] == "review" and retry["reason"] == "用满了步数" and retry["base"] is True + runs = [e["data"]["run_id"] for e in events if e["type"] == "run"] + assert len(runs) == 2 and runs[0] != runs[1] + row = await _row(turn_id) + assert row.status == "done" and row.meta["runId"] == runs[1] + usage = row.meta["usage"] + assert usage["review"]["calls"] == 1 and usage["review"]["cost_usd"] == 0.003, "复核那一次的用量记进这一轮" + assert "earlier" in usage, "换下来的那次运行的用量并进 earlier" + # 重来那一次交给助手的就是复核给的那句话，带着上一次运行的图 + last = model.calls[-2] if len(model.calls) >= 2 else model.calls[-1] + assert any("这个工作流上一次运行没有成功：用满了步数" in str(m[1]) for m in last) + + +async def test_retrying_a_turn_archives_the_previous_attempt(client, monkeypatch, source): + """用户点「重试本轮」：上一次的结局归档进 meta.attempts（照以前页面上的说法），交给助手的话照以前的措辞，那一行回到 running。""" + es.use_copilot(monkeypatch, es.Copilot(source["name"], mode="reply")) + conversation_id = await _conversation(client) + turn_id = await _ask(client, conversation_id, "上周各区域销售额") + await _done(turn_id) + + seen: list[str] = [] + real = chat_job.ChatDriver.generate_in + + def spy(self, instruction, base_graph): + seen.append(instruction) + return real(self, instruction, base_graph) + + monkeypatch.setattr(chat_job.ChatDriver, "generate_in", spy) + r = await client.post("/api/chat/turns", json={"conversation_id": conversation_id, "retry_of": turn_id, + "mode": "retry"}) + assert r.status_code == 202, r.text + assert r.json()["turn"]["status"] == "running" + await _done(turn_id) + row = await _row(turn_id) + [archived] = row.meta["attempts"] + assert archived["n"] == 1 and archived["outcome"] == "done" and archived["summary"] == "已给出答案" + assert archived["answer"] == "销售额是 330。" + assert seen == ["上周各区域销售额\n\n（上一次没有成功：已给出答案。请针对这个原因调整，重新给出完整的工作流。）"] + attempt_events = [e for e in await _events(turn_id) if e["type"] == "turn.attempt"] + assert [e["data"]["mode"] for e in attempt_events] == ["ask", "retry"] + assert attempt_events[-1]["data"]["attempts"] == row.meta["attempts"] + async with SessionLocal() as session: + job = await session.get(ChatTurnJob, turn_id) + assert job.attempt_seq == attempt_events[-1]["seq"], "刷新时从这一次尝试的第一条回放" + + +async def test_one_turn_at_a_time_per_conversation(client, monkeypatch, source): + gate = asyncio.Event() + es.use_copilot(monkeypatch, Gated(source["name"], gate)) + conversation_id = await _conversation(client) + turn_id = await _ask(client, conversation_id) + r = await client.post("/api/chat/turns", json={"conversation_id": conversation_id, "question": "再问一句"}) + assert r.status_code == 409 and r.json()["code"] == "chat_turn_busy" + r = await client.post("/api/chat/turns", json={"conversation_id": conversation_id, "retry_of": turn_id}) + assert r.status_code == 409 + other = await _conversation(client, "另一段对话") + other_turn = await _ask(client, other, "别的对话照常问") + for t in (turn_id, other_turn): + await client.post(f"/api/chat/turns/{t}/cancel") + await _done(t) + + +async def test_continuing_a_finished_run_is_refused_like_the_runs_api(client): + conversation_id = await _conversation(client) + run = Run(status="succeeded", output={"答案": "x"}, graph={"nodes": [], "edges": []}, run_class="exploratory") + async with SessionLocal() as session: + session.add(run) + await session.flush() + turn = ConversationTurn(conversation_id=conversation_id, seq=0, question="q", status="done", answer="x", + run_id=run.id, meta={"v": 1, "runId": run.id}) + session.add(turn) + await session.commit() + turn_id = turn.id + r = await client.post("/api/chat/turns", json={"conversation_id": conversation_id, "retry_of": turn_id, + "mode": "continue"}) + assert r.status_code == 409 and "无需继续运行" in r.json()["detail"], r.text + assert (await _row(turn_id)).status == "done", "被拒时那一行不动" + + +async def test_deleting_a_conversation_stops_its_turn(client, monkeypatch, source): + gate = asyncio.Event() + es.use_copilot(monkeypatch, Gated(source["name"], gate)) + conversation_id = await _conversation(client) + turn_id = await _ask(client, conversation_id) + await _stage(turn_id, "build") + assert (await client.delete(f"/api/conversations/{conversation_id}")).status_code == 204 + assert not chat_job.manager.running(turn_id) + async with SessionLocal() as session: + assert await session.get(ChatTurnJob, turn_id) is None + assert not (await session.execute(select(ChatTurnEvent).where(ChatTurnEvent.turn_id == turn_id))).first() + + +# -------------------------------------------------------------------------- +# 使用者（user 角色）：归属、遮罩、只读、访问密钥 +# -------------------------------------------------------------------------- + + +@pytest.fixture +async def people(): + from tests.fakes import auth_env + + await auth_env.reset() + admin = await auth_env.turn_on_local() + assert (await admin.put("/api/auth/settings", json={"user_can_chat": True})).status_code == 200 + for name, role in (("uma", "user"), ("ula", "user"), ("bob", "builder")): + await auth_env.make_user(name, role) + clients = {name: await auth_env.login(name) for name in ("uma", "ula", "bob")} + try: + yield {"admin": admin, **clients} + finally: + async with SessionLocal() as s: + ids = list((await s.execute(select(Conversation.id).where( + Conversation.created_by.in_(["uma", "ula", "bob"])))).scalars()) + await s.execute(delete(ConversationTurn).where(ConversationTurn.conversation_id.in_(ids))) + await s.execute(delete(Conversation).where(Conversation.id.in_(ids))) + await s.execute(delete(Run).where(Run.started_by.in_(["uma", "ula", "bob"]))) + await s.commit() + await auth_env.reset() + + +async def test_a_users_turns_are_theirs_and_their_runs_stay_masked_and_read_only(people, monkeypatch, source): + from app.tools.datasource import MASK_RESULTS_OPTION, READ_ONLY_OPTION + + es.use_copilot(monkeypatch, es.Copilot(source["name"])) + es.use_report(monkeypatch) + uma, ula, bob = people["uma"], people["ula"], people["bob"] + mine = await _conversation(uma, "uma 的对话") + theirs = await _conversation(ula, "ula 的对话") + turn_id = await _ask(uma, mine, datasource_ids=[source["id"]]) + job = await _done(turn_id) + assert job.status == "succeeded", job.error + async with SessionLocal() as session: + run = await session.get(Run, job.run_id) + assert run.started_by == "uma" and run.origin_kind == "chat" + assert run.options.get(MASK_RESULTS_OPTION) is True and run.options.get(READ_ONLY_OPTION) is True + assert run.memory_scope == "chat:uma", "记忆固定为这位使用者自己的范围" + assert (await _row(turn_id)).status == "done" + + # 只碰得到自己的对话的轮次 + for path in (f"/api/chat/turns/{turn_id}", f"/api/chat/turns/{turn_id}/stream"): + assert (await ula.get(path)).status_code == 403, path + assert (await uma.get(path)).status_code == 200, path + assert (await ula.post(f"/api/chat/turns/{turn_id}/cancel")).status_code == 403 + r = await uma.post("/api/chat/turns", json={"conversation_id": theirs, "question": "偷看别人的对话"}) + assert r.status_code == 403 + r = await ula.post("/api/chat/turns", json={"conversation_id": mine, "retry_of": turn_id}) + assert r.status_code == 403 + # 搭建者照旧看得到（对话按人隔离只收窄使用者） + assert (await bob.get(f"/api/chat/turns/{turn_id}")).status_code == 200 + + # 管理员关了问数据：使用者整组接口都用不了 + assert (await people["admin"].put("/api/auth/settings", json={"user_can_chat": False})).status_code == 200 + r = await uma.post("/api/chat/turns", json={"conversation_id": mine, "question": "再问一句"}) + assert r.status_code == 403 and r.json()["code"] == "chat_forbidden" + + +async def test_a_user_cannot_launch_a_graph_outside_the_chat_whitelist(people, monkeypatch, source): + """使用者的图里有白名单之外的节点：自查挡下（blocked）或发起被拒（start），这一轮照实记 error，不会跑起来。""" + class CodeNode(es.Copilot): + async def astream(self, messages): + from tests.eval_samples import _line + + self.calls.append(list(messages)) + if "上线前自查" in messages[1][1]: + yield _line({"op": "done"}) + return + for op in ({"op": "add_node", "node": {"id": "start", "type": "input", "label": "输入", "config": {}}}, + {"op": "add_node", "node": {"id": "c", "type": "code", "label": "算一下", + "config": {"code": "result = 1"}}}, + {"op": "add_edge", "edge": {"source": "start", "target": "c"}}, + {"op": "add_node", "node": {"id": "out", "type": "output", "label": "答案", "config": {}}}, + {"op": "add_edge", "edge": {"source": "c", "target": "out"}}, + {"op": "done", "explanation": "", "run": True}): + yield _line(op) + + es.use_copilot(monkeypatch, CodeNode(source["name"])) + conversation_id = await _conversation(people["uma"], "uma 的对话") + turn_id = await _ask(people["uma"], conversation_id) + await _done(turn_id) + row = await _row(turn_id) + assert row.status == "error", (row.status, row.error) + cause = row.meta.get("cause") or {} + assert cause.get("kind") in ("blocked", "start"), cause + + +async def test_access_keys_cannot_reach_the_chat_turn_api(people): + """访问密钥照旧调不了问数据（不在 auth/keys.KEY_ROUTES 里）：发起、看、停一律 403 key_route_forbidden。""" + from app.db.models_identity import User + from tests.fakes import auth_env + + await auth_env.personal_keys(True) + async with SessionLocal() as s: + uma = (await s.execute(select(User).where(User.username == "uma"))).scalar_one() + _, secret = await auth_env.make_key(uma) + keyed = auth_env.keyed(secret) + for method, path in (("post", "/api/chat/turns"), ("get", "/api/chat/turns/x"), + ("get", "/api/chat/turns/x/stream"), ("post", "/api/chat/turns/x/cancel")): + r = await getattr(keyed, method)(path, **({"json": {"conversation_id": "c"}} if method == "post" else {})) + assert r.status_code == 403 and r.json().get("code") == "key_route_forbidden", (path, r.text) + + +# -------------------------------------------------------------------------- +# 各种结局照以前页面落库的写法 +# -------------------------------------------------------------------------- + + +async def _seed_events(run_id: str, *events: tuple[str, dict[str, Any]]) -> None: + import time as _time + + from app.db.models import RunEvent + + async with SessionLocal() as session: + for seq, (kind, data) in enumerate(events, start=1): + session.add(RunEvent(run_id=run_id, seq=seq, type=kind, ts=_time.time(), data=data)) + await session.commit() + + +async def test_rejected_failed_and_refused_runs_settle_with_their_cause(client, monkeypatch, source): + conversation_id = await _conversation(client) + rejected = Run(status="cancelled", end_reason="rejected", output={}, graph={"nodes": [], "edges": []}, + run_class="exploratory", error="人工驳回：数字对不上") + failed = Run(status="failed", output={}, graph={"nodes": [], "edges": []}, run_class="exploratory", error="查询超时") + r_id = await _stranded(conversation_id, run=rejected, stage="run") + f_id = await _stranded(conversation_id, run=failed, stage="run") + await _seed_events(rejected.id, ("run.cancelled", {"reason": "rejected", "actor": "张工", "label": "查数", + "note": "数字对不上", "decision": "reject"})) + await _seed_events(failed.id, ("run.failed", {"error": "查询超时", "detail": "TimeoutError: 30s"})) + await chat_job.startup(resume=True) + for t in (r_id, f_id): + await chat_job.manager.wait(t, timeout=30) + row = await _row(r_id) + assert row.status == "error" and row.error == "已取消" and row.meta["outcome"] == "cancelled" + assert row.meta["rejected"] == {"actor": "张工", "label": "查数", "note": "数字对不上", "decision": "reject"} + detail = (await client.get(f"/api/conversations/{conversation_id}")).json() + assert [t for t in detail["turns"] if t["id"] == r_id][0]["meta"]["rejected"]["actor"] == "张工" + row = await _row(f_id) + assert row.status == "error" and row.error == "查询超时" + assert row.meta["cause"] == {"kind": "run", "error": "查询超时", "detail": "TimeoutError: 30s"} + + # 发起被拒：照 LaunchRefused 记下说法、机器码、状态码 + import app.engine.launch as launch + from app.engine.launch import LaunchRefused + + async def refuse(*_a, **_k): + raise LaunchRefused(422, "绑定的工具不存在：「查订单」（调用工具）绑定的 db_query__nope。请到「数据」页接入", "run_tool_missing") + + monkeypatch.setattr(launch, "launch_run", refuse) + es.use_copilot(monkeypatch, es.Copilot(source["name"])) + other = await _conversation(client, "发起被拒") + turn_id = await _ask(client, other, datasource_ids=[source["id"]]) + await _done(turn_id) + row = await _row(turn_id) + assert row.status == "error" and row.meta["cause"]["kind"] == "start" + assert row.meta["cause"]["code"] == "run_tool_missing" and row.meta["cause"]["status"] == 422 + assert row.meta.get("runId") is None + + +async def test_a_late_review_still_counts_toward_the_turn(client, monkeypatch, source): + """复核慢过 REVIEW_WAIT_S：先交原答案（不等它），那一次复核晚到了照样把用量记进这一轮（review.late）。""" + import app.api.copilot as copilot + + es.use_copilot(monkeypatch, es.Copilot(source["name"])) + es.use_report(monkeypatch) + monkeypatch.setattr(chat_job, "REVIEW_WAIT_S", 0.05) + release = asyncio.Event() + + async def slow(session, run, **kw): + await release.wait() + return {"verdict": "ok", "signals": [], "retry_instruction": None, "usage": {"calls": 1, "cost_usd": 0.004}} + + monkeypatch.setattr(copilot, "review_of_run", slow) + conversation_id = await _conversation(client) + turn_id = await _ask(client, conversation_id, datasource_ids=[source["id"]]) + await _done(turn_id) + row = await _row(turn_id) + assert row.status == "done" and "330" in row.answer and row.review == {}, "等不及：原答案照交、没有复核结论" + assert "review" not in row.meta["usage"] + assert chat_job.late_pending(turn_id) and await chat_job.stream_status(turn_id) is None, "实时流等复核回来再关" + release.set() + await _until(lambda: _not_late(turn_id)) + row = await _row(turn_id) + assert row.meta["usage"]["review"] == {"calls": 1, "cost_usd": 0.004} + assert (await _events(turn_id))[-1]["type"] == "review.late" + assert await chat_job.stream_status(turn_id) == "succeeded" + + +async def _not_late(turn_id: str) -> bool: + return not chat_job.late_pending(turn_id) + + +async def test_run_and_rerun_reuse_the_built_graph_without_rebuilding(client, monkeypatch, source): + """只建不跑的那一轮点「运行」：直接跑那张图，不归档、不重建；重新运行本轮：上一次归档，原图再跑一次。两种都不按复核 + 自动重来（同以前页面上 launch 不带 retry）。""" + import app.api.copilot as copilot + + model = es.Copilot(source["name"]) + es.use_copilot(monkeypatch, model) + es.use_report(monkeypatch) + conversation_id = await _conversation(client) + turn_id = await _ask(client, conversation_id, datasource_ids=[source["id"]]) + await _done(turn_id) + built = (await _row(turn_id)).graph + async with SessionLocal() as session: + row = await session.get(ConversationTurn, turn_id) + row.meta = {k: v for k, v in (row.meta or {}).items() if k not in ("runId",)} | {"runId": None} + row.run_id, row.answer = None, "" + await session.commit() + calls = len(model.calls) + real = copilot.review_of_run + + async def review(session, run, *, question, attempt=0, **kw): + out = await real(session, run, question=question, attempt=attempt, **kw) + return {**out, "retry_instruction": "（复核说要重来）"} + + monkeypatch.setattr(copilot, "review_of_run", review) + r = await client.post("/api/chat/turns", json={"conversation_id": conversation_id, "retry_of": turn_id, "mode": "run"}) + assert r.status_code == 202, r.text + await _done(turn_id) + row = await _row(turn_id) + assert row.status == "done" and "330" in row.answer and not row.meta.get("attempts") + assert len(model.calls) == calls, "不重建图" + assert not [e for e in await _events(turn_id) if e["type"] == "turn.retry"], "不按复核自动重来" + + r = await client.post("/api/chat/turns", json={"conversation_id": conversation_id, "retry_of": turn_id, + "mode": "rerun"}) + assert r.status_code == 202, r.text + await _done(turn_id) + row = await _row(turn_id) + assert row.status == "done" and len(row.meta["attempts"]) == 1 and row.graph == built + assert len(model.calls) == calls, "重新运行本轮不重建图" + + r = await client.post("/api/chat/turns", json={"conversation_id": conversation_id, "retry_of": turn_id, + "mode": "max_steps"}) + assert r.status_code == 422 and r.json()["code"] == "chat_turn_invalid", "放宽到几步要写清楚" + + +def test_max_steps_only_touches_agent_nodes(): + from app.chat import record + + graph = {"nodes": [{"id": "a", "type": "agent", "data": {"config": {"max_steps": 8, "tools": ["x"]}}}, + {"id": "o", "type": "output", "data": {"config": {}}}], "edges": []} + out = record.with_max_steps(graph, 40) + assert out["nodes"][0]["data"]["config"] == {"max_steps": 40, "tools": ["x"]} + assert out["nodes"][1] == graph["nodes"][1] and graph["nodes"][0]["data"]["config"]["max_steps"] == 8 + + +def test_attempt_records_say_what_the_page_used_to_say(): + """归档那一次的说法同以前页面上的 attemptOf：已手动停止、服务重启、被驳回（谁在哪一步）、结论不可用、存在缺口、失败原文。""" + from types import SimpleNamespace + + from app.chat import record + + def row(**kw): + base = {"status": "done", "error": "", "answer": "答案", "review": {}, "run_id": "r1", "meta": None, + "updated_at": None} + return SimpleNamespace(**{**base, **kw}) + + assert record.attempt_record(row(status="error", error="已取消"), {"outcome": "cancelled"}, n=1)["summary"] == "已手动停止" + rej = record.attempt_record(row(status="error", error="已取消"), + {"outcome": "cancelled", "rejected": {"actor": "张工", "label": "查数", "note": "不对"}}, n=1) + assert rej["outcome"] == "rejected" and rej["summary"] == "被驳回：张工 驳回了「查数」：不对" + old = record.attempt_record(row(status="error"), {"outcome": "cancelled", "rejected": {"sub": "页面记过的那句"}}, n=1) + assert old["summary"] == "被驳回：页面记过的那句" + assert record.attempt_record(row(status="error"), {"outcome": "suspended"}, n=2)["summary"] == "服务重启，运行中断" + broken = record.attempt_record(row(review={"severity": "broken", "note": "查到一半", + "signals": [{"detail": "用满了步数"}]}), {}, n=1) + assert (broken["outcome"], broken["summary"], broken["detail"]) == ("unusable", "结论不可用：查到一半", "用满了步数") + assert record.attempt_reason(broken) == "结论不可用：查到一半（用满了步数）" + assert record.attempt_record(row(review={"severity": "degraded", "note": "检索降级"}), {}, n=1)["summary"] == "存在缺口：检索降级" + failed = record.attempt_record(row(status="error", error="查询超时"), {"cause": {"kind": "run", "error": "查询超时"}}, n=1) + assert failed["outcome"] == "failed" and failed["summary"] == "查询超时" and failed["cause"]["kind"] == "run" + legacy = record.attempt_record(row(status="error", error="x"), {"failure": {"title": "操作超时", "reason": "查询超时。"}}, n=1) + assert legacy["summary"] == "操作超时：查询超时。" + assert record.attempt_record(row(answer="字" * 5000), {}, n=1)["answer"].endswith("…") From 5ff3e1136d7fae90593a8df991fa3d9d675db920 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 18 Oct 2026 16:49:14 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=E9=97=AE=E6=95=B0=E6=8D=AE=E6=95=B4?= =?UTF-8?q?=E8=BD=AE=E6=94=BE=E5=88=B0=E6=9C=8D=E5=8A=A1=E7=AB=AF=E8=B7=91?= =?UTF-8?q?=EF=BC=88=E5=89=8D=E7=AB=AF=EF=BC=89=EF=BC=9Astore/chat.ts=20?= =?UTF-8?q?=E6=94=B9=E6=88=90=E3=80=8C=E5=8F=91=E8=B5=B7=20+=20=E6=92=AD?= =?UTF-8?q?=E6=94=BE=E3=80=8D=EF=BC=8C=E6=96=AD=E7=BA=BF=E6=8C=89=20seq=20?= =?UTF-8?q?=E9=87=8D=E8=BF=9E=E8=A1=A5=E9=BD=90=EF=BC=8C=E5=88=B7=E6=96=B0?= =?UTF-8?q?=E3=80=81=E5=85=B3=E6=8E=89=E5=86=8D=E6=89=93=E5=BC=80=E6=8E=A5?= =?UTF-8?q?=E7=9D=80=E7=9C=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么：一轮问数据搬到服务端之后（上一个提交），页面不该再推这一轮：建图那条流、发起运行、复核、按复核重来、落库都和页面 绑在一起，页面一断这一轮就停。现在页面只发起、只看，要不要运行、要不要重来、什么时候交付都照服务端说的。 做了什么： - 新的 api/chatTurns.ts：发起、查询、停止（挂在 api.chatTurns），streamChatTurn 订阅这一轮的事件日志——先回放、再接实时； 断了按收到的最后一个 seq 自动重连（收到东西马上接着连，连不上按退避），45 秒什么都没收到（服务端 15 秒一拍心跳）当连接死了。 - store/chat.ts：plan / launch / settle / retry 换成发起（launchTurn）+ 播放（play）。收到的操作照原来的渲染逻辑画（画布边收 边长、过程步骤、思考、查看、用量）；收到 run 就接上运行自己的实时流画过程（watch 只画过程，不再判结局）；stage 说到了 哪一步（停在审批上立刻去拿待审批）；turn.retry 照原来自动重来那样重置；end 照服务端落库的那一行摆结局（和刷新之后从库里 恢复的是同一份 restoreTurn）。「重试本轮」「重新运行本轮」「放宽步数」「跑一下」「继续运行」「停止」都走新接口。 - 页面不再写这一轮的结局（不再 PATCH 轮次、不再调 /copilot/review、不再发 POST /api/runs）；失败的说法照服务端记的原因 （meta.cause）按 lib/explain 讲，老轮次记的 failure 照旧认。 - 刷新、关掉再打开：服务端还在跑的一轮（ConversationTurn.job）从这一次尝试的第一条事件接着播；库里停在半路、服务端又没在跑 的（老轮次、服务重启后停着的）请服务端接着收尾（mode=resume），不在页面上自己收尾。问数据页的显示不变。 - 页面上那份退路（MAX_ATTEMPTS、fillInput、retryInstruction、等复核的计时）去掉：决定只在服务端一处。后端对照测试改成守 「页面上不再有这几样」。 - scripts/check-chat.mjs：伪造的服务端照 backend/app/chat/job.py 的做法用原来那几份剧本把一轮跑完（生成、发起运行、等运行的剧本 放完、复核、拼那一行），原有各段照过；守落库的几处改成守「页面不 PATCH 这一轮」和照服务端那一行摆；新加「断线重连补齐」 「刷新、关掉再打开接着看」「停止（建图中也停得下）」几段，并核对发起、重试、放宽步数、继续运行、停止的请求体。 - 服务端顺带：turn.attempt 带上到目前为止的用量、从什么时候算起（补交付的到运行结束为止），turn.retry 带上换下来的那次运行 并进之后的用量（页面上的运行壳子里没有用量）；接着收尾（resume）不要求运行还在，运行记录删了的照实收成「运行记录不见了」。 --- backend/app/chat/job.py | 20 +- backend/tests/test_chat_turn_policy.py | 57 +- backend/tests/test_chat_turns.py | 23 + frontend/src/api/chatTurns.ts | 153 +++ frontend/src/api/client.ts | 5 + frontend/src/store/chat.ts | 1581 +++++++++--------------- frontend/src/types.ts | 16 + scripts/check-chat.mjs | 1245 +++++++++++++++---- 8 files changed, 1822 insertions(+), 1278 deletions(-) create mode 100644 frontend/src/api/chatTurns.ts diff --git a/backend/app/chat/job.py b/backend/app/chat/job.py index 32bc6f3b..60cd197f 100644 --- a/backend/app/chat/job.py +++ b/backend/app/chat/job.py @@ -310,7 +310,8 @@ class ChatTurnExecution(runtime.JobRun): self.run_id = None await self._job(attempt=attempt, run_id=None) await self.save(run_id=None, meta={"runId": None, "runStatus": None, "usage": dict(self.usage)}) - await self.emit("turn.retry", {"kind": kind, "reason": reason, "base": base, "attempt": attempt}) + await self.emit("turn.retry", {"kind": kind, "reason": reason, "base": base, "attempt": attempt, + "usage": dict(self.usage)}) # ---------------- 执行 ---------------- @@ -751,7 +752,8 @@ async def start(session: AsyncSession, *, conversation: Any, question: str, data raise TurnRefused(422, "放宽到几步要写清楚", TURN_INVALID) request["max_steps"] = int(max_steps) if mode in ("continue", "resume"): - if run is None: + # 接着收尾不要求运行还在：运行记录删了的，服务端照实收成「运行记录不见了」 + if run is None and (mode == "continue" or not run_id): raise TurnRefused(409, "这一轮没有可接着的运行，请重试本轮", TURN_INVALID) if mode == "continue": await _continue_run(run.id, actor) @@ -761,9 +763,10 @@ async def start(session: AsyncSession, *, conversation: Any, question: str, data spent = meta.get("ms") if row.status != "running" and isinstance(spent, (int, float)): request["started_ms"] = now_ms - int(spent) - elif run.started_at is not None: + elif run is not None and run.started_at is not None: request["started_ms"] = int(run.started_at.timestamp() * 1000) - if mode == "resume" and run.status in ("succeeded", "failed", "cancelled") and run.finished_at is not None: + if mode == "resume" and run is not None and run.status in ("succeeded", "failed", "cancelled") \ + and run.finished_at is not None: request["ended_ms"] = int(run.finished_at.timestamp() * 1000) else: # 重试、重新运行、放宽步数、跑一下：服务重启挂起的旧运行没人会再去续它，顺手取消（停在审批上的不算：重试不会从那里发起） @@ -812,8 +815,15 @@ async def start(session: AsyncSession, *, conversation: Any, question: str, data # 这一次尝试从这一条开始：刷新、重开页面时从这里回放 async with SessionLocal() as s: + from app.db.models import ConversationTurn + + fresh = await s.get(ConversationTurn, turn_id) + usage = ((fresh.meta or {}) if fresh is not None and isinstance(fresh.meta, dict) else {}).get("usage") + # 页面据此把这一轮重置成「这一次」的样子：归档、范围、到目前为止的用量、从什么时候算起（补交付的到运行结束为止） seq = await turn_log.append(s, turn_id, "turn.attempt", { - "mode": mode, "attempts": attempts, "scope": request.get("scope") or [], + "mode": mode, "attempts": attempts, "scope": request.get("scope") or [], "usage": usage or {}, + "started_ms": request.get("started_ms"), + **({"ended_ms": request["ended_ms"]} if request.get("ended_ms") else {}), **({"max_steps": request["max_steps"]} if request.get("max_steps") else {})}) await s.execute(update(ChatTurnJob).where(ChatTurnJob.id == turn_id).values(attempt_seq=seq) .execution_options(synchronize_session=False)) diff --git a/backend/tests/test_chat_turn_policy.py b/backend/tests/test_chat_turn_policy.py index f21f7a1c..609c019b 100644 --- a/backend/tests/test_chat_turn_policy.py +++ b/backend/tests/test_chat_turn_policy.py @@ -1,11 +1,10 @@ """问数据一轮的「决定」（app/chat/turn.py；细案 01 做法 1）。 要不要运行、输入怎么填、要不要带着原图重来、重来时说什么，从问数据页下沉到后端：final 带 next，复核带 retry_instruction， -问数据页照它们走；老服务端没给这两个字段时，问数据页退回它自己那份同样的规则。这里守： +问数据的一轮后来整个搬到了服务端（chat/drive.py、chat/job.py），页面只发起、只播放。这里守： - 各条决定的分支（next_step、fill_input、retry_after_review、retry_instruction 的两种措辞）； -- 前后端对照：frontend/src/store/chat.ts 里留着的那份常量（MAX_ATTEMPTS、fillInput 认问题字段的正则、两种重来的措辞） - 和这里逐字一致——写法同 test_frontend_type_unions.py，两边一改就对不上； +- 页面上不再留一份同样的决定（一轮整个搬到服务端之后，frontend/src/store/chat.ts 只发起、只播放）；用户点重试本轮时的措辞； - /copilot/review 按这一轮试过几次给 retry_instruction：可以重来时是那句话，不重来是 null。 问题、字段名都是虚构的。 @@ -112,7 +111,7 @@ def test_retry_instruction_two_wordings(): # -------------------------------------------------------------------------- -# 前后端对照 +# 页面不再留一份同样的决定 # -------------------------------------------------------------------------- @@ -120,30 +119,23 @@ def _ts() -> str: return CHAT_TS.read_text(encoding="utf-8") -def test_max_attempts_matches_the_frontend(): - found = re.search(r"const MAX_ATTEMPTS = (\d+)", _ts()) - assert found and int(found.group(1)) == turn.MAX_ATTEMPTS +def test_the_page_no_longer_keeps_its_own_copy_of_the_decisions(): + """一轮整个搬到服务端之后（docs/design/orchestration-round §1），问数据页只发起、只播放：要不要运行、要不要重来、重来说 + 什么、入口怎么填都只在这里（和 chat/drive.py），页面上那份退路（MAX_ATTEMPTS、fillInput、retryInstruction）去掉了—— + 两份并存迟早走样，界面层也不放逻辑。以前这里逐字对照两边的常量和措辞，现在守的是页面上没有这几样。""" + ts = _ts() + for name in ("const MAX_ATTEMPTS", "function fillInput", "function retryInstruction", "streamCopilot(", + "api.copilot.review(", "api.runs.start(", "patchTurn("): + assert name not in ts, f"chat.ts 里又出现了 {name}：这一轮的决定、发起、落库应在服务端" -def test_the_question_field_pattern_matches_the_frontend(): - found = re.search(r"/(question\|[^/]+)/i\.test", _ts()) - assert found, "chat.ts 的 fillInput 里找不到认问题字段的正则" - assert found.group(1) == turn.QUESTION_FIELD.pattern and turn.QUESTION_FIELD.flags & re.IGNORECASE - - -def _template(source: str) -> str: - """一段 TS 里由 + 连起来的模板字符串 → 一个 Python format 模板（${x} → {x}，\\n → 换行）。""" - pieces = re.findall(r"`([^`]*)`", source) - text = "".join(pieces).replace("\\n", "\n") - return re.sub(r"\$\{(\w+)\}", r"{\1}", text) - - -def test_the_retry_wordings_match_the_frontend(): - body = re.search(r"function retryInstruction\([^)]*\): string \{(.*?)\n\}", _ts(), re.S) - assert body, "chat.ts 里找不到 retryInstruction" - with_base, scratch = re.split(r"\n\s*: ", body.group(1).split("hasBase", 1)[1].split("?", 1)[1], maxsplit=1) - assert _template(with_base) == turn.RETRY_WITH_BASE - assert _template(scratch) == turn.RETRY_FROM_SCRATCH +def test_user_retry_wordings(): + """用户点「重试本轮」时交给助手的话（以前在页面 retryTurn 里拼）：有图时说「没有给出可用的结果」，没图的同从零重建。""" + with_base = turn.user_retry_instruction("上周销售额", "已手动停止", True) + assert with_base == ("这个工作流上一次没有给出可用的结果：已手动停止。\n\n请针对这个原因改它（比如调大最大步数、改正工具参数、" + "换个查询方式），改完继续回答原来的问题：上周销售额") + assert turn.user_retry_instruction("上周销售额", "已手动停止", False) == turn.retry_instruction( + "上周销售额", "已手动停止", False) # -------------------------------------------------------------------------- @@ -172,12 +164,13 @@ async def _run(output: dict, graph: dict, code: str) -> str: return run.id -def test_the_frontend_retries_an_error_the_server_says_to_retry(): - """问数据页照服务端给的 next 走：error 操作带 next=retry（只写了思考、一个操作都没给）时带着原因重来一次，不再直接报 - 「助手本轮未完成」。读 chat.ts 里 error 那一支（行为另由 scripts/check-chat.mjs 的 emptyretry 一节在浏览器里守）。""" - branch = re.search(r"case 'error':(.*?)break\n", _ts(), re.S) - assert branch, "chat.ts 里找不到 error 那一支" - assert re.search(r"op\.next === 'retry'", branch.group(1)) and "retry(" in branch.group(1), branch.group(1) +def test_the_page_says_the_server_is_retrying_an_empty_generation(): + """只写了思考、一个操作都没给（error.next=retry）时，重来一次是服务端定的（chat/drive.py 照 next_step 走，问数据的一轮记一条 + turn.retry，kind=empty）；页面照它把这一轮重置、写「助手上一次没有给出任何操作，正在重新生成工作流…」。读 chat.ts 里 + turn.retry 那一支（行为另由 tests/test_chat_turns.py 和 scripts/check-chat.mjs 的 emptyretry 一节守）。""" + branch = re.search(r"function retryPatch\((.*?)\n\}", _ts(), re.S) + assert branch, "chat.ts 里找不到 retryPatch" + assert re.search(r"d\.kind === 'empty' \? CHAT_RETRY_TEXT\.empty", branch.group(1)), branch.group(1) async def test_answer_ops_says_to_retry_an_empty_generation(client, monkeypatch): diff --git a/backend/tests/test_chat_turns.py b/backend/tests/test_chat_turns.py index 59e6a226..36f554ff 100644 --- a/backend/tests/test_chat_turns.py +++ b/backend/tests/test_chat_turns.py @@ -834,3 +834,26 @@ def test_attempt_records_say_what_the_page_used_to_say(): legacy = record.attempt_record(row(status="error", error="x"), {"failure": {"title": "操作超时", "reason": "查询超时。"}}, n=1) assert legacy["summary"] == "操作超时：查询超时。" assert record.attempt_record(row(answer="字" * 5000), {}, n=1)["answer"].endswith("…") + + +async def test_an_empty_generation_is_retried_once_on_the_server(client, monkeypatch): + """助手只写了思考、一个操作都没给（error.next=retry）：服务端带着原因重来一次（turn.retry kind=empty），不用页面在。""" + import app.api.copilot as copilot + from tests.test_usage_meter import Scripted + + reply = json.dumps({"op": "reply", "text": "上周没有华中区的数据"}, ensure_ascii=False) + model = Scripted(model="claude-opus-5", replies=[[""], [reply]]) + + async def _model(*_a, **_k): + return model, "claude-opus-5" + + monkeypatch.setattr(copilot, "get_chat_model", _model) + conversation_id = await _conversation(client) + turn_id = await _ask(client, conversation_id, "上周华中区销售额") + await _done(turn_id) + events = await _events(turn_id) + [retry] = [e["data"] for e in events if e["type"] == "turn.retry"] + assert retry["kind"] == "empty" and retry["reason"] == "模型没有给出任何修改，也没有说明原因" + row = await _row(turn_id) + assert row.status == "done" and row.answer == "上周没有华中区的数据" and row.meta["noQuery"] is True + assert row.meta["usage"]["copilot"]["calls"] == 2, "两次建图都是这一轮花的" diff --git a/frontend/src/api/chatTurns.ts b/frontend/src/api/chatTurns.ts new file mode 100644 index 00000000..e3990c9b --- /dev/null +++ b/frontend/src/api/chatTurns.ts @@ -0,0 +1,153 @@ +/** + * 问数据的一轮在服务端跑（docs/design/orchestration-round/00-总方案.md §1；服务端 backend/app/api/chat_turns.py），挂在 + * api.chatTurns 上。页面只发起、只看：建图、运行、复核、按复核重来、落库都在服务端，页面断了这一轮也会跑完。 + * + * - start：发起一轮（新问，或者对已有的一轮重试、重新运行、放宽步数、跑一下、继续运行、接着收尾），202 {turn_id, turn}； + * - get：这一轮的任务现在怎样；cancel：停止（取消生成、取消运行）； + * - streamChatTurn：这一轮的事件日志（SSE）。先回放 seq > after 的，再推实时；断了按收到的最后一个 seq 自动重连，补齐、 + * 不重复；45 秒什么都没收到（服务端每 15 秒有一拍心跳）就当连接死了，重连。服务端说这一轮结束了（stream.end）才停。 + * + * 请求函数从 ./http 导入，不从 ./client 导入：client.ts 要导入这个文件把它挂到 api 上，反过来导入就成了循环。 + */ +import type { ConversationTurn } from '../types' +import { ApiError, BASE, NETWORK_MESSAGE, actorHeader, failure, get, parseBody, post, report } from './http' + +/** 发起一轮的方式（服务端 app/chat/job.MODES） */ +export type ChatTurnMode = 'ask' | 'retry' | 'rerun' | 'max_steps' | 'run' | 'continue' | 'resume' + +export interface ChatTurnStart { + conversation_id: string + question?: string + /** 只查这几个数据源。新问不给是不限；重试不给沿用这一轮的，空数组是不限 */ + datasource_ids?: string[] + retry_of?: string + mode?: ChatTurnMode + max_steps?: number +} + +/** 事件日志里的一条（服务端 app/chat/job.py 的模块说明列了各种 type） */ +export interface ChatTurnEvent { + seq: number + type: string + data: Record + ts: number +} + +/** 实时流结束时的情况：status 是服务端说的任务状态（stream.end）；error 是这一轮读不了（不存在、没权限） */ +export interface ChatTurnStreamEnd { + status?: string + error?: string + httpStatus?: number +} + +const turn = (id: string) => `/chat/turns/${encodeURIComponent(id)}` + +export const chatTurns = { + start: (body: ChatTurnStart) => post<{ turn_id: string; turn: ConversationTurn | null }>('/chat/turns', body), + get: (id: string) => get<{ id: string; conversation_id: string; job: ConversationTurn['job']; turn: ConversationTurn }>(turn(id)), + cancel: (id: string) => post<{ ok: boolean }>(`${turn(id)}/cancel`, {}), +} + +/** 多久什么都没收到就当连接死了（服务端心跳 15 秒一拍） */ +const STALL_MS = 45_000 +/** 连不上时重连的退避上限 */ +const RETRY_MAX_MS = 5000 + +/** + * 订阅这一轮的事件日志。返回取消函数（关掉连接、不再重连）。 + * + * 连接正常结束却没收到 stream.end（服务端重启、代理断开）：按最后一个 seq 马上重连；连不上（网络断了、服务端在重启）按 + * 退避重试，直到服务端回来——这一轮在服务端照跑，回来之后补齐中间漏掉的那段 + */ +export function streamChatTurn( + turnId: string, + after: number, + onEvent: (event: ChatTurnEvent) => void, + onEnd: (end: ChatTurnStreamEnd) => void, +): () => void { + let closed = false + let lastSeq = after + let failures = 0 + let controller: AbortController | null = null + let stall: ReturnType | undefined + let again: ReturnType | undefined + + const finish = (end: ChatTurnStreamEnd) => { + if (closed) return + closed = true + clearTimeout(stall) + onEnd(end) + } + const arm = () => { + clearTimeout(stall) + stall = setTimeout(() => controller?.abort(), STALL_MS) + } + + const connect = async () => { + if (closed) return + controller = new AbortController() + /** 这一次连接收到了事件（不只是心跳） */ + let got = false + let alive = false + try { + arm() + const path = `${turn(turnId)}/stream?after=${lastSeq}` + const res = await fetch(BASE + path, { headers: { Accept: 'text/event-stream', ...actorHeader() }, signal: controller.signal }) + if (!res.ok || !res.body) { + const err = failure(res.status, res.statusText, await res.text().catch(() => ''), path) + // 这一轮读不了（删了、不是你的）：再连也一样 + if (err.kind !== 'network' && res.status < 500) { + finish({ error: err.message, httpStatus: res.status }) + return + } + throw err + } + report(true) + alive = true + const reader = res.body.getReader() + const decoder = new TextDecoder() + let buffer = '' + for (;;) { + const { done, value } = await reader.read() + if (done) break + arm() + buffer += decoder.decode(value, { stream: true }) + let idx: number + while ((idx = buffer.indexOf('\n\n')) >= 0) { + const frame = buffer.slice(0, idx) + buffer = buffer.slice(idx + 2) + if (!frame.startsWith('data: ')) continue // 心跳（: ping） + let event: any + try { event = parseBody(frame.slice(6)) } catch { continue } + if (!event || typeof event !== 'object') continue + if (event.type === 'stream.end') { + finish({ status: event.data?.status ?? event.status }) + controller.abort() + return + } + if (typeof event.seq !== 'number' || event.seq <= lastSeq) continue // 重连时重叠的那段只要一份 + lastSeq = event.seq + got = true + if (!closed) onEvent(event as ChatTurnEvent) + } + } + } catch (e: any) { + if (closed) return + if (e instanceof TypeError) report(false, new ApiError(0, NETWORK_MESSAGE, { kind: 'network', raw: String(e) })) + } + if (closed) return + clearTimeout(stall) + // 收到了东西就马上接着连；只有心跳的稍等一下；连不上的按退避 + failures = got ? 0 : alive ? 0 : failures + 1 + const delay = got ? 0 : alive ? 300 : Math.min(300 * 2 ** failures, RETRY_MAX_MS) + again = setTimeout(() => void connect(), delay) + } + void connect() + + return () => { + closed = true + clearTimeout(stall) + clearTimeout(again) + controller?.abort() + } +} diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 13c722d2..b4fed479 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -39,6 +39,8 @@ import { caliberLedger } from './caliberLedger' import { capabilities, endpoints, notify } from './capabilities' // 普通用户建应用（typesBlueprint.ts 是接口约定） import { blueprints, files, personal, templates } from './blueprints' +// 问数据的一轮在服务端跑（编排完善第 1 波）：发起、看、停，实时流在 chatTurns.ts 的 streamChatTurn +import { chatTurns } from './chatTurns' // 请求层搬到了 http.ts（见那边的说明）。原先从这里导出的照旧从这里导出，调用方不用改； // 请求函数也一并转出，给不在 api/ 下、又想直接发请求的地方用。api/ 下的命名空间文件请直接从 http.ts 导入 @@ -986,6 +988,9 @@ export const api = { templates, files, personal, + + // 问数据的一轮（服务端跑整轮：发起、看、停） + chatTurns, } /** diff --git a/frontend/src/store/chat.ts b/frontend/src/store/chat.ts index c50c3252..88511b76 100644 --- a/frontend/src/store/chat.ts +++ b/frontend/src/store/chat.ts @@ -1,17 +1,18 @@ import { create } from 'zustand' -import { ApiError, api, streamCopilot, streamRun } from '../api/client' +import { ApiError, api, streamRun } from '../api/client' +import { streamChatTurn, type ChatTurnEvent, type ChatTurnStart } from '../api/chatTurns' import type { CopilotOp, FixKind, RunFinal } from '../run/decode' -import { copilotContext, copilotLook, decodeRun, looksOf } from '../run/decode' +import { copilotContext, copilotLook, decodeRun } from '../run/decode' import { CHAT_BLUEPRINT_TEXT, CHAT_RETRY_TEXT, LEDGER_TEXT, LOOK_TEXT, REJECTED_TEXT } from '../lib/terms' import type { LookEvent } from '../typesApp' -import { errorMessage, humanizeError, isNetworkError } from '../lib/errors' -import { guardReview, hasEvidence } from '../lib/evidence' +import { errorMessage, humanizeError } from '../lib/errors' +import { hasEvidence } from '../lib/evidence' import { explainRunError, explainStartError } from '../lib/explain' import { parseServerTime } from '../lib/format' import { useCatalog } from './catalog' import { useConversations } from './conversations' import type { - ConversationTurn, GraphSpec, ReviewResult, Run, RunEvent, ValidationIssue, + ChatTurnJob, ConversationTurn, GraphSpec, ReviewResult, Run, RunEvent, } from '../types' /** @@ -24,13 +25,19 @@ import type { * 一轮对话 = 一次「建图 → 跑图 → 出结果」。图对用户是隐藏的（想看可以展开）， * 但它仍然是真实存在的工作流，跑完能存成模板、能在画布里继续改。 * + * **一轮在服务端跑，这里只发起、只播放**（docs/design/orchestration-round/00-总方案.md §1）。以前建图、发起运行、复核、 + * 按复核重来、落库都是这一页在推：网络一抖，建图那条流就断了、建了一半的图丢掉；页面一关，这一轮停在 running，没人复核、 + * 不会重来，15 分钟后才被判成断了。现在一轮是服务端的一个后台任务（backend/app/chat/job.py）：这里 POST /api/chat/turns 发起， + * 再订阅这一轮的事件日志（api/chatTurns 的 streamChatTurn：先回放、再接实时，断了按 seq 自动补齐），收到什么照原来的渲染 + * 逻辑画——画布边收边长、过程步骤、答案、复核说明、用量。要不要运行、要不要重来、什么时候交付都是服务端定的，这一页不再判； + * conversation_turns 那一行也由服务端写，这里不再写这一轮的结局。刷新、关掉再打开，进行中的轮次从这一次尝试的第一条 + * 接着播（ConversationTurn.job.attempt_seq）。运行自己的事件仍经运行的实时流来（watch），只用来画过程。 + * * 这里**只存原始流**（Copilot 的操作、运行的事件），不在 store 里翻译。 * 翻译归 run/decode.ts，全站一份；store 里再译一遍的结果是同一次运行在 * 对话页和画布上讲出两个不同的故事，而用户没法判断哪个是真的。 * - * 轮次按会话分桶存放。之前是单个 turns 数组，于是"对话"只存在于内存里： - * 刷新页面就全没了，而每轮又各自从零建图、互不知情。现在每一轮同时写进 - * 后端的 conversation_turns，刷新能恢复，下一轮也拿得到上一轮的上下文。 + * 轮次按会话分桶存放，刷新从后端的 conversation_turns 恢复。 * * **忙不忙按会话算，由轮次的相位推出来**，store 上不再有全局的 busy / cancel。 * 全局只有一份的时候，会话 A 在跑、切到 B，B 的按钮就成了「停止」，点下去 @@ -99,7 +106,7 @@ export interface ScopeSource { } /** - * 用户点「重试这一轮」之前的那一次。 + * 用户点「重试这一轮」之前的那一次（服务端归档，backend/app/chat/record.attempt_record）。 * * 用户主动重试不能把上一次抹掉：那一次的答案可能只是有缺口、并非全错， * 新的一次也未必更好，两份要能对照。 @@ -111,13 +118,29 @@ export interface Attempt { summary: string detail?: string runId?: string | null - /** 那一次交出来的答案（截断到 ATTEMPT_ANSWER_CAP） */ + /** 那一次交出来的答案（截断到 4000 字） */ answer?: string at: number + /** 失败的那一次的原因（服务端只记原因，说法在这里按 lib/explain 讲，见 causeFailure） */ + cause?: TurnCause } /** - * 刷新之后还得在的可信度信息，存在 conversation_turns.meta。 + * 这一轮为什么失败（服务端 backend/app/chat/record.py 的 cause）：服务端只记发生了什么，说法在界面层按 lib/explain 讲 + * （causeFailure），和记录页、画布右栏同一套。老轮次记的是翻好的 failure，照旧认 + */ +export type TurnCause = + | { kind: 'refused'; message?: string; code?: string | null } + | { kind: 'copilot'; message?: string; hint?: string | null; detail?: string | null } + | { kind: 'blocked'; issues?: string[]; self_checked?: boolean } + | { kind: 'start'; message?: string; code?: string | null; status?: number | null } + | { kind: 'run'; error?: string; detail?: string | null } + | { kind: 'gone' } + | { kind: 'build_interrupted' } + | { kind: 'crash'; message?: string } + +/** + * 刷新之后还得在的可信度信息，存在 conversation_turns.meta（服务端写，backend/app/chat/record.py）。 * * 出具档位、运行类别、「这一条没查库」以前都只活在内存里，刷新就没了——而历史 * 会话恰恰是事后复盘、转发结论时最常看的地方。老轮次把它挂在 review.meta 下， @@ -133,23 +156,28 @@ export interface TurnMeta { issuance?: Record noQuery?: boolean outcome?: 'cancelled' | 'suspended' - /** 运行被人驳回结束（outcome 是 cancelled）：副标题那一句，刷新之后照样写「被驳回」，不退回成「你已停止本轮」 */ - rejected?: Rejected + /** + * 运行被人驳回结束（outcome 是 cancelled）：服务端记 run.cancelled 带的谁、哪一步、备注（REJECTED_TEXT.sub 拼成一句）； + * 老轮次记的是拼好的那一句（sub）。刷新之后照样写「被驳回」，不退回成「你已停止本轮」 + */ + rejected?: Partial & { actor?: string; label?: string; note?: string; decision?: string } + /** 老轮次（页面写的）：翻好的失败说明 */ failure?: Failure + /** 服务端写的：失败的原因 */ + cause?: TurnCause attempts?: Attempt[] /** 这一轮从开问到交付的墙钟毫秒 */ ms?: number /** 查了几次库 */ queries?: number /** - * 答案没取全。不记下来的话，刷新之后只能靠「恰好 2000 字」去猜：多个成果键拼起来的 - * 答案比 2000 长，猜不中，「刷新后会自动补全」的承诺就落空了 + * 答案没取全（老轮次：以前页面落库时用的是事件里截断过的那份）。服务端落库读的是完整成果，新轮次不会有 */ clipped?: 'partial' | 'lost' /** 这一问限定了只查哪些数据源 */ scope?: ScopeSource[] /** 这一轮以助手反问收尾（answer 列里是助手问的那句话，不是答案）：刷新之后照样标「助手反问」，不当成没查库的回答 */ - asked?: true + asked?: true | { question?: string; options?: string[] } /** 这一轮助手查看过的表（look 事件原样，look §2.5.8）：下一轮服务端据此写「上一轮查看过」，追问时直接点名查看 */ looks?: LookEvent[] /** @@ -157,10 +185,7 @@ export interface TurnMeta { * 运行的成果里：有这个记号的轮次恢复时要回运行那里取一次 */ evidence?: boolean - /** - * 这份 meta 写下的时刻（本机时钟）。断在建图阶段的轮次按它和 created_at 里晚的那个算 - * 有多久没动静：后端按 updated_at 算，而重试一轮昨天的问题时 created_at 是昨天的 - */ + /** 这份 meta 写下的时刻 */ at?: number /** 这一轮的花费：助手建图、运行、复核各一份（next-round 细案 01 做法 3），见 TurnUsage */ usage?: TurnUsage @@ -194,7 +219,7 @@ export interface TurnUsage { export interface ChatTurn { id: string - /** 后端 conversation_turns 里的 id。回填靠它 */ + /** 后端 conversation_turns 里的 id（也是服务端这一轮的任务 id）。发起请求还在路上时为 null */ serverId: string | null question: string phase: Phase @@ -224,10 +249,12 @@ export interface ChatTurn { startedAt: number /** 交付、失败、取消或中断的时刻。计时器停在这里 */ endedAt?: number - /** 已收到的最大事件 seq。重新接流时要带上，否则后端把历史整条重推 */ + /** 已收到的最大运行事件 seq。重新接运行的流时要带上，否则后端把历史整条重推 */ lastSeq: number - /** 这一轮自己的取消句柄。放 turn 上而不是 store 上：store 单值会被 - * 下一轮 ask 覆盖，上一轮的 WebSocket 就再也关不掉了 */ + /** 已收到的最大这一轮事件 seq（服务端的事件日志，和运行事件的 seq 是两套） */ + turnSeq?: number + /** 关掉这一轮在这一页上开着的流（这一轮的事件日志、运行的实时流）。放 turn 上而不是 store 上：store 单值会被 + * 下一轮 ask 覆盖，上一轮的流就再也关不掉了。只是不看了，服务端那一轮照跑；要停用 stop */ cancel: (() => void) | null /** 事件之外知道的结局（stream.end、GET 运行查到的状态），交给 decodeRun 收尾 */ final?: RunFinal @@ -245,21 +272,20 @@ export interface ChatTurn { hydrated?: 'pending' | 'done' /** 跑完之后的复核结论。null = 没复核过，和「复核过、没问题」不是一回事 */ review?: ReviewResult | null - /** 复核原样（含 verdict='ok'），落库时和 meta 一起写回 review 字段 */ + /** 复核原样（含 verdict='ok'） */ reviewRaw?: ReviewResult | null /** 被复核重写之前的答案。改写是有损的，原件得留着让用户能对照 */ rawOutput?: Record | null - /** 这一轮自动重建过几次图。上限 1——第二次还不行多半是问题本身没说清， - * 该让用户介入，而不是替他一遍遍烧钱 */ + /** 这一轮自动重建过几次图（服务端的 turn.retry）。上限 1——第二次还不行多半是问题本身没说清 */ attempt?: number /** 用户点「重试」之前的各次尝试 */ attempts?: Attempt[] /** - * 答案不全：partial = 这次只拿到了事件里那份截断的，完整版要回源； + * 答案不全（老轮次）：partial = 当时只拿到了事件里那份截断的，完整版要回源； * lost = 旧版本保存时就被切在 2000 字，对应的运行也删了，补不回来 */ clipped?: 'partial' | 'lost' - /** 刷新前落库的查库次数和耗时（事件没取回来时头部小标签靠它） */ + /** 落库的查库次数和耗时（事件没取回来时头部小标签靠它） */ meta?: TurnMeta | null /** * 模型此刻在出字：只记写了多少、思考想到哪一句，原文不存也不落库。答案要等 @@ -271,10 +297,11 @@ export interface ChatTurn { /** 这一轮到目前为止的助手建图、复核、被换掉的运行的用量（当前这次运行的在 run.usage），见 TurnUsage */ usage?: TurnUsage /** - * 助手问回来的那句话（N11）：这一轮到此结束，问题作为这一轮的内容显示、进历史，输入框提示回答它；用户的回答就是下一轮。 - * 以前这里没有分支：只问不改的一轮流结束时既没有 final 也没有 reply，这一轮一直停在「正在理解需求」 + * 助手问回来的那句话（N11）：这一轮到此结束，问题作为这一轮的内容显示、进历史，输入框提示回答它；用户的回答就是下一轮 */ asked?: string + /** 这一轮在服务端的任务（从库里恢复时带来的）：进行中的从它的 attempt_seq 接着播 */ + job?: ChatTurnJob | null } export type LoadState = 'loading' | 'ready' | 'error' @@ -306,9 +333,9 @@ interface ChatState { * scope 不给就用这个会话眼下的范围 */ ask: (conversationId: string, question: string, opts?: { scope?: ScopeSource[] }) => boolean - /** 只停这个会话自己正在跑的那一轮 */ + /** 只停这个会话自己正在跑的那一轮（服务端那一轮连同它的运行一起停） */ stop: (conversationId: string) => void - /** 审批完成后重新接上运行 */ + /** 审批完成后重新接上运行的实时流 */ reattach: (conversationId: string, turnId: string) => Promise /** 历史轮次的「执行过程」：第一次点去取事件，之后是展开 / 收起 */ toggleSteps: (conversationId: string, turnId: string) => Promise @@ -317,7 +344,7 @@ interface ChatState { /** 跑挂了或被服务重启打断：从断点接着跑。成功接上返回 true，被拒时抛出后端的原因 */ continueTurn: (conversationId: string, turnId: string) => Promise /** - * 用户主动重试这一轮：上一次折叠留档，新的一次接在同一轮里。 + * 用户主动重试这一轮：上一次由服务端折叠留档，新的一次接在同一轮里。 * maxSteps：把图里 agent 的步数上限调到这个值，原图重跑（「放宽步数重跑」）； * rerun：原图原样重跑（取消了、中断了的那种，不需要改）； * scope：换一个数据源范围重来（限定的库不在了），不给就沿用这一轮的 @@ -326,7 +353,7 @@ interface ChatState { conversationId: string, turnId: string, opts?: { maxSteps?: number; rerun?: boolean; scope?: ScopeSource[] }, ) => void - /** 向后端核对一轮历史：真实状态、完整答案、档位。只做一次 */ + /** 向后端核对一轮历史：真实状态、完整答案、档位；服务端还在跑的接着播。只做一次 */ hydrate: (conversationId: string, turnId: string) => Promise /** 上次核对撞上后端出错（unknown）的轮次，再核对一次 */ recheck: (conversationId: string, turnId: string) => Promise @@ -352,15 +379,13 @@ const PHASE_TEXT: Record = { unknown: '无法确认本轮的当前状态', } +/** 复核那一步的进度 */ +const REVIEWING = '正在核对结果…' + const EMPTY: ChatTurn[] = [] -/** 落库成果的上限。只是防跑飞，不是显示上限 */ -const ANSWER_CAP = 20_000 - -/** 等复核的上限。超了就先把原答案交出去，说明后补不了就算了（复核这一次的用量晚到了照记，见 settle） */ -const REVIEW_TIMEOUT_MS = 25_000 -/** 实际等多久：开发期检查脚本可以调短它（window.__chatReviewWait），不用真等 25 秒 */ -let reviewWaitMs = REVIEW_TIMEOUT_MS +/** 服务端这一轮的任务还在跑（排队、进行中） */ +const LIVE_JOB = new Set(['queued', 'running']) /** * 旧版本落库时答案被切在这个长度：run.finished 事件里的 output 按字典每个值 @@ -379,16 +404,6 @@ function atClip(s: string): boolean { return n === LEGACY_CLIP } -/** - * 还停在 running 的轮次、又没有运行：断在建图阶段了。建图跟着页面那条流走，页面一关 - * 就停；这么久还没动静就当它断了（和后端 conversations.BUILD_STALE 一致）。「动静」 - * 按最后一次落库算，见 TurnMeta.at - */ -const BUILD_STALE_MS = 15 * 60_000 - -/** 留档的旧答案存多少。它是用来对照的，不是用来读全文的——全文在运行记录里 */ -const ATTEMPT_ANSWER_CAP = 4000 - /** 事件流里的增量：不落库，decodeRun 一开头就丢掉；存进 events 只会让每条都重解码整个会话 */ const EPHEMERAL = new Set(['llm.token', 'llm.thinking.delta']) @@ -400,7 +415,6 @@ const PEN_MS_REDUCED = 1000 const REPLAY_SKIP = new Set(['run.finished', 'run.failed', 'run.cancelled', 'run.interrupted', 'log']) type Patch = (fn: (t: ChatTurn) => Partial) => void -type Save = (body: Record) => Promise type SetState = (fn: (s: ChatState) => Partial | ChatState) => void /** 会话此刻被哪一轮占着。按相位推，不另记一份——另记的那份迟早和相位对不上 */ @@ -425,11 +439,7 @@ export function stepsVisible(t: ChatTurn): boolean { } /** - * 成果 dict 压成一句话，存进 conversation_turns.answer。 - * - * 它有两个去处：刷新页面后重建界面，以及下一轮拼进 Copilot 的上下文。 - * 后者决定了这里不能只存个 "[object Object]"——模型要靠它知道上一轮 - * 答了什么，才接得住"再按月份拆一下"。 + * 成果 dict 压成一句话（同服务端 chat/record.answer_text）。老轮次补全截断的答案时拿它比对 */ function answerText(output: Record | null | undefined): string { if (!output) return '' @@ -438,32 +448,16 @@ function answerText(output: Record | null | undefined): string { if (key.startsWith('_') || value == null || value === '') continue parts.push(typeof value === 'string' ? value : JSON.stringify(value)) } - // 喂给下一轮的那份另有更狠的截断（HISTORY_ANSWER_CHARS），两回事。 - // 这里的上限只是防跑飞的护栏，真切到了必须说出来 - const text = parts.join('\n') - return text.length > ANSWER_CAP - ? text.slice(0, ANSWER_CAP) + `\n\n…（内容过长，已截断，完整内容请查看运行记录）` - : text + return parts.join('\n') } -/** 事件里那份 output 看起来是不是被切过：老后端不带 output_truncated，只能看长度 */ -function looksClipped(output: Record | null | undefined): boolean { - if (!output) return false - return Object.values(output).some((v) => typeof v === 'string' && atClip(v)) -} - -/** 出具信息存进 meta 前瘦一下身：无法回指的数字可能有上百个，横幅只列得下一行 */ +/** 出具信息瘦一下身：无法回指的数字可能有上百个，横幅只列得下一行 */ function slimIssuance(iss: any): Record | undefined { if (!iss || typeof iss !== 'object' || !iss.tier) return undefined const unmatched = Array.isArray(iss.unmatched_numbers) ? iss.unmatched_numbers.slice(0, 20) : undefined return { ...iss, ...(unmatched ? { unmatched_numbers: unmatched } : {}) } } -/** 查库次数：db_query 的每一次调用 */ -function countQueries(events: RunEvent[]): number { - return events.filter((e) => e.type === 'tool.start' && String(e.data?.tool ?? '').startsWith('db_query')).length -} - export function metaOf(t: ConversationTurn): TurnMeta | null { return t.meta && typeof t.meta === 'object' ? (t.meta as TurnMeta) : null } @@ -476,30 +470,6 @@ function reviewOf(t: ConversationTurn): ReviewResult | null { return rest as ReviewResult } -function buildMeta(t: ChatTurn): TurnMeta { - return { - v: 1, - runId: t.run?.id ?? null, - runClass: t.run?.run_class ?? t.meta?.runClass, - runStatus: t.runStatus, - issuance: slimIssuance(t.output?._issuance ?? t.rawOutput?._issuance) ?? t.meta?.issuance, - noQuery: t.noQuery || undefined, - outcome: t.phase === 'cancelled' ? 'cancelled' : t.phase === 'suspended' ? 'suspended' : undefined, - rejected: t.phase === 'cancelled' && t.rejected ? t.rejected : undefined, - failure: t.failure ?? undefined, - attempts: t.attempts?.length ? t.attempts : undefined, - ms: t.endedAt ? t.endedAt - t.startedAt : t.meta?.ms, - queries: t.events.length ? countQueries(t.events) : t.meta?.queries, - clipped: t.clipped, - scope: t.scope?.length ? t.scope : undefined, - asked: t.asked ? true : undefined, - evidence: hasEvidence(t.output) || t.meta?.evidence || undefined, - looks: lookList(t), - usage: usageOfTurn(t), - at: Date.now(), - } -} - const USAGE_KEYS = ['cost_usd', 'unpriced_calls', 'calls', 'unfinished_calls', 'input_tokens', 'output_tokens'] as const /** 两份用量相加（缺的当 0）。b 不是对象时原样交回 a */ @@ -527,7 +497,7 @@ export function usageOfTurn(t: ChatTurn): TurnUsage | undefined { return Object.keys(out).length ? out : undefined } -/** 当前这次运行换下来了（自动重来、用户点重试）：它的用量并进 earlier，这一轮的花费不丢这一笔 */ +/** 当前这次运行换下来了（自动重来）：它的用量并进 earlier，这一轮的花费不丢这一笔（同服务端 record.retire_run） */ function retireRun(t: ChatTurn): TurnUsage | undefined { const u = usageOfTurn(t) if (!u?.run) return u @@ -547,12 +517,6 @@ export function turnCost(u: TurnUsage | undefined): { usd: number; unpriced: num return { usd: Number(sum.cost_usd ?? 0), unpriced: Number(sum.unpriced_calls ?? 0), calls: Number(sum.calls ?? 0) } } -/** 这一轮的查看记录：操作流里有就取操作流的（重试会重新查看），恢复的历史轮次没有操作流，沿用 meta 里的 */ -function lookList(t: ChatTurn): LookEvent[] | undefined { - const looks = looksOf(t.ops) - return looks.length ? looks : t.meta?.looks?.length ? t.meta.looks : undefined -} - /** * 一行错误文案 → 结构化。后端的 detail 按约定已是人话，humanizeError 只做拆句和去类名。 * 运行本身的失败不走这里，走 runFailure @@ -598,10 +562,9 @@ function runFailure(error: string | null | undefined, detail?: string | null): F const lineOf = (f: Failure) => (f.reason ? `${f.title}：${f.reason}` : f.title) /** - * 发起运行那一下被拒（POST /runs 的 ApiError）的说明。绑定的工具在本机不存在（422 run_tool_missing） - * 按 lib/explain 讲，带上直达入口（数据源工具去数据页，自定义 / MCP 工具去工具页），和画布上 - * 发起被拒（run/RunControl）同一套；这时还没有运行 id，流里的报错块不会再拿原话认一遍， - * 所以入口得在这里就记进 failure。别的照旧按一句报错拆 + * 发起运行那一下被拒（服务端记的 start 原因：状态码、说法、机读码）的说明。绑定的工具在本机不存在（422 run_tool_missing） + * 按 lib/explain 讲，带上直达入口（数据源工具去数据页，自定义 / MCP 工具去工具页），和画布上发起被拒（run/RunControl） + * 同一套；这时还没有运行 id，流里的报错块不会再拿原话认一遍，所以入口得在这里就记进 failure。别的照旧按一句报错拆 */ function startFailure(e: unknown): Failure { const x = explainStartError(e) @@ -631,92 +594,96 @@ const SUSPENDED: Failure = { hint: '点击「继续运行」从断点继续；如需整轮重来，请点击「重新运行本轮」。', } -// ------------------------------------------------------------------------- -// 落库：同一轮的写入排队发，免得「运行中」比「完成」晚到、把它盖回去 -// ------------------------------------------------------------------------- - -/** 还没拿到 serverId 的新轮次：开轮请求在路上，写入要等它 */ -const serverIds = new Map>() -const saveChains = new Map>() - -function saver(conversationId: string, turnId: string): Save { - return (body) => { - const prev = saveChains.get(turnId) ?? Promise.resolve() - const next = prev.then(async () => { - const turn = findTurn(conversationId, turnId) - const serverId = turn?.serverId ?? (await serverIds.get(turnId)) ?? null - if (!serverId) return - try { - await api.conversations.patchTurn(conversationId, serverId, body) - } catch { /* 存不下不该影响正在进行的对话 */ } - }) - saveChains.set(turnId, next) - return next - } +/** 建图时服务重启（或者老轮次：页面被关掉）：没有可恢复的断点 */ +const BUILD_INTERRUPTED: Failure = { + title: '上次未完成', + reason: '生成工作流时页面被关闭或服务重启，本轮没有可恢复的断点。', + hint: '点击「重试本轮」重新开始。', } -function findTurn(conversationId: string, turnId: string): ChatTurn | undefined { - return useChat.getState().byConversation[conversationId]?.find((t) => t.id === turnId) -} - -/** - * 在回收站里的会话只能翻看。从这里发起的运行左栏找不到、没人看着，却照样计费； - * 跑着的时候又不能彻底删除。按钮已经按这个关掉了，这里再守一道 - */ -const inTrash = (conversationId: string) => - useConversations.getState().trash.some((c) => c.id === conversationId) - -/** - * 连同 meta 和复核一起写。两者在后端都是整块替换。review 写 {} 而不是 null：null 等于 - * 「这次不改」，重试之后上一次的复核就留在库里，刷新后挂到新答案头上 - */ -function persist(conversationId: string, turnId: string, body: Record = {}) { - const turn = findTurn(conversationId, turnId) - if (!turn) return Promise.resolve() - return saver(conversationId, turnId)({ ...body, meta: buildMeta(turn), review: turn.reviewRaw ?? {} }) -} - -/** - * 一轮里「当前这一次」的异步流程：建图的 SSE、启动运行、事件流、复核。 - * - * 停止、重试、重新接流都会换一个 epoch，旧流程之后到的东西一律作废。不这样的话， - * 被停掉的 SSE 缓冲里还没派发完的 final 会把这一轮又启动起来；重试之后，上一次 - * 那条流晚到的事件会混进新的一次里。 - */ -interface Flow { - conversationId: string - turnId: string - patch: Patch - save: Save - persist: (body?: Record) => Promise - alive: () => boolean -} - -const epochs = new Map() - -function flowOf(conversationId: string, turnId: string): Flow { - const epoch = (epochs.get(turnId) ?? 0) + 1 - epochs.set(turnId, epoch) - const alive = () => epochs.get(turnId) === epoch - const patch = patcher(useChat.setState as SetState, conversationId, turnId) - const save = saver(conversationId, turnId) +/** 限定的库一个都不在了：补救是「不限数据源重试」，带着原范围重试只会再被拒一次 */ +function scopeGoneFailure(message: string): Failure { + const at = message.indexOf('：') return { - conversationId, - turnId, - alive, - patch: (fn) => { if (alive()) patch(fn) }, - save: (body) => (alive() ? save(body) : Promise.resolve()), - persist: (body) => (alive() ? persist(conversationId, turnId, body) : Promise.resolve()), + title: at > 0 ? message.slice(0, at) : message, + reason: at > 0 ? `${message.slice(at + 1).split('。')[0]}。` : undefined, + hint: '点击「不限数据源重试」；如仍需限定该数据源，请先前往「数据」页确认它存在且已启用。', + detail: message, + scopeGone: true, } } -/** 让这一轮手上所有的旧流程作废 */ -const retire = (turnId: string) => epochs.set(turnId, (epochs.get(turnId) ?? 0) + 1) +/** + * 服务端记的失败原因 → 界面上的说明（说法都在这里和 lib/explain，服务端不写人话标题）。scope 是这一问限定的数据源： + * 开流前被拒时认得出是不是「限定的库都不在了」（认码不认话；不带码的老说法退回按开头认） + */ +function causeFailure(cause: TurnCause, scope?: ScopeSource[]): Failure { + switch (cause.kind) { + case 'refused': { + const message = String(cause.message ?? '') + if (cause.code === 'datasource_scope_empty' || (scope?.length && message.startsWith('限定的数据源'))) { + return scopeGoneFailure(message) + } + return failureOf(message || '助手没有开始') + } + case 'copilot': + // 后端给的是三段：message 人话、hint 怎么办、detail 原始异常。原文收进可展开、可复制的技术细节 + return { + ...failureOf(cause.message || '生成失败'), + ...(cause.hint ? { hint: String(cause.hint) } : {}), + ...(cause.detail ? { detail: String(cause.detail) } : {}), + } + case 'blocked': + // 自查修正过（或重来一次）还是没过。校验消息是写给画布用户的（"先拖一个节点进来"），直接甩给一个在问数据页 + // 打字的人毫无意义——先给一句他能照做的，原文留在技术细节里作线索 + return { + title: '未能生成可运行的工作流', + reason: cause.self_checked === false ? '已自动调整一次，生成的工作流仍未通过检查。' : '助手已按检查结果修正过，生成的工作流仍未通过检查。', + hint: '请补充描述后重新提问，写明要查询的数据库、时间范围和指标。', + detail: (cause.issues ?? []).join('\n') || undefined, + } + case 'start': + return startFailure(new ApiError(Number(cause.status ?? 0), String(cause.message ?? ''), { code: cause.code ?? undefined })) + case 'run': + return runFailure(cause.error ?? '', cause.detail ?? undefined) + case 'gone': + return GONE + case 'build_interrupted': + return BUILD_INTERRUPTED + case 'crash': + return failureOf(cause.message || '发生错误') + } + return failureOf('发生错误') +} + +/** 归档的那一次：失败的照 lib/explain 再讲一遍（服务端只记原话），别的照服务端写的那一句 */ +function withSummary(a: Attempt): Attempt { + if (a.outcome !== 'failed' || !a.cause) return a + const f = causeFailure(a.cause) + return { ...a, summary: lineOf(f), ...(f.detail && !a.detail ? { detail: f.detail } : {}) } +} + +const attemptsOf = (list: unknown): Attempt[] | undefined => + Array.isArray(list) && list.length ? (list as Attempt[]).map(withSummary) : undefined + +/** 被驳回的那一句：服务端记的是谁、哪一步、备注，按 REJECTED_TEXT.sub 拼；老轮次记的是拼好的那一句 */ +function rejectedOf(r: TurnMeta['rejected']): Rejected | null { + if (!r || typeof r !== 'object') return null + return { sub: typeof r.sub === 'string' && r.sub ? r.sub : REJECTED_TEXT.sub(r) } +} // ------------------------------------------------------------------------- // 从库里恢复 // ------------------------------------------------------------------------- +/** 服务端还在跑的这一轮，它在哪一步 → 这一页的相位和进度 */ +function livePhase(job: ChatTurnJob): { phase: Phase; status: string } { + if (job.stage === 'waiting') return { phase: 'waiting', status: PHASE_TEXT.waiting } + if (job.stage === 'run') return { phase: 'running', status: PHASE_TEXT.running } + if (job.stage === 'review') return { phase: 'running', status: REVIEWING } + return { phase: 'planning', status: PHASE_TEXT.planning } +} + function restoreTurn(t: ConversationTurn): ChatTurn { const meta = metaOf(t) const review = reviewOf(t) @@ -724,15 +691,20 @@ function restoreTurn(t: ConversationTurn): ChatTurn { // 老数据没有 meta：停止写的是 status=error + 「已取消」 const outcome = meta?.outcome ?? (t.status === 'error' && t.error === '已取消' ? 'cancelled' : undefined) const answer = t.answer ?? '' - // 老数据没有 meta.failure，error 列里是原始异常：有运行的按运行失败解释 + // 老轮次记的是翻好的 failure；服务端写的记原因（cause）；再老的数据两样都没有，error 列里是原始异常：有运行的按运行失败解释 const failure = meta?.failure + ?? (meta?.cause ? causeFailure(meta.cause, meta.scope) : null) ?? (t.error && t.error !== '已取消' ? (runId ? runFailure(t.error) : failureOf(t.error)) : null) + const live = t.status === 'running' && t.job && LIVE_JOB.has(t.job.status) ? livePhase(t.job) : null let phase: Phase let status: string - if (t.status === 'running') { - // 上次停在半路：页面被关掉、服务重启、或者还在等审批。有运行就去问后端它 - // 现在怎样了；没有运行说明断在建图阶段，没有断点可续 + if (live) { + // 服务端还在跑：hydrate 从这一次尝试的第一条事件接着播 + ;({ phase, status } = live) + } else if (t.status === 'running') { + // 上次停在半路（服务重启后没接着收尾、或者这一波之前页面推的老轮次）：有运行就请服务端接着收尾；没有运行说明断在 + // 建图阶段，没有断点可续 phase = runId ? 'checking' : 'error' status = runId ? PHASE_TEXT.checking : PHASE_TEXT.error } else if (t.status === 'error') { @@ -746,22 +718,18 @@ function restoreTurn(t: ConversationTurn): ChatTurn { status = PHASE_TEXT.done } - const buildInterrupted: Failure = { - title: '上次未完成', - reason: '生成工作流时页面被关闭或服务重启，本轮没有可恢复的断点。', - hint: '点击「重试本轮」重新开始。', - } const output: Record | null = answer ? { answer, ...(meta?.issuance ? { _issuance: meta.issuance } : {}) } : null const graphful = !!t.graph && ((t.graph as GraphSpec).nodes?.length ?? 0) > 0 + const rejected = phase === 'cancelled' ? rejectedOf(meta?.rejected) : null return { id: t.id, serverId: t.id, question: t.question, phase, - status, + status: phase === 'cancelled' && rejected ? REJECTED_TEXT.title : status, thinking: '', ops: [], events: [], @@ -770,9 +738,9 @@ function restoreTurn(t: ConversationTurn): ChatTurn { // 先给个壳：run_class 从 meta 来；usage、完整成果由 hydrate 回源补 run: runId ? ({ id: runId, run_class: meta?.runClass } as Run) : null, output, - error: phase === 'error' ? (failure ? lineOf(failure) : t.status === 'running' ? buildInterrupted.title : '') : '', - failure: phase === 'error' ? (t.status === 'running' ? buildInterrupted : failure) : phase === 'suspended' ? SUSPENDED : null, - rejected: phase === 'cancelled' ? meta?.rejected ?? null : null, + error: phase === 'error' ? (failure ? lineOf(failure) : t.status === 'running' ? BUILD_INTERRUPTED.title : '') : '', + failure: phase === 'error' ? (t.status === 'running' ? BUILD_INTERRUPTED : failure) : phase === 'suspended' ? SUSPENDED : null, + rejected, startedAt: Date.parse(t.created_at ?? '') || Date.now(), endedAt: meta?.ms != null ? (Date.parse(t.created_at ?? '') || 0) + meta.ms : undefined, lastSeq: 0, @@ -790,7 +758,7 @@ function restoreTurn(t: ConversationTurn): ChatTurn { review: review && review.verdict !== 'ok' ? review : null, reviewRaw: review, rawOutput: review?.original ? { answer: review.original } : null, - attempts: meta?.attempts, + attempts: attemptsOf(meta?.attempts), // 上次记下没取全的：运行还在就等 hydrate 补，运行没了就补不回来了。 // 老数据没记过，只能看是不是恰好切在 2000 字——运行也删了的，至少别装作完整 clipped: meta?.clipped === 'partial' ? (runId ? 'partial' : 'lost') @@ -799,11 +767,16 @@ function restoreTurn(t: ConversationTurn): ChatTurn { meta, scope: meta?.scope, asked: meta?.asked && answer ? answer : undefined, + usage: meta?.usage, + job: t.job ?? null, } } /** 恢复出来的这一轮要不要马上向后端核对（而不是等它滚进视口） */ function needsEagerHydrate(t: ChatTurn): boolean { + // 服务端还在跑：马上接着播 + if (t.job && LIVE_JOB.has(t.job.status) && BUSY_PHASES.has(t.phase)) return true + if (t.phase === 'waiting' && t.job && LIVE_JOB.has(t.job.status)) return true if (!t.run?.id) return false if (t.phase === 'checking' || t.phase === 'suspended') return true // 「接着跑」给不给取决于运行现在的状态；老数据没存过 @@ -815,6 +788,53 @@ function needsEagerHydrate(t: ChatTurn): boolean { && (t.clipped === 'partial' || (typeof t.output?.answer === 'string' && atClip(t.output.answer))) } +function findTurn(conversationId: string, turnId: string): ChatTurn | undefined { + return useChat.getState().byConversation[conversationId]?.find((t) => t.id === turnId) +} + +/** + * 在回收站里的会话只能翻看。从这里发起的运行左栏找不到、没人看着，却照样计费； + * 跑着的时候又不能彻底删除。按钮已经按这个关掉了，这里再守一道 + */ +const inTrash = (conversationId: string) => + useConversations.getState().trash.some((c) => c.id === conversationId) + +// ------------------------------------------------------------------------- +// 这一页上的「当前这一次」：播放这一轮的事件日志、接运行的实时流 +// ------------------------------------------------------------------------- + +/** + * 一轮里「当前这一次」在这一页上开着的流。 + * + * 停止、重试、重新接流都会换一个 epoch，旧的流之后到的东西一律作废：重试之后，上一次那条流晚到的事件不能混进新的一次里 + */ +interface Flow { + conversationId: string + turnId: string + patch: Patch + alive: () => boolean +} + +const epochs = new Map() + +function flowOf(conversationId: string, turnId: string): Flow { + const epoch = (epochs.get(turnId) ?? 0) + 1 + epochs.set(turnId, epoch) + const alive = () => epochs.get(turnId) === epoch + const patch = patcher(useChat.setState as SetState, conversationId, turnId) + return { conversationId, turnId, alive, patch: (fn) => { if (alive()) patch(fn) } } +} + +/** 让这一轮手上所有的旧流程作废 */ +const retire = (turnId: string) => epochs.set(turnId, (epochs.get(turnId) ?? 0) + 1) + +/** 正在播放的轮次：能从外面让它重新接上运行的实时流（审批过了） */ +interface Player { + flow: Flow + openRun: (runId: string, from?: number) => void + closeRun: () => void +} +const players = new Map() export const useChat = create((set, get) => ({ byConversation: {}, @@ -829,7 +849,7 @@ export const useChat = create((set, get) => ({ forget: (conversationId) => set((s) => { - s.byConversation[conversationId]?.forEach((t) => { retire(t.id); t.cancel?.() }) + s.byConversation[conversationId]?.forEach((t) => { retire(t.id); t.cancel?.(); players.delete(t.id) }) const { [conversationId]: _gone, ...rest } = s.byConversation const { [conversationId]: _state, ...states } = s.loadState return { byConversation: rest, loadState: states } @@ -860,22 +880,10 @@ export const useChat = create((set, get) => ({ ...(lastScope && !(conversationId in s.scopes) ? { scopes: { ...s.scopes, [conversationId]: lastScope } } : {}), } }) - // 内存里已有的那几轮没有并进来，下面对它们的 hydrate / persist 按 id 找不到，自然跳过 - const now = Date.now() - detail.turns.forEach((row, i) => { - const t = restored[i] + // 内存里已有的那几轮没有并进来，下面对它们的 hydrate 按 id 找不到，自然跳过 + for (const t of restored) { if (needsEagerHydrate(t)) void get().hydrate(conversationId, t.id) - // 断在建图阶段、早就没动静的：库里那行一直是 running，每次进来都当它还在跑， - // Copilot 的上下文也把它当成没答完的一轮漏掉。认定断了就写回去。 - // 动静按最后一次落库算：重试昨天那一轮时 created_at 是昨天，而它正在别的页面里建图 - // 后端判「断了」按 updated_at（conversations.BUILD_STALE），有就用同一只钟；老后端 - // 没有这一列，退到 created_at 和本机记下的 meta.at 里晚的那个 - const updated = parseServerTime(row.updated_at ?? null)?.getTime() - const at = updated ?? Math.max(parseServerTime(row.created_at ?? null)?.getTime() ?? 0, t.meta?.at ?? 0) - if (row.status === 'running' && !t.run && t.phase === 'error' && at && now - at > BUILD_STALE_MS) { - void persist(conversationId, t.id, { status: 'error', error: lineOf(t.failure ?? GONE) }) - } - }) + } } catch (e) { set((s) => ({ loadState: { ...s.loadState, [conversationId]: 'error' }, @@ -929,21 +937,11 @@ export const useChat = create((set, get) => ({ // 没取过的会话能走到这里，只可能是刚建的空会话：它没有历史可丢 loadState: state ? s.loadState : { ...s.loadState, [conversationId]: 'ready' }, })) - const flow = flowOf(conversationId, id) - useConversations.getState().touch(conversationId, question) - - // 先开一轮，拿到 serverId 再开始建图。顺序不能反：Copilot 那边要读 - // 这次会话的历史，而"这一轮"必须已经是 running 状态才不会被算进自己的上下文 - const started = api.conversations.startTurn(conversationId, question) - .then((turn) => { - patcher(set, conversationId, id)(() => ({ serverId: turn.id })) - return turn.id - }) - .catch(() => null) - serverIds.set(id, started) - - plan(flow, { question, instruction: question, attempt: 0, scope }) + void launchTurn(flowOf(conversationId, id), { + conversation_id: conversationId, question, + ...(scope.length ? { datasource_ids: scope.map((d) => d.id) } : {}), + }) return true }, @@ -954,41 +952,41 @@ export const useChat = create((set, get) => ({ if (!target) return retire(target.id) target.cancel?.() - // 上面只是把自己这头的事件流关了。运行在后端，断开订阅不会让它停： - // 以前点了「停止」，模型和 SQL 其实还在跑、还在计费，界面上却已经显示"已取消" - if (target.run?.id) { - void api.runs.cancel(target.run.id).catch(() => undefined) // 已经跑完会回 409，无妨 - } + players.delete(target.id) + // 上面只是把自己这头的流关了。这一轮在服务端跑，断开订阅不会让它停：停止要告诉服务端，它连同运行一起停。 + // 发起请求还在路上的（没有 serverId），launchTurn 拿到 id 后看到这一轮已作废，会补一个停止过去 + if (target.serverId) void api.chatTurns.cancel(target.serverId).catch(() => undefined) // 已经结束会回 409，无妨 patcher(set, conversationId, target.id)(() => ({ phase: 'cancelled', status: PHASE_TEXT.cancelled, cancel: null, error: '', failure: null, endedAt: Date.now(), writing: null, ...(target.run ? { runStatus: 'cancelled', final: { status: 'cancelled' } } : {}), })) - void persist(conversationId, target.id, { status: 'error', error: '已取消' }) }, runNow: (conversationId, turnId) => { const turn = findTurn(conversationId, turnId) - if (!turn?.graph || isBusy(get().byConversation[conversationId]) || inTrash(conversationId)) return + if (!turn?.graph || !turn.serverId || isBusy(get().byConversation[conversationId]) || inTrash(conversationId)) return const flow = flowOf(conversationId, turnId) flow.patch(() => ({ phase: 'running', status: PHASE_TEXT.running, pendingRun: false, startedAt: Date.now(), endedAt: undefined, })) - void launch(flow, turn.graph, turn.question) + void launchTurn(flow, { conversation_id: conversationId, retry_of: turn.serverId, mode: 'run' }) }, continueTurn: async (conversationId, turnId) => { // 从断点接着跑。**不带图**——问数据页的用户改不了图，这里就是"用原来的 // 配置重试失败的那一步"，对超时、限流、网络抖、服务重启这类瞬时失败有用。 // 配置本身错了（模型 id 写错、缺必填输入）要到画布上改，那边的「接着跑」 - // 会把改过的图一起带上。 - // - // 接上之后 run.finished 照常触发 settle，复核层自动覆盖这一次 + // 会把改过的图一起带上。接上之后服务端照常等它跑完、复核、收尾 const turn = findTurn(conversationId, turnId) - if (!turn?.run || isBusy(get().byConversation[conversationId]) || inTrash(conversationId)) return false + if (!turn?.run || !turn.serverId || isBusy(get().byConversation[conversationId]) || inTrash(conversationId)) return false try { - await api.runs.continue(turn.run.id) + const res = await api.chatTurns.start({ conversation_id: conversationId, retry_of: turn.serverId, mode: 'continue' }) + const flow = flowOf(conversationId, turnId) + flow.patch(() => ({ phase: 'running', status: PHASE_TEXT.running, error: '', failure: null, job: res.turn?.job ?? null })) + play(flow, turn.serverId, Math.max(0, (res.turn?.job?.attempt_seq ?? 1) - 1)) + return true } catch (e) { // 409 的 detail 是人话（「这次运行已取消，不能接着跑…」），交给调用方弹出来。 // 顺手再核对一次：按钮是照着过时的状态给的，别让它留在那里被点第二次 @@ -996,8 +994,6 @@ export const useChat = create((set, get) => ({ void get().hydrate(conversationId, turnId) throw e } - await get().reattach(conversationId, turnId) - return true }, retryTurn: (conversationId, turnId, opts) => { @@ -1005,69 +1001,54 @@ export const useChat = create((set, get) => ({ const turn = turns?.find((t) => t.id === turnId) if (!turn || isBusy(turns) || inTrash(conversationId)) return turn.cancel?.() - // 服务重启挂起的旧运行没人会再去续它：不取消的话，它在记录里一直是「已挂起 · 可续跑」。 - // 停在审批上的不算——重试不会从那里发起 - if (turn.run?.id && turn.phase !== 'waiting' && (turn.phase === 'suspended' || turn.runStatus === 'interrupted')) { - void api.runs.cancel(turn.run.id).catch(() => undefined) - } const flow = flowOf(conversationId, turnId) - const base = turn.graph && (turn.graph.nodes?.length ?? 0) > 0 ? turn.graph : null - const record = attemptOf(turn) - const reason = record.detail ? `${record.summary}（${record.detail.slice(0, 300)}）` : record.summary - const direct = !!base && !!(opts?.maxSteps || opts?.rerun) + const base = turn.graph && (turn.graph.nodes?.length ?? 0) > 0 + const mode = base && opts?.maxSteps ? 'max_steps' as const : base && opts?.rerun ? 'rerun' as const : 'retry' as const const scope = opts?.scope ?? turn.scope - - // 上一次原样留档，这一次从头记：旧的事件、答案、复核留在这里只会让人分不清 - // 哪段是最终那次 - flow.patch((t) => ({ - phase: direct ? 'running' : 'planning', - status: opts?.maxSteps ? `正在将步数上限放宽到 ${opts.maxSteps} 并重新运行…` - : direct ? '正在重新运行本轮…' : '正在按原问题重新生成…', - attempts: [...(t.attempts ?? []), record], - ops: [], events: [], run: null, output: null, review: null, reviewRaw: null, - rawOutput: null, failure: null, error: '', lastSeq: 0, final: undefined, - runStatus: undefined, clipped: undefined, pendingRun: false, steps: undefined, - hydrated: 'done', startedAt: Date.now(), endedAt: undefined, attempt: 0, cancel: null, - noQuery: false, thinking: '', writing: null, scope: scope?.length ? scope : undefined, - // 上一次的运行换下来了，它花的钱还算在这一轮里 - usage: retireRun(t), + // 先把相位摆出来；上一次怎么归档、这一次从哪开始，等服务端的 turn.attempt 来了照它重置 + flow.patch(() => ({ + phase: mode === 'retry' ? 'planning' : 'running', status: retryStatus(mode, opts?.maxSteps), + failure: null, error: '', pendingRun: false, hydrated: 'done', endedAt: undefined, cancel: null, + startedAt: Date.now(), writing: null, scope: scope?.length ? scope : undefined, })) - void flow.persist({ status: 'running', error: '', answer: '' }) - - if (direct && base) { - const graph = opts?.maxSteps ? withMaxSteps(base, opts.maxSteps) : base - flow.patch(() => ({ graph })) - void launch(flow, graph, turn.question) + if (!turn.serverId) { + // 发起那一下就没到服务端（这一轮在库里不存在）：原样再问一次 + void launchTurn(flow, { + conversation_id: conversationId, question: turn.question, + ...(scope?.length ? { datasource_ids: scope.map((d) => d.id) } : {}), + }) return } - plan(flow, { - question: turn.question, attempt: 0, baseGraph: base, scope, - instruction: base - ? `这个工作流上一次没有给出可用的结果：${reason}。\n\n` - + `请针对这个原因改它（比如调大最大步数、改正工具参数、换个查询方式），` - + `改完继续回答原来的问题：${turn.question}` - : `${turn.question}\n\n（上一次没有成功：${reason}。请针对这个原因调整，重新给出完整的工作流。）`, + void launchTurn(flow, { + conversation_id: conversationId, retry_of: turn.serverId, mode, + ...(mode === 'max_steps' ? { max_steps: opts!.maxSteps } : {}), + ...(opts?.scope ? { datasource_ids: opts.scope.map((d) => d.id) } : {}), }) }, hydrate: async (conversationId, turnId) => { const turn = findTurn(conversationId, turnId) - const runId = turn?.run?.id - if (!turn || !runId || turn.hydrated) return + if (!turn || turn.hydrated) return + const patch = patcher(set, conversationId, turnId) + // 服务端还在跑这一轮（刷新、关掉再打开）：从这一次尝试的第一条事件接着播 + if (turn.serverId && turn.job && LIVE_JOB.has(turn.job.status)) { + patch(() => ({ hydrated: 'done' })) + play(flowOf(conversationId, turnId), turn.serverId, Math.max(0, turn.job.attempt_seq - 1)) + return + } + // 库里那一行停在半路、服务端又没在跑它（服务重启后没接着收尾、这一波之前页面推的老轮次）：请服务端接着等那次运行、 + // 照常复核收尾，不在这一页上自己收尾 + if (turn.phase === 'checking' && turn.serverId && turn.run?.id) { + patch(() => ({ hydrated: 'pending' })) + await resumeTurn(conversationId, turnId) + return + } + const runId = turn.run?.id + if (!runId) return // 核对期间用户点了重试或停止：拿回来的是上一次的状态，作废 const epoch = epochs.get(turnId) ?? 0 const stale = () => (epochs.get(turnId) ?? 0) !== epoch - const patch = patcher(set, conversationId, turnId) patch(() => ({ hydrated: 'pending' })) - /** - * 查到了库里那行还不知道的结局：内存里改，库里也改。只改内存的话那行永远是 - * running——每次进来都要再核对一遍，Copilot 的上下文把它当没答完的一轮漏掉， - * 运行一删它就只能退成「上次没有跑完」，连「已取消」都说不出来了 - */ - const conclude = (fields: Partial, error: string) => { - patch(() => fields) - void persist(conversationId, turnId, { status: 'error', error }) - } let run: Run try { run = await api.runs.get(runId) @@ -1075,33 +1056,10 @@ export const useChat = create((set, get) => ({ const cur = findTurn(conversationId, turnId) if (stale() || !cur) return if (e instanceof ApiError && e.status === 404) { - if (cur.phase === 'checking') { - conclude({ - hydrated: 'done', runStatus: 'gone', phase: 'error', status: PHASE_TEXT.error, - error: GONE.title, failure: GONE, - }, GONE.title) - return - } - // 没取全的答案，运行又删了：补不回来，但得说出来，也记下来 + // 没取全的答案，运行又删了：补不回来，但得说出来 const lost = cur.phase === 'done' && !cur.review?.answer && (cur.clipped === 'partial' || atClip(String(cur.output?.answer ?? ''))) patch((t) => ({ hydrated: 'done', runStatus: 'gone', clipped: lost ? 'lost' : t.clipped })) - if (lost) void persist(conversationId, turnId) - } else if (cur.phase === 'checking') { - // 后端出错、超时：不知道它现在怎样，但不能一直停在「正在核对」转圈——那之后 - // 再也没有人来核对（重连只在断开过之后才触发）。说清楚，给「重新核对」 - const said = errorMessage(e) - const raw = humanizeError(e).raw - const failure: Failure = { - title: PHASE_TEXT.unknown, - reason: `查询本轮运行状态时出错（${said}）。运行可能仍在进行，也可能已结束。`, - hint: '请稍后点击「重新核对」；服务端重新连接后会自动再核对一次。', - // 原话已经在括号里的，技术细节不再重复一遍 - detail: raw && !said.includes(raw) ? raw : undefined, - } - patch(() => ({ - hydrated: undefined, phase: 'unknown', status: PHASE_TEXT.unknown, failure, error: lineOf(failure), - })) } else { patch(() => ({ hydrated: undefined })) // 已经有个结局摆着，下次再补 } @@ -1111,63 +1069,16 @@ export const useChat = create((set, get) => ({ const cur = findTurn(conversationId, turnId) if (!cur || stale()) return patch(() => ({ run, runStatus: run.status, final: { status: run.status, pending }, hydrated: 'done' })) - const unsettled = cur.phase === 'checking' || cur.phase === 'suspended' || cur.phase === 'error' - - // 这一轮其实还活着：接回去，别让它在历史里装死 - if (run.status === 'running' || run.status === 'queued') { - if (unsettled) { - const flow = flowOf(conversationId, turnId) - // 计时不从提问那天算：中断过的接着上次停下的地方走，停在半路的从运行开始算 - const spent = cur.endedAt ? Math.max(0, cur.endedAt - cur.startedAt) : null - const startedAt = spent != null ? Date.now() - spent - : parseServerTime(run.started_at ?? null)?.getTime() ?? cur.startedAt - flow.patch(() => ({ - phase: 'running', status: PHASE_TEXT.running, failure: null, error: '', startedAt, endedAt: undefined, - })) - watch(flow, run.id, cur.lastSeq, cur.question) - } + const settled = cur.phase === 'suspended' || cur.phase === 'error' + // 已经收过尾的这一轮其实又活了（别处接着跑了、通过了审批、跑完了）：请服务端接着等它、照常复核收尾 + const revived = run.status === 'running' || run.status === 'queued' || (run.status === 'interrupted' && pending) + || run.status === 'succeeded' + if (settled && revived && cur.serverId) { + await resumeTurn(conversationId, turnId) return } - if (run.status === 'interrupted') { - if (pending) { - // 还在等人：库里那行保持 running 就是对的，它确实没结束 - patch(() => ({ phase: 'waiting', status: PHASE_TEXT.waiting, failure: null, error: '' })) - void useCatalog.getState().refreshApprovals() - } else if (cur.phase !== 'done' && cur.phase !== 'suspended') { - conclude({ phase: 'suspended', status: PHASE_TEXT.suspended, failure: SUSPENDED, error: '' }, PHASE_TEXT.suspended) - } - return - } - if (run.status === 'cancelled') { - if (cur.phase !== 'done' && cur.phase !== 'cancelled') { - // 被驳回只看服务端的 end_reason：核对时手上只有运行记录，副标题用报错原文（它就是给人看的那句） - const rejected = run.end_reason === 'rejected' ? { sub: run.error || REJECTED_TEXT.title } : null - conclude({ phase: 'cancelled', status: rejected ? REJECTED_TEXT.title : PHASE_TEXT.cancelled, failure: null, - error: '', rejected }, '已取消') - } - return - } - if (run.status === 'failed') { - if (cur.phase === 'error' && cur.failure) return - if (cur.phase !== 'done') { - const failure = runFailure(run.error || '') - conclude({ phase: 'error', status: PHASE_TEXT.error, failure, error: lineOf(failure) }, lineOf(failure)) - } - return - } - if (run.status !== 'succeeded') return - if (unsettled) { - // 跑完了，但交付没赶上（复核期间关了页面、服务重启）：把这一轮补完， - // 走一遍和实时一样的复核与落库。相位停在「正在核对」而不是「运行中」：运行 - // 早就结束了，按实时算的计时会从提问那天一路数到现在（「72:00:01」） - const flow = flowOf(conversationId, turnId) - flow.patch(() => ({ phase: 'checking', status: '正在核对结果…', failure: null, error: '' })) - // 耗时按运行真正结束的时刻算：补交付可能发生在几天之后 - const endedAt = parseServerTime(run.finished_at ?? null)?.getTime() - void settle(flow, run.id, cur.question, run.output ?? null, { run, endedAt }) - return - } - // 已经交付过的：答案没取全就换成完整的，顺手把档位补回来 + if (run.status !== 'succeeded' || cur.phase !== 'done') return + // 已经交付过的：答案没取全就换成完整的（老轮次），顺手把档位补回来 const full = answerText(run.output) const stored = String(cur.output?.answer ?? '') const repaired = !cur.review?.answer && stored.length >= 200 @@ -1175,7 +1086,6 @@ export const useChat = create((set, get) => ({ const issuance = slimIssuance(run.output?._issuance) if (repaired) { patch(() => ({ output: run.output, clipped: undefined })) - void persist(conversationId, turnId, { answer: full }) } else if (hasEvidence(run.output) && !hasEvidence(cur.output) && !cur.review?.answer) { // 库里只存了答案的文字，逐段证据的标注（_evidence）和字段名都在运行的成果里： // 换回运行那份，刷新之后报告里的数字照样能点开。复核改写过的不换（有证据的不会被改写） @@ -1197,27 +1107,81 @@ export const useChat = create((set, get) => ({ reattach: async (conversationId, turnId) => { const turn = findTurn(conversationId, turnId) if (!turn?.run) return - // 等审批时的那条流后端不关；不先关掉它，恢复后的事件会从两条流各来一遍 - turn.cancel?.() - const flow = flowOf(conversationId, turnId) - // 计时接着上次停下的地方走，不从提问那一刻算：中断、失败之后隔了一夜才点 - // 「接着跑」，头部不该写「已运行 14 小时」。同一次会话里等审批的那段照算， - // 它是这一轮过程的一部分 - const spent = turn.endedAt ? Math.max(0, turn.endedAt - turn.startedAt) : 0 - const startedAt = turn.endedAt || turn.restored ? Date.now() - spent : turn.startedAt - // 上一轮的错要清掉：接着跑之后它还挂在那里，就成了一条已经不成立的红字 - flow.patch(() => ({ - phase: 'running', status: PHASE_TEXT.running, error: '', failure: null, startedAt, - final: undefined, runStatus: 'running', endedAt: undefined, cancel: null, writing: null, - })) - watch(flow, turn.run.id, turn.lastSeq, turn.question) + // 这一页正在播这一轮：审批过了，服务端接着等那次运行，这里把运行的实时流重新接上 + const player = players.get(turnId) + if (player?.flow.alive()) { + player.openRun(turn.run.id, turn.lastSeq) + return + } + // 没在播（老轮次、刷新前停在审批上的）：请服务端接着等这次运行、照常收尾 + if (turn.serverId) await resumeTurn(conversationId, turnId) }, })) if (import.meta.env.DEV) { ;(window as any).__chat = useChat - // 检查脚本用：把等复核的上限调短（毫秒），量「复核晚于上限时用量照记」 - ;(window as any).__chatReviewWait = (ms: number) => { reviewWaitMs = ms } +} + +/** 用户点重试时先摆出来的那一句（服务端的 turn.attempt 来了照它再说一遍） */ +function retryStatus(mode: 'retry' | 'rerun' | 'max_steps' | string, maxSteps?: number): string { + return mode === 'max_steps' ? `正在将步数上限放宽到 ${maxSteps} 并重新运行…` + : mode === 'rerun' ? '正在重新运行本轮…' + : mode === 'retry' ? '正在按原问题重新生成…' + : mode === 'run' || mode === 'continue' || mode === 'resume' ? PHASE_TEXT.running + : PHASE_TEXT.planning +} + +/** + * 发起一轮（新问、重试本轮、重新运行、跑一下），拿到 id 就开始播。发起那一下就被拒（409 这段对话正忙、网络断了……）： + * 说清楚，给「重试本轮」——服务端没有这一轮的话就原样再问一次 + */ +async function launchTurn(flow: Flow, body: ChatTurnStart) { + try { + const res = await api.chatTurns.start(body) + // 发起请求在路上时用户点了停止或重试：服务端那一轮已经起来了，得追一个停止过去 + if (!flow.alive()) { + void api.chatTurns.cancel(res.turn_id).catch(() => undefined) + return + } + flow.patch(() => ({ serverId: res.turn_id, job: res.turn?.job ?? null })) + play(flow, res.turn_id, Math.max(0, (res.turn?.job?.attempt_seq ?? 1) - 1)) + } catch (e) { + if (!flow.alive()) return + const failure = failureOf(e) + flow.patch(() => ({ + phase: 'error', status: PHASE_TEXT.error, error: lineOf(failure), failure, cancel: null, endedAt: Date.now(), + })) + } +} + +/** + * 请服务端接着等这一轮的那次运行、照常复核收尾（mode=resume），再接着播。核对撞上后端出错、超时：不知道它现在怎样， + * 但不能一直停在「正在核对」转圈——说清楚，给「重新核对」 + */ +async function resumeTurn(conversationId: string, turnId: string) { + const turn = findTurn(conversationId, turnId) + if (!turn?.serverId) return + const flow = flowOf(conversationId, turnId) + try { + const res = await api.chatTurns.start({ conversation_id: conversationId, retry_of: turn.serverId, mode: 'resume' }) + if (!flow.alive()) return + flow.patch(() => ({ hydrated: 'done', job: res.turn?.job ?? null })) + play(flow, turn.serverId, Math.max(0, (res.turn?.job?.attempt_seq ?? 1) - 1)) + } catch (e) { + if (!flow.alive()) return + const said = errorMessage(e) + const raw = humanizeError(e).raw + const failure: Failure = { + title: PHASE_TEXT.unknown, + reason: `查询本轮运行状态时出错（${said}）。运行可能仍在进行，也可能已结束。`, + hint: '请稍后点击「重新核对」；服务端重新连接后会自动再核对一次。', + // 原话已经在括号里的，技术细节不再重复一遍 + detail: raw && !said.includes(raw) ? raw : undefined, + } + flow.patch(() => ({ + hydrated: undefined, phase: 'unknown', status: PHASE_TEXT.unknown, failure, error: lineOf(failure), + })) + } } /** 定点更新某个会话里的某一轮。认的是 turn id，所以切走看别的会话也不受影响 */ @@ -1257,35 +1221,6 @@ async function pendingApproval(runId: string): Promise { } } -/** 这一轮现在的样子，折成一条留档 */ -function attemptOf(t: ChatTurn): Attempt { - const n = (t.attempts?.length ?? 0) + 1 - const answer = answerText(t.output) || undefined - const base = { - n, at: t.endedAt ?? Date.now(), runId: t.run?.id ?? null, - answer: answer && answer.length > ATTEMPT_ANSWER_CAP ? answer.slice(0, ATTEMPT_ANSWER_CAP) + '…' : answer, - } - const signals = (t.review?.signals ?? []).map((s) => s.detail).join('\n') || undefined - // 被驳回的那次照实写谁在哪一步驳回（以前写「已手动停止」，用户其实没有停过它） - if (t.phase === 'cancelled' && t.rejected) { - const sub = t.rejected.sub && t.rejected.sub !== REJECTED_TEXT.title ? `：${t.rejected.sub}` : '' - return { ...base, outcome: 'rejected', summary: `${REJECTED_TEXT.title}${sub}` } - } - if (t.phase === 'cancelled') return { ...base, outcome: 'cancelled', summary: '已手动停止' } - if (t.phase === 'suspended') return { ...base, outcome: 'suspended', summary: '服务重启，运行中断' } - if (t.phase === 'error' || t.failure) { - const f = t.failure ?? failureOf(t.error || '发生错误') - return { ...base, outcome: 'failed', summary: lineOf(f), detail: f.detail } - } - if (t.review?.severity === 'broken') { - return { ...base, outcome: 'unusable', summary: `结论不可用：${t.review.note}`, detail: signals } - } - if (t.review?.severity === 'degraded') { - return { ...base, outcome: 'partial', summary: `存在缺口：${t.review.note}`, detail: signals } - } - return { ...base, outcome: 'done', summary: '已给出答案' } -} - /** * 图里 agent 节点的步数上限（取最大的那个）。没配的跟随设置里的默认值 fallback； * 以前画布自动写进去的 12 后端按「没配」处理（engine/guards.py），这里口径一致 @@ -1299,381 +1234,255 @@ export function agentSteps(graph: GraphSpec | null | undefined, fallback: number })) } -function withMaxSteps(graph: GraphSpec, steps: number): GraphSpec { +// ------------------------------------------------------------------------- +// 播放：服务端这一轮的事件日志 → 这一轮的样子 +// ------------------------------------------------------------------------- + +/** 这一次尝试开始了（turn.attempt）：上一次的过程、答案、复核都清掉，归档照服务端给的 */ +function attemptReset(t: ChatTurn, d: Record, ts: number): Partial { + const mode = String(d.mode ?? 'ask') + const attempts = attemptsOf(d.attempts) ?? (mode === 'run' || mode === 'continue' || mode === 'resume' ? t.attempts : undefined) + const scope = Array.isArray(d.scope) && d.scope.length ? d.scope as ScopeSource[] : t.scope + const usage = d.usage && typeof d.usage === 'object' ? d.usage as TurnUsage : t.usage + const startedAt = typeof d.started_ms === 'number' ? d.started_ms : ts ? ts * 1000 : Date.now() + if (mode === 'continue' || mode === 'resume') { + // 接着等同一次运行：过程照旧留着，运行的流接着上次收到的地方。运行早就跑完了的（补交付）停在「正在核对」、计时停在 + // 运行结束的那一刻：按实时算会从提问那天一路数到现在（「72:00:01」） + const ended = typeof d.ended_ms === 'number' ? d.ended_ms : undefined + return { + phase: ended ? 'checking' : mode === 'resume' && t.phase === 'waiting' ? 'waiting' : 'running', + status: ended ? REVIEWING : PHASE_TEXT.running, + error: '', failure: null, final: undefined, endedAt: ended, writing: null, startedAt, attempts, scope, usage, + hydrated: 'done', + } + } + if (mode === 'run') { + // 只建了图、用户点「运行」：建图的过程照旧留着，只换上这一次的运行 + return { + phase: 'running', status: PHASE_TEXT.running, attempts, scope, usage, + events: [], run: null, output: null, review: null, reviewRaw: null, rawOutput: null, failure: null, error: '', + lastSeq: 0, final: undefined, rejected: null, runStatus: undefined, pendingRun: false, hydrated: 'done', + startedAt, endedAt: undefined, writing: null, noQuery: false, + } + } return { - ...graph, - nodes: graph.nodes.map((n: any) => (n.type === 'agent' - ? { ...n, data: { ...n.data, config: { ...(n.data?.config ?? {}), max_steps: steps } } } - : n)), + phase: mode === 'ask' || mode === 'retry' ? 'planning' : 'running', + status: mode === 'ask' ? PHASE_TEXT.planning : retryStatus(mode, Number(d.max_steps) || undefined), + attempts, scope, usage, + ops: [], events: [], run: null, output: null, review: null, reviewRaw: null, + rawOutput: null, failure: null, error: '', lastSeq: 0, final: undefined, rejected: null, + runStatus: undefined, clipped: undefined, pendingRun: false, steps: undefined, + hydrated: 'done', startedAt, endedAt: undefined, attempt: 0, + noQuery: false, thinking: '', writing: null, asked: undefined, + } +} + +/** 助手的一条操作：原样收下（思考增量并进上一条），照原来的样子更新进度。要不要运行、要不要重来都等服务端说 */ +function opPatch(t: ChatTurn, op: CopilotOp): Partial { + const last = t.ops[t.ops.length - 1] + const ops = op.op === 'thinking' && last?.op === 'thinking' + ? [...t.ops.slice(0, -1), { ...last, delta: String(last.delta ?? '') + String(op.delta ?? '') }] + : [...t.ops, op] + switch (op.op) { + case 'thinking': + return { ops, phase: 'planning', thinking: (t.thinking + (op.delta ?? '')).slice(-2000) } + case 'heartbeat': + // 查看进行中（looking）不是这一页的阶段：浮条直说「正在查看表结构…」 + return { ops, status: op.phase === 'looking' ? LOOK_TEXT.running + : PHASE_TEXT[(op.phase as Phase) ?? 'planning'] ?? '正在处理…' } + case 'look': + case 'context': { + // 查看结束：浮条换成查看了什么 + const look = op.op === 'look' ? copilotLook(op) : copilotContext(op) + return look ? { ops, status: look.title } : { ops } + } + case 'plan': + return { ops, phase: 'building', status: op.summary || '正在生成工作流…' } + case 'add_node': + return { ops, phase: 'building', status: `正在生成工作流…（${ops.filter((o) => o.op === 'add_node').length} 步）` } + case 'use_caliber': + // 助手选用了一份已发布口径（取数节点和口径卡由服务端生成，过程里那一行由 decodeCopilot 写） + return { ops, phase: 'building', status: LEDGER_TEXT.use.status } + case 'done': + return { ops, explanation: op.explanation ?? '' } + case 'ask': { + // 助手要先问清楚才能动手（N11）：记下问题。这一轮怎么收尾（反问、摆着等运行）等服务端的 end + const asked = String(op.question ?? '').trim() + return asked ? { ops, status: CHAT_BLUEPRINT_TEXT.asked, asked } : { ops } + } + case 'final': + // 图建好了：先摆出来（画布边收边长到这里收尾），跑不跑由服务端定（stage run / end ready / end error） + return { ops, ...(op.graph ? { graph: op.graph as GraphSpec } : {}), explanation: op.explanation ?? t.explanation } + case 'usage': { + // 这一轮助手建图花了多少（服务端 answer_ops 最后一条）：累加进这一轮的用量 + const base = t.usage ?? t.meta?.usage + return { ops, usage: { ...base, copilot: addUsage(base?.copilot, op as UsageTotal) } } + } + } + return { ops } +} + +/** 自动重来一次（turn.retry）：这一次的过程和答案清掉（它是自动补救，用户还没拿到过答案），换下来的运行的用量并进这一轮 */ +function retryPatch(t: ChatTurn, d: Record): Partial { + const status = d.kind === 'empty' ? CHAT_RETRY_TEXT.empty + : d.base ? '上次运行未成功，正在调整工作流…' : '上次运行未成功，正在重新生成工作流…' + return { + attempt: Number(d.attempt ?? (t.attempt ?? 0) + 1), phase: 'planning', status, + ops: [], events: [], run: null, output: null, final: undefined, runStatus: undefined, + review: null, reviewRaw: null, rawOutput: null, lastSeq: 0, failure: null, error: '', + // 服务端带着换下来的那次运行的用量（页面上的运行壳子里没有用量）；老服务端没带就按手上的算 + usage: d.usage && typeof d.usage === 'object' ? d.usage as TurnUsage : retireRun(t), } } /** - * 用户的问题填进这张图的入口字段。 - * - * 在此之前这里是硬编码的 `input: {}`——问题根本没进图。Copilot 只要给入口 - * 声明了一个必填字段（它有时候会，比如 `topic`），这次运行就在第一个节点 - * 上立刻挂掉："缺少必填输入：topic"。用户看到的是自己刚问完就报错，而且 - * 错误里提的那个字段名他从来没见过——那是生成的图里的东西。 - * - * 之前"能跑"纯属运气：Copilot 那一次恰好没声明字段。 - * - * history 是另一件事：图跑起来的时候，模型得知道"它们"指的是上一轮那批数据。 - * Copilot 侧的上下文只影响图长什么样，管不到这个。 + * 这一轮到此为止（end）：照服务端落库的那一行摆结局（和刷新之后从库里恢复的是同一份），过程留着。跑完了的成果用 end 带来的 + * 完整那份（报告的逐段证据在里面）；复核改写了的照那一行 */ -export function fillInput( - graph: GraphSpec, question: string, history = '', -): Record { - const entry = (graph.nodes ?? []).find((n: any) => n.type === 'input') - const fields: any[] = (entry?.data?.config?.fields ?? []) as any[] - if (!fields.length) return { question, history } // 没声明字段也带上，下游可能用 input.question - - const payload: Record = { history } - for (const f of fields) { - if (f?.name) payload[f.name] = f.default ?? '' +function endPatch(t: ChatTurn, d: Record): Partial { + const row = d.turn as ConversationTurn | null + if (!row) return { cancel: null, writing: null, endedAt: Date.now() } + const r = restoreTurn(row) + const run = d.run && typeof d.run === 'object' ? d.run as Partial & { id: string } : null + const output = d.output && typeof d.output === 'object' && r.phase === 'done' && !r.review?.answer && !r.noQuery && !r.asked + ? d.output as Record : r.output + return { + phase: r.phase === 'checking' ? 'unknown' : r.phase, + status: r.status, error: r.error, failure: r.failure, rejected: r.rejected, + review: r.review, reviewRaw: r.reviewRaw, rawOutput: r.rawOutput, output, + meta: r.meta, usage: r.meta?.usage ?? t.usage, noQuery: r.noQuery, asked: r.asked, pendingRun: r.pendingRun, + graph: r.graph ?? t.graph, explanation: r.explanation || t.explanation, attempts: r.attempts, + run: run ? ({ ...(t.run ?? {}), ...run } as Run) : r.run ? (t.run?.id === r.run.id ? t.run : r.run) : null, + runStatus: run?.status ?? r.runStatus, + final: run?.status ? { status: run.status } : r.final ?? t.final, + // 计时停在服务端记的那一刻（补交付的停在运行结束时） + clipped: undefined, writing: null, endedAt: r.meta?.ms != null ? t.startedAt + r.meta.ms : Date.now(), + job: row.job ?? null, } - // 问题往哪个字段放：名字像"问题"的优先，其次第一个必填的，再不然第一个。 - // 只有一个字段时不用挑——那必然就是它 - const named = fields.find((f) => - /question|query|topic|input|问题|需求|主题/i.test(String(f?.name ?? ''))) - const target = named ?? fields.find((f) => f?.required) ?? fields[0] - if (target?.name) payload[target.name] = question - return payload -} - -/** 把这次会话之前几轮压成纯文本，塞进 input.history */ -function historyOf(conversationId: string, upToTurnId: string): string { - const turns = useChat.getState().byConversation[conversationId] ?? [] - const lines: string[] = [] - for (const t of turns) { - if (t.id === upToTurnId) break - const answer = answerText(t.output) || t.explanation - if (t.question && answer) lines.push(`问：${t.question}\n答：${answer.slice(0, 600)}`) - } - return lines.slice(-6).join('\n\n') } /** - * 一轮最多自动再试一次。第二次还不行多半是问题本身没说清，该让用户介入。 - * 这几个决定（要不要运行、要不要重来、重来说什么）已经下沉到服务端 backend/app/chat/turn.py：final 带 next，复核带 - * retry_instruction，这里照它们走；下面这份常量和两种重来的措辞只在老服务端（没给这两个字段）时用，和服务端那份逐字 - * 对照（backend/tests/test_chat_turn_policy.py） + * 播放这一轮在服务端的事件日志（发起 + 播放）。after 是从哪一条之后开始（这一次尝试的第一条之前）；断线时 streamChatTurn + * 按收到的最后一个 seq 自动重连、补齐。收到发起了运行（run）就接上运行自己的实时流画过程；运行停在审批上、审批过了接着跑 + * 时（stage waiting → run）重新接上 */ -const MAX_ATTEMPTS = 2 - -/** 重来时交给助手的话（同服务端 chat/turn.py 的 RETRY_WITH_BASE / RETRY_FROM_SCRATCH），只在老服务端时自己拼 */ -function retryInstruction(question: string, reason: string, hasBase: boolean): string { - return hasBase - ? `这个工作流上一次运行没有成功：${reason}。\n\n` - + `请针对这个原因改它（比如调大最大步数、改正工具参数、换个查询方式），` - + `改完继续回答原来的问题：${question}` - : `${question}\n\n（上一次没有成功：${reason}。请针对这个原因调整，重新给出完整的工作流。）` -} - -interface PlanArgs { - /** 用户原本问的是什么。跑图取数、复核都要用它，不能被重试的补充说明污染 */ - question: string - /** 真正发给 Copilot 的那句话。重试时会在后面缀上失败原因 */ - instruction: string - /** 重试时把上一次那张图交回去，让它改而不是重建。null = 从零建 */ - baseGraph?: GraphSpec | null - attempt: number - /** 这一问限定只查的数据源。空或不给 = 不限 */ - scope?: ScopeSource[] -} - -/** - * 建图 → 跑图。从 ask 里拆出来，是为了能带着失败原因原地重来一次。 - * - * 重试必须回到**同一轮**：用户问了一个问题，中间重建了一次图是过程，不是 - * 两次对话。之前这件事全靠用户自己发现不对、再打一句「重来」——他那句话 - * 本来就是在手动做这件事。 - */ -function plan(flow: Flow, args: PlanArgs) { - const { question, instruction, baseGraph, attempt, scope } = args +function play(flow: Flow, serverId: string, after: number) { const { patch } = flow - - /** 这一轮建出来的图。retry 要把它交回去，所以得在闭包里留一份 */ - let built: GraphSpec | null = baseGraph ?? null - /** 助手问回来的那句话：流结束时没有别的结局（final / reply / error）就按反问收尾 */ - let asked = '' - /** 这一轮已经有了结局（final、reply、error）：流结束时不再按反问收尾 */ - let settled = false - /** - * 服务端这一轮已经按运行时同一套规则自查过（收到了 check 的结论）：没过的问题已经交回模型在同一轮里改过了（N11、G12）。 - * 这时前端不再整轮重来——那是再烧一次完整生成，改的还是同一批问题；只有老服务端（不自查）才由前端重来一次 - */ - let selfChecked = false - - /** - * 带着原因自动重来一次。超过上限就把原因如实留在这一轮上，不再烧钱。 - * - * 关键是**把失败的那张图一起交回去**。只给一句"上次没成功：工具参数错了" - * 而不给图，模型只能照着原问题从零再建一遍——很可能建出一模一样的那张， - * 白跑一个来回。而且它连自己上次建了什么都看不到：会话历史按 - * `status != running` 过滤，正在重试的这一轮恰好被排除在外。 - * - * 交回去之后走的是后端的"改图"分支：图预置进累加器，模型只输出改动操作， - * 于是调大最大步数、改正工具参数这种事就是改一个字段，不是推倒重来。 - * - * 这里会清空这一次的过程和答案——它是自动补救，用户还没拿到过答案。用户主动 - * 点的重试走 retryTurn，上一次要留档。 - */ - const retry = (reason: string, instruction?: string, status?: string) => { - // 服务端给了那句话（复核的 retry_instruction）就是它已经按次数判过可以重来；老服务端由这里按次数判 - if ((!instruction && attempt + 1 >= MAX_ATTEMPTS) || !flow.alive()) return false - const base = built && (built.nodes?.length ?? 0) > 0 ? built : null - patch((t) => ({ - attempt: attempt + 1, - phase: 'planning', - status: status ?? (base ? '上次运行未成功，正在调整工作流…' : '上次运行未成功，正在重新生成工作流…'), - ops: [], events: [], run: null, output: null, final: undefined, runStatus: undefined, - review: null, reviewRaw: null, rawOutput: null, lastSeq: 0, failure: null, error: '', - // 换下来的那次运行花的钱还是这一轮的 - usage: retireRun(t), - })) - plan(flow, { - question, - attempt: attempt + 1, - baseGraph: base, - scope, - instruction: instruction ?? retryInstruction(question, reason, !!base), - }) - return true + players.get(flow.turnId)?.closeRun() + let watching: { runId: string; stop: () => void } | null = null + const openRun = (runId: string, from = 0) => { + if (!flow.alive()) return + if (watching?.runId === runId) return + watching?.stop() + watching = { runId, stop: watch(flow, runId, from) } + } + const closeRun = () => { + watching?.stop() + watching = null + } + /** 运行停在审批上：运行的实时流会自己结束（stream.end interrupted），审批过了要重新接 */ + const parked = () => { + const w = watching + watching = null + w?.stop() } - /** 建图阶段就停下了：没有运行，没有断点，只能重试或换个说法 */ - const fail = (failure: Failure, graph?: GraphSpec) => { - patch(() => ({ - ...(graph ? { graph } : {}), - phase: 'error', status: PHASE_TEXT.error, error: lineOf(failure), failure, - cancel: null, endedAt: Date.now(), - })) - void flow.persist({ status: 'error', error: lineOf(failure) }) - } - - const cancelCopilot = streamCopilot( - { - instruction, base_graph: baseGraph ?? null, intent: 'answer', conversation_id: flow.conversationId, - ...(scope?.length ? { datasource_ids: scope.map((d) => d.id) } : {}), - }, - (op) => { - // 停止之后缓冲里还没派发完的操作：一条 final 就能把这一轮又跑起来 - if (!flow.alive()) return - // 先原样收下，翻译交给 decodeCopilot。thinking 合并在写入时做： - // 一次生成几百上千条 delta，逐条存下来光数组就比图大一个量级 - patch((t) => { - const last = t.ops[t.ops.length - 1] - if (op.op === 'thinking' && last?.op === 'thinking') { - const merged = { ...last, delta: String(last.delta ?? '') + String(op.delta ?? '') } - return { ops: [...t.ops.slice(0, -1), merged] } + const apply = (e: ChatTurnEvent) => { + const d = e.data ?? {} + switch (e.type) { + case 'turn.attempt': + closeRun() + patch((t) => ({ ...attemptReset(t, d, e.ts), turnSeq: e.seq })) + if ((d.mode === 'continue' || d.mode === 'resume')) { + const runId = findTurn(flow.conversationId, flow.turnId)?.run?.id + if (runId) openRun(runId, findTurn(flow.conversationId, flow.turnId)?.lastSeq ?? 0) } - return { ops: [...t.ops, op] } - }) - switch (op.op) { - case 'thinking': - patch((t) => ({ - phase: 'planning', - thinking: (t.thinking + (op.delta ?? '')).slice(-2000), - })) - break - case 'heartbeat': - // 查看进行中（looking）不是这一页的阶段：浮条直说「正在查看表结构…」 - patch(() => ({ status: op.phase === 'looking' ? LOOK_TEXT.running - : PHASE_TEXT[(op.phase as Phase) ?? 'planning'] ?? '正在处理…' })) - break - case 'look': - case 'context': { - // 查看结束：浮条换成查看了什么。记录随这一轮的 meta 落库（buildMeta 从操作流取） - const look = op.op === 'look' ? copilotLook(op) : copilotContext(op) - if (look) patch(() => ({ status: look.title })) - break - } - case 'plan': - patch(() => ({ phase: 'building', status: op.summary || '正在生成工作流…' })) - break - case 'add_node': - patch((t) => ({ - phase: 'building', - status: `正在生成工作流…（${t.ops.filter((o) => o.op === 'add_node').length} 步）`, - })) - break - case 'use_caliber': - // 助手选用了一份已发布口径（取数节点和口径卡由服务端生成，过程里那一行由 decodeCopilot 写） - patch(() => ({ phase: 'building', status: LEDGER_TEXT.use.status })) - break - case 'done': - patch(() => ({ explanation: op.explanation ?? '' })) - break - case 'check': - // 自查的结论（通过、没修好）：同一轮里已经交回模型改过了 - if (op.status === 'passed' || op.status === 'failed') selfChecked = true - break - case 'ask': - // 助手要先问清楚才能动手（N11）：记下问题，这一轮到此为止。后面要是还有 final（问之前已经改了图），按 final 收尾、 - // 问题一并带上；流就此结束的话，结束时按反问收尾 - asked = String(op.question ?? '').trim() - if (asked) patch(() => ({ status: CHAT_BLUEPRINT_TEXT.asked, asked })) - break - case 'reply': { - // 这句话不需要工作流——问的是前几轮已经查出来的东西，或者是 - // 对过程说的话（「重试」「换个说法」）。以前这些也各自重建一整张 - // 图、跑一遍 153 张表的库，纯属白花钱 - settled = true - const text = String(op.text ?? '') - patch(() => ({ - phase: 'done', status: PHASE_TEXT.done, cancel: null, endedAt: Date.now(), - output: { answer: text }, noQuery: true, - })) - void flow.persist({ status: 'done', answer: text }) - break - } - case 'final': { - settled = true - const graph = op.graph as GraphSpec - built = graph // 要重试的话，交回去的就是它 - // 连 meta 一起写：它的 at 是「建图还有动静」的凭据，后端这时也会刷新 updated_at - void flow.persist({ graph, explanation: op.explanation ?? '' }) - - // 没过校验的图不能拿去跑：引擎会照实抛出它自己的措辞（"图是空的， - // 先拖一个节点进来"），而用户根本不在画布上，这句话对他毫无意义。 - // 这条路真的把它当答案交到用户面前过 - const blockers = ((op.issues ?? []) as ValidationIssue[]) - .filter((i) => i.level === 'error') - .map((i) => i.message) - // 建完之后怎么走由服务端定（final.next，backend/app/chat/turn.py 的 next_step）：run / ready / fail。 - // 老服务端没给 next：照原来的规则——有 error 级问题时自查过就判失败、没自查过重来一次；autorun=false 只建不跑 - const next: string = op.next === 'run' || op.next === 'ready' || op.next === 'fail' ? op.next - : blockers.length ? (selfChecked ? 'fail' : 'retry') - : op.autorun === false ? 'ready' : 'run' - if (next === 'retry' && retry(`生成的工作流没有通过校验：${blockers.join('；')}`)) break - if (next === 'fail' || next === 'retry') { - // 自查修正过（或重来一次）还是没过。校验消息是写给画布用户的（"先拖一个节点进来"）， - // 直接甩给一个在问数据页打字的人毫无意义——先给一句他能照做的， - // 原文留在技术细节里作线索 - fail({ - title: '未能生成可运行的工作流', - reason: selfChecked ? '助手已按检查结果修正过，生成的工作流仍未通过检查。' : '已自动调整一次，生成的工作流仍未通过检查。', - hint: '请补充描述后重新提问，写明要查询的数据库、时间范围和指标。', - detail: blockers.join('\n'), - }, graph) - break + return + case 'op': + patch((t) => ({ ...opPatch(t, d as CopilotOp), turnSeq: e.seq })) + return + case 'stage': { + const stage = String(d.stage ?? '') + patch((t) => { + // 补交付（运行早就跑完了）停在「正在核对」，不按实时运行画 + if (t.phase === 'checking' && t.endedAt) return { turnSeq: e.seq, ...(stage === 'review' ? { runStatus: 'succeeded' } : {}) } + switch (stage) { + case 'build': + return { turnSeq: e.seq, ...(t.phase === 'planning' || t.phase === 'building' ? {} : { phase: 'planning' as Phase }) } + case 'run': + return { turnSeq: e.seq, phase: 'running' as Phase, final: undefined, + ...(t.phase !== 'running' ? { status: PHASE_TEXT.running } : {}) } + case 'waiting': + return { turnSeq: e.seq, phase: 'waiting' as Phase, status: PHASE_TEXT.waiting, + final: { status: 'interrupted', pending: true }, runStatus: 'interrupted' } + case 'review': + return { turnSeq: e.seq, phase: 'running' as Phase, status: REVIEWING, runStatus: 'succeeded' } } - - // 用户要的是流程本身时（「设计一个每天跑的工作流」），建完就跑 - // 等于替他多花一次钱，而他要的是那张图 - if (next === 'ready') { - // 问了用户的不自动运行（服务端 autorun 为假）：图摆着等「运行」，问题照样作为这一轮的内容、进历史 - patch(() => ({ - graph, phase: 'ready', status: asked ? CHAT_BLUEPRINT_TEXT.asked : PHASE_TEXT.ready, cancel: null, - explanation: op.explanation ?? '', pendingRun: true, endedAt: Date.now(), - ...(asked ? { output: { answer: asked } } : {}), - })) - void flow.persist({ status: 'done', ...(asked ? { answer: asked } : {}) }) - break - } - patch(() => ({ - graph, phase: 'running', status: PHASE_TEXT.running, - explanation: op.explanation ?? '', - })) - void launch(flow, graph, question, retry, attempt) - break + return { turnSeq: e.seq } + }) + if (stage === 'waiting') { + parked() + // 审批列表平时 4 秒才轮询一次，头部已经写着「等待审批」，通过/驳回的按钮却要晚几秒才出现——这一下立刻去拿 + void useCatalog.getState().refreshApprovals() } - case 'usage': - // 这一轮助手建图花了多少（服务端 answer_ops 最后一条）：累加进这一轮的用量、随 meta 落库。结局（final、reply） - // 那一下已经落过库了，这里再写一次 meta - patch((t) => ({ usage: { ...(t.usage ?? t.meta?.usage), copilot: addUsage((t.usage ?? t.meta?.usage)?.copilot, op) } })) - void flow.persist() - break - case 'error': - settled = true - // 生成过程一个操作都没给（模型只写了思考）：服务端说重来一次（error.next = retry，backend/app/chat/turn.py 的 - // next_step），照它带着原因重来，次数照 MAX_ATTEMPTS；试满了才照实报。这一次建图的用量随后那条 usage 照样记进这一轮 - if (op.next === 'retry' && retry(String(op.reason ?? op.message ?? ''), undefined, CHAT_RETRY_TEXT.empty)) break - // 后端给的是三段：message 人话、hint 怎么办、detail 原始异常。 - // 原文不再截成一行甩给用户，收进可展开、可复制的技术细节 - fail({ - ...failureOf(op.message ?? '生成失败'), - ...(op.hint ? { hint: String(op.hint) } : {}), - ...(op.detail ? { detail: String(op.detail) } : {}), - }) - break - } - }, - (error, info?: { code?: string }) => { - if (!error) { - // 流正常结束、只问了一句（ask 之后没有 final / reply）：这一轮记成「助手反问」，问题作为内容显示、落库进历史 - if (asked && !settled && flow.alive()) { - patch(() => ({ - phase: 'done', status: CHAT_BLUEPRINT_TEXT.asked, cancel: null, endedAt: Date.now(), - output: { answer: asked }, asked, - })) - void flow.persist({ status: 'done', answer: asked }) + if (stage === 'run') { + const cur = findTurn(flow.conversationId, flow.turnId) + if (cur?.run?.id) openRun(cur.run.id, cur.lastSeq) } return } - if (!flow.alive()) return - const turn = findTurn(flow.conversationId, flow.turnId) - if (!turn || !BUSY_PHASES.has(turn.phase) || turn.run) return - // 限定的库一个都不在了：后端在开流之前就 400，带机读码 datasource_scope_empty。 - // 认出来了，补救得是「不限数据源重试」，带着原范围重试只会再被拒一次。 - // 认码不认话；不带码的老后端退回按 detail 的开头认 - if (scope?.length && (info?.code === 'datasource_scope_empty' || error.startsWith('限定的数据源'))) { - const at = error.indexOf('：') - fail({ - title: at > 0 ? error.slice(0, at) : error, - reason: at > 0 ? `${error.slice(at + 1).split('。')[0]}。` : undefined, - hint: '点击「不限数据源重试」；如仍需限定该数据源，请先前往「数据」页确认它存在且已启用。', - detail: error, - scopeGone: true, + case 'run': { + const runId = String(d.run_id ?? '') + if (!runId) return + patch((t) => ({ + turnSeq: e.seq, phase: 'running', status: PHASE_TEXT.running, runStatus: String(d.status ?? 'queued'), + run: t.run?.id === runId ? t.run : ({ id: runId, status: d.status, run_class: d.run_class, output: {}, usage: {} } as unknown as Run), + ...(t.run?.id === runId ? {} : { events: [], lastSeq: 0 }), + })) + openRun(runId, 0) + return + } + case 'review': + // 复核这一次的用量也是这一轮的花费；结论跟着 end 一起摆出来（复核说明和结论同时出现，而不是结论先读完） + patch((t) => { + const base = t.usage ?? t.meta?.usage + return { turnSeq: e.seq, usage: { ...base, review: addUsage(base?.review, d.result?.usage) } } }) return - } - fail(failureOf(error)) - }, - ) - patch(() => ({ cancel: cancelCopilot })) -} - -async function launch( - flow: Flow, graph: GraphSpec, question: string, retry?: (reason: string, instruction?: string) => boolean, - attempt = 0, -) { - const { patch } = flow - try { - // 这一轮在服务端的 id（开轮请求可能还在路上，等它）：运行记下属于哪段对话、哪一轮（服务端核对后封存进运行来源）。 - // 开轮没成的（serverId 为空）不带，运行照常发起 - const turnId = findTurn(flow.conversationId, flow.turnId)?.serverId ?? (await serverIds.get(flow.turnId)) ?? null - const body = { graph, input: fillInput(graph, question, historyOf(flow.conversationId, flow.turnId)), origin: 'chat' as const } - // 来源核对不过（对话刚被删、这一轮已不在）：来源只是留痕，不挡这一轮的回答，不带来源照常发起 - const run = await (turnId - ? api.runs.start({ ...body, origin_ref: { conversation_id: flow.conversationId, turn_id: turnId } }) - .catch((e) => (e instanceof ApiError && e.code === 'origin_ref_invalid' ? api.runs.start(body) : Promise.reject(e))) - : api.runs.start(body)) - // 启动请求在路上时用户点了停止或重试：运行已经起来了，得追一个取消过去 - if (!flow.alive()) { - void api.runs.cancel(run.id).catch(() => undefined) - return + case 'review.late': + // 等不及的那次复核晚到了：答案已经照原样交了，它的用量照样记进这一轮 + patch((t) => { + const base = t.usage ?? t.meta?.usage + return { turnSeq: e.seq, usage: { ...base, review: addUsage(base?.review, d.usage) }, + meta: t.meta ? { ...t.meta, usage: { ...base, review: addUsage(base?.review, d.usage) } } : t.meta } + }) + return + case 'turn.retry': + closeRun() + patch((t) => ({ ...retryPatch(t, d), turnSeq: e.seq })) + return + case 'end': + patch((t) => ({ ...endPatch(t, d), turnSeq: e.seq })) + if (d.outcome === 'cancelled' || d.outcome === 'suspended' || d.outcome === 'error') { + // 运行已经停了：它的实时流会把最后几条补完再结束；被取消的不再看 + if (d.outcome === 'cancelled') closeRun() + } + return } - patch(() => ({ run, runStatus: run.status })) - // meta 里的 runId 优先于 run_id 列（重试建图失败时那一列清不掉），所以得一起改： - // 只写列的话，重试出来的这次运行在库里一直挂着 runId=null，跑着的时候刷新或者 - // 另开一页，它就成了「断在建图阶段」，放久了还会被当成断了写回 error。 - // 「跑一下」发起时库里那行还是 done，一并改成 running - void flow.persist({ run_id: run.id, status: 'running' }) - watch(flow, run.id, 0, question, retry, attempt) - } catch (e) { - // 发起那一下没到后端：流程是好的，原样再发起就行。通用的「稍等几秒会自动重试」 - // 在这里不成立——这一页没有谁会替它重发 - const failure: Failure = isNetworkError(e) - ? { ...failureOf(e, { hint: '服务端恢复连接后，点击「重新运行本轮」，已生成的工作流无需重新构建。' }), rerun: true } - : startFailure(e) - patch(() => ({ - phase: 'error', status: PHASE_TEXT.error, error: lineOf(failure), failure, - cancel: null, endedAt: Date.now(), - })) - void flow.persist({ status: 'error', error: lineOf(failure) }) } + + const stopTurn = streamChatTurn(serverId, after, (e) => { + if (!flow.alive()) return + apply(e) + }, () => { + // 服务端说这一轮的事件都发完了（或者这一轮读不了了）。运行的实时流还开着的让它自己收尾 + }) + const close = () => { + stopTurn() + closeRun() + } + players.set(flow.turnId, { flow, openRun, closeRun }) + patch(() => ({ cancel: close })) } /** 一帧之内到的事件并成一次写入。后台标签页不跑 rAF，另挂一个定时器兜底 */ @@ -1732,48 +1541,27 @@ function penOf(patch: Patch): { add: (e: RunEvent) => void; clear: () => void } } /** - * 订阅运行事件。 - * - * 只做两件事：把事件原样存进这一轮，以及更新那几个驱动 UI 状态机的相位。 + * 订阅运行事件，只为画过程：把事件原样存进这一轮，顺带更新「此刻在做什么」那一句和出字计数。 * "查了哪张表、跑了什么 SQL"不在这里翻译——decodeRun 会从同一批事件里读出来。 * - * 结局有两个来源：事件本身（run.finished / failed / cancelled、服务关停的 log）， - * 和流结束时后端带来的 stream.end {status}。后者兜住"事件没来齐就断了"的情况—— - * 服务重启后重连，后端回放完就发 stream.end status=interrupted；以前前端不认它， - * 这一轮就一直转圈。 + * 这一轮的相位和结局不看这里：跑完了要复核、要不要重来、什么时候交付都是服务端定的，经这一轮的事件日志来（play）。 + * 只有停在审批上那一下例外：run.interrupted 一到就去拿待审批（头部写着「等待审批」，通过 / 驳回的按钮不该晚几秒才出现） */ -function watch( - flow: Flow, runId: string, after = 0, - question = '', retry?: (reason: string, instruction?: string) => boolean, attempt = 0, -) { - const { patch, conversationId, turnId } = flow - /** 这条流上已经见到结局事件了，stream.end 不用再判一次 */ - let ended = false +function watch(flow: Flow, runId: string, after = 0): () => void { + const { patch } = flow let buffer: RunEvent[] = [] const batch = batcher(() => { if (!buffer.length) return const got = buffer buffer = [] - patch((t) => ({ + patch((t) => (t.run?.id !== runId ? {} : { events: mergeEvents(t.events, got), // 记下进度：审批恢复要重新接流，带上它才不会把历史再收一遍 lastSeq: Math.max(t.lastSeq ?? 0, ...got.map((e) => e.seq ?? 0)), })) }) - const current = () => findTurn(conversationId, turnId) const pen = penOf(patch) - const suspend = () => { - batch.now() - pen.clear() - patch(() => ({ - phase: 'suspended', status: PHASE_TEXT.suspended, failure: SUSPENDED, error: '', - final: { status: 'interrupted', pending: false }, runStatus: 'interrupted', - endedAt: Date.now(), - })) - void flow.persist({ status: 'error', error: PHASE_TEXT.suspended }) - } - const stop = streamRun(runId, (event: RunEvent) => { if (!flow.alive()) return // token 和思考增量：后端不落库，decodeRun 也一开头就丢掉。存进 events 的唯一 @@ -1787,14 +1575,12 @@ function watch( buffer.push(event) batch.push() const d: any = event.data ?? {} - // 补发的历史里的结局是"当时"的结局：一条失败后又接着跑的运行，回放里先有 - // run.failed、后有 run.resumed。照着它改相位、落库，这一轮会先闪成失败再 - // 变回来。回放完后端会用 stream.end 说出现在的状态，结局以那个为准 + // 补发的历史里的结局是"当时"的结局，不据此改什么 if (event.replay && REPLAY_SKIP.has(event.type)) return switch (event.type) { case 'tool.start': { const tool = String(d.tool ?? '') - patch(() => ({ + patch((t) => (t.phase !== 'running' ? {} : { status: tool.startsWith('db_query') ? '正在查询数据…' : tool.startsWith('db_schema') ? '正在读取数据结构…' : tool ? `正在调用 ${tool}…` : '正在运行…', @@ -1802,203 +1588,28 @@ function watch( break } case 'llm.start': - patch(() => ({ status: '正在分析…' })) + patch((t) => (t.phase !== 'running' ? {} : { status: '正在分析…' })) break case 'run.resumed': - patch(() => ({ phase: 'running', status: PHASE_TEXT.running, final: undefined })) + patch(() => ({ final: undefined })) break case 'run.interrupted': - patch(() => ({ phase: 'waiting', status: PHASE_TEXT.waiting })) - // 审批列表平时 4 秒才轮询一次，头部已经写着「等待审批」，通过/驳回的 - // 按钮却要晚几秒才出现——这一下立刻去拿 + patch((t) => (t.phase !== 'running' ? {} : { phase: 'waiting', status: PHASE_TEXT.waiting })) void useCatalog.getState().refreshApprovals() break - case 'log': - // 服务在关停：这一轮会停在断点上。停在审批上的仍然是在等人 - if (d.code === 'server_shutdown' && current()?.phase !== 'waiting') suspend() - break - case 'run.cancelled': { - ended = true - batch.now() - // 人的驳回（reason=rejected）也收成 cancelled：不是故障、不给继续运行，说法写「被驳回」和谁、哪一步、备注 - const rejected = d.reason === 'rejected' ? { sub: REJECTED_TEXT.sub(d) } : null - patch(() => ({ - phase: 'cancelled', status: rejected ? REJECTED_TEXT.title : PHASE_TEXT.cancelled, cancel: null, - endedAt: Date.now(), runStatus: 'cancelled', final: { status: 'cancelled' }, rejected, - })) - void flow.persist({ status: 'error', error: '已取消' }) - break - } - case 'run.failed': { - ended = true - batch.now() - const failure = runFailure(d.error ? String(d.error) : '', d.detail ? String(d.detail) : undefined) - patch(() => ({ - phase: 'error', status: PHASE_TEXT.error, error: lineOf(failure), failure, - cancel: null, endedAt: Date.now(), runStatus: 'failed', final: { status: 'failed' }, - })) - void flow.persist({ status: 'error', error: lineOf(failure) }) - break - } case 'run.finished': - ended = true - batch.now() - patch(() => ({ status: '正在核对结果…', runStatus: 'succeeded' })) - // 跑完不等于答对：先接住这次编排，确认没问题再交付 - void settle(flow, runId, question, d.output ?? null, { - truncated: d.output_truncated === true, retry, attempt, - }) + patch((t) => (t.phase !== 'running' ? {} : { status: REVIEWING })) break } - }, (end) => { + }, () => { batch.now() pen.clear() - if (ended || !flow.alive()) return - const status = String(end?.status ?? '') - const turn = current() - if (!turn) return - if (status === 'interrupted' || status === 'suspended') { - // 连到一条已经停下的运行：分清是停在审批上，还是服务重启挂起的 - void (async () => { - const pending = status === 'suspended' ? false : await pendingApproval(runId) - if (!flow.alive()) return - if (pending) { - patch(() => ({ phase: 'waiting', status: PHASE_TEXT.waiting, final: { status, pending } })) - void useCatalog.getState().refreshApprovals() - } else { - suspend() - } - })() - return - } - if (status === 'succeeded') { - // 结局事件在断线时丢了：成果以库里为准 - ended = true - void settle(flow, runId, question, null, { retry, attempt }) - return - } - // 失败、取消、或者运行不见了：状态和原因让 hydrate 去读 - patch(() => ({ hydrated: undefined, phase: 'checking', status: PHASE_TEXT.checking, cancel: null })) - void useChat.getState().hydrate(conversationId, turnId) }, after) - patch(() => ({ cancel: stop })) -} - - -/** - * 接住一次编排的产出，确认能不能就这么交给用户。 - * - * 「跑完了」和「答得对」是两回事。库里能查到大量 status=succeeded 的运行， - * 过程中检索降级了、工具报错了、agent 步数用满了，而交到用户手上的答案对此 - * 只字不提——最糟的一次交出去的干脆是一句内部管道文案。 - * - * 后端先用规则扫一遍事件流，干净的运行直接返回 ok，一次模型都不调；扫出异常 - * 才叫模型重组答案并把话说破（backend/app/engine/review.py）。 - * - * 这个函数必须**在任何情况下都把答案交出去**：复核是补一层说明，它自己失败了 - * 不能把已经到手的结果一起带走，更不能让这一轮永远停在"正在核对"。 - */ -async function settle( - flow: Flow, runId: string, question: string, - eventOutput: Record | null, - opts: { - truncated?: boolean; retry?: (reason: string, instruction?: string) => boolean; run?: Run; endedAt?: number - /** 这一轮已经自动试了几次（从 0 数）：交给复核，服务端据此决定给不给 retry_instruction */ - attempt?: number - }, -) { - const { patch } = flow - - // 成果以库里那份为准。run.finished 事件里的 output 为了控制推送体积被切过 - // （字典每个值 2000 字），以前落库、显示的都是它：几千字的报告停在半句话上， - // 看起来却是完整的，刷新之后还是那半截 - let run: Run | null = opts.run ?? null - if (!run) { - try { run = await api.runs.get(runId) } catch { /* 取不到就用事件里那份，下面会标出来 */ } + return () => { + batch.now() + pen.clear() + stop() } - const full = run?.output && Object.keys(run.output).length ? run.output : null - const output = full ?? eventOutput - const clipped = !full && (opts.truncated || looksClipped(eventOutput)) ? 'partial' as const : undefined - patch((t) => ({ run: run ?? t.run, runStatus: 'succeeded', final: { status: 'succeeded' } })) - - // 成果**先落库**再显示。两件事顺序不能并： - // - 落库要马上，否则复核期间关掉标签页，这次跑出来的东西就没了； - // - 显示要等复核，否则用户已经把结论读完、信了，说明才姗姗来迟—— - // 实测复核要十秒上下，这十秒里他看到的是一个没人核对过的答案。 - void flow.save({ status: 'done', answer: answerText(output) }) - - let result: ReviewResult | null = null - const reviewing = api.copilot.review({ run_id: runId, question, attempt: opts.attempt ?? 0 }) - let late = false - try { - // 复核慢到一定程度就不等了。它是补一层说明，不该反过来把答案扣住不放—— - // 干净的运行后端 15ms 就返回，会走到这个上限的都是真叫了模型的那些 - result = await Promise.race([ - reviewing, - new Promise((r) => setTimeout(() => { late = true; r(null) }, reviewWaitMs)), - ]) - } catch { /* 复核用不了就按原样交付，下面照常走 */ } - if (late && !result) { - // 没等到复核：答案照原样交付（下面照常收尾）。复核那一次照样调了模型、花了钱：它晚到时把用量记进这一轮、再落一次库， - // 这一轮的花费才照实是助手 + 运行 + 复核（不再改答案：说明晚到了就不补） - void reviewing.then((r) => { - if (!r?.usage || !flow.alive()) return - const spent = r.usage - patch((t) => ({ usage: { ...(t.usage ?? t.meta?.usage), review: addUsage((t.usage ?? t.meta?.usage)?.review, spent) } })) - void flow.persist() - }, () => { /* 复核失败了就没有用量可记 */ }) - } - // 复核期间用户点了停止或者重试：这一轮已经不归这次运行管了 - if (!flow.alive()) return - - // 成果带逐段证据时复核只能加说明：改写会把成果整个换成 {answer}，报告文档的每个片段 - // 就都对不上了。后端也拦，这里再拦一道，老后端回来的改写同样不落到界面和库里 - result = guardReview(result, output) - // 库里存完整结论（包括 verdict='ok'），store 里只留值得摆到界面上的那部分。 - // 两者分开，"这一轮没复核过"和"复核过、没发现问题"才是两件可区分的事—— - // 事后排查"它当时为什么没警告我"，靠的就是这个区别 - const review = result && result.verdict !== 'ok' ? result : null - - // 复核这一次的用量也是这一轮的花费（服务端 review_of_run 的 usage；老服务端没有） - if (result?.usage) { - const spent = result.usage - patch((t) => ({ usage: { ...(t.usage ?? t.meta?.usage), review: addUsage((t.usage ?? t.meta?.usage)?.review, spent) } })) - } - - // 答案根本不可信，而且原因说得清楚——重建一次图再跑，别让用户自己发现。要不要重来、重来说什么由服务端定（复核的 - // retry_instruction，backend/app/chat/turn.py）：给了那句话就照它重来，给的是 null 就不重来。老服务端没有这个字段时 - // 照原来的规则自己判 - if (opts.retry && result && 'retry_instruction' in result) { - if (result.retry_instruction) { - const reason = review ? review.signals.map((s) => s.detail).join('；') || review.note : '' - if (opts.retry(reason, result.retry_instruction)) return - } - } else if (review?.retry && review.severity === 'broken' && opts.retry) { - const reason = review.signals.map((s) => s.detail).join('；') || review.note - if (opts.retry(reason)) return - } - - const rewritten = review?.answer ?? null - const issuance = output?._issuance - patch(() => ({ - phase: 'done', - status: PHASE_TEXT.done, - review, - reviewRaw: result, - cancel: null, - endedAt: opts.endedAt ?? Date.now(), - // 复核改写时读的是库里那份完整成果，改写出来的答案不缺尾巴 - clipped: rewritten ? undefined : clipped, - // 到这一步才把成果摆出来：复核说明和结论同时出现，而不是结论先读完 - output, - // 改写是有损的，原件得留着——界面上可以展开对照。它跟着 review 一起 - // 落库，所以刷新之后还在 - ...(rewritten ? { - rawOutput: { answer: review?.original ?? answerText(output) }, - output: { answer: rewritten, ...(issuance ? { _issuance: issuance } : {}) }, - } : {}), - })) - void flow.persist({ status: 'done', answer: rewritten || answerText(output) }) } export { decodeRun } diff --git a/frontend/src/types.ts b/frontend/src/types.ts index cce85f52..3bdc7109 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -1646,6 +1646,22 @@ export interface ConversationTurn { * （conversations.BUILD_STALE），前端判断同一件事时用同一只钟。老后端没有 */ updated_at?: string + /** 这一轮在服务端的任务（问数据整轮在服务端跑，backend/app/chat/job.job_view）。这一波之前的老轮次没有，为 null */ + job?: ChatTurnJob | null +} + +/** 问数据一轮在服务端的任务现在怎样。status 用后台任务的状态词；stage 是在哪一步；attempt_seq 是这一次尝试从哪一条事件开始 */ +export interface ChatTurnJob { + status: 'queued' | 'running' | 'succeeded' | 'failed' | 'cancelled' | 'interrupted' + stage: 'build' | 'run' | 'waiting' | 'review' | null + mode: string + run_id: string | null + attempt: number + last_seq: number + attempt_seq: number + error: string | null + /** 等不及的那次复核还在后台跑 */ + late_review?: boolean } /** diff --git a/scripts/check-chat.mjs b/scripts/check-chat.mjs index a09dd895..de7582c6 100644 --- a/scripts/check-chat.mjs +++ b/scripts/check-chat.mjs @@ -1,6 +1,12 @@ // 问数据页的检查：会话、轮次、运行全部在浏览器这一侧伪造，页面走的是真的 // chat store / ChatPage / AssistantStream，只有后端换成了脚本。 // +// 一轮问数据在服务端跑（docs/design/orchestration-round/00-总方案.md §1）：页面只发起（POST /api/chat/turns）、只播放 +// （GET /api/chat/turns/{id}/stream）。这里伪造的服务端（startTurn / driveTurn，照 backend/app/chat/job.py 的做法）用原来那几份 +// 剧本把一轮跑完：生成（copilotScript）→ 发起运行（launchScript）→ 等运行的剧本放完（fakeStream）→ 复核（reviewScript）→ +// 照服务端的写法拼那一行（end）。log.gens / runStarts / reviews 照旧记下服务端这一侧做了什么，log.patches 用来守「页面不再写 +// 这一轮的结局」。 +// // 为什么不真跑：沙箱没有模型密钥，而且要撞上的恰恰是真跑很难稳定复现的那几种 // 时刻——两个会话同时在跑、答案超过 2000 字被事件截断、跑到一半服务重启、 // 历史取得慢或者取失败。这里每一种都由脚本精确地摆出来。 @@ -8,7 +14,7 @@ // 守的事： // 1. 忙不忙按会话算：A 在跑时 B 的按钮是「发送」，停止只停 A 自己，删除保护 // 对准正在跑的那个； -// 2. run.finished 的 output 被截断时，显示和落库都用 GET 运行拿到的完整版； +// 2. run.finished 的 output 被截断时，显示用服务端按库里完整成果写的那一份；页面不 PATCH 这一轮； // llm.token 不进 events； // 3. 服务重启：stream.end status=interrupted 且没有待审批，这一轮收成 // 「服务重启，本轮已中断」，给「继续运行」，不再转圈； @@ -16,10 +22,11 @@ // 5. 已取消的、人驳回这类原样续上还会失败的轮次没有「继续运行」，续跑被拒时说人话； // 6. 1440×900 下长会话的输入框和发送键完整可见，整页不多滚； // 7. 首屏例句不拿系统表造句；可信度信息刷新后还在，且写进单独的 meta，不夹在 review 里； -// 8. 从库里恢复的半路轮次：核对撞上 500 有出路、补交付不走实时计时、查到的终态落库； -// 9. 长答案没取全要记进 meta，刷新后补全；2000 字按码点数，不按 UTF-16； -// 10. 发起运行失败：报错不糊成一整行粗体，不许诺不存在的自动重试；重跑挂起的轮次 -// 顺手取消旧运行；断在建图阶段按最后一次落库算有没有动静，发起运行时 meta 跟上新运行； +// 8. 从库里恢复的半路轮次（老轮次、服务重启后停着的）：交给服务端接着收尾（mode=resume），核对撞上 500 有出路、 +// 补交付不走实时计时，结局照服务端写的那一行摆； +// 9. 老轮次的长答案没取全：刷新后补全；2000 字按码点数，不按 UTF-16；新轮次按库里的完整成果交付； +// 10. 发起那一下没到服务端：报错不糊成一整行粗体，不许诺不存在的自动重试；重跑挂起的轮次 +// 顺手取消旧运行；断在建图阶段的老轮次页面不替它写库，重试本轮交给服务端； // 11. 左栏：没打开过的会话也标得出状态；筛选框在有筛选词时不消失；删掉的会话进回收站， // 能预览、恢复、彻底删除（要确认）；回收站里只能看，不发起任何运行，正在跑的不给 // 彻底删除（清空也跳过它）； @@ -38,6 +45,8 @@ // 19. 答案身上的召回标记（账本接口的 recall）写在轮次底部：「已按口径「X」vN 核对，不影响」、依据有变（目录 / 表结构，几项、 // 按新目录新出现几条错误级问题），看的人手上有说的是这个答案的召回待办时链到待办；没比对的答案照样写依据有变；做完的 // 对账卡上写「差在哪」、问说明有没有帮助（说明有用 / 看不懂）。 +// 20. 一轮在服务端跑：实时流断了按收到的最后一个 seq 重连，补齐、不重复；刷新、关掉再打开，服务端那一轮照跑，页面从 +// 这一次尝试的第一条接着播；停止发给服务端的这一轮（建图中也停得下）；发起、重试、继续运行的请求体照实。 // // 所有写请求都被拦在浏览器里，不碰任何库。 // 跑之前前端得起着（./scripts/dev.sh），默认连 5273。对别的实例（比如一份沙箱拷贝）跑时 @@ -63,6 +72,8 @@ const opened = new Set() async function section(name, title, fn) { if (ONLY.length && !ONLY.includes(name)) return console.log(`\n=== ${title} ===`) + // 伪造的服务端：上一段里还在跑的几轮就此放下，不再往下写（各段互不牵连） + srv.gen += 1 try { await fn() } catch (e) { @@ -217,6 +228,10 @@ const REJECT_CONVS = { rejS: conv('c0rejs', '会话：没打开过的那一问', 1, 60 * 24 * 36, { last_status: 'rejected' }), } /** 只在「助手反问」那段出现：库里存着一轮以反问收尾的（answer 列里是助手问的那句话，meta.asked 标着） */ +/** 只在「刷新后接着看」那段出现：一开始是空的，问的那一轮在伪造的服务端里跑，刷新、关掉再打开时从那里取回来 */ +const REPLAY_CONVS = { + replay: conv('c0replay', '会话：刷新后接着看', 0, 60 * 24 * 41), +} const ASKBACK_CONVS = { askBack: conv('c0askbk', '助手反问过的会话', 1, 60 * 24 * 34), } @@ -229,6 +244,7 @@ const ASK_CONVS = { askZ: conv('c0askz', '问数据：各区域销售额和最大的一单', 1, 60 * 24 * 33), } const DETAIL = { + c0replay: [], c0askbk: [turn('tab1', '上周销售额是多少', { answer: '销售额按下单日还是支付日算？', meta: { v: 1, asked: true } })], c0ledgr: [turn('tl1', '上周的周报', { answer: '（周报原文）', graph: GRAPH, run_id: 'run-led', @@ -412,6 +428,8 @@ const TABLES = { const log = { patches: [], cancels: [], continues: [], runStarts: [], archived: [], posts: [], approvalGets: [], runGets: [], deleted: [], gens: [], sourceGets: 0, catalogPatches: [], reviews: [], + /** 问数据的一轮：发起（POST /api/chat/turns 的请求体）、停止、实时流的每一次连接（从哪个 seq 之后） */ + turnStarts: [], turnCancels: [], streams: [], } /** 新服务端复核给的重来那句话：前端得原样交回去，不自己拼 */ const DECIDE_RETRY = '（服务端拼好的重来说法）照服务端的决定重新回答：有几家门店' @@ -439,6 +457,20 @@ const ctl = { rejectConvs: false, /** 列表里多摆「口径账本」那段的会话（LEDGER_CONVS） */ ledgerConvs: false, + /** 伪造的服务端：运行的剧本放完了（`${runId}@${after}`）、继续运行过的运行、等复核最多多久（毫秒） */ + wsDone: {}, continued: {}, reviewWaitMs: 25_000, + /** POST /api/chat/turns 根本没到后端（网断了） */ + startFail: false, + /** 接下来几次实时流连接直接断掉（测断线重连补齐） */ + dropStreams: 0, + /** 对话详情里带上服务端写的那几行（测刷新后接着看） */ + liveDetail: false, + /** 助手每条操作之间隔多久（毫秒） */ + opGapMs: 10, + /** 列表里多摆「刷新后接着看」那段的会话（REPLAY_CONVS）；那段里建图、运行各停在一道闸上 */ + replayConvs: false, replayBuildGo: false, replayRunGo: false, + /** 「停止」那段：建图停在一道闸上 */ + stopBuildGo: false, } /** 会话的归档 / 删除状态。各段各自 resetDb，免得上一段删掉的会话影响下一段 */ const db = { archived: new Set(), purged: new Set() } @@ -449,7 +481,7 @@ const resetDb = () => { resetDb() const allConvs = () => [...Object.values(CONVS), ...Object.values(TRASHED), ...(ctl.trashMore ? Object.values(TRASH_MORE) : []), ...(ctl.guardConvs ? Object.values(GUARD_CONVS) : []), ...(ctl.askConvs ? Object.values(ASK_CONVS) : []), - ...(ctl.askBack ? Object.values(ASKBACK_CONVS) : []), + ...(ctl.askBack ? Object.values(ASKBACK_CONVS) : []), ...(ctl.replayConvs ? Object.values(REPLAY_CONVS) : []), ...(ctl.fixConvs ? Object.values(FIX_CONVS) : []), ...(ctl.rejectConvs ? Object.values(REJECT_CONVS) : []), ...(ctl.ledgerConvs ? Object.values(LEDGER_CONVS) : [])] .filter((c) => !db.purged.has(c.id)) @@ -479,8 +511,13 @@ async function fakeApi(route) { if (id === 'c0faile' && ctl.failE) return json({ detail: '数据库暂时不可用' }, 500) const known = allConvs().find((c) => c.id === id) if (!known && id !== 'c0new') return json({ detail: '这个对话不存在，可能已经被删了' }, 404) - const turns = typeof DETAIL[id] === 'function' ? DETAIL[id]() : DETAIL[id] - return json({ ...(known ?? conv('c0new', '新对话', 0, 0)), turns: turns ?? [] }) + let turns = (typeof DETAIL[id] === 'function' ? DETAIL[id]() : DETAIL[id]) ?? [] + if (ctl.liveDetail) { + // 服务端写的那几行（刷新、关掉再打开时取回来的就是它们）：夹具里有的换成服务端现在的，新问的接在后面 + const live = [...srv.turns.values()].filter((T) => T.conversation_id === id && T.gen === srv.gen).map(rowOut) + turns = [...turns.map((t) => live.find((x) => x.id === t.id) ?? t), ...live.filter((x) => !turns.some((t) => t.id === x.id))] + } + return json({ ...(known ?? conv('c0new', '新对话', 0, 0)), turns }) } if (m && method === 'PATCH') { const b = body() @@ -502,109 +539,35 @@ async function fakeApi(route) { m = path.match(/^\/conversations\/([^/]+)\/turns\/([^/]+)$/) if (m && method === 'PATCH') { log.patches.push({ conv: m[1], turn: m[2], body: body() }); return json(turn(m[2], '', body())) } + // 问数据的一轮（伪造的服务端，见 startTurn / driveTurn） + if (path === '/chat/turns' && method === 'POST') { + const b = body() + if (ctl.startFail) { + log.turnStarts.push({ ...b, failed: true }) + return route.abort('connectionrefused') + } + const got = await startTurn(b) + return json(got.json, got.status) + } + m = path.match(/^\/chat\/turns\/([^/]+)(\/stream|\/cancel)?$/) + if (m) { + const T = srv.turns.get(decodeURIComponent(m[1])) + if (!T) return json({ detail: '该轮对话不存在，可能已被删除' }, 404) + if (m[2] === '/stream' && method === 'GET') return streamTurn(route, T, Number(url.searchParams.get('after') ?? 0)) + if (m[2] === '/cancel' && method === 'POST') { + log.turnCancels.push(T.id) + return cancelTurn(T) ? json({ ok: true }) : json({ detail: '这一轮已经结束，无需停止', code: 'chat_turn_finished' }, 409) + } + if (!m[2] && method === 'GET') return json({ id: T.id, conversation_id: T.conversation_id, job: { ...T.job }, turn: rowOut(T) }) + } if (path === '/copilot/generate-stream' && method === 'POST') { + // 画布助手照旧走这条流（问数据的一轮在伪造的服务端里走同一份剧本，见 copilotScript） const b = body() log.gens.push(b) - if (ctl.scope400 && b.datasource_ids?.length) { - // 和后端 copilot._sources 同一个形状：detail 是一句话，code 是稳定的机读码 - // reworded：后端换了说法，只剩机读码认得出来 - return json({ detail: ctl.scope400 === 'reworded' ? '这一轮圈定的库一个也找不到了：可能已经被删掉或停用' - : '限定的数据源都已不可用，可能已被删除或停用。请取消限定后重试，或到「数据」页确认数据源存在且已启用', code: 'datasource_scope_empty' }, 400) - } - if (String(b.instruction ?? '').includes('建流程就失败')) { - // 后端的 error 操作是三段：人话、怎么办、原始异常 - const ops = [ - { op: 'heartbeat', phase: 'planning', elapsed_ms: 400 }, - { op: 'error', message: '模型返回的工作流结构有误：第 2 个节点的类型「sql」不存在', - hint: '请重试，或把需求描述得更具体', - detail: "2 validation errors for GraphSpec\nnodes.1.type\n Input should be 'input', 'output', 'llm' [type=literal_error]" }, - ] - return route.fulfill({ status: 200, contentType: 'text/event-stream', - body: ops.map((o) => `data: ${JSON.stringify(o)}\n\n`).join('') }) - } - if (String(b.instruction ?? '').includes('自查没修好')) { - // 服务端自查交回模型改过、还是没过（check failed），final 带着同一批问题：前端不该再整轮重来 - const ops = [ - { op: 'heartbeat', phase: 'planning', elapsed_ms: 300 }, - ...GRAPH.nodes.map((n) => ({ op: 'add_node', node: n })), - { op: 'check', status: 'failed', issues: ['「查数」节点还没有选择工具'] }, - { op: 'final', graph: GRAPH, explanation: '', issues: [{ level: 'error', node_id: 'ag', message: '「查数」节点还没有选择工具' }] }, - ] - return route.fulfill({ status: 200, contentType: 'text/event-stream', - body: ops.map((o) => `data: ${JSON.stringify(o)}\n\n`).join('') }) - } - if (String(b.instruction ?? '').includes('只写了思考')) { - // 生成过程一个操作都没给（deepseek 偶发只出思考）：服务端的 error 带 next=retry（backend/app/chat/turn.py 的 next_step）。 - // 重来那一次（指令里缀了「上一次没有成功」）照常直接回答；「两次都」那一问每次都只写思考 - const again = String(b.instruction).includes('上一次没有成功') && !String(b.instruction).includes('两次都') - const spent = { by_model: { m: { input_tokens: 500, output_tokens: 40, calls: 1, cost_usd: 0.001, unpriced_calls: 0 } }, - input_tokens: 500, output_tokens: 40, calls: 1, cost_usd: 0.001, unpriced_calls: 0 } - const ops = again ? [ - { op: 'heartbeat', phase: 'planning', elapsed_ms: 200 }, - { op: 'reply', text: '已经查过了：一共 3 家门店。' }, - { op: 'usage', ...spent }, - ] : [ - { op: 'heartbeat', phase: 'planning', elapsed_ms: 200 }, - { op: 'thinking', delta: '先想一想要查哪张表……' }, - { op: 'error', message: '助手本轮未完成：模型没有给出任何修改，也没有说明原因', hint: '画布未修改。可以重试，或把需求描述得更具体', - detail: '', reason: '模型没有给出任何修改，也没有说明原因', code: 'copilot_empty', next: 'retry' }, - { op: 'usage', ...spent }, - ] - return route.fulfill({ status: 200, contentType: 'text/event-stream', - body: ops.map((o) => `data: ${JSON.stringify(o)}\n\n`).join('') }) - } - if (String(b.instruction ?? '').includes('照服务端的决定') || String(b.instruction ?? '').includes(DECIDE_RETRY)) { - // 新服务端：final 带 next（backend/app/chat/turn.py），最后一条是这一轮建图的用量（answer_ops 的 usage）。 - // 「只建不跑」那一问：autorun 没写 false，next 却是 ready——前端得照 next 走 - const ready = String(b.instruction ?? '').includes('只建不跑') - const ops = [ - { op: 'heartbeat', phase: 'planning', elapsed_ms: 300 }, - ...GRAPH.nodes.map((n) => ({ op: 'add_node', node: n })), - { op: 'done', explanation: '一张三步的图' }, - { op: 'check', status: 'passed', repaired: 0, warnings: [] }, - { op: 'final', graph: GRAPH, issues: [], explanation: '一张三步的图', autorun: true, next: ready ? 'ready' : 'run' }, - { op: 'usage', by_model: { m: { input_tokens: 4000, output_tokens: 300, calls: 1, cost_usd: 0.012, unpriced_calls: 0 } }, - input_tokens: 4000, output_tokens: 300, calls: 1, cost_usd: 0.012, unpriced_calls: 0 }, - ] - return route.fulfill({ status: 200, contentType: 'text/event-stream', - body: ops.map((o) => `data: ${JSON.stringify(o)}\n\n`).join('') }) - } - if (String(b.instruction ?? '').includes('先问清楚')) { - // 助手要先问清楚才能动手（服务端 copilot_gen 的 ask）：问题转出去，这一轮到此为止——没有 final，也没有 reply - const ops = [ - { op: 'heartbeat', phase: 'planning', elapsed_ms: 300 }, - { op: 'ask', question: '销售额按下单日还是支付日算？' }, - ] - return route.fulfill({ status: 200, contentType: 'text/event-stream', - body: ops.map((o) => `data: ${JSON.stringify(o)}\n\n`).join('') }) - } - if (String(b.instruction ?? '').includes('表示作废')) { - // 用户说了一条数据事实：服务端核对后转出目录修改提案，只回答、不建图 - const ops = [ - { op: 'heartbeat', phase: 'planning', elapsed_ms: 300 }, - { op: 'catalog_patch', source: 'shop', source_id: 'src-shop', table: 'orders', table_label: '订单', version: 2, changes: [ - { path: 'columns.status.codes', before: { 1: '已支付' }, before_status: 'confirmed', after: { 1: '已支付', 9: '作废' }, - value: { 9: '作废' }, reason: '用户说明 status=9 表示作废', state: 'change' }] }, - { op: 'reply', text: '已整理成数据目录的修改建议，确认后保存即可。' }, - ] - return route.fulfill({ status: 200, contentType: 'text/event-stream', - body: ops.map((o) => `data: ${JSON.stringify(o)}\n\n`).join('') }) - } - const ops = [ - { op: 'heartbeat', phase: 'planning', elapsed_ms: 800 }, - { op: 'heartbeat', phase: 'looking', elapsed_ms: 1000 }, - // 助手按需查看了表（look 事件）：过程里多一行「查看了 N 张表」，查看记录存进这一轮的 meta.looks。 - // 这个库还没有数据目录（catalog_tables: 0）：那一行下面提示先去起草目录 - { op: 'look', id: 'L1', by: 'model', need: '查订单和明细', status: 'ok', elapsed_ms: 1200, objects: [ - { kind: 'table', source: 'shop', source_id: 'src-shop', named: [], picked: ['orders', 'order_items'], joined: [], - missing: [], total: 48, catalog_tables: 0 }] }, - { op: 'plan', summary: '查一下再回答' }, - ...GRAPH.nodes.map((n) => ({ op: 'add_node', node: n })), - { op: 'done', explanation: '一张三步的图' }, - { op: 'final', graph: GRAPH, issues: [], explanation: '一张三步的图' }, - ] + const got = copilotScript(b) + if (got.status) return json(got.json, got.status) return route.fulfill({ status: 200, contentType: 'text/event-stream', - body: ops.map((o) => `data: ${JSON.stringify(o)}\n\n`).join('') }) + body: got.ops.map((o) => `data: ${JSON.stringify(o)}\n\n`).join('') }) } if (path === '/datasources/src-shop/catalog/orders/patch' && method === 'POST') { log.catalogPatches.push(body()) @@ -624,60 +587,12 @@ async function fakeApi(route) { if (path === '/datasources/src-shop/catalog/orders/impact' && method === 'GET') { return json({ table: 'orders', templates: [] }) } - if (path === '/copilot/review' && method === 'POST') { - const wait = ctl.reviewDelay[body().run_id] - if (wait) await new Promise((r) => setTimeout(r, wait)) - if (ctl.lateReview && String(body().run_id).startsWith('run-decide')) { - // 复核慢过问数据页等的上限（检查里调短成几百毫秒）：答案先交出去，这一次复核的用量晚到了照样要记进这一轮 - await new Promise((r) => setTimeout(r, ctl.lateReview)) - log.reviews.push(body()) - return json({ verdict: 'ok', note: '', answer: null, retry: false, severity: '', signals: [], retry_instruction: null, - usage: { cost_usd: 0.004, unpriced_calls: 0, calls: 1 } }) - } - if (String(body().run_id).startsWith('run-decide')) { - // 新服务端：第一次判不可用、给出重来的那句话；重来之后的那次没问题、不再重来。各带复核的用量 - log.reviews.push(body()) - const first = body().attempt === 0 - return json({ verdict: first ? 'annotated' : 'ok', note: first ? '查到一半就停了' : '', answer: null, retry: first, - severity: first ? 'broken' : '', signals: first ? [{ kind: 'step_limit', detail: '用满了步数', severity: 'broken' }] : [], - retry_instruction: first ? DECIDE_RETRY : null, - usage: { cost_usd: first ? 0.003 : 0, unpriced_calls: 0, calls: first ? 1 : 0 } }) - } - // 老后端的复核不认 _evidence，照样回一次改写 - if (body().run_id === 'run-evid' || body().run_id === 'run-evq') { - return json({ verdict: 'rewritten', note: '检索降级过，结论请对照原始数据', answer: '改写后的答案：销售额大约四万多', - original: body().run_id === 'run-evq' ? EVQ.output.answer : EVIDENCE.output['周报'], retry: false, severity: 'degraded', - signals: [{ kind: 'retrieval_degraded', detail: '检索退回关键词', severity: 'degraded' }] }) - } - return json({ verdict: 'ok', note: '', answer: null, retry: false, severity: '', signals: [] }) - } + if (path === '/copilot/review' && method === 'POST') return json(await reviewScript(body())) if (path === '/runs' && method === 'POST') { - const b = body() - const conversation = String(b.input?.question ?? '') - // 发起请求根本没到后端（网断了、后端正在重启） - if (conversation.includes('启动失败') && ctl.launchFail) return route.abort('connectionrefused') - // 发起就被拒：绑定的工具不存在（后端 runs.TOOL_MISSING，{detail, code} 的形状） - if (conversation.includes('工具不在')) { - return json({ detail: '绑定的工具不存在：「查数」（调用工具）绑定的 db_query__nope。请到「数据」页接入，或在节点中重新选择', - code: 'run_tool_missing' }, 422) - } - const id = conversation.includes('整形解析失败') ? (conversation.includes('现在的说法') - ? (conversation.includes('上游') ? 'run-jsonup-now' : 'run-jsonerr-now') - : conversation.includes('节点名带超时') ? 'run-jsonslow' - : conversation.includes('上游') ? 'run-jsonup' - : conversation.includes('改写') ? 'run-jsonauth' : 'run-jsonerr') - : conversation.includes('问数据带证据') ? 'run-evq' - : conversation.includes('带证据') ? 'run-evid' - : conversation.includes('长答案') ? (conversation.includes('取不到运行') ? 'run-noget' : 'run-long') - : conversation.includes('审批') ? 'run-wait' - : conversation.includes('重启') ? 'run-restart' - : conversation.includes('正在写') ? 'run-writing' - : conversation.includes('强杀') ? 'run-killed' - : conversation.includes('步数用满') ? `run-steps-${++runSeq}` - : conversation.includes('照服务端的决定') ? `run-decide-${++runSeq}` - : `run-busy-${++runSeq}` - log.runStarts.push({ id, graph: b.graph, origin_ref: b.origin_ref }) - return json({ id, status: 'queued', run_class: 'exploratory', output: {}, usage: {} }) + const got = launchScript(body()) + if (got.abort) return route.abort('connectionrefused') + if (got.status) return json(got.json, got.status) + return json({ id: got.id, status: 'queued', run_class: 'exploratory', output: {}, usage: {} }) } m = path.match(/^\/runs\/([^/]+)\/(cancel|continue)$/) if (m && method === 'POST') { @@ -688,19 +603,9 @@ async function fakeApi(route) { } m = path.match(/^\/runs\/([^/]+)$/) if (m && method === 'GET') { - const id = m[1] - log.runGets.push(id) - if (id === 'run-long') return json({ id, status: 'succeeded', run_class: 'exploratory', output: { answer: FULL }, usage: {} }) - if (id === 'run-noget' || (id === 'run-unk' && ctl.run500)) return json({ detail: '服务端错误（500），详情请查看服务日志' }, 500) - if (id === 'run-unk') return json({ id, status: 'cancelled', run_class: 'exploratory', output: {}, usage: {} }) - if (id.startsWith('run-404')) return json({ detail: '运行不存在' }, 404) - if (RUNS[id]) return json({ id, run_class: 'exploratory', usage: {}, ...RUNS[id] }) - if (id.startsWith('run-c') || id === 'run-b1') return json({ id, status: 'succeeded', run_class: 'exploratory', output: {}, usage: {} }) - if (id.startsWith('run-decide')) { - return json({ id, status: 'succeeded', run_class: 'exploratory', output: { answer: '三家门店。' }, - usage: { cost_usd: 0.02, calls: 2, input_tokens: 6000, output_tokens: 400 } }) - } - return json({ id, status: 'interrupted', run_class: 'exploratory', output: {}, usage: {} }) + log.runGets.push(m[1]) + const got = runRecord(m[1]) + return json(got.json, got.status) } if (path.startsWith('/approvals')) { log.approvalGets.push(Date.now()) @@ -732,6 +637,197 @@ async function fakeApi(route) { return route.abort() // 其余写操作一律不放 } +/** + * 复核的剧本（服务端 api/copilot.review_of_run 的结果），按运行 id 挑；有的故意慢（reviewDelay、lateReview）。问数据的一轮由伪造的 + * 服务端（fakeTurns）调它，/copilot/review 照旧答它 + */ +async function reviewScript(b) { + const wait = ctl.reviewDelay[b.run_id] + if (wait) await new Promise((r) => setTimeout(r, wait)) + if (ctl.lateReview && String(b.run_id).startsWith('run-decide')) { + // 复核慢过问数据页等的上限（检查里调短成几百毫秒）：答案先交出去，这一次复核的用量晚到了照样要记进这一轮 + await new Promise((r) => setTimeout(r, ctl.lateReview)) + log.reviews.push(b) + return { verdict: 'ok', note: '', answer: null, retry: false, severity: '', signals: [], retry_instruction: null, + usage: { cost_usd: 0.004, unpriced_calls: 0, calls: 1 } } + } + if (String(b.run_id).startsWith('run-decide')) { + // 新服务端：第一次判不可用、给出重来的那句话；重来之后的那次没问题、不再重来。各带复核的用量 + log.reviews.push(b) + const first = b.attempt === 0 + return { verdict: first ? 'annotated' : 'ok', note: first ? '查到一半就停了' : '', answer: null, retry: first, + severity: first ? 'broken' : '', signals: first ? [{ kind: 'step_limit', detail: '用满了步数', severity: 'broken' }] : [], + retry_instruction: first ? DECIDE_RETRY : null, + usage: { cost_usd: first ? 0.003 : 0, unpriced_calls: 0, calls: first ? 1 : 0 } } + } + // 老后端的复核不认 _evidence，照样回一次改写 + if (b.run_id === 'run-evid' || b.run_id === 'run-evq') { + return { verdict: 'rewritten', note: '检索降级过，结论请对照原始数据', answer: '改写后的答案：销售额大约四万多', + original: b.run_id === 'run-evq' ? EVQ.output.answer : EVIDENCE.output['周报'], retry: false, severity: 'degraded', + signals: [{ kind: 'retrieval_degraded', detail: '检索退回关键词', severity: 'degraded' }] } + } + return { verdict: 'ok', note: '', answer: null, retry: false, severity: '', signals: [] } +} + +/** 发起一次运行（服务端 engine/launch.launch_run）：按问题挑这次运行的 id，记下发起时的图和来源。被拒的给 {status, json} */ +function launchScript(b) { + const conversation = String(b.input?.question ?? '') + // 发起请求根本没到后端（网断了、后端正在重启） + if (conversation.includes('启动失败') && ctl.launchFail) return { abort: true } + // 发起就被拒：绑定的工具不存在（后端 runs.TOOL_MISSING，{detail, code} 的形状） + if (conversation.includes('工具不在')) { + return { status: 422, json: { detail: '绑定的工具不存在：「查数」（调用工具）绑定的 db_query__nope。请到「数据」页接入，或在节点中重新选择', + code: 'run_tool_missing' } } + } + const id = conversation.includes('整形解析失败') ? (conversation.includes('现在的说法') + ? (conversation.includes('上游') ? 'run-jsonup-now' : 'run-jsonerr-now') + : conversation.includes('节点名带超时') ? 'run-jsonslow' + : conversation.includes('上游') ? 'run-jsonup' + : conversation.includes('改写') ? 'run-jsonauth' : 'run-jsonerr') + : conversation.includes('问数据带证据') ? 'run-evq' + : conversation.includes('带证据') ? 'run-evid' + : conversation.includes('长答案') ? (conversation.includes('取不到运行') ? 'run-noget' : 'run-long') + : conversation.includes('刷新后接着看') ? 'run-replay' + : conversation.includes('审批') ? 'run-wait' + : conversation.includes('重启') ? 'run-restart' + : conversation.includes('正在写') ? 'run-writing' + : conversation.includes('强杀') ? 'run-killed' + : conversation.includes('步数用满') ? `run-steps-${++runSeq}` + : conversation.includes('照服务端的决定') ? `run-decide-${++runSeq}` + : `run-busy-${++runSeq}` + log.runStarts.push({ id, graph: b.graph, origin_ref: b.origin_ref }) + return { id } +} + +/** 库里的运行（GET /api/runs/{id}）：{status, json} */ +function runRecord(id) { + if (id === 'run-long') return { status: 200, json: { id, status: 'succeeded', run_class: 'exploratory', output: { answer: FULL }, usage: {} } } + if (id === 'run-noget' || (id === 'run-unk' && ctl.run500)) return { status: 500, json: { detail: '服务端错误（500），详情请查看服务日志' } } + if (id === 'run-unk') return { status: 200, json: { id, status: 'cancelled', run_class: 'exploratory', output: {}, usage: {} } } + if (id.startsWith('run-404')) return { status: 404, json: { detail: '运行不存在' } } + if (RUNS[id]) return { status: 200, json: { id, run_class: 'exploratory', usage: {}, ...RUNS[id] } } + if (id.startsWith('run-c') || id === 'run-b1') return { status: 200, json: { id, status: 'succeeded', run_class: 'exploratory', output: {}, usage: {} } } + if (id.startsWith('run-decide')) { + return { status: 200, json: { id, status: 'succeeded', run_class: 'exploratory', output: { answer: '三家门店。' }, + usage: { cost_usd: 0.02, calls: 2, input_tokens: 6000, output_tokens: 400 } } } + } + return { status: 200, json: { id, status: 'interrupted', run_class: 'exploratory', output: {}, usage: {} } } +} + +/** + * 助手生成过程的剧本（服务端 api/copilot.answer_ops 交出的那一串操作），按问法挑。开流之前就被拒的给 {status, json}。 + * 画布助手经 /copilot/generate-stream 收它；问数据的一轮由伪造的服务端（fakeTurns）照它跑 + */ +function copilotScript(b) { + if (ctl.scope400 && b.datasource_ids?.length) { + // 和后端 copilot._sources 同一个形状：detail 是一句话，code 是稳定的机读码 + // reworded：后端换了说法，只剩机读码认得出来 + return { status: 400, json: { detail: ctl.scope400 === 'reworded' ? '这一轮圈定的库一个也找不到了：可能已经被删掉或停用' + : '限定的数据源都已不可用，可能已被删除或停用。请取消限定后重试，或到「数据」页确认数据源存在且已启用', code: 'datasource_scope_empty' } } + } + if (String(b.instruction ?? '').includes('建流程就失败')) { + // 后端的 error 操作是三段：人话、怎么办、原始异常 + const ops = [ + { op: 'heartbeat', phase: 'planning', elapsed_ms: 400 }, + { op: 'error', message: '模型返回的工作流结构有误：第 2 个节点的类型「sql」不存在', + hint: '请重试，或把需求描述得更具体', + detail: "2 validation errors for GraphSpec\nnodes.1.type\n Input should be 'input', 'output', 'llm' [type=literal_error]" }, + ] + return { ops } + } + if (String(b.instruction ?? '').includes('自查没修好')) { + // 服务端自查交回模型改过、还是没过（check failed），final 带着同一批问题：前端不该再整轮重来 + const ops = [ + { op: 'heartbeat', phase: 'planning', elapsed_ms: 300 }, + ...GRAPH.nodes.map((n) => ({ op: 'add_node', node: n })), + { op: 'check', status: 'failed', issues: ['「查数」节点还没有选择工具'] }, + { op: 'final', graph: GRAPH, explanation: '', issues: [{ level: 'error', node_id: 'ag', message: '「查数」节点还没有选择工具' }] }, + ] + return { ops } + } + if (String(b.instruction ?? '').includes('只写了思考')) { + // 生成过程一个操作都没给（deepseek 偶发只出思考）：服务端的 error 带 next=retry（backend/app/chat/turn.py 的 next_step）。 + // 重来那一次（指令里缀了「上一次没有成功」）照常直接回答；「两次都」那一问每次都只写思考 + const again = String(b.instruction).includes('上一次没有成功') && !String(b.instruction).includes('两次都') + const spent = { by_model: { m: { input_tokens: 500, output_tokens: 40, calls: 1, cost_usd: 0.001, unpriced_calls: 0 } }, + input_tokens: 500, output_tokens: 40, calls: 1, cost_usd: 0.001, unpriced_calls: 0 } + const ops = again ? [ + { op: 'heartbeat', phase: 'planning', elapsed_ms: 200 }, + { op: 'reply', text: '已经查过了：一共 3 家门店。' }, + { op: 'usage', ...spent }, + ] : [ + { op: 'heartbeat', phase: 'planning', elapsed_ms: 200 }, + { op: 'thinking', delta: '先想一想要查哪张表……' }, + { op: 'error', message: '助手本轮未完成：模型没有给出任何修改，也没有说明原因', hint: '画布未修改。可以重试，或把需求描述得更具体', + detail: '', reason: '模型没有给出任何修改，也没有说明原因', code: 'copilot_empty', next: 'retry' }, + { op: 'usage', ...spent }, + ] + return { ops } + } + if (String(b.instruction ?? '').includes('照服务端的决定') || String(b.instruction ?? '').includes(DECIDE_RETRY)) { + // 新服务端：final 带 next（backend/app/chat/turn.py），最后一条是这一轮建图的用量（answer_ops 的 usage）。 + // 「只建不跑」那一问：autorun 没写 false，next 却是 ready——前端得照 next 走 + const ready = String(b.instruction ?? '').includes('只建不跑') + const ops = [ + { op: 'heartbeat', phase: 'planning', elapsed_ms: 300 }, + ...GRAPH.nodes.map((n) => ({ op: 'add_node', node: n })), + { op: 'done', explanation: '一张三步的图' }, + { op: 'check', status: 'passed', repaired: 0, warnings: [] }, + { op: 'final', graph: GRAPH, issues: [], explanation: '一张三步的图', autorun: true, next: ready ? 'ready' : 'run' }, + { op: 'usage', by_model: { m: { input_tokens: 4000, output_tokens: 300, calls: 1, cost_usd: 0.012, unpriced_calls: 0 } }, + input_tokens: 4000, output_tokens: 300, calls: 1, cost_usd: 0.012, unpriced_calls: 0 }, + ] + return { ops } + } + if (/刷新后接着看|停止测试/.test(String(b.instruction ?? ''))) { + // 建到一半停在闸上：刷新、关掉再打开、停止都要在建图中途发生 + const gate = String(b.instruction).includes('停止测试') ? 'stopBuildGo' : 'replayBuildGo' + const ops = [ + { op: 'heartbeat', phase: 'planning', elapsed_ms: 300 }, + { op: 'plan', summary: '查一下再回答' }, + ...GRAPH.nodes.slice(0, 2).map((n) => ({ op: 'add_node', node: n })), + { gate }, + { op: 'add_node', node: GRAPH.nodes[2] }, + { op: 'done', explanation: '一张三步的图' }, + { op: 'final', graph: GRAPH, issues: [], explanation: '一张三步的图', autorun: true, next: 'run' }, + ] + return { ops } + } + if (String(b.instruction ?? '').includes('先问清楚')) { + // 助手要先问清楚才能动手（服务端 copilot_gen 的 ask）：问题转出去，这一轮到此为止——没有 final，也没有 reply + const ops = [ + { op: 'heartbeat', phase: 'planning', elapsed_ms: 300 }, + { op: 'ask', question: '销售额按下单日还是支付日算？' }, + ] + return { ops } + } + if (String(b.instruction ?? '').includes('表示作废')) { + // 用户说了一条数据事实：服务端核对后转出目录修改提案，只回答、不建图 + const ops = [ + { op: 'heartbeat', phase: 'planning', elapsed_ms: 300 }, + { op: 'catalog_patch', source: 'shop', source_id: 'src-shop', table: 'orders', table_label: '订单', version: 2, changes: [ + { path: 'columns.status.codes', before: { 1: '已支付' }, before_status: 'confirmed', after: { 1: '已支付', 9: '作废' }, + value: { 9: '作废' }, reason: '用户说明 status=9 表示作废', state: 'change' }] }, + { op: 'reply', text: '已整理成数据目录的修改建议，确认后保存即可。' }, + ] + return { ops } + } + const ops = [ + { op: 'heartbeat', phase: 'planning', elapsed_ms: 800 }, + { op: 'heartbeat', phase: 'looking', elapsed_ms: 1000 }, + // 助手按需查看了表（look 事件）：过程里多一行「查看了 N 张表」，查看记录存进这一轮的 meta.looks。 + // 这个库还没有数据目录（catalog_tables: 0）：那一行下面提示先去起草目录 + { op: 'look', id: 'L1', by: 'model', need: '查订单和明细', status: 'ok', elapsed_ms: 1200, objects: [ + { kind: 'table', source: 'shop', source_id: 'src-shop', named: [], picked: ['orders', 'order_items'], joined: [], + missing: [], total: 48, catalog_tables: 0 }] }, + { op: 'plan', summary: '查一下再回答' }, + ...GRAPH.nodes.map((n) => ({ op: 'add_node', node: n })), + { op: 'done', explanation: '一张三步的图' }, + { op: 'final', graph: GRAPH, issues: [], explanation: '一张三步的图' }, + ] + return { ops } +} + /** 每条运行的事件剧本。done=true 时剧本放完就发 stream.end */ const ev = (seq, type, node_id, data = {}) => ({ seq, type, node_id, data, ts: Date.now() / 1000 }) function script(runId, after = 0) { @@ -814,6 +910,19 @@ function script(runId, after = 0) { end: 'succeeded', } } + if (runId === 'run-replay') { + // 跑到一半停在闸上：关掉再打开时运行还在跑 + return { + events: [ + ev(1, 'run.started', null, { nodes: 3 }), + ev(2, 'node.started', 'ag', { node_type: 'agent', label: '查数' }), + { gate: 'replayRunGo' }, + ev(3, 'node.finished', 'ag', { duration_ms: 900 }), + ev(4, 'run.finished', null, { output: { answer: '刷新之后接着跑完了：两家门店。' } }), + ], + end: 'succeeded', + } + } if (runId === 'run-restart' && after >= 4) { // 接着跑：从断点往下，这次跑完 return { @@ -905,7 +1014,8 @@ function script(runId, after = 0) { async function fakeStream(ws) { const runId = new URL(ws.url()).pathname.split('/')[3] ctl.wsClosed[runId] = false - ws.onClose(() => { ctl.wsClosed[runId] = true }) + let closed = false + ws.onClose(() => { closed = true; ctl.wsClosed[runId] = true }) const after = Number(new URL(ws.url()).searchParams.get('after') ?? 0) const { events, end } = script(runId, after) for (const e of events) { @@ -914,6 +1024,7 @@ async function fakeStream(ws) { continue } if (e.seq <= after) continue + if (closed) return if (runId === 'run-writing' && e.type === 'llm.token' && !ctl.writingAt) ctl.writingAt = Date.now() // 审批和中断同时产生：后端在停下的那一刻才建出这条待审批 if (runId === 'run-wait' && e.type === 'run.interrupted') { @@ -923,8 +1034,474 @@ async function fakeStream(ws) { ws.send(JSON.stringify(e)) await new Promise((r) => setTimeout(r, 15)) } + if (closed) return if (end === 'close') ws.close() else if (end) ws.send(JSON.stringify({ type: 'stream.end', status: end, data: { status: end } })) + // 剧本放完了：伪造的服务端据此认这次运行到了终态（真服务端按库里的状态，事件先落库） + if (end) ctl.wsDone[`${runId}@${after}`] = true +} + +// ---------------------------------------------------------------- 伪造的服务端一轮 + +/** + * 问数据的一轮在服务端跑（backend/app/chat/job.py；接口 backend/app/api/chat_turns.py）。页面只发起（POST /api/chat/turns）、 + * 只看（GET /api/chat/turns/{id}/stream 先回放、再推实时，断了按 seq 重连），这里照服务端的做法把一轮跑完： + * 生成（copilotScript 的剧本，原样记进 log.gens）→ final.next → 发起运行（launchScript，记进 log.runStarts）→ 等运行的剧本放完 + * （fakeStream）→ 复核（reviewScript，最多等 ctl.reviewWaitMs，晚到的照记用量）→ 按复核重来一次 → 照服务端的写法拼那一行（end）。 + * 实时流是长轮询：没有新事件就等一会儿再答，答完连接就关；页面按收到的最后一个 seq 马上接着连（和服务端断线时一样）。 + */ +const srv = { turns: new Map(), seq: 0, launched: new Set(), gen: 0 } +const ACTIVE_JOB = new Set(['queued', 'running']) +const turnActive = (T) => ACTIVE_JOB.has(T.job.status) || !!T.late +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)) +const BUSY_TEXT = '这段对话正有一轮在进行：等它结束或先停止，再提问' + +function emit(T, type, data = {}) { + const e = { seq: T.events.length + 1, type, data, ts: Date.now() / 1000 } + T.events.push(e) + T.job.last_seq = e.seq + return e.seq +} + +/** 那一行（GET /api/conversations/{id} 里的一轮、end 里带的 turn） */ +const rowOut = (T) => ({ ...T.row, meta: T.row.meta ? { ...T.row.meta } : T.row.meta, job: { ...T.job } }) + +const addTotal = (a, b) => { + const out = { ...(a ?? {}) } + for (const k of ['cost_usd', 'unpriced_calls', 'calls', 'unfinished_calls', 'input_tokens', 'output_tokens']) { + const v = Number(b?.[k] ?? 0) + if (Number.isFinite(v) && v) out[k] = Math.round((Number(out[k] ?? 0) + v) * 1e6) / 1e6 + } + return out +} +const retireRun = (usage) => { + const { run, ...rest } = usage ?? {} + return run && Object.keys(run).length ? { ...rest, earlier: addTotal(rest.earlier, run) } : rest +} +const answerText = (output) => Object.entries(output ?? {}) + .filter(([k, v]) => !k.startsWith('_') && v != null && v !== '').map(([, v]) => (typeof v === 'string' ? v : JSON.stringify(v))).join('\n') + +/** 用户点重试：上一次的结局归档（同服务端 chat/record.attempt_record） */ +function attemptRecord(row, n) { + const meta = row.meta ?? {} + const review = row.review ?? {} + const base = { n, at: Date.now(), runId: meta.runId !== undefined ? meta.runId : row.run_id ?? null, + ...(row.answer ? { answer: row.answer.length > 4000 ? `${row.answer.slice(0, 4000)}…` : row.answer } : {}) } + const signals = (review.signals ?? []).map((x) => x.detail).join('\n') || undefined + if (meta.outcome === 'cancelled' && meta.rejected) { + const sub = meta.rejected.sub ?? '' + return { ...base, outcome: 'rejected', summary: `被驳回${sub ? `：${sub}` : ''}` } + } + if (meta.outcome === 'cancelled' || (row.status === 'error' && row.error === '已取消')) return { ...base, outcome: 'cancelled', summary: '已手动停止' } + if (meta.outcome === 'suspended') return { ...base, outcome: 'suspended', summary: '服务重启，运行中断' } + if (row.status === 'error') { + const f = meta.failure + return { ...base, outcome: 'failed', summary: f ? (f.reason ? `${f.title}：${f.reason}` : f.title) : row.error || '发生错误', + ...(meta.cause ? { cause: meta.cause } : {}) } + } + if (review.severity === 'broken') return { ...base, outcome: 'unusable', summary: `结论不可用：${review.note ?? ''}`, detail: signals } + if (review.severity === 'degraded') return { ...base, outcome: 'partial', summary: `存在缺口：${review.note ?? ''}`, detail: signals } + return { ...base, outcome: 'done', summary: '已给出答案' } +} +const attemptReason = (a) => (a.detail ? `${a.summary}（${String(a.detail).slice(0, 300)}）` : a.summary) + +/** 种一轮：库里已经有的（夹具里的那一行），或者新问的 */ +function seedTurn(conversationId, id, row) { + const T = { + id, conversation_id: conversationId, row: { ...row, meta: row.meta ? { ...row.meta } : null }, + job: { status: 'succeeded', stage: null, mode: 'ask', run_id: null, attempt: 0, last_seq: 0, attempt_seq: 0, error: null }, + events: [], cancelled: false, late: null, runId: null, usage: {}, looks: [], + } + srv.turns.set(id, T) + return T +} +function knownRow(conversationId, turnId) { + const rows = typeof DETAIL[conversationId] === 'function' ? DETAIL[conversationId]() : DETAIL[conversationId] ?? [] + return rows.find((t) => t.id === turnId) ?? null +} +const scopeOf = (ids) => (ids ?? []).map((id) => { + const hit = [...SOURCES, EXTRA_SOURCE].find((x) => x.id === id || x.name === id) + return hit ? { id: hit.id, name: hit.name } : { id, name: id } +}) + +/** 这一轮还在跑时每一步之间看一眼：停了、这一段检查结束了，就不再往下写 */ +const gone = (T, epoch) => T.cancelled || T.epoch !== epoch || T.gen !== srv.gen + +async function startTurn(b) { + const mode = b.mode ?? (b.retry_of ? 'retry' : 'ask') + let T + if (mode === 'ask') { + log.posts.push(b.conversation_id) + const id = `srv-${b.conversation_id}-${++srv.seq}` + const scope = scopeOf(b.datasource_ids) + T = seedTurn(b.conversation_id, id, turn(id, b.question, { status: 'running', meta: { v: 1, ...(scope.length ? { scope } : {}) } })) + } else { + T = srv.turns.get(b.retry_of) + // 夹具里的那一行：每一段检查从库里原样的那一行起（上一段、上一套主题对它做过什么都不算） + const row = !T || T.gen !== srv.gen ? knownRow(b.conversation_id, b.retry_of) : null + if (row) T = seedTurn(b.conversation_id, b.retry_of, row) + if (!T) return { status: 404, json: { detail: '该轮对话不存在，可能已被删除' } } + } + const others = [...srv.turns.values()].filter((x) => x !== T && x.conversation_id === b.conversation_id + && ACTIVE_JOB.has(x.job.status) && x.job.stage !== 'waiting' && x.gen === srv.gen) + if (mode !== 'resume' && others.length) return { status: 409, json: { detail: BUSY_TEXT, code: 'chat_turn_busy' } } + if (mode !== 'ask' && ACTIVE_JOB.has(T.job.status)) return { status: 409, json: { detail: BUSY_TEXT, code: 'chat_turn_busy' } } + const meta = { ...(T.row.meta ?? {}) } + const runId = meta.runId !== undefined ? meta.runId : T.row.run_id ?? null + const now = Date.now() + const request = { mode, started_ms: now } + let attempts = [...(meta.attempts ?? [])] + const scope = b.datasource_ids ? scopeOf(b.datasource_ids) : meta.scope ?? [] + if (mode === 'continue') { + log.continues.push(runId) + if (runId !== 'run-restart' && runId !== 'run-susp') { + return { status: 409, json: { detail: '这次运行已完成，无需继续运行：只有失败的运行或被服务重启打断的运行可以继续运行。如需再次运行，请重新发起。' } } + } + ctl.continued[runId] = true + delete ctl.wsDone[`${runId}@4`] + } + if (mode === 'resume' && ctl.run500 && runId === 'run-unk') { + // 服务端核对那一下出错（查库超时之类）：页面说没核对上，给「重新核对」 + return { status: 500, json: { detail: '服务端错误（500），详情请查看服务日志' } } + } + if (mode === 'continue' || mode === 'resume') { + request.follow = runId + if (T.row.status !== 'running' && typeof meta.ms === 'number') request.started_ms = now - meta.ms + const rec = runRecord(String(runId ?? '')) + if (T.row.status === 'running' && rec.json?.started_at) request.started_ms = Date.parse(rec.json.started_at) + if (mode === 'resume' && ['succeeded', 'failed', 'cancelled'].includes(rec.json?.status) && rec.json?.finished_at) { + request.ended_ms = Date.parse(rec.json.finished_at) + } + T.row = { ...T.row, status: 'running', error: '', meta: { ...meta, scope: scope.length ? scope : undefined } } + } else { + if (['rerun', 'max_steps', 'run'].includes(mode) && !(T.row.graph?.nodes?.length)) { + return { status: 409, json: { detail: '这一轮没有可运行的工作流，请重试本轮', code: 'chat_turn_invalid' } } + } + if (runId && mode !== 'run') { + // 服务重启挂起的旧运行没人会再去续它：顺手取消（停在审批上的不算） + const rec = runRecord(runId) + const pending = ctl.approvals.some((a) => a.run_id === runId && a.status === 'pending') + if (rec.json?.status === 'interrupted' && !pending) log.cancels.push(runId) + } + if (mode !== 'run') { + const archived = attemptRecord(T.row, attempts.length + 1) + attempts.push(archived) + if (mode === 'retry') { + const reason = attemptReason(archived) + request.base = !!T.row.graph?.nodes?.length + request.instruction = request.base + ? `这个工作流上一次没有给出可用的结果：${reason}。\n\n请针对这个原因改它（比如调大最大步数、改正工具参数、换个查询方式），改完继续回答原来的问题：${T.row.question}` + : `${T.row.question}\n\n（上一次没有成功：${reason}。请针对这个原因调整，重新给出完整的工作流。）` + } + if (mode === 'max_steps') request.max_steps = b.max_steps + } + const usage = mode === 'run' ? meta.usage : retireRun(meta.usage) + T.row = { ...T.row, status: 'running', error: '', answer: mode === 'run' ? T.row.answer : '', review: {}, run_id: null, + meta: mode === 'run' ? { ...meta, runId: null } : { v: 1, ...(attempts.length ? { attempts } : {}), + ...(scope.length ? { scope } : {}), ...(usage && Object.keys(usage).length ? { usage } : {}), runId: null } } + } + T.epoch = (T.epoch ?? 0) + 1 + T.gen = srv.gen + T.cancelled = false + T.late = null + T.request = request + T.scope = scope + T.job = { status: 'running', stage: null, mode, run_id: request.follow ?? null, attempt: 0, last_seq: T.events.length, + attempt_seq: 0, error: null } + T.job.attempt_seq = emit(T, 'turn.attempt', { + mode, attempts, scope, usage: T.row.meta?.usage ?? {}, started_ms: request.started_ms, + ...(request.ended_ms ? { ended_ms: request.ended_ms } : {}), ...(request.max_steps ? { max_steps: request.max_steps } : {}), + }) + log.turnStarts.push({ ...b, _turn: T.id }) + void driveTurn(T, T.epoch).catch((e) => console.log(` · 伪造的服务端出错：${e?.stack ?? e}`)) + return { status: 202, json: { turn_id: T.id, turn: rowOut(T) } } +} + +function stage(T, name) { + T.job.stage = name + emit(T, 'stage', { stage: name }) +} + +function saveRow(T, fields, meta) { + const merged = { ...(T.row.meta ?? {}), v: 1 } + for (const [k, v] of Object.entries(meta ?? {})) { + if (v === undefined || v === null) delete merged[k] + else merged[k] = v + } + merged.at = Date.now() + T.row = { ...T.row, ...fields, meta: merged } +} + +/** 这一轮到此为止：写那一行、发 end（同服务端 ChatTurnExecution._conclude） */ +function settle(T, outcome, fields, meta, output = null, run = null) { + const ms = Math.max(0, (T.request.ended_ms ?? Date.now()) - T.request.started_ms) + saveRow(T, fields, { looks: T.looks.length ? T.looks : undefined, ms, ...meta }) + emit(T, 'end', { outcome, turn: rowOut(T), output, run }) + T.job = { ...T.job, status: 'succeeded', stage: null } +} + +async function driveTurn(T, epoch) { + const req = T.request + const mode = req.mode + const autoRetry = mode === 'ask' || mode === 'retry' + let attempts = 0 + let instruction = req.instruction ?? T.row.question + let base = req.base ? T.row.graph : null + let direct = ['rerun', 'max_steps', 'run'].includes(mode) ? T.row.graph : null + if (mode === 'max_steps') { + direct = { ...direct, nodes: direct.nodes.map((n) => (n.type === 'agent' + ? { ...n, data: { ...n.data, config: { ...(n.data?.config ?? {}), max_steps: req.max_steps } } } : n)) } + T.row = { ...T.row, graph: direct } + } + let follow = req.follow ?? null + T.usage = { ...(T.row.meta?.usage ?? {}) } + T.looks = [] + while (true) { + if (!direct && !follow) { + stage(T, 'build') + const gen = { instruction, base_graph: base, intent: 'answer', conversation_id: T.conversation_id, + ...(T.scope?.length ? { datasource_ids: T.scope.map((x) => x.id) } : {}) } + log.gens.push(gen) + const got = copilotScript(gen) + if (got.status) { + const message = got.json.detail + return settle(T, 'error', { status: 'error', error: message, review: {} }, + { cause: { kind: 'refused', message, code: got.json.code ?? null }, runId: null, usage: T.usage }) + } + const ops = [] + for (const op of got.ops) { + if (op.gate) { await until(() => ctl[op.gate] || gone(T, epoch), 15000); continue } + if (gone(T, epoch)) return + emit(T, 'op', op) + if (op.op !== 'thinking' && op.op !== 'heartbeat') ops.push(op) + if (op.op === 'look') T.looks.push(op) + if (op.op === 'usage') T.usage.copilot = addTotal(T.usage.copilot, op) + if (op.op === 'final' && !op.asked) T.row = { ...T.row, graph: op.graph, explanation: op.explanation ?? '' } + await sleep(ctl.opGapMs ?? 10) + } + if (gone(T, epoch)) return + const terminal = ops.filter((o) => ['final', 'reply', 'error'].includes(o.op)).at(-1) + if (!terminal) { + const ask = ops.filter((o) => o.op === 'ask').at(-1) + const q = String(ask?.question ?? '') + return settle(T, 'asked', { status: 'done', answer: q, review: {} }, + { asked: { question: q, options: ask?.options ?? [] }, runId: null, usage: T.usage }, q ? { answer: q } : null) + } + if (terminal.op === 'error') { + if (terminal.next === 'retry' && attempts + 1 < 2) { + attempts += 1 + const reason = terminal.reason ?? String(terminal.message ?? '').replace(/^助手本轮未完成：/, '') + instruction = base ? `这个工作流上一次运行没有成功：${reason}。\n\n请针对这个原因改它（比如调大最大步数、改正工具参数、换个查询方式），改完继续回答原来的问题：${T.row.question}` + : `${T.row.question}\n\n（上一次没有成功：${reason}。请针对这个原因调整，重新给出完整的工作流。）` + emit(T, 'turn.retry', { kind: 'empty', reason, base: !!base, attempt: attempts, usage: T.usage }) + continue + } + const message = String(terminal.message ?? '助手本轮未完成') + return settle(T, 'error', { status: 'error', error: message, review: {} }, + { cause: { kind: 'copilot', message, hint: terminal.hint, detail: terminal.detail }, runId: null, usage: T.usage }) + } + if (terminal.op === 'reply') { + const text = String(terminal.text ?? '') + return settle(T, 'reply', { status: 'done', answer: text, review: {} }, { noQuery: true, runId: null, usage: T.usage }, + { answer: text }) + } + const blockers = (terminal.issues ?? []).filter((i) => i.level === 'error').map((i) => i.message) + const selfChecked = ops.some((o) => o.op === 'check') + const asked = ops.filter((o) => o.op === 'ask').at(-1) + const next = terminal.next ?? (blockers.length ? (selfChecked ? 'fail' : 'retry') : terminal.autorun === false ? 'ready' : 'run') + if (next === 'fail' || next === 'retry') { + return settle(T, 'error', { status: 'error', error: `生成的工作流没有通过校验：${blockers.join('；')}`, graph: terminal.graph, + explanation: terminal.explanation ?? '', review: {} }, + { cause: { kind: 'blocked', issues: blockers, self_checked: selfChecked }, runId: null, usage: T.usage }) + } + if (next === 'ready') { + const q = String(asked?.question ?? '').trim() + return settle(T, 'ready', { status: 'done', graph: terminal.graph, explanation: terminal.explanation ?? '', review: {}, + ...(q ? { answer: q } : {}) }, + { runId: null, usage: T.usage, ...(q ? { asked: { question: q, options: [] } } : {}) }, q ? { answer: q } : null) + } + direct = terminal.graph + } + + stage(T, 'run') + let runId = follow + if (!runId) { + const launched = launchScript({ graph: direct, input: { question: T.row.question, history: '' }, origin: 'chat', + origin_ref: { conversation_id: T.conversation_id, turn_id: T.id } }) + if (launched.abort || launched.status) { + const message = launched.json?.detail ?? '运行没有发起' + return settle(T, 'error', { status: 'error', error: message, review: {} }, + { cause: { kind: 'start', message, code: launched.json?.code ?? null, status: launched.status ?? 0 }, runId: null, usage: T.usage }) + } + runId = launched.id + srv.launched.add(runId) + // 同一个运行 id 各段、各套主题会再用：这一次从头算（剧本还没放、没有接着跑过） + delete ctl.wsDone[`${runId}@0`] + delete ctl.continued[runId] + delete T.usage.run + T.runId = runId + T.job.run_id = runId + saveRow(T, { run_id: runId, status: 'running' }, { runId, runClass: 'exploratory', runStatus: 'queued' }) + emit(T, 'run', { run_id: runId, status: 'queued', run_class: 'exploratory' }) + } else { + T.runId = runId + } + const graphRun = direct + direct = follow = null + const result = await followRun(T, epoch, runId) + if (gone(T, epoch)) return + const rec = runRecord(runId) + const run = { id: runId, status: result.status, run_class: 'exploratory', usage: rec.json?.usage ?? {}, + error: result.error ?? null, end_reason: result.endReason ?? null } + T.usage.run = addTotal({}, rec.json?.usage) + const meta = { runId, runClass: 'exploratory', runStatus: result.status, usage: T.usage } + if (result.status === 'gone') { + return settle(T, 'error', { status: 'error', error: '运行记录不见了', review: {} }, { ...meta, cause: { kind: 'gone' }, runStatus: 'gone' }) + } + if (result.status === 'rejected') { + return settle(T, 'cancelled', { status: 'error', error: '已取消', review: {} }, + { ...meta, runStatus: 'cancelled', outcome: 'cancelled', rejected: result.rejected }, null, { ...run, status: 'cancelled' }) + } + if (result.status === 'cancelled') { + return settle(T, 'cancelled', { status: 'error', error: '已取消', review: {} }, { ...meta, outcome: 'cancelled' }, null, run) + } + if (result.status === 'suspended') { + return settle(T, 'suspended', { status: 'error', error: '服务重启，本轮已中断', review: {} }, + { ...meta, outcome: 'suspended', runStatus: 'interrupted' }, null, { ...run, status: 'interrupted' }) + } + if (result.status === 'failed') { + return settle(T, 'error', { status: 'error', error: result.error ?? '运行失败', review: {} }, + { ...meta, cause: { kind: 'run', error: result.error ?? '', ...(result.detail ? { detail: result.detail } : {}) } }, null, run) + } + + stage(T, 'review') + // 成果带逐段证据时复核只能加说明（服务端 review_of_run 按 locked 不改写）：剧本里老后端那种改写在这里收成「加说明」 + const locked = !!(result.output ?? {})._evidence + const reviewing = reviewScript({ run_id: runId, question: T.row.question, attempt: attempts }).then((r) => (locked && r?.answer + ? { ...r, answer: null, original: null, verdict: r.verdict === 'rewritten' ? 'annotated' : r.verdict } : r)) + const review = await Promise.race([reviewing, sleep(ctl.reviewWaitMs ?? 25_000).then(() => null)]) + if (gone(T, epoch)) return + if (review) { + emit(T, 'review', { result: review }) + T.usage.review = addTotal(T.usage.review, review.usage) + if (autoRetry && review.retry_instruction) { + attempts += 1 + T.usage = retireRun(T.usage) + T.runId = null + saveRow(T, { run_id: null }, { runId: null, runStatus: null, usage: T.usage }) + const reason = (review.signals ?? []).map((x) => x.detail).join('；') || review.note + emit(T, 'turn.retry', { kind: 'review', reason, base: !!graphRun?.nodes?.length, attempt: attempts, usage: T.usage }) + instruction = review.retry_instruction + base = graphRun?.nodes?.length ? graphRun : null + continue + } + } else { + // 等不及：先交原答案；那一次复核晚到了照样把用量记进这一轮 + T.late = reviewing.then((r) => { + if (T.epoch !== epoch) return + if (r?.usage) { + const usage = { ...(T.row.meta?.usage ?? {}), review: addTotal(T.row.meta?.usage?.review, r.usage) } + saveRow(T, {}, { usage }) + emit(T, 'review.late', { usage: r.usage }) + } + }).finally(() => { if (T.epoch === epoch) T.late = null }) + } + const output = result.output ?? {} + const rewritten = review && review.verdict !== 'ok' && !output._evidence ? review.answer : null + const issuance = output._issuance + return settle(T, 'done', { status: 'done', answer: rewritten || answerText(output), review: review ?? {} }, + { ...meta, runStatus: 'succeeded', queries: result.queries, + ...(issuance ? { issuance } : {}), ...(output._evidence ? { evidence: true } : {}) }, output, { ...run, status: 'succeeded' }) + } +} + +/** + * 等这次运行到终态（同服务端 _follow：停在审批上照旧等人，没有待审批的断点收成中断）。这一段检查里发起的运行按它的剧本： + * 剧本有结局的等实时流放完再收尾，没结局的（在跑、等审批、出字中）一直等到被停止；库里早就有的运行（老轮次接着收尾）按 + * GET 运行的状态 + */ +async function followRun(T, epoch, runId) { + const after = ctl.continued[runId] ? 4 : 0 + const sc = script(runId, after) + const rec = runRecord(runId) + if (rec.status === 404) return { status: 'gone' } + const pending = () => ctl.approvals.some((a) => a.run_id === runId && a.status === 'pending') + const terminal = (status) => { + const fin = sc.events.filter((e) => e.type === 'run.finished').at(-1)?.data?.output + const failed = sc.events.filter((e) => e.type === 'run.failed').at(-1)?.data + const queries = sc.events.filter((e) => e.type === 'tool.start' && String(e.data?.tool ?? '').startsWith('db_query')).length + const db = rec.status === 200 && rec.json.status === 'succeeded' && Object.keys(rec.json.output ?? {}).length ? rec.json.output : null + if (status === 'succeeded') { + // 成果以库里那份为准（事件里那份为了推送体积被切过） + return { status, queries, output: db ?? (runId === 'run-noget' ? { answer: FULL, note: '附注' } : fin ?? {}) } + } + if (status === 'failed') return { status, error: failed?.error ?? rec.json?.error ?? '', detail: failed?.detail } + if (status === 'cancelled' && rec.json?.end_reason === 'rejected') { + return { status: 'rejected', endReason: 'rejected', rejected: { sub: rec.json?.error || '被驳回' } } + } + return { status } + } + /** 停在审批上：等人（stage waiting），审批过了接着等 */ + const waitApproval = async () => { + if (T.job.stage !== 'waiting') stage(T, 'waiting') + while (!gone(T, epoch) && pending()) await sleep(80) + if (!gone(T, epoch)) stage(T, 'run') + } + const deadline = Date.now() + 120_000 + if (srv.launched.has(runId) || ctl.continued[runId]) { + const ended = sc.end === 'close' ? 'interrupted' : sc.end + if (ended) { + await until(() => ctl.wsDone[`${runId}@${after}`] || gone(T, epoch), 3000) + if (ended !== 'interrupted') return terminal(ended) + if (!pending()) return { status: 'suspended' } + } + // 剧本里不结束的：一直在跑，停在审批上的等人 + while (!gone(T, epoch) && Date.now() < deadline) { + if (pending()) await waitApproval() + await sleep(80) + } + return { status: 'running' } + } + // 库里早就有的运行 + while (!gone(T, epoch) && Date.now() < deadline) { + const status = runRecord(runId).json?.status + if (['succeeded', 'failed', 'cancelled'].includes(status)) return terminal(status) + if (status === 'interrupted') { + if (!pending()) return { status: 'suspended' } + await waitApproval() + } + await sleep(80) + } + return { status: 'running' } +} + +/** 停止（服务端 chat/job.cancel）：取消生成、取消运行，那一行收成已取消 */ +function cancelTurn(T) { + if (!ACTIVE_JOB.has(T.job.status)) return false + T.cancelled = true + if (T.runId) log.cancels.push(T.runId) + saveRow(T, { status: 'error', error: '已取消', review: {} }, { outcome: 'cancelled', runStatus: T.runId ? 'cancelled' : undefined }) + emit(T, 'end', { outcome: 'cancelled', turn: rowOut(T), output: null, + run: T.runId ? { id: T.runId, status: 'cancelled', run_class: 'exploratory', usage: {} } : null }) + T.job = { ...T.job, status: 'cancelled', stage: null } + return true +} + +/** 实时流（长轮询）：没有新事件就等一会儿；这一轮不在进行中、也都给完了，补一条 stream.end */ +async function streamTurn(route, T, after) { + log.streams.push({ turn: T.id, after }) + if (ctl.dropStreams > 0) { + // 断线：这一次连接什么都没给就断了（网络抖一下）。页面按收到的最后一个 seq 退避重连 + ctl.dropStreams -= 1 + return route.abort('connectionreset') + } + await until(() => T.events.length > after || !turnActive(T) || T.gen !== srv.gen, 6000) + const fresh = T.events.filter((e) => e.seq > after) + let body = fresh.map((e) => `data: ${JSON.stringify(e)}\n\n`).join('') + if (!turnActive(T) && after + fresh.length >= T.events.length) { + body += `data: ${JSON.stringify({ type: 'stream.end', status: T.job.status, data: { status: T.job.status } })}\n\n` + } + return route.fulfill({ status: 200, contentType: 'text/event-stream', body: body || ': ping\n\n' }) } // ---------------------------------------------------------------- 浏览器 @@ -945,7 +1522,7 @@ async function open(theme = 'light', { width = 1440, height = 900, reducedMotion return route.fulfill({ json: { ui: { theme }, run: {}, limits: { max_agent_steps: 25 }, copilot: {}, embedding: {}, ...(settings ?? {}) } }) }) - await page.route(/\/api\/(conversations|copilot\/(generate-stream|review)|runs|approvals|datasources)(\/|\?|$)/, fakeApi) + await page.route(/\/api\/(conversations|chat\/turns|copilot\/(generate-stream|review)|runs|approvals|datasources)(\/|\?|$)/, fakeApi) await page.routeWebSocket(/\/api\/runs\/[^/]+\/stream/, fakeStream) return { page, ctx, errors } } @@ -1192,25 +1769,20 @@ for (const theme of THEMES) { const t = (await chatState(page)).byConversation.c0trunc.at(-1) check('显示的是完整答案', String(t.output?.answer ?? '').length === FULL.length, `${String(t.output?.answer ?? '').length} / ${FULL.length}`) check('llm.token 没进 events', !t.events.some((e) => e.type === 'llm.token'), `${t.events.length} 条`) - const saved = log.patches.slice(patchesBefore).filter((p) => typeof p.body.answer === 'string' && p.body.answer) - check('落库的也是完整答案', saved.length > 0 && saved.every((p) => p.body.answer.length === FULL.length), - saved.map((p) => p.body.answer.length).join(',')) - // 落库是排队异步发的，交付之后再等它一下 - await until(() => log.patches.slice(patchesBefore).some((p) => p.body.meta)) - const metas = log.patches.slice(patchesBefore).map((p) => p.body.meta).filter(Boolean) - const meta = metas.at(-1) - check('可信度信息跟着落库（运行、类别、查库次数）', meta?.runId === 'run-long' && meta?.runClass === 'exploratory', - JSON.stringify(metas)) + // 一轮在服务端跑（docs/design/orchestration-round §1）：那一行由服务端写，页面不再 PATCH 这一轮的结局 + check('页面不写这一轮的结局（不 PATCH 轮次）', log.patches.slice(patchesBefore).length === 0, + JSON.stringify(log.patches.slice(patchesBefore).map((p) => Object.keys(p.body)))) + const meta = t.meta + check('服务端写的那一行带着可信度信息（运行、类别），页面照它摆', meta?.runId === 'run-long' && meta?.runClass === 'exploratory', + JSON.stringify(meta)) // 后端有了单独的 meta 列：review 只放复核结论，不再夹带 - check('meta 单独写，不再塞进 review', log.patches.slice(patchesBefore) - .every((p) => !p.body.review || !('meta' in p.body.review)), - JSON.stringify(log.patches.slice(patchesBefore).map((p) => Object.keys(p.body.review ?? {})))) + check('meta 单独存，不夹在 review 里', !t.reviewRaw || !('meta' in t.reviewRaw), JSON.stringify(Object.keys(t.reviewRaw ?? {}))) check('完整答案取到了，meta 里没有「没取全」', !meta?.clipped, JSON.stringify(meta?.clipped)) await page.getByText('展开全部').first().click().catch(() => {}) check('完整结尾在页面上', await shows(page, '【完整结尾】')) // 建图时助手查看了哪些表：收在「规划」里，点开是一行，再点开看表名 check('建图的 look 事件原样记进这一轮', t.ops.some((o) => o.op === 'look')) - check('查看记录存进这一轮的 meta.looks（look §2.5.8）', Array.isArray(meta?.looks) && meta.looks.length === 1 + check('查看记录随那一行存进 meta.looks（look §2.5.8）', Array.isArray(meta?.looks) && meta.looks.length === 1 && meta.looks[0].objects?.[0]?.source === 'shop', JSON.stringify(meta?.looks)) const turnCard = page.locator('[data-turn]').last() await turnCard.locator('button', { hasText: '规划' }).first().click() @@ -1252,12 +1824,8 @@ for (const theme of THEMES) { check('页面上是逐段可点的报告，不是改写后的文字', await page.locator('[data-evidence-doc] [data-seg]').count() > 0 && !(await shows(page, '改写后的答案'))) check('没有「复核改写过此答案」的对照', await page.getByText('复核改写过此答案').count() === 0) - await until(() => log.patches.slice(patchesBefore).some((p) => p.body.meta)) - const saved = log.patches.slice(patchesBefore) - check('落库的是报告原文，不是改写', saved.filter((p) => typeof p.body.answer === 'string' && p.body.answer) - .every((p) => p.body.answer === EVIDENCE.output['周报']), saved.map((p) => String(p.body.answer ?? '').slice(0, 12)).join(' | ')) - check('meta 记下这一轮有证据（刷新后回运行那里取标注）', saved.map((p) => p.body.meta).filter(Boolean).at(-1)?.evidence === true) - check('落库的复核也不带改写', saved.filter((p) => p.body.review?.verdict).every((p) => !p.body.review.answer)) + check('页面不写这一轮的结局（不 PATCH 轮次）', log.patches.slice(patchesBefore).length === 0) + check('meta 记下这一轮有证据（刷新后回运行那里取标注）', t.meta?.evidence === true, JSON.stringify(t.meta)) check('没有运行时报错', errors.length === 0, errors[0] ?? '') await ctx.close() }) @@ -1304,10 +1872,8 @@ for (const theme of THEMES) { await page.waitForTimeout(300) await page.screenshot({ path: `${SHOTS}/askdata-evidence-${theme}.png` }) await page.keyboard.press('Escape') - await until(() => log.patches.slice(patchesBefore).some((p) => p.body.meta)) - const saved = log.patches.slice(patchesBefore) - check('落库的是报告原文，meta 记下这一轮有证据', saved.filter((p) => typeof p.body.answer === 'string' && p.body.answer) - .every((p) => p.body.answer === EVQ.output.answer) && saved.map((p) => p.body.meta).filter(Boolean).at(-1)?.evidence === true) + check('页面不写这一轮的结局；服务端写的 meta 记下这一轮有证据', log.patches.slice(patchesBefore).length === 0 + && t.meta?.evidence === true, JSON.stringify(t.meta)) check('没有运行时报错（实时）', errors.length === 0, errors[0] ?? '') await ctx.close() } @@ -1429,6 +1995,9 @@ for (const theme of THEMES) { () => window.__chat.getState().byConversation.c0resti?.at(-1)?.phase === 'done', null, { timeout: 8000 }) .then(() => true, () => false), (await chatState(page)).byConversation.c0resti.at(-1).phase) check('续跑发给的是这一轮自己的运行', log.continues.at(-1) === 'run-restart', log.continues.join(',')) + const cont = log.turnStarts.filter((b) => b.mode === 'continue').at(-1) + check('继续运行的请求：带着这一轮（retry_of、mode=continue），不带图', cont?.conversation_id === 'c0resti' + && cont?.retry_of === t.serverId && !cont?.graph, JSON.stringify(cont)) check('续上之后答案出来了', await shows(page, '接着跑完了：三条记录。')) check('续上之后补救按钮收起', await page.getByRole('button', { name: '继续运行' }).count() === 0) await page.waitForTimeout(300) @@ -1508,6 +2077,9 @@ for (const theme of THEMES) { (n) => window.__chat.getState().byConversation.c0histf.at(-1).phase === 'done' && window.__chat.getState().byConversation.c0histf.at(-1).attempts?.length === 1, startsBefore, { timeout: 8000 }) .then(() => true, () => false)) + const asked = log.turnStarts.filter((b) => b.retry_of === 'tf4').at(-1) + check('放宽步数的请求：带着这一轮、方式和目标步数（mode=max_steps、max_steps=24），图由服务端改', + asked?.conversation_id === 'c0histf' && asked?.mode === 'max_steps' && asked?.max_steps === 24 && !asked?.graph, JSON.stringify(asked)) const started = log.runStarts.slice(startsBefore).at(-1) check('发起的图里 agent 步数是 24', started?.graph?.nodes?.find((n) => n.type === 'agent')?.data?.config?.max_steps === 24, JSON.stringify(started?.graph?.nodes?.map((n) => n.data?.config))) @@ -1844,9 +2416,9 @@ for (const theme of THEMES) { check('不标「本条未查询数据库」（这不是回答）', !(await card.innerText()).includes('本条未查询数据库')) const box = page.getByRole('textbox', { name: '向数据提问' }) check('输入框提示回答助手的问题', (await box.getAttribute('placeholder')).includes('回答助手的问题')) - const saved = await until(() => log.patches.slice(patches).some((x) => x.body.answer === '销售额按下单日还是支付日算？')) - const body = log.patches.slice(patches).find((x) => x.body.answer)?.body - check('问题落库进历史（answer 列），meta 标着反问', saved && body.status === 'done' && body.meta?.asked === true, JSON.stringify(body)) + // 那一行由服务端写（答案列是助手问的那句话，meta 标着反问）：页面不 PATCH，照 end 带来的那一行摆 + check('页面不写这一轮的结局；服务端写的那一行 meta 标着反问', log.patches.slice(patches).length === 0 + && !!turn.meta?.asked, JSON.stringify(turn.meta?.asked)) check('没有运行时报错', errors.length === 0, errors[0] ?? '') await ctx.close() @@ -1924,7 +2496,8 @@ for (const theme of THEMES) { check('重来时交回去的就是服务端给的那句话', log.gens.at(-1)?.instruction === DECIDE_RETRY, log.gens.at(-1)?.instruction) check('第二次复核带着这一轮试过的次数', log.reviews.slice(reviews).map((r) => r.attempt).join(',') === '0,1', JSON.stringify(log.reviews.slice(reviews))) - const last = log.patches.filter((x) => x.body?.meta?.usage).at(-1)?.body?.meta?.usage + // 服务端写的那一行（end 带来的） + const last = turn.meta?.usage check('meta.usage 记下助手、复核、换下来的运行和这一次运行', !!last && Math.abs(last.copilot?.cost_usd - 0.024) < 1e-9 && Math.abs(last.review?.cost_usd - 0.003) < 1e-9 && Math.abs(last.earlier?.cost_usd - 0.02) < 1e-9 && Math.abs(last.run?.cost_usd - 0.02) < 1e-9, JSON.stringify(last)) @@ -1936,7 +2509,8 @@ for (const theme of THEMES) { // 复核慢过等待上限：答案先交出去，复核这一次的用量晚到了照记（这一轮的花费照实是助手 + 运行 + 复核，复核不落下） ctl.lateReview = 900 - await page.evaluate(() => window.__chatReviewWait(200)) + // 服务端最多等复核这么久（检查里调短） + ctl.reviewWaitMs = 200 const patchesBefore = log.patches.length await send(page, '照服务端的决定：复核晚到的那一轮') const delivered = await page.waitForFunction(() => { @@ -1947,12 +2521,13 @@ for (const theme of THEMES) { check('复核没等到：答案先交出去，这时还没有复核的用量', delivered && !early.usage?.review, JSON.stringify(early.usage)) await page.waitForTimeout(1200) const lateTurn = (await chatState(page)).byConversation.c0emptg.at(-1) - const lateMeta = log.patches.slice(patchesBefore).filter((x) => x.body?.meta?.usage).at(-1)?.body?.meta?.usage - check('复核晚到：它的用量照记进这一轮并落库', Math.abs((lateTurn.usage?.review?.cost_usd ?? 0) - 0.004) < 1e-9 + const lateMeta = lateTurn.meta?.usage + check('页面不写这一轮的结局', log.patches.slice(patchesBefore).length === 0) + check('复核晚到：它的用量照记进这一轮（服务端并进那一行的 meta.usage）', Math.abs((lateTurn.usage?.review?.cost_usd ?? 0) - 0.004) < 1e-9 && Math.abs((lateMeta?.review?.cost_usd ?? 0) - 0.004) < 1e-9 && Math.abs((lateMeta?.copilot?.cost_usd ?? 0) - 0.012) < 1e-9, JSON.stringify(lateMeta)) ctl.lateReview = 0 - await page.evaluate(() => window.__chatReviewWait(25_000)) + ctl.reviewWaitMs = 25_000 // 只建不跑：autorun 没写 false，next 是 ready，前端照 next 走、不发起运行 const before = log.runStarts.length @@ -2028,6 +2603,144 @@ for (const theme of THEMES) { await ctx.close() }) + await section('reconnect', '实时流断了：按收到的最后一个 seq 重连，补齐、不重复（一轮在服务端照跑）', async () => { + const { page, ctx, errors } = await open(theme) + await goto(page, 'c0emptg') + ctl.opGapMs = 120 + const streamsBefore = log.streams.length + const patchesBefore = log.patches.length + await send(page, '断线重连：查一下订单和明细') + await page.waitForFunction(() => (window.__chat.getState().byConversation.c0emptg?.at(-1)?.ops.length ?? 0) >= 2, + null, { timeout: 6000 }).catch(() => {}) + // 接下来两次连接都直接断掉（网络抖了两下）：这一轮在服务端照跑，页面退避着重连 + ctl.dropStreams = 2 + const ran = await page.waitForFunction(() => { + const t = window.__chat.getState().byConversation.c0emptg?.at(-1) + return !!t?.run?.id && t.phase === 'running' + }, null, { timeout: 12000 }).then(() => true, () => false) + ctl.opGapMs = 10 + await page.waitForTimeout(300) + const t = (await chatState(page)).byConversation.c0emptg.at(-1) + const T = srv.turns.get(t.serverId) + const afters = log.streams.slice(streamsBefore).filter((x) => x.turn === t.serverId).map((x) => x.after) + const resumedAt = afters.findIndex((a, i) => i >= 2 && a > 0 && afters[i - 1] === a && afters[i - 2] === a) + check('断了之后接着连：每次都带上收到的最后一个 seq（断掉的那两次和接上的那一次是同一个）', ran && resumedAt > 0, + afters.join(',')) + check('从不往回要：重连的 seq 只增不减', afters.every((a, i) => i === 0 || a >= afters[i - 1]), afters.join(',')) + check('补齐：服务端这一轮记下的事件，页面一条不少地收到了', !!T && t.turnSeq === T.events.length, + `${t.turnSeq} / ${T?.events.length}`) + const ops = T?.events.filter((e) => e.type === 'op').length ?? -1 + check('不重复：助手的操作一条只收一份', t.ops.length === ops && t.ops.filter((o) => o.op === 'add_node').length === 3, + `${t.ops.length} / ${ops}`) + check('画布边收边长到收尾：图有 3 个节点，运行接上了', t.graph?.nodes?.length === 3 && !!t.run?.id) + check('页面不写这一轮的结局（不 PATCH）', log.patches.slice(patchesBefore).length === 0) + await page.getByRole('button', { name: '停止这一轮' }).click().catch(() => {}) + check('没有运行时报错', errors.length === 0, errors[0] ?? '') + await ctx.close() + }) + + await section('replay', '刷新、关掉再打开：服务端那一轮照跑，页面从这一次尝试的第一条接着播', async () => { + resetDb() + ctl.replayConvs = true + ctl.liveDetail = true + ctl.replayBuildGo = false + ctl.replayRunGo = false + try { + const patchesBefore = log.patches.length + let { page, ctx, errors } = await open(theme) + await goto(page, 'c0replay') + const startsBefore = log.turnStarts.length + await send(page, '刷新后接着看：各门店的订单数') + await page.waitForFunction(() => (window.__chat.getState().byConversation.c0replay?.at(-1)?.ops + .filter((o) => o.op === 'add_node').length ?? 0) === 2, null, { timeout: 6000 }).catch(() => {}) + const serverId = (await chatState(page)).byConversation.c0replay.at(-1).serverId + + // 刷新：库里那一行是 running、带着服务端的任务（建图中） + await page.reload() + await ready(page, () => !!window.__chat, '问数据页的 store（window.__chat）') + const live = await page.waitForFunction((id) => { + const t = window.__chat.getState().byConversation.c0replay?.find((x) => x.serverId === id) + return !!t && (t.phase === 'planning' || t.phase === 'building') && t.ops.filter((o) => o.op === 'add_node').length === 2 + }, serverId, { timeout: 6000 }).then(() => true, () => false) + const t1 = (await chatState(page)).byConversation.c0replay?.find((x) => x.serverId === serverId) + check('刷新之后：这一轮还在服务端建图，照那个样子摆（不当成断了）', live, `${t1?.phase} · ${t1?.ops?.length} 条操作`) + check('……从这一次尝试的第一条回放，已经建出的两步一步不少、一步不重', + t1?.ops.filter((o) => o.op === 'add_node').length === 2 && t1?.ops.filter((o) => o.op === 'plan').length === 1) + check('……刷新没有再发起一次（只发起过一次）', log.turnStarts.length - startsBefore === 1, + `${log.turnStarts.length - startsBefore} 次`) + check('……回放从这一次尝试开始（after 是 attempt_seq 之前）', log.streams.some((x) => x.turn === serverId && x.after === 0)) + + ctl.replayBuildGo = true + const running = await page.waitForFunction((id) => { + const t = window.__chat.getState().byConversation.c0replay?.find((x) => x.serverId === id) + return t?.phase === 'running' && t.run?.id === 'run-replay' && t.graph?.nodes?.length === 3 + }, serverId, { timeout: 6000 }).then(() => true, () => false) + check('接着播：建完、服务端发起了运行，画布长全', running) + await page.screenshot({ path: `${SHOTS}/replay-running-${theme}.png` }) + check('没有运行时报错（刷新后）', errors.length === 0, errors[0] ?? '') + await ctx.close() + + // 关掉再打开：运行还在跑（停在闸上） + ;({ page, ctx, errors } = await open(theme)) + await goto(page, 'c0replay') + const reopened = await page.waitForFunction((id) => { + const t = window.__chat.getState().byConversation.c0replay?.find((x) => x.serverId === id) + return t?.phase === 'running' && t.run?.id === 'run-replay' && t.ops.filter((o) => o.op === 'add_node').length === 3 + }, serverId, { timeout: 6000 }).then(() => true, () => false) + check('关掉再打开：照样接着看（运行中，建图的过程还在）', reopened, + JSON.stringify((await chatState(page)).byConversation.c0replay?.map((x) => x.phase))) + check('……运行的过程也接回来了', await page.waitForFunction((id) => + (window.__chat.getState().byConversation.c0replay?.find((x) => x.serverId === id)?.events.length ?? 0) >= 2, + serverId, { timeout: 4000 }).then(() => true, () => false)) + ctl.replayRunGo = true + const done = await page.waitForFunction((id) => + window.__chat.getState().byConversation.c0replay?.find((x) => x.serverId === id)?.phase === 'done', + serverId, { timeout: 8000 }).then(() => true, () => false) + check('跑完、复核完：答案出来了', done && await shows(page, '刷新之后接着跑完了：两家门店。')) + check('页面一个结局都没写（不 PATCH）', log.patches.slice(patchesBefore).length === 0) + await page.screenshot({ path: `${SHOTS}/replay-done-${theme}.png` }) + check('没有运行时报错（重新打开后）', errors.length === 0, errors[0] ?? '') + await ctx.close() + } finally { + ctl.replayConvs = false + ctl.liveDetail = false + ctl.replayBuildGo = true + ctl.replayRunGo = true + } + }) + + await section('stopreq', '停止：建图中也停得下，停止请求发给服务端的这一轮', async () => { + const { page, ctx, errors } = await open(theme) + await goto(page, 'c0emptg') + ctl.stopBuildGo = false + const patchesBefore = log.patches.length + const startsBefore = log.turnStarts.length + const runsBefore = log.runStarts.length + await send(page, '停止测试：各门店的订单数') + await page.waitForFunction(() => (window.__chat.getState().byConversation.c0emptg?.at(-1)?.ops.length ?? 0) >= 2, + null, { timeout: 6000 }).catch(() => {}) + const asked = log.turnStarts.slice(startsBefore).at(-1) + check('发起的请求：这段对话、这个问题，不限数据源时不带 datasource_ids', asked?.conversation_id === 'c0emptg' + && asked?.question === '停止测试：各门店的订单数' && !asked?.retry_of && !('datasource_ids' in (asked ?? {})), JSON.stringify(asked)) + const t0 = (await chatState(page)).byConversation.c0emptg.at(-1) + await page.getByRole('button', { name: '停止这一轮' }).click() + await until(() => log.turnCancels.includes(t0.serverId), 3000) + check('停止请求发给服务端的这一轮（POST /api/chat/turns/{id}/cancel）', log.turnCancels.at(-1) === t0.serverId, + `${log.turnCancels.at(-1)} / ${t0.serverId}`) + await page.waitForTimeout(300) + const t = (await chatState(page)).byConversation.c0emptg.at(-1) + check('这一轮收成已取消，不画红', t.phase === 'cancelled' && !t.failure, t.phase) + ctl.stopBuildGo = true + await page.waitForTimeout(400) + const T = srv.turns.get(t0.serverId) + check('服务端那一轮也停了：不再往下建、不发起运行', T?.job.status === 'cancelled' && log.runStarts.length === runsBefore + && T.events.filter((e) => e.type === 'op' && e.data.op === 'add_node').length === 2, `${T?.job.status} · ${log.runStarts.length - runsBefore} 次运行`) + check('建图中停止：补救区说「你在搭建工作流时停止了本轮」', await shows(page, '你在搭建工作流时停止了本轮')) + check('页面不写这一轮的结局（不 PATCH）', log.patches.slice(patchesBefore).length === 0) + check('没有运行时报错', errors.length === 0, errors[0] ?? '') + await ctx.close() + }) + await section('steps', '历史轮次的执行过程：点开、收起', async () => { const { page, ctx, errors } = await open(theme) await goto(page, 'c0idleb') @@ -2396,26 +3109,35 @@ for (const theme of THEMES) { check('补交付完成', await shows(page, '补交付的答案：三条记录。', 5000)) delete ctl.reviewDelay['run-late'] - // 停在半路的几轮：各自的终态查到之后要落库，下次进来不再核对 + // 停在半路的几轮（这一波之前页面推的老轮次，服务端没有它们的任务）：页面不再自己收尾、不写库，请服务端接着等各自的 + // 那次运行、照常收尾（POST /api/chat/turns mode=resume），结局照服务端写的那一行摆 ctl.approvals = [{ id: 'ap-t', run_id: 'run-t-wait', node_id: 'ag', mode: 'approve', title: '确认一下', payload: {}, status: 'pending', response: {}, created_at: new Date().toISOString() }] const patchesBefore = log.patches.length + const resumesBefore = log.turnStarts.length await goto(page, 'c0termr') - await until(() => ['tr1', 'tr2', 'tr3', 'tr4', 'tr6'].every((id) => - log.patches.slice(patchesBefore).some((p) => p.turn === id && p.body.status === 'error')), 6000) - await page.waitForTimeout(300) - const saved = (id) => log.patches.slice(patchesBefore).filter((p) => p.turn === id).at(-1)?.body - check('在别处取消的：落库成已取消', saved('tr1')?.status === 'error' && saved('tr1')?.error === '已取消' - && saved('tr1')?.meta?.outcome === 'cancelled', JSON.stringify(saved('tr1'))) - check('在别处失败的：落库成失败，原因跟着写', saved('tr2')?.status === 'error' && /鉴权/.test(saved('tr2')?.error ?? ''), - JSON.stringify(saved('tr2'))?.slice(0, 160)) - check('服务重启挂起的：落库成中断', saved('tr3')?.status === 'error' && saved('tr3')?.meta?.outcome === 'suspended', - JSON.stringify(saved('tr3'))?.slice(0, 160)) - check('运行被删掉的：落库成「上次未完成」', saved('tr4')?.status === 'error' && saved('tr4')?.error === '上次未完成', - JSON.stringify(saved('tr4'))?.slice(0, 160)) - check('……后端现在的说法（鉴权失败（401））：同样认成鉴权失败落库', saved('tr6')?.status === 'error' - && (saved('tr6')?.error ?? '').startsWith('模型鉴权失败'), JSON.stringify(saved('tr6'))?.slice(0, 160)) - check('还在等审批的不落库（它还没结束）', !saved('tr5'), JSON.stringify(saved('tr5'))) + const phaseOf = async (id) => (await chatState(page)).byConversation.c0termr?.find((t) => t.id === id) + for (let i = 0; i < 80; i++) { + const turns = (await chatState(page)).byConversation.c0termr ?? [] + if (['tr1', 'tr2', 'tr3', 'tr4', 'tr5', 'tr6'].every((id) => ['cancelled', 'error', 'suspended', 'waiting'] + .includes(turns.find((t) => t.id === id)?.phase))) break + await page.waitForTimeout(100) + } + const resumed = log.turnStarts.slice(resumesBefore).filter((b) => b.mode === 'resume').map((b) => b.retry_of) + check('停在半路的几轮都交给服务端接着收尾（mode=resume，带着各自这一轮）', + ['tr1', 'tr2', 'tr3', 'tr4', 'tr5', 'tr6'].every((id) => resumed.includes(id)) + && log.turnStarts.slice(resumesBefore).every((b) => b.conversation_id === 'c0termr'), resumed.join(',')) + check('页面不写这几轮的结局（不 PATCH 轮次）', log.patches.slice(patchesBefore).length === 0, + JSON.stringify(log.patches.slice(patchesBefore).map((p) => p.turn))) + const tr = Object.fromEntries(await Promise.all(['tr1', 'tr2', 'tr3', 'tr4', 'tr5', 'tr6'].map(async (id) => [id, await phaseOf(id)]))) + check('在别处取消的：收成已取消', tr.tr1?.phase === 'cancelled', tr.tr1?.phase) + check('在别处失败的：收成失败，原因说人话', tr.tr2?.phase === 'error' && /鉴权/.test(tr.tr2?.error ?? ''), tr.tr2?.error) + check('服务重启挂起的：收成中断', tr.tr3?.phase === 'suspended', tr.tr3?.phase) + check('运行被删掉的：收成「上次未完成」', tr.tr4?.phase === 'error' && tr.tr4?.failure?.title === '上次未完成', + JSON.stringify(tr.tr4?.failure)) + check('……后端现在的说法（鉴权失败（401））：同样认成鉴权失败', tr.tr6?.phase === 'error' + && (tr.tr6?.error ?? '').startsWith('模型鉴权失败'), tr.tr6?.error) + check('还在等审批的：照旧等人（它还没结束）', tr.tr5?.phase === 'waiting', tr.tr5?.phase) ctl.approvals = [] check('没有运行时报错', errors.length === 0, errors[0] ?? '') await ctx.close() @@ -2428,10 +3150,10 @@ for (const theme of THEMES) { await send(page, '一份取不到运行的长答案') await page.waitForFunction(() => window.__chat.getState().byConversation.c0clips?.at(-1)?.phase === 'done', null, { timeout: 10000 }).catch(() => {}) const t = (await chatState(page)).byConversation.c0clips.at(-1) - check('GET 运行失败：先交截断的那份并标出来', t.clipped === 'partial', t.clipped) - await until(() => log.patches.slice(patchesBefore).some((p) => p.body.meta?.clipped === 'partial')) - check('「没取全」写进 meta，刷新后还认得出来', log.patches.slice(patchesBefore).some((p) => p.body.meta?.clipped === 'partial'), - JSON.stringify(log.patches.slice(patchesBefore).map((p) => p.body.meta?.clipped))) + // 一轮在服务端跑：交付读的是库里的完整成果，页面 GET 运行失败也不影响，新轮次不会再有「没取全」 + check('服务端按库里的完整成果交付：答案完整、不标「没取全」', !t.clipped && String(t.output?.answer ?? '').length === FULL.length, + `${t.clipped} · ${String(t.output?.answer ?? '').length} / ${FULL.length}`) + check('页面不写这一轮的结局（不 PATCH 轮次）', log.patches.slice(patchesBefore).length === 0) const runGetsBefore = log.runGets.length await goto(page, 'c0clipt') @@ -2453,9 +3175,12 @@ for (const theme of THEMES) { await ctx.close() }) - await section('launch', '发起运行失败：报错分得清标题和原因，补救说真话', async () => { + await section('launch', '发起那一下没到服务端：报错分得清标题和原因，补救说真话', async () => { + // 一轮在服务端跑之后，发起运行是服务端的事，「发起运行的请求没到后端」不会再发生；会断的是发起这一轮的那一下 + //（POST /api/chat/turns）。这时服务端那边还没有这一轮，补救就是原样再问一次 const { page, ctx, errors } = await open(theme) - ctl.launchFail = true + ctl.startFail = true + ctl.launchFail = false await goto(page, 'c0launu') await send(page, '一个启动失败的问题') const alert = page.locator('[data-turn] [role="alert"]').first() @@ -2464,17 +3189,19 @@ for (const theme of THEMES) { check('标题只是「无法连接服务」，原因不糊进粗体', title === '无法连接服务', title) const text = await alert.innerText().catch(() => '') check('不许诺页面上不存在的自动重试', !text.includes('自动重试'), text.replace(/\n/g, ' / ')) - const rerun = page.locator('[data-remedy="failed"]').getByRole('button', { name: '重新运行本轮' }) - check('流程已经搭好：给「重新运行本轮」，不让它重新搭', await rerun.count() === 1) - check('说出怎么办：连上之后点「重新运行本轮」', text.includes('重新运行本轮'), text.replace(/\n/g, ' / ')) + const retry = page.locator('[data-remedy="failed"]').getByRole('button', { name: '重试本轮' }) + check('给「重试本轮」', await retry.count() === 1) + check('说出怎么办：连上之后点「重试本轮」', text.includes('重试本轮'), text.replace(/\n/g, ' / ')) await page.screenshot({ path: `${SHOTS}/launch-failed-${theme}.png` }) - ctl.launchFail = false + ctl.startFail = false const gensBefore = log.gens.length - const startsBefore = log.runStarts.length - await rerun.click().catch(() => {}) - await until(() => log.runStarts.length > startsBefore, 5000) - check('重跑：原图直接发起，不再去问助手', log.runStarts.length > startsBefore && log.gens.length === gensBefore, - `${log.runStarts.length - startsBefore} 次发起 · ${log.gens.length - gensBefore} 次建图`) + const startsBefore = log.turnStarts.length + await retry.click().catch(() => {}) + await until(() => log.gens.length > gensBefore, 5000) + const again = log.turnStarts.slice(startsBefore).at(-1) + check('重试：服务端还没有这一轮，原样再问一次（同一个问题，不带 retry_of）', again?.question === '一个启动失败的问题' + && !again?.retry_of && again?.conversation_id === 'c0launu', JSON.stringify(again)) + check('这一次照常建图', log.gens.length > gensBefore, `${log.gens.length - gensBefore} 次建图`) await page.getByRole('button', { name: '停止这一轮' }).click().catch(() => {}) ctl.launchFail = true check('没有运行时报错', errors.length === 0, errors[0] ?? '') @@ -2542,39 +3269,45 @@ for (const theme of THEMES) { await until(() => log.runStarts.length > startsBefore, 5000) await until(() => log.cancels.slice(before).includes('run-susp'), 2000) check('挂起的旧运行被取消，不会一直挂在记录里', log.cancels.slice(before).includes('run-susp'), log.cancels.slice(before).join(',')) + const rerun = log.turnStarts.filter((b) => b.retry_of === 'to1').at(-1) + check('重新运行本轮的请求：带着这一轮、mode=rerun', rerun?.mode === 'rerun' && rerun?.conversation_id === 'c0suspo', + JSON.stringify(rerun)) check('新的一次照常发起', log.runStarts.length > startsBefore) await page.getByRole('button', { name: '停止这一轮' }).click().catch(() => {}) check('没有运行时报错', errors.length === 0, errors[0] ?? '') await ctx.close() }) - await section('stale', '断在建图阶段：按最后一次落库算有没有动静；发起运行时 meta 跟上新运行', async () => { + await section('stale', '断在建图阶段的老轮次：页面不替它写库；重试本轮交给服务端', async () => { + // 这一波之前页面推的老轮次停在 running、又没有运行：建图那条流随页面、随上次的进程一起没了。以前页面按「多久没动静」 + // 替它写回 error；现在服务端启动时整理它们（chat/job.startup），页面只照实摆「上次未完成」，不写库。 + // 服务端还在建图的一轮带着任务（ConversationTurn.job），照它在跑的样子接着播（见「刷新后接着看」那一段） const { page, ctx, errors } = await open(theme) const patchesBefore = log.patches.length await goto(page, 'c0buildx') await shows(page, '重试之后二十分钟没动静的问题') - await until(() => log.patches.slice(patchesBefore).some((p) => p.turn === 'tx8'), 3000) - await page.waitForTimeout(400) - const mine = () => log.patches.slice(patchesBefore) - check('两天前问的、两分钟前刚重试过：不当成断了写回 error', - !mine().some((p) => p.turn === 'tx7' && p.body.status === 'error'), JSON.stringify(mine().filter((p) => p.turn === 'tx7'))) - check('重试之后二十分钟没动静的：认定断了，写回 error', - mine().some((p) => p.turn === 'tx8' && p.body.status === 'error'), JSON.stringify(mine().filter((p) => p.turn === 'tx8')).slice(0, 200)) + await page.waitForTimeout(600) + const turns = (await chatState(page)).byConversation.c0buildx ?? [] + check('页面不替老轮次写库（不 PATCH）', log.patches.slice(patchesBefore).length === 0, + JSON.stringify(log.patches.slice(patchesBefore).map((p) => p.turn))) + check('断在建图阶段的老轮次：照实说上次未完成', turns.every((t) => t.phase === 'error' && t.failure?.title === '上次未完成'), + turns.map((t) => `${t.id}:${t.phase}`).join(',')) - // 重试一次：搭好图时连 meta 一起写（有动静的凭据），发起运行时 meta.runId 换成新运行 + // 重试一次：交给服务端（带着这一轮、mode=retry），这一次的运行挂在这一轮上 const startsBefore = log.runStarts.length - const retryAt = log.patches.length - await page.locator('[data-remedy="failed"]').getByRole('button', { name: '重试本轮' }).click() + const turnStarts = log.turnStarts.length + await page.locator('[data-remedy="failed"]').last().getByRole('button', { name: '重试本轮' }).click() await until(() => log.runStarts.length > startsBefore, 6000) + const body = log.turnStarts.slice(turnStarts).at(-1) + check('重试本轮：请求带着这一轮和方式（retry_of、mode=retry）', body?.retry_of === 'tx8' && body?.mode === 'retry' + && body?.conversation_id === 'c0buildx', JSON.stringify(body)) const newRun = log.runStarts.at(-1)?.id - await until(() => log.patches.slice(retryAt).some((p) => p.turn === 'tx8' && p.body.run_id === newRun), 3000) - const ps = log.patches.slice(retryAt).filter((p) => p.turn === 'tx8').map((p) => p.body) - const graphSave = ps.find((b) => b.graph) - check('搭好图时连 meta 一起写，带上此刻', !!graphSave?.meta && Math.abs(Date.now() - (graphSave.meta.at ?? 0)) < 30_000, - JSON.stringify(graphSave?.meta ?? null)?.slice(0, 120)) - const started = ps.find((b) => b.run_id === newRun) - check('发起运行：run_id 列和 meta.runId 一起换成新运行，状态是 running', - !!newRun && started?.meta?.runId === newRun && started?.status === 'running', JSON.stringify(started ?? null)?.slice(0, 160)) + await page.waitForFunction((id) => window.__chat.getState().byConversation.c0buildx?.find((t) => t.id === 'tx8')?.run?.id === id, + newRun, { timeout: 4000 }).catch(() => {}) + const tx8 = (await chatState(page)).byConversation.c0buildx.find((t) => t.id === 'tx8') + check('服务端发起的这次运行挂在这一轮上，照常在跑', !!newRun && tx8?.run?.id === newRun && tx8?.phase === 'running', + `${tx8?.run?.id} · ${tx8?.phase}`) + check('页面照旧不写库', log.patches.slice(patchesBefore).length === 0) await page.getByRole('button', { name: '停止这一轮' }).click().catch(() => {}) check('没有运行时报错', errors.length === 0, errors[0] ?? '') await ctx.close() From fa9725adf26ce6b8026ddf82a25239bee5f6a903 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 18 Oct 2026 16:55:30 +0800 Subject: [PATCH 3/7] =?UTF-8?q?=E9=97=AE=E6=95=B0=E6=8D=AE=E4=B8=80?= =?UTF-8?q?=E8=BD=AE=E7=9A=84=E6=94=B6=E5=B0=BE=E8=A1=A5=E6=BC=8F=EF=BC=9A?= =?UTF-8?q?=E6=8E=A5=E7=9D=80=E6=94=B6=E5=B0=BE=E4=B8=8D=E7=AE=97=E5=8D=A0?= =?UTF-8?q?=E7=9D=80=E5=AF=B9=E8=AF=9D=EF=BC=8C=E5=AE=9E=E6=97=B6=E6=B5=81?= =?UTF-8?q?=E6=B2=A1=E7=AD=89=E5=88=B0=E7=BB=93=E5=B1=80=E4=B8=8D=E5=86=8D?= =?UTF-8?q?=E8=BD=AC=E5=9C=88=EF=BC=8C=E5=88=B7=E6=96=B0=E5=90=8E=E6=8E=A5?= =?UTF-8?q?=E7=9D=80=E6=92=AD=E7=9A=84=E8=BF=87=E7=A8=8B=E7=85=A7=E5=AE=9E?= =?UTF-8?q?=E6=97=B6=E7=9A=84=E6=A0=B7=E5=AD=90=E6=91=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 一段对话里停在半路的几轮（老轮次、服务重启后停着的）要一起交给服务端收尾：resume 不建图、不发起运行，不再按「这段对话正忙」 拒掉第二轮（以前打开这样的对话，只有第一轮收得了尾）。测试先写、去掉修复回跑会失败。 - 实时流结束了却没见到这一轮的 end（服务端那一轮出了岔子、这一轮被删了或没权限了）：回服务端问一次那一行，照它摆；问不到就 照实说「无法确认本轮的当前状态」，给「重新核对」。重新核对一轮还没有运行的：问服务端它在不在跑，在跑接着播。 - 刷新、关掉再打开后接着播的那一轮就是正在跑的一轮：建图的过程照实时的样子一直摆着，不当成要点开才取的历史轮次 （check-chat「刷新后接着看」一段多守这一条，去掉修复回跑会失败）。 --- backend/app/chat/job.py | 3 +- backend/tests/test_chat_turns.py | 24 ++++++++++++++++ frontend/src/store/chat.ts | 48 ++++++++++++++++++++++++++++++-- scripts/check-chat.mjs | 2 ++ 4 files changed, 73 insertions(+), 4 deletions(-) diff --git a/backend/app/chat/job.py b/backend/app/chat/job.py index 60cd197f..747ef734 100644 --- a/backend/app/chat/job.py +++ b/backend/app/chat/job.py @@ -716,7 +716,8 @@ async def start(session: AsyncSession, *, conversation: Any, question: str, data if mode not in MODES or (mode == "ask") != (retry_of is None): raise TurnRefused(422, "发起方式不对：新问不带 retry_of，重试、重新运行、继续运行要带上是哪一轮", TURN_INVALID) besides = retry_of.id if retry_of is not None else None - if await _busy(session, conversation.id, besides=besides): + # 接着收尾（resume）不建图、不发起运行，不算占着这段对话：一段对话里停在半路的几轮可以一起交给服务端收尾 + if mode != "resume" and await _busy(session, conversation.id, besides=besides): raise TurnRefused(409, BUSY, TURN_BUSY) now_ms = int(time.time() * 1000) request: dict[str, Any] = {"started_ms": now_ms} diff --git a/backend/tests/test_chat_turns.py b/backend/tests/test_chat_turns.py index 36f554ff..cc57e0ad 100644 --- a/backend/tests/test_chat_turns.py +++ b/backend/tests/test_chat_turns.py @@ -857,3 +857,27 @@ async def test_an_empty_generation_is_retried_once_on_the_server(client, monkeyp row = await _row(turn_id) assert row.status == "done" and row.answer == "上周没有华中区的数据" and row.meta["noQuery"] is True assert row.meta["usage"]["copilot"]["calls"] == 2, "两次建图都是这一轮花的" + + +async def test_several_stranded_turns_of_one_conversation_can_be_resumed_together(client): + """一段对话里停在半路的几轮（老轮次、服务重启后停着的）一起交给服务端收尾：接着收尾不建图、不发起运行，不算占着这段对话。""" + conversation_id = await _conversation(client) + ids = [] + for answer in ("甲", "乙"): + async with SessionLocal() as session: + run = Run(status="succeeded", output={"答案": answer}, graph={"nodes": [], "edges": []}, run_class="exploratory") + session.add(run) + await session.flush() + turn = ConversationTurn(conversation_id=conversation_id, seq=len(ids), question=f"问{answer}", status="running", + run_id=run.id, meta={"v": 1, "runId": run.id}) + session.add(turn) + await session.commit() + ids.append(turn.id) + for turn_id in ids: + r = await client.post("/api/chat/turns", json={"conversation_id": conversation_id, "retry_of": turn_id, + "mode": "resume"}) + assert r.status_code == 202, r.text + for turn_id, answer in zip(ids, ("甲", "乙")): + await _done(turn_id) + row = await _row(turn_id) + assert row.status == "done" and row.answer == answer diff --git a/frontend/src/store/chat.ts b/frontend/src/store/chat.ts index 88511b76..2a42744b 100644 --- a/frontend/src/store/chat.ts +++ b/frontend/src/store/chat.ts @@ -1043,6 +1043,27 @@ export const useChat = create((set, get) => ({ await resumeTurn(conversationId, turnId) return } + // 重新核对一轮还没有运行的（实时流断在建图中、读不到了）：问服务端它现在怎样，在跑就接着播，收了尾就照那一行摆 + if (turn.phase === 'checking' && turn.serverId) { + patch(() => ({ hydrated: 'pending' })) + try { + const res = await api.chatTurns.get(turn.serverId) + if (res.job && LIVE_JOB.has(res.job.status)) { + patch(() => ({ hydrated: 'done', job: res.job })) + play(flowOf(conversationId, turnId), turn.serverId, Math.max(0, res.job.attempt_seq - 1)) + } else { + patch((t) => ({ ...endPatch(t, { turn: res.turn }), hydrated: 'done' })) + } + } catch (e) { + const failure: Failure = { + title: PHASE_TEXT.unknown, + reason: `查询本轮运行状态时出错（${errorMessage(e)}）。运行可能仍在进行，也可能已结束。`, + hint: '请稍后点击「重新核对」；服务端重新连接后会自动再核对一次。', + } + patch(() => ({ hydrated: undefined, phase: 'unknown', status: PHASE_TEXT.unknown, failure, error: lineOf(failure) })) + } + return + } const runId = turn.run?.id if (!runId) return // 核对期间用户点了重试或停止：拿回来的是上一次的状态，作废 @@ -1262,7 +1283,7 @@ function attemptReset(t: ChatTurn, d: Record, ts: number): Partial< phase: 'running', status: PHASE_TEXT.running, attempts, scope, usage, events: [], run: null, output: null, review: null, reviewRaw: null, rawOutput: null, failure: null, error: '', lastSeq: 0, final: undefined, rejected: null, runStatus: undefined, pendingRun: false, hydrated: 'done', - startedAt, endedAt: undefined, writing: null, noQuery: false, + startedAt, endedAt: undefined, writing: null, noQuery: false, restored: false, steps: undefined, } } return { @@ -1274,6 +1295,8 @@ function attemptReset(t: ChatTurn, d: Record, ts: number): Partial< runStatus: undefined, clipped: undefined, pendingRun: false, steps: undefined, hydrated: 'done', startedAt, endedAt: undefined, attempt: 0, noQuery: false, thinking: '', writing: null, asked: undefined, + // 刷新后接着播的那一轮就是正在跑的一轮：过程照实时的样子一直摆着（不是从库里恢复、要点开才取的历史轮次） + restored: false, } } @@ -1474,8 +1497,27 @@ function play(flow: Flow, serverId: string, after: number) { const stopTurn = streamChatTurn(serverId, after, (e) => { if (!flow.alive()) return apply(e) - }, () => { - // 服务端说这一轮的事件都发完了（或者这一轮读不了了）。运行的实时流还开着的让它自己收尾 + }, (end) => { + // 服务端说这一轮的事件都发完了（或者这一轮读不了了）。运行的实时流还开着的让它自己收尾。 + // 正常情况下 end 早就到了；没到（服务端那一轮出了岔子、这一轮被删了、没权限了）就别让它一直转圈：照库里那一行摆， + // 读不到就照实说没核对上 + const cur = findTurn(flow.conversationId, flow.turnId) + if (!flow.alive() || !cur || !(BUSY_PHASES.has(cur.phase) || cur.phase === 'checking' || cur.phase === 'waiting')) return + const unknown = (said: string) => { + const failure: Failure = { + title: PHASE_TEXT.unknown, + reason: `查询本轮运行状态时出错（${said}）。运行可能仍在进行，也可能已结束。`, + hint: '请稍后点击「重新核对」；服务端重新连接后会自动再核对一次。', + } + patch(() => ({ phase: 'unknown', status: PHASE_TEXT.unknown, failure, error: lineOf(failure), cancel: null, + writing: null, hydrated: undefined })) + } + if (end.error) return unknown(end.error) + void api.chatTurns.get(serverId).then((res) => { + if (!flow.alive()) return + if (res.turn && res.turn.status !== 'running') patch((t) => endPatch(t, { turn: res.turn })) + else if (!res.job || !LIVE_JOB.has(res.job.status)) unknown(PHASE_TEXT.unknown) + }, (e) => { if (flow.alive()) unknown(errorMessage(e)) }) }) const close = () => { stopTurn() diff --git a/scripts/check-chat.mjs b/scripts/check-chat.mjs index de7582c6..c0abb8a6 100644 --- a/scripts/check-chat.mjs +++ b/scripts/check-chat.mjs @@ -2666,6 +2666,8 @@ for (const theme of THEMES) { check('刷新之后：这一轮还在服务端建图，照那个样子摆（不当成断了）', live, `${t1?.phase} · ${t1?.ops?.length} 条操作`) check('……从这一次尝试的第一条回放，已经建出的两步一步不少、一步不重', t1?.ops.filter((o) => o.op === 'add_node').length === 2 && t1?.ops.filter((o) => o.op === 'plan').length === 1) + check('……建图的过程照实时的样子摆着（不是要点开才取的历史轮次）', !t1?.restored + && await page.locator('[data-turn]').last().locator('button', { hasText: '规划' }).count() > 0) check('……刷新没有再发起一次（只发起过一次）', log.turnStarts.length - startsBefore === 1, `${log.turnStarts.length - startsBefore} 次`) check('……回放从这一次尝试开始（after 是 attempt_seq 之前）', log.streams.some((x) => x.turn === serverId && x.after === 0)) From 8b74c0d5320c366e5dcb5da8e498d93e21e44f7b Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 18 Oct 2026 17:03:10 +0800 Subject: [PATCH 4/7] =?UTF-8?q?=E9=97=AE=E6=95=B0=E6=8D=AE=E9=A1=B5?= =?UTF-8?q?=E7=9A=84=E7=8A=B6=E6=80=81=EF=BC=9A=E5=8E=BB=E6=8E=89=E6=92=AD?= =?UTF-8?q?=E6=94=BE=E6=94=B9=E9=80=A0=E5=90=8E=E7=94=A8=E4=B8=8D=E4=B8=8A?= =?UTF-8?q?=E7=9A=84=E5=AF=BC=E5=85=A5=EF=BC=88parseServerTime=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 一轮搬到服务端之后，运行开始时刻由服务端随 turn.attempt 给（started_ms），页面不再自己解析运行的 started_at。 --- frontend/src/store/chat.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/src/store/chat.ts b/frontend/src/store/chat.ts index 2a42744b..ff0e3a0e 100644 --- a/frontend/src/store/chat.ts +++ b/frontend/src/store/chat.ts @@ -8,7 +8,6 @@ import type { LookEvent } from '../typesApp' import { errorMessage, humanizeError } from '../lib/errors' import { hasEvidence } from '../lib/evidence' import { explainRunError, explainStartError } from '../lib/explain' -import { parseServerTime } from '../lib/format' import { useCatalog } from './catalog' import { useConversations } from './conversations' import type { From 124131b393414cbae59d57c5674bb66b65b6a07d Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 18 Oct 2026 17:10:01 +0800 Subject: [PATCH 5/7] =?UTF-8?q?=E9=97=AE=E6=95=B0=E6=8D=AE=E4=B8=80?= =?UTF-8?q?=E8=BD=AE=EF=BC=9A=E5=81=9C=E6=AD=A2=E6=AD=A3=E5=A5=BD=E8=90=BD?= =?UTF-8?q?=E5=9C=A8=E5=8F=91=E8=B5=B7=E8=BF=90=E8=A1=8C=E7=9A=84=E9=82=A3?= =?UTF-8?q?=E4=B8=80=E4=B8=8B=EF=BC=8C=E4=B9=9F=E4=B8=8D=E4=BC=9A=E7=95=99?= =?UTF-8?q?=E4=B8=8B=E6=B2=A1=E4=BA=BA=E7=AE=A1=E7=9A=84=E8=BF=90=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 发起运行（launch_run）是这一轮后台任务里的一步，停止就是取消这个任务：取消落在发起的半路上，运行可能已经建好、开始执行，这一轮 却还没记下它的 id——停止取消不到它，它照样跑、照样计费，那一行也不知道有过这次运行。现在发起那一下不让停止打断：等它发起完、 记下运行，再照常取消它，那一行收成已取消、meta.runId 是这次运行。 测试：停止落在发起的半路上（发起被拦住、放行之前先停），停下之后这次运行是已取消、记在这一轮上；去掉修复回跑会失败。顺手把 「运行中停止」那一条改成等任务上记下运行 id 再停（stage=run 在发起之前就记下了，满载跑全量时读到的 run_id 偶尔还是空的）。 --- backend/app/chat/job.py | 12 +++++++ backend/tests/test_chat_turns.py | 60 ++++++++++++++++++++++++++++++-- 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/backend/app/chat/job.py b/backend/app/chat/job.py index 747ef734..611ff17e 100644 --- a/backend/app/chat/job.py +++ b/backend/app/chat/job.py @@ -36,6 +36,7 @@ from __future__ import annotations import asyncio +import contextlib import logging import time from collections.abc import Mapping @@ -173,6 +174,17 @@ class ChatDriver(TurnDriver): raise LaunchRefused(403, str(e)) from e return args + async def _launch(self, graph: dict[str, Any], payload: dict[str, Any] | None = None) -> str | None: + """发起那一下不让停止打断到一半：运行建好了、这一轮却不知道它，就成了没人管、照样计费的运行。停止正好落在这时， + 等它发起完、记下运行（on_launched），再照常去取消它（_stopped）。""" + task = asyncio.ensure_future(super()._launch(graph, payload)) + try: + return await asyncio.shield(task) + except asyncio.CancelledError: + with contextlib.suppress(Exception): + await task + raise + async def on_launched(self, run: Any) -> None: await self.execution.launched(run) diff --git a/backend/tests/test_chat_turns.py b/backend/tests/test_chat_turns.py index cc57e0ad..fff8502e 100644 --- a/backend/tests/test_chat_turns.py +++ b/backend/tests/test_chat_turns.py @@ -104,6 +104,17 @@ async def _stage(turn_id: str, stage: str) -> None: await _until(hit) +async def _launched(turn_id: str) -> str: + """等这一轮发起了运行，交回运行 id（stage=run 在发起之前就记下了，要等的是任务上的 run_id）。""" + async def hit() -> bool: + async with SessionLocal() as session: + job = await session.get(ChatTurnJob, turn_id) + return job is not None and bool(job.run_id) + await _until(hit) + async with SessionLocal() as session: + return (await session.get(ChatTurnJob, turn_id)).run_id + + def _sse(text: str) -> list[dict[str, Any]]: return [json.loads(line[6:]) for line in text.splitlines() if line.startswith("data: ")] @@ -303,9 +314,7 @@ async def test_stop_while_running_cancels_the_run(client, monkeypatch, source): monkeypatch.setattr(mock_model.MockChatModel, "_astream", slow_stream) conversation_id = await _conversation(client) turn_id = await _ask(client, conversation_id, datasource_ids=[source["id"]]) - await _stage(turn_id, "run") - async with SessionLocal() as session: - run_id = (await session.get(ChatTurnJob, turn_id)).run_id + run_id = await _launched(turn_id) assert (await client.post(f"/api/chat/turns/{turn_id}/cancel")).status_code == 200 job = await _done(turn_id) assert job.status == "cancelled" @@ -881,3 +890,48 @@ async def test_several_stranded_turns_of_one_conversation_can_be_resumed_togethe await _done(turn_id) row = await _row(turn_id) assert row.status == "done" and row.answer == answer + + +async def test_a_stop_that_lands_while_launching_still_cancels_the_run(client, monkeypatch, source): + """停止正好落在发起运行的那一下：发起不能被打断到一半（运行建好了、这一轮却不知道它，就成了没人管的运行）——等它发起完、 + 记下运行，再照常取消它，那一行收成已取消。""" + import app.engine.launch as launch + from app.providers import mock_model + + es.use_copilot(monkeypatch, es.Copilot(source["name"])) + hold = asyncio.Event() + + async def slow(self, messages, stop=None, run_manager=None, **kwargs): + await hold.wait() + raise AssertionError("不该放行") + + async def slow_stream(self, messages, stop=None, run_manager=None, **kwargs): + await hold.wait() + raise AssertionError("不该放行") + yield # noqa: unreachable - 让它成为异步生成器 + + monkeypatch.setattr(mock_model.MockChatModel, "_agenerate", slow) + monkeypatch.setattr(mock_model.MockChatModel, "_astream", slow_stream) + entered, release = asyncio.Event(), asyncio.Event() + real = launch.launch_run + + async def gated(session, **kw): + entered.set() + await release.wait() + return await real(session, **kw) + + monkeypatch.setattr(launch, "launch_run", gated) + conversation_id = await _conversation(client) + turn_id = await _ask(client, conversation_id, datasource_ids=[source["id"]]) + await asyncio.wait_for(entered.wait(), timeout=20) + stopping = asyncio.ensure_future(client.post(f"/api/chat/turns/{turn_id}/cancel")) + await asyncio.sleep(0.2) + release.set() + assert (await stopping).status_code == 200 + job = await _done(turn_id) + assert job.status == "cancelled" and job.run_id, "发起完了才停：运行记在这一轮上" + await run_manager.wait_idle(job.run_id) + async with SessionLocal() as session: + assert (await session.get(Run, job.run_id)).status == "cancelled", "这次运行跟着取消了，没有没人管的运行" + row = await _row(turn_id) + assert row.status == "error" and row.error == "已取消" and row.meta["runId"] == job.run_id From 85eee897996ed412b500558ff19588f560bcd47e Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 18 Oct 2026 18:07:03 +0800 Subject: [PATCH 6/7] =?UTF-8?q?=E9=97=AE=E6=95=B0=E6=8D=AE=E4=B8=80?= =?UTF-8?q?=E8=BD=AE=E7=9A=84=E5=A4=8D=E6=A0=B8=E4=BF=AE=E6=AD=A3=EF=BC=88?= =?UTF-8?q?=E5=90=8E=E7=AB=AF=EF=BC=89=EF=BC=9A=E5=90=8C=E6=97=B6=E6=9D=A5?= =?UTF-8?q?=E7=9A=84=E5=87=A0=E6=AC=A1=E5=8F=91=E8=B5=B7=E5=8F=AA=E8=B5=B7?= =?UTF-8?q?=E5=BE=97=E6=9D=A5=E4=B8=80=E6=AC=A1=E3=80=81=E4=BA=8B=E4=BB=B6?= =?UTF-8?q?=20seq=20=E4=B8=8D=E6=92=9E=E5=8F=B7=E3=80=81=E6=99=9A=E5=88=B0?= =?UTF-8?q?=E7=9A=84=E5=A4=8D=E6=A0=B8=E8=AE=B0=E5=BE=97=E4=B8=8A=E4=B9=9F?= =?UTF-8?q?=E9=80=81=E5=BE=97=E5=88=B0=E3=80=81=E4=BD=BF=E7=94=A8=E8=80=85?= =?UTF-8?q?=E4=B8=8D=E8=83=BD=E7=BB=A7=E7=BB=AD=E8=BF=90=E8=A1=8C=E3=80=81?= =?UTF-8?q?=20=E5=85=A5=E5=8F=A3=E7=9A=84=20history=20=E5=8F=AA=E5=8F=96?= =?UTF-8?q?=E8=BF=99=E4=B8=80=E8=BD=AE=E4=B9=8B=E5=89=8D=E7=9A=84=E3=80=81?= =?UTF-8?q?=E5=8F=AA=E6=9C=89=E9=97=AE=E6=95=B0=E6=8D=AE=E7=9A=84=E5=AF=B9?= =?UTF-8?q?=E8=AF=9D=E8=83=BD=E5=8F=91=E8=B5=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A1 并发发起不原子（复核实测：同一对话同时两次新问都 202、两轮 seq 都是 0；同一轮同时 3 次重试 [500, 202, 202]，同一轮建两次图、 停止只停了后一个；写 turn.attempt 撞号时任务已提交成排队却没有后台任务，这段对话从此一直「正忙」）： - 发起的核对和写入（这段对话忙不忙、下一轮排第几、轮次、任务、这一次尝试的第一条事件 turn.attempt）放进一个事务，第一句先占住 这段对话（hold_conversation：SQLite 上拿到写锁、PostgreSQL 上锁住这一行），同时来的几次一个接一个，后来的看得见先来的写下的； 重试、接着收尾的那一轮占住之后重读，任务按「不在排队、不在跑」条件更新、看 rowcount；写不进去整个事务回滚，什么都不留下。 继续运行要先续上那次运行（它用自己的会话写库，不能放进占着对话的事务），续上了却没排上队的把它取消；挂起旧运行的取消挪到 提交之后（只有排上队的那一次去取消）。页面打开时自动发的接着收尾占锁时不改活跃时间。 - conversations.new_turn 先记活跃时间再数 seq（第一句写就占住这段对话），老页面的开轮接口也不再撞号。 - 事件的 seq 由任务行上的计数器发（UPDATE … SET last_seq = last_seq + 1 RETURNING），同一轮同时追加的几处各拿不同的号。 - ChatTurnManager.start：同一轮在这个进程里还有后台任务活着就不起第二个、不覆盖登记；上一次的任务刚记完结局还没退出时等它退出再起。 A2 使用者（user 角色）现在能「继续运行」了：以前 POST /api/runs/{id}/continue 对它是 403（只放行停止），问数据这里照原说法回 403 role_required，运行不动；重试本轮、重新运行本轮照常。 A3 晚到的复核：先写库、发 review.late，再出登记、叫醒订阅方（以前反过来，开着的实时流先收到 stream.end 走掉）；那一行的几处写入 （任务、晚到的复核、下一次尝试）先占住那一行再读改写（hold_turn），用量改成按增量记在库里那一份上（以前任务开始时拿一份拷贝、 之后整块盖回去，重试时晚到复核记的那笔会被盖掉）；自动重来时换下来的那次运行交出去之后不再当这一次的运行记一遍。 A4 运行入口的 history 只取这一轮之前的：新的 conversations.run_history 照以前页面 historyOf 的规则（答案没有用说明、都没有的跳过、 每轮截 600 字、最多 6 轮），重试、重新运行较早的一轮时后面几轮不再进它的入口。 A5 只有问数据的对话（kind=chat）能发起一轮，别的种类 422 chat_turn_invalid。 测试：tests/test_chat_turns_races.py（asyncio.gather 并发两次新问、三次重试、两个标签页接着收尾、二十次并发追加事件；发起写到一半 出错什么都不留下；后台任务不起第二个；使用者继续运行 403 且说法同 /runs；开着的流收到 review.late；重试不丢晚到复核的用量； history 只取之前的、截 600 字、跳过没答案的；重新运行较早的一轮不带后面几轮；只认问数据的对话）。先写测试，12 条都失败；去掉 占锁、计数器、不起第二个、先写库再叫醒、增量记用量、同一事务、按 seq 截这几处修复分别回跑，对应的测试都会失败。 --- backend/app/api/conversations.py | 38 ++- backend/app/chat/job.py | 376 ++++++++++++++++--------- backend/app/chat/log.py | 32 ++- backend/tests/test_chat_turns_races.py | 351 +++++++++++++++++++++++ 4 files changed, 657 insertions(+), 140 deletions(-) create mode 100644 backend/tests/test_chat_turns_races.py diff --git a/backend/app/api/conversations.py b/backend/app/api/conversations.py index b3d54cf0..4d969ba3 100644 --- a/backend/app/api/conversations.py +++ b/backend/app/api/conversations.py @@ -24,7 +24,7 @@ from typing import Any from fastapi import APIRouter, Depends, Header, HTTPException from pydantic import BaseModel, Field -from sqlalchemy import and_, delete, func, select +from sqlalchemy import and_, delete, func, select, update from sqlalchemy.ext.asyncio import AsyncSession from app.api.deps import actor_of, current_principal @@ -531,7 +531,13 @@ async def create_turn( async def new_turn(session: AsyncSession, conversation: Conversation, question: str) -> ConversationTurn: - """开一轮（status=running，只有问题），不提交：开轮接口和服务端发起的一轮（app/chat/job.start）共用。""" + """开一轮（status=running，只有问题），不提交：开轮接口和服务端发起的一轮（app/chat/job.start）共用。 + + 先记活跃时间、再数下一轮排第几：记活跃时间是这个事务的第一句写，占住这段对话（SQLite 上拿到写锁、PostgreSQL 上锁住 + 这一行），同时开的两轮一个接一个数，不会都拿到同一个 seq。""" + now = utcnow() + await session.execute(update(Conversation).where(Conversation.id == conversation.id).values(last_active_at=now) + .execution_options(synchronize_session=False)) next_seq = int((await session.execute( select(func.coalesce(func.max(ConversationTurn.seq), -1)) .where(ConversationTurn.conversation_id == conversation.id) @@ -543,7 +549,7 @@ async def new_turn(session: AsyncSession, conversation: Conversation, question: ) session.add(turn) - conversation.last_active_at = utcnow() + conversation.last_active_at = now # 第一轮的问题就是这次对话的标题。用户改过名就不要再覆盖回去 if next_seq == 0 and conversation.title in ("", "新对话"): conversation.title = title_from(question) @@ -624,6 +630,32 @@ async def recent_turns( return list(reversed(rows)) +#: 运行入口的 history 里每轮的答案截到多少字（同以前问数据页 historyOf 的 slice(0, 600)） +RUN_HISTORY_CHARS = 600 + + +async def run_history(session: AsyncSession, conversation_id: str | None, *, before_seq: int, + limit: int = HISTORY_TURNS) -> str: + """问数据一轮发起运行时填进入口的 history（`{{ input.history }}`）：只取这一轮之前的（seq < before_seq），照以前问数据页 + historyOf 的规则——答案没有就用说明，都没有的跳过，每轮截 600 字，最多 limit 轮，「问：…\n答：…」之间空一行。 + + 以前这一份是页面按它内存里的轮次拼的；一轮搬到服务端之后在这里拼。和 history_text 不一样的地方是只看这一轮之前的： + 重试、重新运行较早的一轮时，后面接着问过的几轮不该进它的入口——那时候它还没被问过。""" + if not conversation_id: + return "" + rows = list((await session.execute( + select(ConversationTurn.question, ConversationTurn.answer, ConversationTurn.explanation) + .where(ConversationTurn.conversation_id == conversation_id, ConversationTurn.seq < before_seq) + .order_by(ConversationTurn.seq) + )).all()) + lines = [] + for question, answer, explanation in rows: + text = answer or explanation or "" + if question and text: + lines.append(f"问：{question}\n答：{text[:RUN_HISTORY_CHARS]}") + return "\n\n".join(lines[-limit:]) + + async def history_text( session: AsyncSession, conversation_id: str | None, limit: int = HISTORY_TURNS ) -> str: diff --git a/backend/app/chat/job.py b/backend/app/chat/job.py index 611ff17e..2d04bf4d 100644 --- a/backend/app/chat/job.py +++ b/backend/app/chat/job.py @@ -8,7 +8,7 @@ **流程和评测是同一份**（chat/drive.TurnDriver）。问数据这边的差异写在 ChatDriver： - 生成带着对话（历史、上一轮的图）、限定的数据源；每一条操作写进事件日志（思考按 THINKING_FLUSH_S 合并成一条，几百条增量 不逐条落库）； -- 发起：来源 chat（origin 记对话和这一轮），入口填问题和前几轮的问答（conversations.history_text）；开登录后 user 角色照 +- 发起：来源 chat（origin 记对话和这一轮），入口填问题和这一轮之前的几轮问答（conversations.run_history）；开登录后 user 角色照 chat/launch.user_chat_run（白名单、自己的记忆范围、遮罩、只读）——和以前浏览器发 POST /api/runs 时一字不差； - 停在审批上：照旧等人（评测是取消），审批过了接着等这次运行；没有待审批的断点（服务重启挂起）这一轮收成「服务重启，本轮已 中断」，可以「继续运行」； @@ -32,6 +32,10 @@ **并发。** 归「交互」类（总方案 §5 W2-b）：这一波还是进程内的 core/job_runtime（一个 asyncio 任务一轮，不占数据源的队）， 和以前浏览器各推各的一样不设上限；第 2 波按库里的计数认领、按类别给并发上限和优先级（问数据优先于评测），租约到期别的进程接手。 +同时来的几次发起（两个标签页、连点几下）在库里排队：发起的核对和写入在一个事务里、第一句先占住这段对话（hold_conversation）， +同一段对话只起得来一轮、同一轮只排得上一次，第一条事件也在这个事务里写；事件的 seq 由任务行上的计数器发（chat/log.add）； +同一行的几处写入（任务、晚到的复核、下一次尝试）先占住那一行再读改写（hold_turn），用量按增量记在库里那一份上，谁也不盖掉谁； +同一轮在这个进程里不起第二个后台任务（ChatTurnManager.start）。 """ from __future__ import annotations @@ -43,6 +47,7 @@ from collections.abc import Mapping from typing import Any from sqlalchemy import delete, select, update +from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession from app.chat import log as turn_log @@ -150,14 +155,19 @@ class ChatDriver(TurnDriver): async def on_retry(self, reason: str, *, kind: str, base_graph: dict[str, Any] | None) -> None: from app.chat import turn as policy + # 换下来的那次运行交出去并进 earlier；这一次从「还没有运行」起，收尾时不再把它当这一次的运行记一遍 + run, self.last_run = (self.last_run if kind == "review" else None), None await self.execution.retrying(reason, kind=kind, base=policy.has_nodes(base_graph), attempt=self.turn.attempts, - run=self.last_run if kind == "review" else None) + run=run) async def run_input(self, session: AsyncSession, graph: dict[str, Any]) -> dict[str, Any]: - from app.api.conversations import history_text + """入口填问题和这一轮之前的几轮问答（conversations.run_history，同以前页面上的 historyOf）：重试、重新运行较早的 + 一轮时，后面接着问过的几轮不算进来。""" + from app.api.conversations import run_history from app.chat import turn as policy - return policy.fill_input(graph, self.question, await history_text(session, self.execution.conversation_id)) + history = await run_history(session, self.execution.conversation_id, before_seq=self.execution.turn_seq) + return policy.fill_input(graph, self.question, history) async def launch_args(self, session: AsyncSession, graph: dict[str, Any]) -> dict[str, Any]: from app.api.deps import restricted @@ -265,8 +275,8 @@ class ChatTurnExecution(runtime.JobRun): self.actor: str | None = None self.mode = "ask" self.request: dict[str, Any] = {} - #: 这一轮的用量 {copilot, review, earlier, run}（UsageTotal），从库里那一行接着记 - self.usage: dict[str, Any] = {} + #: 这一轮在对话里排第几（运行入口的 history 只取它之前的几轮） + self.turn_seq = 0 self.run_id: str | None = None self.run_class: str | None = None self.driver: ChatDriver | None = None @@ -282,9 +292,11 @@ class ChatTurnExecution(runtime.JobRun): """追加一条事件（和这一轮的库写入排在同一条队里，停止时不会写到一半）。""" return await self._write(turn_log.append, self.turn_id, type_, data or {}) - async def save(self, *, meta: Mapping[str, Any] | None = None, **fields: Any) -> None: - """改 conversation_turns 那一行：fields 照写，meta 并进去（值为 None 的键删掉），顺手记 meta.at。""" - await self._write(_save_turn, self.turn_id, fields, dict(meta or {})) + async def save(self, *, meta: Mapping[str, Any] | None = None, usage: list[tuple[Any, ...]] | None = None, + **fields: Any) -> dict[str, Any]: + """改 conversation_turns 那一行：fields 照写，meta 并进去（值为 None 的键删掉），顺手记 meta.at；usage 是对 meta.usage 的 + 几步改动（见 _apply_usage），在库里现有的那一份上做。交回改完之后的 meta.usage。""" + return await self._write(_save_turn, self.turn_id, fields, dict(meta or {}), list(usage or [])) async def _job(self, **values: Any) -> None: async def go(session: AsyncSession) -> None: @@ -297,10 +309,10 @@ class ChatTurnExecution(runtime.JobRun): turn_log.notify(self.turn_id) async def add_usage(self, part: str, usage: Mapping[str, Any] | None) -> None: + """这一轮又花了一笔（助手建图、复核）：在库里那一份上加，不拿内存里的拷贝盖回去——晚到的复核也往同一份上记""" if not isinstance(usage, Mapping): return - self.usage[part] = record.add_total(self.usage.get(part), usage) - await self.save(meta={"usage": dict(self.usage)}) + await self.save(usage=[("add", part, dict(usage))]) async def on_phase(self, phase: str) -> None: await self._job(stage=phase) @@ -308,22 +320,18 @@ class ChatTurnExecution(runtime.JobRun): async def launched(self, run: Any) -> None: self.run_id, self.run_class = run.id, run.run_class - self.usage.pop("run", None) await self._job(run_id=run.id) - await self.save(run_id=run.id, status="running", + await self.save(run_id=run.id, status="running", usage=[("drop", "run")], meta={"runId": run.id, "runClass": run.run_class, "runStatus": run.status}) await self.emit("run", {"run_id": run.id, "status": run.status, "run_class": run.run_class}) async def retrying(self, reason: str, *, kind: str, base: bool, attempt: int, run: Any = None) -> None: """自动重来一次：换下来的那次运行的用量并进 earlier；这一行回到「还没有运行」。""" - if run is not None: - self.usage["run"] = record.usage_total(run.usage) or {} - self.usage = record.retire_run(self.usage) + ops: list[tuple[Any, ...]] = [("set", "run", record.usage_total(run.usage) or {})] if run is not None else [] self.run_id = None await self._job(attempt=attempt, run_id=None) - await self.save(run_id=None, meta={"runId": None, "runStatus": None, "usage": dict(self.usage)}) - await self.emit("turn.retry", {"kind": kind, "reason": reason, "base": base, "attempt": attempt, - "usage": dict(self.usage)}) + usage = await self.save(run_id=None, meta={"runId": None, "runStatus": None}, usage=[*ops, ("retire",)]) + await self.emit("turn.retry", {"kind": kind, "reason": reason, "base": base, "attempt": attempt, "usage": usage}) # ---------------- 执行 ---------------- @@ -342,8 +350,7 @@ class ChatTurnExecution(runtime.JobRun): return self.conversation_id, self.actor, self.mode = job.conversation_id, job.actor, job.mode self.request = dict(job.request or {}) - meta = dict(row.meta or {}) if isinstance(row.meta, dict) else {} - self.usage = dict(meta.get("usage") or {}) + self.turn_seq = row.seq self.run_id = job.run_id # 这一轮按发起的人的身份跑：answer_ops 里的 restricted()、对话归属，发起运行时的遮罩、只读都看它 set_principal(principal_of(job.principal)) @@ -371,8 +378,8 @@ class ChatTurnExecution(runtime.JobRun): """按这一轮的结局写那一行、发 end、任务记成功。""" driver = self.driver assert driver is not None - fields, meta, outcome, output = await _settled(driver, self) - await self.save(meta=meta, **fields) + fields, meta, outcome, output, usage = await _settled(driver, self) + await self.save(meta=meta, usage=usage, **fields) await self._end(outcome, output) # 晚到的复核先登记、再改任务状态：实时流看到任务结束时，要知道还得等这一次复核回来 if driver.late_review is not None: @@ -406,7 +413,7 @@ class ChatTurnExecution(runtime.JobRun): if not self.ended: await self.save(status="error", error=record.CANCELLED, meta={"outcome": "cancelled", "runStatus": "cancelled" if run_id else None, - "usage": dict(self.usage), **_timing(self.request)}) + **_timing(self.request)}) await self._end("cancelled") await self._job(status="cancelled", stage=None, finished_at=utcnow()) @@ -423,8 +430,7 @@ class ChatTurnExecution(runtime.JobRun): await self._settle() if not self.ended: await self.save(status="error", error=message, - meta={"cause": {"kind": "crash", "message": message}, "usage": dict(self.usage), - **_timing(self.request)}) + meta={"cause": {"kind": "crash", "message": message}, **_timing(self.request)}) await self._end("error") await self._job(status="failed", stage=None, error=message, finished_at=utcnow()) @@ -433,6 +439,14 @@ class ChatTurnManager(runtime.JobManager): def __init__(self) -> None: super().__init__(ChatTurnExecution, label="问数据的一轮", name="chat-turn") + def start(self, job_id: str) -> bool: # type: ignore[override] + """同一轮在这个进程里还有一个后台任务活着就不起第二个、也不把登记换成新的（换了的话，停止只停得到后起的那个， + 先起的那个跑完把「已取消」盖成「已完成」）。起了交回 True。""" + if self.running(job_id): + return False + super().start(job_id) + return True + manager = ChatTurnManager() @@ -442,32 +456,30 @@ _late: dict[str, asyncio.Task[Any]] = {} def _track_late(turn_id: str, task: asyncio.Task[Any]) -> None: async def finish() -> None: + # 先把用量记进那一行、发 review.late，再出登记、叫醒订阅方：反过来的话，开着的实时流醒来看到「不在进行中了」， + # 先发了 stream.end 走掉，review.late 就到不了它 try: - result = await task - except asyncio.CancelledError: - return - except Exception: # noqa: BLE001 - logger.exception("问数据：晚到的复核出错") - return + try: + result = await task + except asyncio.CancelledError: + return + except Exception: # noqa: BLE001 + logger.exception("问数据：晚到的复核出错") + return + usage = (result or {}).get("usage") if isinstance(result, Mapping) else None + if not usage: + return + async with SessionLocal() as session: + # 加在库里现有的那一份上（这一轮可能已经在重试，那一次的用量也在往同一份上记） + await _save_turn(session, turn_id, {}, {}, [("add", "review", dict(usage))], touch_meta=False) + async with SessionLocal() as session: + await turn_log.append(session, turn_id, "review.late", {"usage": usage}) + except Exception: # noqa: BLE001 - 记账失败只记日志 + logger.exception("问数据：记晚到的复核用量出错") finally: if _late.get(turn_id) is watcher: _late.pop(turn_id, None) turn_log.notify(turn_id) - usage = (result or {}).get("usage") if isinstance(result, Mapping) else None - if not usage: - return - async with SessionLocal() as session: - from app.db.models import ConversationTurn - - row = await session.get(ConversationTurn, turn_id) - if row is None: - return - meta = dict(row.meta or {}) if isinstance(row.meta, dict) else {} - spent = dict(meta.get("usage") or {}) - spent["review"] = record.add_total(spent.get("review"), usage) - row.meta = {**meta, "usage": spent} - await session.commit() - await turn_log.append(session, turn_id, "review.late", {"usage": usage}) watcher = asyncio.ensure_future(finish()) _late[turn_id] = watcher @@ -483,12 +495,41 @@ def late_pending(turn_id: str) -> bool: # -------------------------------------------------------------------------- -async def _save_turn(session: AsyncSession, turn_id: str, fields: Mapping[str, Any], meta: Mapping[str, Any]) -> None: +def _apply_usage(usage: Mapping[str, Any] | None, ops: list[tuple[Any, ...]]) -> dict[str, Any]: + """在一份 meta.usage 上做几步改动：("add", 哪一份, 用量) 加上；("set", 哪一份, 用量) 换成；("drop", 哪一份) 去掉； + ("retire",) 当前这次运行换下来了（run 并进 earlier，同 record.retire_run）。""" + out = dict(usage or {}) + for op in ops: + kind = op[0] + if kind == "add": + out[op[1]] = record.add_total(out.get(op[1]), op[2]) + elif kind == "set": + out[op[1]] = dict(op[2] or {}) + elif kind == "drop": + out.pop(op[1], None) + elif kind == "retire": + out = record.retire_run(out) + return out + + +async def hold_turn(session: AsyncSession, turn_id: str) -> None: + """占住这一行再读改写（同一行的几处写入——这一轮的任务、晚到的复核、发起下一次尝试——一个接一个来，谁也不盖掉谁）： + 第一句就写它，SQLite 上拿到写锁、PostgreSQL 上锁住这一行，别的写入等它提交。""" from app.db.models import ConversationTurn - row = await session.get(ConversationTurn, turn_id) + await session.execute(update(ConversationTurn).where(ConversationTurn.id == turn_id).values(updated_at=utcnow()) + .execution_options(synchronize_session=False)) + + +async def _save_turn(session: AsyncSession, turn_id: str, fields: Mapping[str, Any], meta: Mapping[str, Any], + usage: list[tuple[Any, ...]] | None = None, *, touch_meta: bool = True) -> dict[str, Any]: + from app.db.models import ConversationTurn + + await hold_turn(session, turn_id) + row = await session.get(ConversationTurn, turn_id, populate_existing=True) if row is None: - return + await session.rollback() + return {} for key, value in fields.items(): setattr(row, key, value) merged = dict(row.meta or {}) if isinstance(row.meta, dict) else {} @@ -497,10 +538,14 @@ async def _save_turn(session: AsyncSession, turn_id: str, fields: Mapping[str, A merged.pop(key, None) else: merged[key] = value + if usage: + merged["usage"] = _apply_usage(merged.get("usage"), usage) merged["v"] = 1 - merged["at"] = int(time.time() * 1000) + if touch_meta: + merged["at"] = int(time.time() * 1000) row.meta = merged await session.commit() + return dict(merged.get("usage") or {}) async def _snapshot(session: AsyncSession, turn_id: str, run_id: str | None) -> tuple[dict[str, Any] | None, @@ -568,8 +613,9 @@ async def _run_failed_detail(run_id: str) -> str | None: async def _settled(driver: ChatDriver, execution: ChatTurnExecution - ) -> tuple[dict[str, Any], dict[str, Any], str, Any]: - """这一轮的结局 → (那一行要写的列, 并进 meta 的, outcome, 完整成果)。和以前页面在各个结局处落库写的一一对应。""" + ) -> tuple[dict[str, Any], dict[str, Any], str, Any, list[tuple[Any, ...]]]: + """这一轮的结局 → (那一行要写的列, 并进 meta 的, outcome, 完整成果, 对 meta.usage 的改动)。和以前页面在各个结局处落库写的 + 一一对应。用量里助手、复核那几份是随做随记的（add_usage），这里只记这次运行的那一份。""" from app.engine.runner import END_REJECTED t = driver.turn @@ -583,7 +629,7 @@ async def _settled(driver: ChatDriver, execution: ChatTurnExecution text = t.reply or "" fields.update(status="done", answer=text, review={}) meta.update(noQuery=True, runId=None) - return fields, {**meta, "usage": dict(execution.usage)}, "reply", {"answer": text} + return fields, meta, "reply", {"answer": text}, [] if path == "asked": asked = t.asked or {} question = str(asked.get("question") or "").strip() @@ -594,11 +640,11 @@ async def _settled(driver: ChatDriver, execution: ChatTurnExecution # 改了图又问了用户、或者只问不改（图原样交回）：图摆着等「运行」，问题作为这一轮的内容 fields.update(graph=driver.graph or term.get("graph"), explanation=str(term.get("explanation") or "")) outcome = "ready" - return fields, {**meta, "usage": dict(execution.usage)}, outcome, {"answer": question} if question else None + return fields, meta, outcome, {"answer": question} if question else None, [] if path == "ready": fields.update(status="done", graph=driver.graph, explanation=str(term.get("explanation") or ""), review={}) meta["runId"] = None - return fields, {**meta, "usage": dict(execution.usage)}, "ready", None + return fields, meta, "ready", None, [] if path == "build_failed": if driver.refusal is not None: message = str(getattr(driver.refusal, "detail", "") or driver.refusal) @@ -616,15 +662,13 @@ async def _settled(driver: ChatDriver, execution: ChatTurnExecution fields.update(graph=driver.graph, explanation=str(term.get("explanation") or "")) fields.update(status="error", error=error, review={}) meta.update(cause={k: v for k, v in cause.items() if v is not None}, runId=None) - return fields, {**meta, "usage": dict(execution.usage)}, "error", None + return fields, meta, "error", None, [] - # 发起过运行的结局 - if run is not None: - execution.usage["run"] = record.usage_total(run.usage) or {} - usage = dict(execution.usage) + # 发起过运行的结局：这次运行的用量记成 run（换下来的早在重来时并进了 earlier） + usage: list[tuple[Any, ...]] = [("set", "run", record.usage_total(run.usage) or {})] if run is not None else [] run_id = run.id if run is not None else execution.run_id meta.update(runId=run_id, runClass=getattr(run, "run_class", None) or execution.run_class, - runStatus=getattr(run, "status", None), queries=await _queries(run_id), usage=usage) + runStatus=getattr(run, "status", None), queries=await _queries(run_id)) if path == "run_failed": refused = driver.launch_refused if refused is not None: @@ -632,27 +676,27 @@ async def _settled(driver: ChatDriver, execution: ChatTurnExecution fields.update(status="error", error=message, review={}) meta.update(cause={"kind": "start", "message": message, "code": getattr(refused, "code", None), "status": getattr(refused, "status_code", None)}, runId=None, runStatus=None, queries=None) - return fields, meta, "error", None + return fields, meta, "error", None, usage if run is None: fields.update(status="error", error=record.GONE, review={}) meta.update(cause={"kind": "gone"}, runStatus="gone") - return fields, meta, "error", None + return fields, meta, "error", None, usage if run.status == "cancelled" and run.end_reason != END_REJECTED: fields.update(status="error", error=record.CANCELLED, review={}) meta["outcome"] = "cancelled" - return fields, meta, "cancelled", None + return fields, meta, "cancelled", None, usage fields.update(status="error", error=run.error or "运行失败", review={}) meta["cause"] = {k: v for k, v in {"kind": "run", "error": run.error or "", "detail": await _run_failed_detail(run.id)}.items() if v is not None} - return fields, meta, "error", None + return fields, meta, "error", None, usage if path == "rejected": fields.update(status="error", error=record.CANCELLED, review={}) meta.update(outcome="cancelled", rejected=await _rejected_by(run.id) or {"decision": "reject"}) - return fields, meta, "cancelled", None + return fields, meta, "cancelled", None, usage if path == "suspended": fields.update(status="error", error=record.SUSPENDED, review={}) meta.update(outcome="suspended", runStatus="interrupted") - return fields, meta, "suspended", None + return fields, meta, "suspended", None, usage # 跑完了：成果以库里那份为准（事件里那份为了推送体积被切过） output = run.output if isinstance(run.output, dict) else {} @@ -661,7 +705,7 @@ async def _settled(driver: ChatDriver, execution: ChatTurnExecution fields.update(status="done", answer=rewritten or record.answer_text(output), review=review or {}) meta.update(issuance=record.slim_issuance(output.get("_issuance")), evidence=True if record.has_evidence(output) else None) - return fields, meta, "done", output + return fields, meta, "done", output, usage # -------------------------------------------------------------------------- @@ -712,43 +756,132 @@ async def _scope(session: AsyncSession, ids: list[str] | None) -> list[dict[str, return out +#: 只有问数据的对话能发起一轮（画布助手、应用润色、蓝图的对话各有各的助手；启动整理也只认问数据的对话） +NOT_CHAT = "这段对话不是问数据的对话，不能在这里发起问数据" +#: 使用者（user 角色）不能继续运行：以前 POST /api/runs/{id}/continue 对它就是 403（api/deps._run_scope 只放行停止），说法同那边 +ROLE_ONLY = "没有权限：这项操作需要搭建者及以上的角色" + + +async def hold_conversation(session: AsyncSession, conversation_id: str, *, touch: bool = True) -> None: + """占住这段对话：发起一轮的核对和写入（这段对话忙不忙、下一轮排第几、任务、第一条事件）在同一个事务里，第一句就写它—— + SQLite 上拿到写锁、PostgreSQL 上锁住这一行，同一段对话同时来的几次发起一个接一个，后来的看得见先来的写下的。 + + touch：顺手记活跃时间（人在这里开始做事）。页面打开时自动发的「接着收尾」不算，原样写回，不然旧对话一打开就顶到列表最上面。""" + from app.db.models import Conversation + + stamp = utcnow() if touch else Conversation.last_active_at + await session.execute(update(Conversation).where(Conversation.id == conversation_id) + .values(last_active_at=stamp).execution_options(synchronize_session=False)) + + async def start(session: AsyncSession, *, conversation: Any, question: str, datasource_ids: list[str] | None, actor: str | None, mode: str | None = None, retry_of: Any = None, max_steps: int | None = None) -> str: """发起一轮（新问，或者对已有的一轮重试、重新运行、继续运行、接着收尾）。交回这一轮的 id（conversation_turns.id）。 新问：建 conversation_turns 那一行（同 POST /api/conversations/{id}/turns）和任务。已有的一轮：上一次的结局归档进 - meta.attempts（重试、重新运行、放宽步数），那一行回到 running，任务重新排队。不发起抛 TurnRefused。""" - from app.api.conversations import new_turn, turn_run_id - from app.chat import turn as policy - from app.db.models import Approval, Run - from app.db.models_chat import ChatTurnJob + meta.attempts（重试、重新运行、放宽步数），那一行回到 running，任务重新排队。不发起抛 TurnRefused。 + + **同时来的几次发起只起得来一次。** 核对和写入在同一个事务里、先占住这段对话（hold_conversation）：同一段对话同时两次新问 + 只建一轮（另一次 409），同一轮同时几次重试、两个标签页同时请服务端接着收尾只有一次排得上队（任务按状态条件更新）； + 这一次尝试的第一条事件（turn.attempt）也在这个事务里写，写不进去就什么都不留下。继续运行要先续上那次运行（它自己按运行的 + 状态认领，同一次运行同时只续得上一次），它用自己的会话写库，不能放进占着对话的事务里；续上了、这里却没排上队的，把续上的 + 那次运行取消掉。服务重启挂起的旧运行在提交之后才取消（只有排上队的那一次去取消）。""" + from app.api.coded import ROLE_REQUIRED + from app.api.conversations import turn_run_id + from app.api.deps import restricted + from app.db.models import Run mode = mode or ("retry" if retry_of is not None else "ask") if mode not in MODES or (mode == "ask") != (retry_of is None): raise TurnRefused(422, "发起方式不对：新问不带 retry_of，重试、重新运行、继续运行要带上是哪一轮", TURN_INVALID) - besides = retry_of.id if retry_of is not None else None + if getattr(conversation, "kind", "chat") != "chat": + raise TurnRefused(422, NOT_CHAT, TURN_INVALID) + if mode == "continue" and restricted(): + raise TurnRefused(403, ROLE_ONLY, ROLE_REQUIRED) + if retry_of is None and not question.strip(): + raise TurnRefused(422, "问题不能为空", TURN_INVALID) + if mode == "max_steps" and (not max_steps or max_steps < 1): + raise TurnRefused(422, "放宽到几步要写清楚", TURN_INVALID) + + continued: str | None = None + if mode == "continue": + run_id = turn_run_id(retry_of) + run = await session.get(Run, run_id) if run_id else None + if run is None: + raise TurnRefused(409, "这一轮没有可接着的运行，请重试本轮", TURN_INVALID) + await _continue_run(run.id, actor) + continued = run.id + try: + turn_id, stale_run = await _claim(session, conversation=conversation, question=question, + datasource_ids=datasource_ids, actor=actor, mode=mode, retry_of=retry_of, + max_steps=max_steps) + except BaseException: + await session.rollback() + if continued: + from app.engine.runner import run_manager + + with contextlib.suppress(Exception): + await run_manager.cancel(continued, actor=actor) + raise + if stale_run: + # 服务重启挂起的旧运行没人会再去续它，顺手取消（停在审批上的不算：重试不会从那里发起） + from app.engine.runner import run_manager + + try: + await run_manager.cancel(stale_run, actor=actor) + except Exception: # noqa: BLE001 + logger.exception("问数据：重试时取消挂起的旧运行 %s 出错", stale_run) + if not manager.start(turn_id): + # 上一次的后台任务刚记完结局、还没退出：等它退出再起 + await manager.wait(turn_id, timeout=STOP_GRACE_S) + if not manager.start(turn_id): + async with SessionLocal() as s2: + from app.db.models_chat import ChatTurnJob + + await s2.execute(update(ChatTurnJob).where(ChatTurnJob.id == turn_id).values( + status="failed", error="上一次的后台任务还没有退出", finished_at=utcnow()) + .execution_options(synchronize_session=False)) + await s2.commit() + raise TurnRefused(409, BUSY, TURN_BUSY) + return turn_id + + +async def _claim(session: AsyncSession, *, conversation: Any, question: str, datasource_ids: list[str] | None, + actor: str | None, mode: str, retry_of: Any, max_steps: int | None) -> tuple[str, str | None]: + """在一个事务里核对、写轮次、排队、写这一次尝试的第一条事件，提交。交回 (这一轮的 id, 提交之后要取消的挂起旧运行)。""" + from app.api.conversations import new_turn, turn_run_id + from app.chat import turn as policy + from app.db.models import Approval, ConversationTurn, Run + from app.db.models_chat import ChatTurnJob + + await hold_conversation(session, conversation.id, touch=mode != "resume") + if retry_of is not None: + await hold_turn(session, retry_of.id) # 接着收尾（resume）不建图、不发起运行，不算占着这段对话：一段对话里停在半路的几轮可以一起交给服务端收尾 + besides = retry_of.id if retry_of is not None else None if mode != "resume" and await _busy(session, conversation.id, besides=besides): raise TurnRefused(409, BUSY, TURN_BUSY) now_ms = int(time.time() * 1000) request: dict[str, Any] = {"started_ms": now_ms} + stale_run: str | None = None if retry_of is None: - if not question.strip(): - raise TurnRefused(422, "问题不能为空", TURN_INVALID) scope = await _scope(session, datasource_ids) row = await new_turn(session, conversation, question) row.meta = {"v": 1, **({"scope": scope} if scope else {}), "at": now_ms} request["scope"] = scope - session.add(ChatTurnJob(id=row.id, conversation_id=conversation.id, status="queued", mode=mode, request=request, - principal=principal_snapshot(), actor=actor, started_at=None)) + job = ChatTurnJob(id=row.id, conversation_id=conversation.id, status="queued", mode=mode, request=request, + principal=principal_snapshot(), actor=actor, started_at=None) + session.add(job) await session.flush() - turn_id = row.id attempts: list[Any] = [] else: - row = retry_of - job = await session.get(ChatTurnJob, row.id) + # 占住之后重读：发起之前读到的那一份可能已经被同时来的另一次发起改过了 + row = await session.get(ConversationTurn, retry_of.id, populate_existing=True) + if row is None: + raise TurnRefused(404, "该轮对话不存在，可能已被删除") + job = await session.get(ChatTurnJob, row.id, populate_existing=True) if job is not None and job.status in runtime.ACTIVE: raise TurnRefused(409, BUSY, TURN_BUSY) meta = dict(row.meta or {}) if isinstance(row.meta, dict) else {} @@ -761,15 +894,11 @@ async def start(session: AsyncSession, *, conversation: Any, question: str, data if mode in ("rerun", "max_steps", "run") and not graphful: raise TurnRefused(409, "这一轮没有可运行的工作流，请重试本轮", TURN_INVALID) if mode == "max_steps": - if not max_steps or max_steps < 1: - raise TurnRefused(422, "放宽到几步要写清楚", TURN_INVALID) - request["max_steps"] = int(max_steps) + request["max_steps"] = int(max_steps or 0) if mode in ("continue", "resume"): # 接着收尾不要求运行还在：运行记录删了的，服务端照实收成「运行记录不见了」 - if run is None and (mode == "continue" or not run_id): + if not run_id: raise TurnRefused(409, "这一轮没有可接着的运行，请重试本轮", TURN_INVALID) - if mode == "continue": - await _continue_run(run.id, actor) request["follow"] = True # 计时：收过尾的（失败、中断之后点「继续运行」）接着上次停下的地方走——隔了一夜再点，头部不该写「已运行 14 小时」； # 停在半路的（页面以前没来得及收尾的）从运行开始算；运行早就结束了的（补交付）按它真正结束的时刻算，不按现在 @@ -781,18 +910,14 @@ async def start(session: AsyncSession, *, conversation: Any, question: str, data if mode == "resume" and run is not None and run.status in ("succeeded", "failed", "cancelled") \ and run.finished_at is not None: request["ended_ms"] = int(run.finished_at.timestamp() * 1000) + row.status, row.error = "running", "" + row.meta = {**meta, "at": now_ms} else: - # 重试、重新运行、放宽步数、跑一下：服务重启挂起的旧运行没人会再去续它，顺手取消（停在审批上的不算：重试不会从那里发起） if run is not None and run.status == "interrupted" and mode != "run": pending = (await session.execute(select(Approval.id).where( Approval.run_id == run.id, Approval.status.in_(("pending", "answered"))).limit(1))).first() if pending is None: - from app.engine.runner import run_manager - - try: - await run_manager.cancel(run.id, actor=actor) - except Exception: # noqa: BLE001 - logger.exception("问数据：重试时取消挂起的旧运行 %s 出错", run.id) + stale_run = run.id if mode != "run": archived = record.attempt_record(row, meta, n=len(attempts) + 1, at_ms=int(row.updated_at.timestamp() * 1000) if row.updated_at else now_ms) @@ -810,39 +935,38 @@ async def start(session: AsyncSession, *, conversation: Any, question: str, data row.meta = {k: v for k, v in new_meta.items() if v is not None} row.status, row.error, row.answer, row.review = "running", "", "", {} row.run_id = None - if mode in ("continue", "resume"): - row.status, row.error = "running", "" - row.meta = {**meta, "at": now_ms} + values = {"status": "queued", "stage": None, "mode": mode, "request": request, + "run_id": run_id if mode in ("continue", "resume") else None, "attempt": 0, + "principal": principal_snapshot(), "actor": actor, "error": None, "started_at": None, + "finished_at": None} if job is None: - job = ChatTurnJob(id=row.id, conversation_id=conversation.id) - session.add(job) - job.status, job.stage, job.mode, job.request = "queued", None, mode, request - job.run_id = run_id if mode in ("continue", "resume") else None - job.attempt = 0 - job.principal, job.actor, job.error, job.started_at, job.finished_at = ( - principal_snapshot(), actor, None, None, None) - conversation.last_active_at = utcnow() + session.add(ChatTurnJob(id=row.id, conversation_id=conversation.id, **values)) + else: + # 只在它不在排队、不在跑的时候排上：同时来的另一次已经排上了的，这一次不算 + claimed = await session.execute(update(ChatTurnJob).where( + ChatTurnJob.id == row.id, ChatTurnJob.status.not_in(runtime.ACTIVE)).values(**values) + .execution_options(synchronize_session=False)) + if claimed.rowcount != 1: + raise TurnRefused(409, BUSY, TURN_BUSY) await session.flush() - turn_id = row.id - await session.commit() - - # 这一次尝试从这一条开始：刷新、重开页面时从这里回放 - async with SessionLocal() as s: - from app.db.models import ConversationTurn - - fresh = await s.get(ConversationTurn, turn_id) - usage = ((fresh.meta or {}) if fresh is not None and isinstance(fresh.meta, dict) else {}).get("usage") - # 页面据此把这一轮重置成「这一次」的样子：归档、范围、到目前为止的用量、从什么时候算起（补交付的到运行结束为止） - seq = await turn_log.append(s, turn_id, "turn.attempt", { - "mode": mode, "attempts": attempts, "scope": request.get("scope") or [], "usage": usage or {}, - "started_ms": request.get("started_ms"), - **({"ended_ms": request["ended_ms"]} if request.get("ended_ms") else {}), - **({"max_steps": request["max_steps"]} if request.get("max_steps") else {})}) - await s.execute(update(ChatTurnJob).where(ChatTurnJob.id == turn_id).values(attempt_seq=seq) - .execution_options(synchronize_session=False)) - await s.commit() - manager.start(turn_id) - return turn_id + turn_id = row.id + usage_now = (row.meta or {}).get("usage") if isinstance(row.meta, dict) else None + # 这一次尝试从这一条开始：刷新、重开页面时从这里回放。页面据此把这一轮重置成「这一次」的样子：归档、范围、到目前为止的 + # 用量、从什么时候算起（补交付的到运行结束为止） + seq = await turn_log.add(session, turn_id, "turn.attempt", { + "mode": mode, "attempts": attempts, "scope": request.get("scope") or [], "usage": usage_now or {}, + "started_ms": request.get("started_ms"), + **({"ended_ms": request["ended_ms"]} if request.get("ended_ms") else {}), + **({"max_steps": request["max_steps"]} if request.get("max_steps") else {})}) + await session.execute(update(ChatTurnJob).where(ChatTurnJob.id == turn_id).values(attempt_seq=seq) + .execution_options(synchronize_session=False)) + try: + await session.commit() + except IntegrityError as e: + # 不占锁的写入（老页面开轮）正好同时：照「这段对话正忙」答 + raise TurnRefused(409, BUSY, TURN_BUSY) from e + turn_log.notify(turn_id) + return turn_id, stale_run async def _continue_run(run_id: str, actor: str | None) -> None: diff --git a/backend/app/chat/log.py b/backend/app/chat/log.py index 4b120a25..9e73c8b1 100644 --- a/backend/app/chat/log.py +++ b/backend/app/chat/log.py @@ -67,19 +67,29 @@ def notify(turn_id: str) -> None: waiters.notify(turn_id) -async def append(session: AsyncSession, turn_id: str, type_: str, data: dict[str, Any] | None = None) -> int: - """追加一条，提交，叫醒订阅方。交回它的 seq。 +async def add(session: AsyncSession, turn_id: str, type_: str, data: dict[str, Any] | None = None) -> int: + """在调用方的事务里加一条（不提交），交回它的 seq。 - seq 按这一轮库里已有的最大值往下接：同一轮同一时刻只有一个写的人（它的后台任务；任务不在跑时才由停止、启动整理来写）， - 唯一约束 (turn_id, seq) 兜底。任务行的 last_seq 顺手跟上（GET /api/chat/turns/{id} 照它说现在到哪了）。""" + seq 由任务行上的计数器发：UPDATE chat_turn_jobs SET last_seq = last_seq + 1 … RETURNING 是一句原子的写，同一轮同时追加的 + 几处（这一轮的任务、停止、晚到的复核、发起下一次尝试）各拿到不同的号，唯一约束 (turn_id, seq) 兜底。以前按库里已有的最大值 + 往下接，同时两处追加就撞号。任务行的 last_seq 也就是 GET /api/chat/turns/{id} 说的「现在到哪了」。没有任务行的（不该有） + 退回按最大值往下接。""" from app.db.models_chat import ChatTurnEvent, ChatTurnJob - last = int(await session.scalar(select(func.coalesce(func.max(ChatTurnEvent.seq), 0)) - .where(ChatTurnEvent.turn_id == turn_id)) or 0) - seq = last + 1 - session.add(ChatTurnEvent(turn_id=turn_id, seq=seq, type=type_, ts=time.time(), data=dict(data or {}))) - await session.execute(update(ChatTurnJob).where(ChatTurnJob.id == turn_id).values(last_seq=seq) - .execution_options(synchronize_session=False)) + seq = (await session.execute(update(ChatTurnJob).where(ChatTurnJob.id == turn_id) + .values(last_seq=ChatTurnJob.last_seq + 1).returning(ChatTurnJob.last_seq) + .execution_options(synchronize_session=False))).scalar_one_or_none() + if seq is None: + seq = int(await session.scalar(select(func.coalesce(func.max(ChatTurnEvent.seq), 0)) + .where(ChatTurnEvent.turn_id == turn_id)) or 0) + 1 + session.add(ChatTurnEvent(turn_id=turn_id, seq=int(seq), type=type_, ts=time.time(), data=dict(data or {}))) + await session.flush() + return int(seq) + + +async def append(session: AsyncSession, turn_id: str, type_: str, data: dict[str, Any] | None = None) -> int: + """追加一条，提交，叫醒订阅方。交回它的 seq（见 add）。""" + seq = await add(session, turn_id, type_, data) await session.commit() notify(turn_id) return seq @@ -134,5 +144,5 @@ async def stream(turn_id: str, after: int, *, active: Callable[[], Awaitable[str yield PING -__all__ = ["HEARTBEAT_S", "PING", "READ_BATCH", "STREAM_END", "append", "frame", "notify", "read", "stream", "waiters", +__all__ = ["HEARTBEAT_S", "PING", "READ_BATCH", "STREAM_END", "add", "append", "frame", "notify", "read", "stream", "waiters", "wire"] diff --git a/backend/tests/test_chat_turns_races.py b/backend/tests/test_chat_turns_races.py new file mode 100644 index 00000000..0ded95cb --- /dev/null +++ b/backend/tests/test_chat_turns_races.py @@ -0,0 +1,351 @@ +"""问数据的一轮在服务端跑：并发发起、晚到的复核、使用者能做的事、入口的历史、对话的种类（复核意见 A1–A5）。 + +守的是： +- 同一段对话同时两次新问：只起得了一轮（另一次 409），两轮不会拿到同一个 seq、不会有两个任务同时跑；同一轮同时几次重试、 + 两个标签页同时请服务端接着收尾：只有一次起得来，别的 409，不会 500，同一轮不会建两次图；事件的 seq 并发追加也不撞； + 发起写到一半出错时什么都不留下（这段对话不会从此一直「正忙」）；后台任务对同一轮不起第二个； +- 使用者（user 角色）不能「继续运行」：以前 POST /api/runs/{id}/continue 对它是 403（只放行停止），问数据这里照原说法； +- 晚到的复核：开着的实时流先收到 review.late 再收到 stream.end；它记的用量不会被紧接着的重试覆盖掉； +- 运行入口的 history 只取这一轮之前的几轮（照以前页面 historyOf 的规则：截 600 字、跳过没答案的、最多 6 轮）； +- 只有问数据的对话（kind=chat）能发起一轮。 + +库、问题、名字都是虚构的；助手和报告撰写都是剧本。 +""" +from __future__ import annotations + +import asyncio +import json +from typing import Any + +from httpx import ASGITransport, AsyncClient +from sqlalchemy import func, select + +from app.chat import job as chat_job +from app.chat import log as turn_log +from app.db.base import SessionLocal +from app.db.models import Conversation, ConversationTurn, Run +from app.db.models_chat import ChatTurnJob +from app.main import app +from tests import eval_samples as es +from tests.test_chat_turns import ( # noqa: F401 - 夹具照用 + _ask, _conversation, _done, _events, _row, _stage, _until, client, engine_up, people, source, +) + + +class Counting(es.Copilot): + """数一数生成过程起了几次（一次生成里的「接着写」「自查」不算）；gate 给了就每次起草前停在闸上。""" + + def __init__(self, source: str, *, gate: asyncio.Event | None = None, **kw: Any) -> None: + super().__init__(source, **kw) + self.gate = gate + self.drafts = 0 + + async def astream(self, messages): + if len(messages) == 2 and "上线前自查" not in messages[1][1]: + self.drafts += 1 + if self.gate is not None: + await self.gate.wait() + async for chunk in super().astream(messages): + yield chunk + + +async def _jobs_of(conversation_id: str) -> list[ChatTurnJob]: + async with SessionLocal() as session: + return list((await session.execute(select(ChatTurnJob).where( + ChatTurnJob.conversation_id == conversation_id))).scalars()) + + +# -------------------------------------------------------------------------- +# A1 并发发起 +# -------------------------------------------------------------------------- + + +async def test_two_asks_at_once_start_only_one_turn(client, monkeypatch, source): + gate = asyncio.Event() + model = Counting(source["name"], gate=gate, mode="reply") + es.use_copilot(monkeypatch, model) + conversation_id = await _conversation(client) + body = {"conversation_id": conversation_id, "question": "上周各区域销售额"} + got = await asyncio.gather(*(client.post("/api/chat/turns", json=body) for _ in range(2))) + codes = sorted(r.status_code for r in got) + assert codes == [202, 409], [r.text for r in got] + assert [r for r in got if r.status_code == 409][0].json()["code"] == "chat_turn_busy" + async with SessionLocal() as session: + rows = list((await session.execute(select(ConversationTurn).where( + ConversationTurn.conversation_id == conversation_id))).scalars()) + assert len(rows) == 1 and rows[0].seq == 0, [(r.id, r.seq) for r in rows] + assert len(await _jobs_of(conversation_id)) == 1 + gate.set() + await _done(rows[0].id) + assert model.drafts == 1 and (await _row(rows[0].id)).status == "done" + + +async def test_retrying_one_turn_from_several_places_at_once_starts_it_once(client, monkeypatch, source): + model = Counting(source["name"], mode="reply") + es.use_copilot(monkeypatch, model) + conversation_id = await _conversation(client) + turn_id = await _ask(client, conversation_id, "上周各区域销售额") + await _done(turn_id) + gate = asyncio.Event() + model.gate, model.drafts = gate, 0 + body = {"conversation_id": conversation_id, "retry_of": turn_id, "mode": "retry"} + async with AsyncClient(transport=ASGITransport(app=app, raise_app_exceptions=False), base_url="http://test") as raw: + got = await asyncio.gather(*(raw.post("/api/chat/turns", json=body) for _ in range(3))) + codes = sorted(r.status_code for r in got) + assert codes == [202, 409, 409], [r.text for r in got] + gate.set() + job = await _done(turn_id) + assert job.status == "succeeded" and model.drafts == 1, "同一轮只建了一次图" + events = await _events(turn_id) + assert [e["seq"] for e in events] == list(range(1, len(events) + 1)) + assert [e["data"]["mode"] for e in events if e["type"] == "turn.attempt"] == ["ask", "retry"] + assert len((await _row(turn_id)).meta["attempts"]) == 1, "只归档了一次" + + +async def test_two_tabs_resuming_the_same_turn_start_it_once(client): + conversation_id = await _conversation(client) + async with SessionLocal() as session: + run = Run(status="succeeded", output={"答案": "华东 300。"}, graph={"nodes": [], "edges": []}, run_class="exploratory") + session.add(run) + await session.flush() + turn = ConversationTurn(conversation_id=conversation_id, seq=0, question="q", status="running", run_id=run.id, + meta={"v": 1, "runId": run.id}) + session.add(turn) + await session.commit() + turn_id = turn.id + body = {"conversation_id": conversation_id, "retry_of": turn_id, "mode": "resume"} + got = await asyncio.gather(*(client.post("/api/chat/turns", json=body) for _ in range(2))) + assert sorted(r.status_code for r in got) == [202, 409], [r.text for r in got] + await _done(turn_id) + assert [e["type"] for e in await _events(turn_id)].count("end") == 1 + assert (await _row(turn_id)).answer == "华东 300。" + + +async def test_event_seqs_never_collide_under_concurrent_appends(client): + conversation_id = await _conversation(client) + async with SessionLocal() as session: + turn = ConversationTurn(conversation_id=conversation_id, seq=0, question="q", status="done") + session.add(turn) + await session.flush() + session.add(ChatTurnJob(id=turn.id, conversation_id=conversation_id, status="succeeded")) + await session.commit() + turn_id = turn.id + + async def one(i: int) -> int: + async with SessionLocal() as session: + return await turn_log.append(session, turn_id, "op", {"op": "heartbeat", "i": i}) + + seqs = await asyncio.gather(*(one(i) for i in range(20))) + assert sorted(seqs) == list(range(1, 21)), seqs + async with SessionLocal() as session: + job = await session.get(ChatTurnJob, turn_id) + assert job.last_seq == 20 + + +async def test_a_start_that_fails_halfway_leaves_nothing_behind(client, monkeypatch, source): + """写 turn.attempt 时出错：轮次、任务、事件一样都不留下（不会留着一个排队中、却没有后台任务的任务，让这段对话从此一直正忙）。""" + es.use_copilot(monkeypatch, es.Copilot(source["name"], mode="reply")) + conversation_id = await _conversation(client) + real = turn_log.add + calls = {"n": 0} + + async def flaky(session, turn_id, type_, data=None): + calls["n"] += 1 + if calls["n"] == 1: + raise RuntimeError("写事件时库出错了") + return await real(session, turn_id, type_, data) + + monkeypatch.setattr(turn_log, "add", flaky) + async with AsyncClient(transport=ASGITransport(app=app, raise_app_exceptions=False), base_url="http://test") as raw: + r = await raw.post("/api/chat/turns", json={"conversation_id": conversation_id, "question": "第一次"}) + assert r.status_code == 500 + async with SessionLocal() as session: + assert not (await session.execute(select(ConversationTurn).where( + ConversationTurn.conversation_id == conversation_id))).first() + assert not await _jobs_of(conversation_id) + turn_id = await _ask(client, conversation_id, "再问一次") + await _done(turn_id) + assert (await _row(turn_id)).status == "done" + + +async def test_the_job_manager_does_not_start_a_turn_twice(client, monkeypatch, source): + gate = asyncio.Event() + es.use_copilot(monkeypatch, Counting(source["name"], gate=gate, mode="reply")) + conversation_id = await _conversation(client) + turn_id = await _ask(client, conversation_id) + await _stage(turn_id, "build") + first = chat_job.manager._runs[turn_id] + assert chat_job.manager.start(turn_id) is False, "还在跑的同一轮不起第二个" + assert chat_job.manager._runs[turn_id] is first, "也不把登记换成新的（停止要停得到正在跑的那个）" + gate.set() + await _done(turn_id) + + +# -------------------------------------------------------------------------- +# A2 使用者不能继续运行 +# -------------------------------------------------------------------------- + + +async def test_a_user_cannot_continue_a_run_from_chat_either(people): + """以前 POST /api/runs/{id}/continue 对使用者是 403（role_required，只放行停止）；问数据这里照原说法，运行不动。""" + uma = people["uma"] + conversation_id = await _conversation(uma, "uma 的对话") + async with SessionLocal() as session: + run = Run(status="failed", output={}, graph={"nodes": [], "edges": []}, run_class="exploratory", + origin_kind="chat", started_by="uma", error="查询超时") + session.add(run) + await session.flush() + turn = ConversationTurn(conversation_id=conversation_id, seq=0, question="q", status="error", error="查询超时", + run_id=run.id, meta={"v": 1, "runId": run.id, "runStatus": "failed"}) + session.add(turn) + await session.commit() + turn_id, run_id = turn.id, run.id + old = await uma.post(f"/api/runs/{run_id}/continue", json={}) + r = await uma.post("/api/chat/turns", json={"conversation_id": conversation_id, "retry_of": turn_id, "mode": "continue"}) + assert r.status_code == 403 and r.json()["code"] == "role_required", r.text + assert (old.status_code, old.json()["detail"]) == (r.status_code, r.json()["detail"]), "说法同 POST /api/runs/{id}/continue" + async with SessionLocal() as session: + assert (await session.get(Run, run_id)).status == "failed" + assert await session.get(ChatTurnJob, turn_id) is None + # 重试本轮、重新运行本轮照旧能用（它们另起一次问数据运行） + r = await uma.post("/api/chat/turns", json={"conversation_id": conversation_id, "retry_of": turn_id, "mode": "rerun"}) + assert r.status_code != 403, r.text + + +# -------------------------------------------------------------------------- +# A3 晚到的复核 +# -------------------------------------------------------------------------- + + +async def _late_turn(client, monkeypatch, source) -> tuple[str, str, asyncio.Event]: + import app.api.copilot as copilot + + es.use_copilot(monkeypatch, es.Copilot(source["name"])) + es.use_report(monkeypatch) + monkeypatch.setattr(chat_job, "REVIEW_WAIT_S", 0.05) + release = asyncio.Event() + + async def slow(session, run, **kw): + await release.wait() + return {"verdict": "ok", "signals": [], "retry_instruction": None, "usage": {"calls": 1, "cost_usd": 0.004}} + + monkeypatch.setattr(copilot, "review_of_run", slow) + conversation_id = await _conversation(client) + turn_id = await _ask(client, conversation_id, datasource_ids=[source["id"]]) + await _done(turn_id) + assert chat_job.late_pending(turn_id) + return conversation_id, turn_id, release + + +async def test_an_open_stream_gets_the_late_review_before_it_ends(client, monkeypatch, source): + _, turn_id, release = await _late_turn(client, monkeypatch, source) + last = (await _events(turn_id))[-1]["seq"] + + async def active() -> str | None: + return await chat_job.stream_status(turn_id) + + async def follow() -> list[dict[str, Any]]: + got = [] + async for frame in turn_log.stream(turn_id, last, active=active, heartbeat=0.2): + if frame.startswith("data: "): + got.append(json.loads(frame[6:])) + return got + + listening = asyncio.ensure_future(follow()) + await asyncio.sleep(0.3) + release.set() + frames = await asyncio.wait_for(listening, timeout=10) + assert [f["type"] for f in frames] == ["review.late", "stream.end"], frames + assert frames[0]["data"]["usage"] == {"calls": 1, "cost_usd": 0.004} + + +async def test_a_retry_right_after_does_not_lose_the_late_review_usage(client, monkeypatch, source): + """晚到的复核还没回来就点了重试：它回来时记的用量不会被新的一次尝试覆盖（这一轮的花费照实是两次加起来）。""" + conversation_id, turn_id, release = await _late_turn(client, monkeypatch, source) + gate = asyncio.Event() + es.use_copilot(monkeypatch, Counting(source["name"], gate=gate, mode="reply")) + r = await client.post("/api/chat/turns", json={"conversation_id": conversation_id, "retry_of": turn_id, "mode": "retry"}) + assert r.status_code == 202, r.text + await _stage(turn_id, "build") + release.set() + await _until(lambda: _not_late(turn_id)) + gate.set() + await _done(turn_id) + row = await _row(turn_id) + assert row.status == "done" and row.meta["usage"].get("review", {}).get("cost_usd") == 0.004, row.meta["usage"] + + +async def _not_late(turn_id: str) -> bool: + return not chat_job.late_pending(turn_id) + + +# -------------------------------------------------------------------------- +# A4 运行入口的 history 只取这一轮之前的 +# -------------------------------------------------------------------------- + + +async def test_history_only_takes_the_turns_before_this_one(): + """照以前页面 historyOf 的规则：这一轮之前的、有答案（答案或说明）的，每轮截 600 字，最多 6 轮。""" + from app.api.conversations import run_history + + async with SessionLocal() as session: + conversation = Conversation(kind="chat", title="历史") + session.add(conversation) + await session.flush() + rows = [("第一问", "答一", "", "done"), ("没答上的一问", "", "", "error"), ("第三问", "", "只有说明", "error"), + ("长的一问", "长" * 700, "", "done"), ("这一轮", "", "", "running"), ("后面的一问", "后面的答案", "", "done")] + for seq, (q, a, e, st) in enumerate(rows): + session.add(ConversationTurn(conversation_id=conversation.id, seq=seq, question=q, answer=a, explanation=e, + status=st)) + await session.commit() + text = await run_history(session, conversation.id, before_seq=4) + assert text == "\n\n".join(["问：第一问\n答：答一", "问：第三问\n答：只有说明", "问：长的一问\n答：" + "长" * 600]) + assert await run_history(session, conversation.id, before_seq=0) == "" + for seq in range(6, 14): + session.add(ConversationTurn(conversation_id=conversation.id, seq=seq, question=f"问{seq}", answer=f"答{seq}", + status="done")) + await session.commit() + assert (await run_history(session, conversation.id, before_seq=14)).count("问：") == 6 + + +async def test_rerunning_an_earlier_turn_does_not_feed_it_later_turns(client, monkeypatch, source): + es.use_copilot(monkeypatch, es.Copilot(source["name"], mode="reply")) + conversation_id = await _conversation(client) + first = await _ask(client, conversation_id, "第一问") + await _done(first) + es.use_copilot(monkeypatch, es.Copilot(source["name"])) + es.use_report(monkeypatch) + second = await _ask(client, conversation_id, "第二问", datasource_ids=[source["id"]]) + await _done(second) + es.use_copilot(monkeypatch, es.Copilot(source["name"], mode="reply")) + third = await _ask(client, conversation_id, "第三问") + await _done(third) + r = await client.post("/api/chat/turns", json={"conversation_id": conversation_id, "retry_of": second, "mode": "rerun"}) + assert r.status_code == 202, r.text + job = await _done(second) + async with SessionLocal() as session: + run = await session.get(Run, job.run_id) + history = run.input.get("history", "") + assert "第一问" in history and "第三问" not in history, history + + +# -------------------------------------------------------------------------- +# A5 只有问数据的对话能发起一轮 +# -------------------------------------------------------------------------- + + +async def test_only_chat_conversations_can_start_a_turn(client): + async with SessionLocal() as session: + canvas = Conversation(kind="canvas", title="画布助手的对话") + app_conv = Conversation(kind="app", title="应用润色") + session.add_all([canvas, app_conv]) + await session.commit() + ids = [canvas.id, app_conv.id] + for conversation_id in ids: + r = await client.post("/api/chat/turns", json={"conversation_id": conversation_id, "question": "q"}) + assert r.status_code == 422 and r.json()["code"] == "chat_turn_invalid", r.text + async with SessionLocal() as session: + n = await session.scalar(select(func.count()).select_from(ConversationTurn).where( + ConversationTurn.conversation_id == conversation_id)) + assert n == 0 + From d6318b639db1ecf7aa66f229b4052967d34f3366 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 18 Oct 2026 18:07:03 +0800 Subject: [PATCH 7/7] =?UTF-8?q?=E9=97=AE=E6=95=B0=E6=8D=AE=E9=A1=B5?= =?UTF-8?q?=E7=9A=84=E5=A4=8D=E6=A0=B8=E4=BF=AE=E6=AD=A3=EF=BC=88=E5=89=8D?= =?UTF-8?q?=E7=AB=AF=EF=BC=89=EF=BC=9A=E4=BD=BF=E7=94=A8=E8=80=85=E4=B8=8D?= =?UTF-8?q?=E7=BB=99=E3=80=8C=E7=BB=A7=E7=BB=AD=E8=BF=90=E8=A1=8C=E3=80=8D?= =?UTF-8?q?=EF=BC=8C=E4=B8=A4=E4=B8=AA=E6=A0=87=E7=AD=BE=E9=A1=B5=E5=90=8C?= =?UTF-8?q?=E6=97=B6=E6=89=93=E5=BC=80=E5=81=9C=E5=9C=A8=E5=8D=8A=E8=B7=AF?= =?UTF-8?q?=E7=9A=84=E4=B8=80=E8=BD=AE=E6=97=B6=E5=90=8E=E5=88=B0=E7=9A=84?= =?UTF-8?q?=E9=82=A3=E4=B8=AA=E6=8E=A5=E7=9D=80=E7=9C=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 使用者（user 角色）续不了运行（服务端照 POST /api/runs/{id}/continue 的老规矩回 403）：补救区不摆「继续运行」，服务重启打断的 那一轮说明里也不叫它去点，改说「如需整轮重来，请点击「重新运行本轮」」（lib/terms 的 CHAT_TURN_TEXT）；重试本轮、重新运行本轮照常。 以前这个按钮对使用者也摆着，点了只弹一句 403。 - 两个标签页同时打开停在半路的一轮：都会请服务端接着收尾，服务端现在只起得来一次，后到的那个收到 409 chat_turn_busy 时不再说 「无法确认本轮的当前状态」，改成问一下这一轮现在怎样——在跑就从这一次尝试的第一条接着播，收了尾就照那一行摆。 - scripts/check-chat.mjs 新加两段：「使用者（user 角色）」（伪造 /api/auth/me 认出使用者：中断的、跑挂的都不给继续运行，一次续跑都 不发）、「两个标签页同时打开」（服务端只接着收尾一次，两边都收尾、答案一样、都不写库）。去掉修复回跑，这两段都会失败。 --- frontend/src/lib/terms.ts | 6 ++++ frontend/src/pages/ChatPage.tsx | 10 +++++-- frontend/src/store/chat.ts | 17 +++++++++++ scripts/check-chat.mjs | 53 ++++++++++++++++++++++++++++++++- 4 files changed, 82 insertions(+), 4 deletions(-) diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 8f01ebb6..1a05b4d2 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -768,6 +768,12 @@ export function runCostText(usd: number | null | undefined, unpriced: number, ap return `${(approx || unpriced > 0) && usd > 0 ? '≥ ' : ''}${formatCost(usd)}` } +/** 问数据的一轮在服务端跑（编排完善第 1 波 W1-a）：使用者（user 角色）不能继续运行时，补救区里那句「怎么办」 */ +export const CHAT_TURN_TEXT = { + /** 服务重启打断的一轮：使用者没有「继续运行」，只说重新运行 */ + suspendedHintForUser: '如需整轮重来，请点击「重新运行本轮」。', +} as const + /** * 问数据一轮的花费：助手建图（含挑表、接着写、自查修正）+ 运行 + 复核（next-round 细案 01 做法 3）。以前只算运行那份， * 这一轮的花费是低报的 diff --git a/frontend/src/pages/ChatPage.tsx b/frontend/src/pages/ChatPage.tsx index d3790806..3f4c1a33 100644 --- a/frontend/src/pages/ChatPage.tsx +++ b/frontend/src/pages/ChatPage.tsx @@ -18,7 +18,7 @@ import { humanizeError } from '../lib/errors' import { explainRunError } from '../lib/explain' import { canLeave, leavePass } from '../lib/leave' import { formatNumber, formatTime } from '../lib/format' -import { CHAT_BLUEPRINT_TEXT, REJECTED_TEXT, TURN_COST_TEXT, runCostText } from '../lib/terms' +import { CHAT_BLUEPRINT_TEXT, CHAT_TURN_TEXT, REJECTED_TEXT, TURN_COST_TEXT, runCostText } from '../lib/terms' import { useCatalog, useCatalogListState, useDatasources, useOnReconnect, type CatalogListState } from '../store/catalog' import { useAtLeast } from '../store/auth' import { useBlueprint } from '../store/blueprint' @@ -665,6 +665,9 @@ function Remedies({ conversationId, turnId, last, onRephrase, readOnly }: { const { retryTurn, continueTurn, ask, recheck, setScope } = useChat.getState() const { cap, fallback } = useStepCap() const [resuming, setResuming] = useState(false) + // 继续运行只给搭建者及以上：使用者（user 角色）续不了运行（服务端照 POST /api/runs/{id}/continue 的老规矩回 403）， + // 摆着只会点一下报一次错；重试本轮、重新运行本轮另起一次问数据运行，照常给 + const builder = useAtLeast('builder') if (!turn) return null const kind = remedyOf(turn) @@ -674,7 +677,7 @@ function Remedies({ conversationId, turnId, last, onRephrase, readOnly }: { && (!s.reason || s.reason === 'steps')) const current = fallback ? agentSteps(turn.graph, fallback) : null const target = current && cap ? Math.min(cap, Math.max(current * 2, current + 8)) : null - const canContinue = !!turn.run?.id && (kind === 'suspended' + const canContinue = builder && !!turn.run?.id && (kind === 'suspended' || (kind === 'failed' && turn.runStatus === 'failed' && turn.failure?.continuable !== false)) const offNote = readOnly ? IN_TRASH : busy ? '本对话仍有一轮在运行，请等待其结束' : undefined const off = busy || readOnly @@ -775,7 +778,8 @@ function Remedies({ conversationId, turnId, last, onRephrase, readOnly }: { } // 中断、取消、没核对上在流里不出红框（它们不是故障），为什么停、停了意味着什么在这里说 - const note = kind === 'suspended' || kind === 'unknown' ? turn.failure + const note = kind === 'suspended' && !builder && turn.failure ? { ...turn.failure, hint: CHAT_TURN_TEXT.suspendedHintForUser } + : kind === 'suspended' || kind === 'unknown' ? turn.failure : kind === 'cancelled' && turn.rejected ? { title: '', reason: REJECTED_TEXT.chatReason, hint: turn.rejected.sub } : kind === 'cancelled' ? { title: '', reason: turn.run ? '你已停止本轮，对应的运行也已一并取消。' : '你在搭建工作流时停止了本轮。' } : null diff --git a/frontend/src/store/chat.ts b/frontend/src/store/chat.ts index ff0e3a0e..cabac70e 100644 --- a/frontend/src/store/chat.ts +++ b/frontend/src/store/chat.ts @@ -1189,6 +1189,23 @@ async function resumeTurn(conversationId: string, turnId: string) { play(flow, turn.serverId, Math.max(0, (res.turn?.job?.attempt_seq ?? 1) - 1)) } catch (e) { if (!flow.alive()) return + // 别处（另一个标签页）刚请服务端接着收尾了这一轮：不再起一次，接着看它（服务端在跑就从这一次尝试的第一条播，收了尾就照那一行摆） + if (e instanceof ApiError && e.status === 409 && e.code === 'chat_turn_busy') { + try { + const res = await api.chatTurns.get(turn.serverId) + if (!flow.alive()) return + if (res.job && LIVE_JOB.has(res.job.status)) { + flow.patch(() => ({ hydrated: 'done', job: res.job })) + play(flow, turn.serverId, Math.max(0, res.job.attempt_seq - 1)) + return + } + if (res.turn && res.turn.status !== 'running') { + flow.patch((t) => ({ ...endPatch(t, { turn: res.turn }), hydrated: 'done' })) + return + } + } catch { /* 问不到就照下面说没核对上 */ } + if (!flow.alive()) return + } const said = errorMessage(e) const raw = humanizeError(e).raw const failure: Failure = { diff --git a/scripts/check-chat.mjs b/scripts/check-chat.mjs index c0abb8a6..ddd6238d 100644 --- a/scripts/check-chat.mjs +++ b/scripts/check-chat.mjs @@ -46,7 +46,9 @@ // 按新目录新出现几条错误级问题），看的人手上有说的是这个答案的召回待办时链到待办；没比对的答案照样写依据有变；做完的 // 对账卡上写「差在哪」、问说明有没有帮助（说明有用 / 看不懂）。 // 20. 一轮在服务端跑：实时流断了按收到的最后一个 seq 重连，补齐、不重复；刷新、关掉再打开，服务端那一轮照跑，页面从 -// 这一次尝试的第一条接着播；停止发给服务端的这一轮（建图中也停得下）；发起、重试、继续运行的请求体照实。 +// 这一次尝试的第一条接着播；停止发给服务端的这一轮（建图中也停得下）；发起、重试、继续运行的请求体照实； +// 使用者（user 角色）不给「继续运行」（服务端照 POST /api/runs/{id}/continue 的老规矩不让它续跑）；两个标签页同时打开 +// 停在半路的一轮，服务端只接着收尾一次，后到的那个接着看同一轮。 // // 所有写请求都被拦在浏览器里，不碰任何库。 // 跑之前前端得起着（./scripts/dev.sh），默认连 5273。对别的实例（比如一份沙箱拷贝）跑时 @@ -2743,6 +2745,55 @@ for (const theme of THEMES) { await ctx.close() }) + await section('userrole', '使用者（user 角色）：不给「继续运行」（服务端照老规矩不让使用者续跑），重试、重新运行本轮照常', async () => { + const { page, ctx, errors } = await open(theme) + // 开了登录、认出来是使用者、管理员开了问数据 + await page.route('**/api/auth/me', (route) => route.fulfill({ json: { mode: 'local', username: 'uma', display_name: 'uma', + role: 'user', groups: [], verified: true, user_can_chat: true } })) + const continuesBefore = log.continues.length + await goto(page, 'c0suspo') + await shows(page, '服务重启，本轮已中断') + const card = page.locator('[data-remedy="suspended"]') + await card.waitFor({ timeout: 6000 }).catch(() => {}) + const cardText = (await card.innerText().catch(() => '')).replace(/\n/g, ' / ') + check('中断的一轮：使用者看不到「继续运行」，说明里也不叫它去点', await card.getByRole('button', { name: '继续运行' }).count() === 0 + && !cardText.includes('继续运行') && cardText.includes('重新运行本轮'), cardText) + check('……「重新运行本轮」照常给', await card.getByRole('button', { name: '重新运行本轮' }).count() === 1) + await goto(page, 'c0histf') + await shows(page, '步数用满的那一轮') + await page.waitForTimeout(600) + const failed = page.locator('[data-turn="tf2"] + * [data-remedy="failed"]') + check('跑挂了的一轮：使用者也看不到「继续运行」，「重试本轮」照常给', + await page.getByRole('button', { name: '继续运行' }).count() === 0 + && await failed.getByRole('button', { name: /重试本轮|重新问一次/ }).count() === 1, + (await failed.innerText().catch(() => '')).replace(/\n/g, ' / ')) + check('一次续跑都没发', log.continues.length === continuesBefore) + check('没有运行时报错', errors.length === 0, errors[0] ?? '') + await ctx.close() + }) + + await section('twotabs', '两个标签页同时打开停在半路的一轮：服务端只接着收尾一次，两边都接着看同一轮', async () => { + const a = await open(theme) + const b = await open(theme) + ctl.reviewDelay['run-late'] = 1200 + const startsBefore = log.turnStarts.length + const postsBefore = log.patches.length + await Promise.all([goto(a.page, 'c0lateq'), goto(b.page, 'c0lateq')]) + const done = (page) => page.waitForFunction(() => window.__chat.getState().byConversation.c0lateq?.[0]?.phase === 'done', + null, { timeout: 8000 }).then(() => true, () => false) + const [okA, okB] = await Promise.all([done(a.page), done(b.page)]) + check('两边都收尾了，答案一样', okA && okB && await shows(a.page, '补交付的答案：三条记录。') && await shows(b.page, '补交付的答案：三条记录。')) + const resumed = log.turnStarts.slice(startsBefore).filter((x) => x.mode === 'resume' && x.retry_of === 'tq1') + check('服务端只接着收尾了一次（后到的那个 409，改成接着看）', resumed.length === 1, `${resumed.length} 次`) + const phases = await Promise.all([a.page, b.page].map((p) => p.evaluate(() => window.__chat.getState().byConversation.c0lateq?.[0]?.phase))) + check('后到的那个不说「无法确认本轮的当前状态」', !phases.includes('unknown'), phases.join(',')) + check('两边都不写库（不 PATCH）', log.patches.length === postsBefore) + delete ctl.reviewDelay['run-late'] + check('没有运行时报错', a.errors.length === 0 && b.errors.length === 0, a.errors[0] ?? b.errors[0] ?? '') + await a.ctx.close() + await b.ctx.close() + }) + await section('steps', '历史轮次的执行过程：点开、收起', async () => { const { page, ctx, errors } = await open(theme) await goto(page, 'c0idleb')