From 7820556ab6ed5612b26ebddde9b229ce3516c230 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 00:03:24 +0800 Subject: [PATCH 1/7] =?UTF-8?q?=E5=8F=91=E8=B5=B7=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E7=9A=84=E8=A7=84=E5=88=99=E6=8A=BD=E6=88=90=20engine/launch.p?= =?UTF-8?q?y=20=E7=9A=84=20launch=5Frun=EF=BC=9A=E4=BB=A5=E5=89=8D?= =?UTF-8?q?=E5=85=A8=E5=86=99=E5=9C=A8=20api/runs.py=20=E7=9A=84=20start?= =?UTF-8?q?=5Frun=20=E9=87=8C=E3=80=81=E5=8F=AA=E8=AE=A4=20HTTP=20?= =?UTF-8?q?=E8=AF=B7=E6=B1=82=EF=BC=8C=E5=90=8E=E9=9D=A2=E7=9A=84=E5=AE=9A?= =?UTF-8?q?=E6=97=B6=E4=BB=BB=E5=8A=A1=E8=A6=81=E7=94=A8=E5=90=8C=E4=B8=80?= =?UTF-8?q?=E5=A5=97=E8=A7=84=E5=88=99=E5=8F=91=E8=B5=B7=E6=AD=A3=E5=BC=8F?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=EF=BC=8C=E5=8F=A6=E6=8A=84=E4=B8=80=E4=BB=BD?= =?UTF-8?q?=E8=BF=9F=E6=97=A9=E8=B5=B0=E6=A0=B7=E3=80=82=E6=AD=A3=E5=BC=8F?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E5=8F=AA=E8=83=BD=E4=BB=8E=E5=BD=93=E5=89=8D?= =?UTF-8?q?=E5=8F=91=E5=B8=83=E7=89=88=E6=9C=AC=E5=8F=91=E8=B5=B7=E3=80=81?= =?UTF-8?q?=E5=8F=A3=E5=BE=84=E5=8D=87=E7=89=88=E6=9C=AA=E5=A3=B0=E6=98=8E?= =?UTF-8?q?=E7=AD=96=E7=95=A5=E5=B0=B1=E6=8B=A6=E3=80=81=E7=BB=91=E5=AE=9A?= =?UTF-8?q?=E7=9A=84=E5=B7=A5=E5=85=B7=E4=B8=8D=E5=AD=98=E5=9C=A8=E5=B0=B1?= =?UTF-8?q?=E6=8B=A6=E3=80=81=E5=8F=91=E8=B5=B7=E5=90=8E=E8=AE=B0=E5=8F=97?= =?UTF-8?q?=E7=AE=A1=E7=BA=A7=E5=88=AB=E9=82=A3=E6=9D=A1=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=92=8C=E7=9B=AE=E5=BD=95=E5=8F=98=E5=8C=96=E3=80=81=E5=8F=A3?= =?UTF-8?q?=E5=BE=84=E5=8D=87=E7=89=88=E5=A4=84=E7=BD=AE=EF=BC=8C=E5=85=A8?= =?UTF-8?q?=E9=83=A8=E6=90=AC=E8=BF=9B=E6=9C=8D=E5=8A=A1=EF=BC=9B=E6=8B=92?= =?UTF-8?q?=E7=BB=9D=E6=97=B6=E6=8A=9B=20LaunchRefused(status=5Fcode,=20me?= =?UTF-8?q?ssage,=20code)=EF=BC=8Cstart=5Frun=20=E5=8F=AA=E5=89=A9?= =?UTF-8?q?=E4=B8=80=E5=B1=82=E5=8C=85=E8=A3=85=EF=BC=8C=E6=8A=8A=E5=AE=83?= =?UTF-8?q?=E6=8D=A2=E5=9B=9E=E5=92=8C=E4=BB=A5=E5=89=8D=E9=80=90=E5=AD=97?= =?UTF-8?q?=E7=9B=B8=E5=90=8C=E7=9A=84=E5=93=8D=E5=BA=94=EF=BC=88=E5=B8=A6?= =?UTF-8?q?=E6=9C=BA=E5=99=A8=E7=A0=81=E7=9A=84=E4=BB=8D=E5=9B=9E=20{detai?= =?UTF-8?q?l,=20code}=EF=BC=89=E3=80=82=E6=AD=A3=E5=BC=8F=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E7=9A=84=E5=BF=AB=E7=85=A7=E5=9B=BE=E4=B8=8D=E6=88=90?= =?UTF-8?q?=E5=BD=A2=EF=BC=8C=E4=BB=A5=E5=89=8D=E7=94=B1=20main.py=20?= =?UTF-8?q?=E7=9A=84=20ValueError=20=E5=85=9C=E5=BA=95=E5=9B=9E=20400?= =?UTF-8?q?=EF=BC=8C=E7=8E=B0=E5=9C=A8=E5=9C=A8=E6=9C=8D=E5=8A=A1=E9=87=8C?= =?UTF-8?q?=E6=8D=A2=E6=88=90=E5=90=8C=E6=A0=B7=E7=9A=84=20400=20=E6=8B=92?= =?UTF-8?q?=E7=BB=9D=EF=BC=8C=E8=B0=83=E5=BA=A6=E5=99=A8=E4=B9=9F=E6=8B=BF?= =?UTF-8?q?=E5=BE=97=E5=88=B0=E6=98=8E=E7=A1=AE=E7=9A=84=E6=8B=92=E7=BB=9D?= =?UTF-8?q?=E3=80=82node=5Fsetting=E3=80=81tool=5Fbindings=E3=80=81missing?= =?UTF-8?q?=5Ftools=20=E8=B7=9F=E7=9D=80=E6=90=AC=E8=BF=87=E5=8E=BB?= =?UTF-8?q?=EF=BC=8Cruns.py=20=E5=86=8D=E5=AF=BC=E5=87=BA=EF=BC=8C?= =?UTF-8?q?=E6=97=A7=E7=9A=84=E5=AF=BC=E5=85=A5=E8=B7=AF=E5=BE=84=E7=85=A7?= =?UTF-8?q?=E6=97=A7=E5=8F=AF=E7=94=A8=EF=BC=9Brunner=20=E5=9B=BA=E5=AE=9A?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E7=89=88=E6=9C=AC=E5=8F=96=E7=BB=91=E5=AE=9A?= =?UTF-8?q?=E6=97=B6=E6=94=B9=E4=BB=8E=20engine/launch=20=E5=AF=BC?= =?UTF-8?q?=E5=85=A5=EF=BC=8C=E4=B8=8D=E5=86=8D=E4=BE=9D=E8=B5=96=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E5=B1=82=E3=80=82=E6=96=B0=E6=B5=8B=E8=AF=95=E5=85=88?= =?UTF-8?q?=E6=8A=8A=20HTTP=20=E7=9A=84=2017=20=E7=A7=8D=E6=8B=92=E7=BB=9D?= =?UTF-8?q?=EF=BC=88=E5=90=AB=E5=B7=A5=E5=85=B7=E7=BC=BA=E5=A4=B1=E7=9A=84?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E3=80=81=E5=B7=A5=E5=85=B7=E3=80=81=E6=B7=B7?= =?UTF-8?q?=E5=90=88=E3=80=81=E8=B6=85=E8=BF=87=208=20=E5=A4=84=E5=9B=9B?= =?UTF-8?q?=E7=A7=8D=E8=AF=B4=E6=B3=95=EF=BC=89=E6=8C=89=E5=8E=9F=E8=AF=9D?= =?UTF-8?q?=E5=86=99=E6=AD=BB=E3=80=81=E5=9C=A8=E6=97=A7=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E4=B8=8A=E9=80=9A=E8=BF=87=EF=BC=8C=E5=86=8D=E9=80=90=E6=9D=A1?= =?UTF-8?q?=E6=AF=94=E5=AF=B9=E7=9B=B4=E6=8E=A5=E8=B0=83=20launch=5Frun=20?= =?UTF-8?q?=E7=9A=84=E7=BB=93=E6=9E=9C=EF=BC=9B=E5=8F=A6=E6=B5=8B=E6=9C=8D?= =?UTF-8?q?=E5=8A=A1=E5=8F=91=E8=B5=B7=E7=9A=84=E6=AD=A3=E5=BC=8F=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E7=85=A7=E6=A0=B7=E5=86=99=E6=B2=BB=E7=90=86=E4=BA=8B?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/runs.py | 243 +++------------------- backend/app/engine/launch.py | 273 +++++++++++++++++++++++++ backend/app/engine/runner.py | 5 +- backend/tests/test_run_launch.py | 337 +++++++++++++++++++++++++++++++ 4 files changed, 636 insertions(+), 222 deletions(-) create mode 100644 backend/app/engine/launch.py create mode 100644 backend/tests/test_run_launch.py diff --git a/backend/app/api/runs.py b/backend/app/api/runs.py index d61bd8a0..058ad518 100644 --- a/backend/app/api/runs.py +++ b/backend/app/api/runs.py @@ -2,7 +2,6 @@ from __future__ import annotations import asyncio import contextlib -import logging from datetime import datetime from typing import Any from urllib.parse import unquote @@ -12,24 +11,17 @@ from pydantic import BaseModel, Field from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy import select as _select - from app.api.coded import CodedHTTPException from app.core.bus import bus -from app.core import artifact_store -from app.core.events import EventType -from app.data.catalog_impact import catalog_drift from app.db.base import SessionLocal, get_session -from app.db.models import Approval, Artifact, Run, RunEvent, Workflow, WorkflowVersion -from app.engine.governance import governance_note, unresolved_caliber_upgrades +from app.db.models import Approval, Artifact, Run, RunEvent +# 发起规则和发起前检查绑定的工具搬到了 engine/launch.py。这几个名字以前从这里导入,照旧可用 +from app.engine.launch import ( # noqa: F401 + TOOL_MISSING, LaunchRefused, launch_run, missing_tools, node_setting, tool_bindings, +) from app.engine.runner import run_manager -from app.engine.schema import GraphSpec, NodeType, type_label router = APIRouter(prefix="/api/runs", tags=["runs"]) -logger = logging.getLogger(__name__) - -#: 发起时发现绑定的工具在本机不存在(见 missing_tools) -TOOL_MISSING = "run_tool_missing" def actor_of(header: str | None) -> str | None: @@ -90,218 +82,29 @@ async def start_run( session: AsyncSession = Depends(get_session), x_actor: str | None = Header(default=None), ) -> Run: - actor = actor_of(x_actor) - # 没挂在工作流上(画布没保存就跑、问数据当场搭的)。老数据里是「临时图」,前端两种都认 - name = "未保存的工作流" - version: int | None = None - version_hash: str | None = None - governed = False - - if payload.run_class == "formal": - # 正式运行的全部语义就这一条规则:必须引用一个不可变的已发布版本。 - # 传裸 graph、或工作流还没发布过,都进不了 formal。 - if not payload.workflow_id: - raise HTTPException(400, "正式运行需要从已保存的工作流发起:请求中没有指定工作流") - if payload.graph is not None: - raise HTTPException(400, "正式运行只能使用已发布的版本,画布上未发布的修改不会生效。请先保存并发布") - workflow = await session.get(Workflow, payload.workflow_id) - if not workflow: - raise HTTPException(404, "工作流不存在,可能已被删除") - # 只看 published_version:status 反映的是当前画布(改过图就退回 draft), - # 而 formal 跑的是已发布的那一版,两者本来就可以不一致 - if not workflow.published_version: - raise HTTPException(409, f"「{workflow.name}」还没有发布版本,请先在编排页发布") - version = payload.version or workflow.published_version - # "不可变"和"过过闸"是两件事。快照确实不可变,但 PATCH 保存出来的 - # 草稿版本同样有快照——只认 version 存不存在的话,工作流只要发布过 - # 任意一个版本,之后所有草稿都能以 formal 身份启动,发布期的 - # validate_graph + lint_for_publish 全部绕过。 - if version != workflow.published_version: - raise HTTPException( - 409, - f"v{version} 不是当前发布版本(v{workflow.published_version})。" - "正式运行只能使用当前发布版本;如需运行这一版,请先发布它。", - ) - snapshot = ( - await session.execute( - _select(WorkflowVersion).where( - WorkflowVersion.workflow_id == workflow.id, - WorkflowVersion.version == version, - ) - ) - ).scalar_one_or_none() - if not snapshot: - raise HTTPException(404, f"版本 v{version} 不存在") - graph = snapshot.graph - version_hash = snapshot.graph_hash or artifact_store.graph_hash(graph) - name = workflow.name - # 按这一版发布时记下的级别;记级别之前发布的老版本没有,照旧看工作流现在的 status - governed = (snapshot.level == "governed") if snapshot.level else workflow.status == "governed" - - # 口径升版是被迫处置的事件:钉住的方法卡有新版本而未声明策略,拒绝启动 - spec = GraphSpec.model_validate(graph) - errors, upgrade_events = await unresolved_caliber_upgrades(session, spec) - if errors: - raise HTTPException(409, ";".join(errors)) - # 数据目录和口径卡不同:目录是给人和助手看的说明,不是运行的输入。发布之后变了只提醒、不拦—— - # 拦下来等于让一次文字修订卡住正式出具 - catalog_changes = await catalog_drift(session, snapshot.catalog_versions) - else: - graph = payload.graph - upgrade_events = [] - catalog_changes = [] - if payload.workflow_id: - workflow = await session.get(Workflow, payload.workflow_id) - if not workflow: - raise HTTPException(404, "工作流不存在,可能已被删除") - graph = graph or workflow.graph - name = workflow.name - if not graph: - raise HTTPException(400, "没有要运行的内容:请求中既没有工作流 ID,也没有工作流内容") - await _refuse_missing_tools(session, graph) - - from app.api.settings import resolve_run_scope - - memory_scope, collection = await resolve_run_scope( - session, payload.memory_scope, payload.collection - ) + """发起规则全在 engine/launch.py(定时任务到点发起也用它),这里只把拒绝换回 HTTP 响应。""" try: - run = await run_manager.start( - graph=graph, - input_payload=payload.input, + return await launch_run( + session, workflow_id=payload.workflow_id, - workflow_name=name, - memory_scope=memory_scope, - collection=collection, - run_class="formal" if payload.run_class == "formal" else "exploratory", - version=version, - version_hash=version_hash, - started_by=actor, + graph=payload.graph, + input_payload=payload.input, + memory_scope=payload.memory_scope, + collection=payload.collection, + run_class=payload.run_class, + version=payload.version, + actor=actor_of(x_actor), ) - except ValueError as e: - raise HTTPException(400, str(e)) from e - - if payload.run_class == "formal": - # 按不按受管出具在发起这一刻定下来、记进封存范围:status 反映的是当前画布,中途改图、 - # 审批停下再恢复时再去读它,报告节点和出口复核可能各读到不同的值 - await run_manager.note(run.id, "log", **governance_note( - governed=governed, workflow_name=name, version=version)) - if catalog_changes: - # 落在封存范围内:之后回看这次正式出具,能知道当时的目录和发布时不一样、差在哪几张表。 - # 每张表带 impact:direct 写死的 SQL 用到的,possible Agent 可能查询的(界面分开说) - await run_manager.note(run.id, EventType.CATALOG_DRIFT, version=version, count=len(catalog_changes), - tables=catalog_changes) - # 已声明策略的升版记入事件流,出具物上能看到"这期换口径了、怎么处置的" - for ev in upgrade_events: - payload_ev = dict(ev) - node = payload_ev.pop("node_id", None) - await run_manager.note(run.id, "caliber.upgrade", node_id=node, **payload_ev) - return run + except LaunchRefused as e: + raise _refused(e) from e -# -------------------------------------------------------------------------- -# 发起前检查绑定的工具 -# -------------------------------------------------------------------------- - - -def node_setting(spec: GraphSpec, node: Any, key: str) -> Any: - """节点配置的实际取值,和运行时 NodeContext.cfg 同一个规则:节点上没写(None、空串)就取图级 defaults。""" - value = node.config.get(key) - return spec.defaults.get(key) if value in (None, "") else value - - -def tool_bindings(spec: GraphSpec) -> list[tuple[str, str]]: - """图里每一处绑定的工具:(谁绑的, 工具名)。调用工具节点的 tool、Agent 的 tools、协作成员的 tools。 - - 取值和运行时一致(node_setting):Agent 节点上没写 tools、协作节点上没写成员,运行时用的是 - 图级 defaults 里的,这里也算它们——不然发起前检查放过了缺的工具,发起时固定数据版本也漏了源。 - - 模板拼出来的名字({{ vars.tool }})也在内:发起前检查(_bindings)跳过它们,发起时固定数据 - 版本(engine/runner.py)要知道有没有这种名字——有的话说不准会用到哪个数据源。 - """ - out: list[tuple[str, str]] = [] - for node in spec.nodes: - who = f"「{node.title}」({type_label(node.type)})" - if node.type == NodeType.TOOL: - names = [node_setting(spec, node, "tool")] - out.extend((who, n) for n in names if isinstance(n, str) and n.strip()) - elif node.type == NodeType.AGENT: - out.extend((who, n) for n in node_setting(spec, node, "tools") or [] - if isinstance(n, str) and n.strip()) - elif node.type == NodeType.SUPERVISOR: - for member in node_setting(spec, node, "agents") or []: - if not isinstance(member, dict): - continue - label = f"「{node.title}」的成员「{member.get('name') or '?'}」" - out.extend((label, n) for n in member.get("tools") or [] if isinstance(n, str) and n.strip()) - return [(who, name.strip()) for who, name in out] - - -def _bindings(spec: GraphSpec) -> list[tuple[str, str]]: - """发起前要检查的绑定。模板拼出来的工具名({{ vars.tool }})发起时还不知道是谁,交给运行时。""" - return [(who, name) for who, name in tool_bindings(spec) if "{{" not in name] - - -async def missing_tools(session: AsyncSession, spec: GraphSpec) -> list[tuple[str, str, str]]: - """本机不存在的绑定:(谁绑的, 工具名, 去哪接入:data / tools)。 - - 解析和运行时 build_tools 一致:内置(静态注册)、mcp:<服务>/<工具>、db_query__ / db_schema__ - <数据源>、其余按自定义工具。数据源和自定义工具只认启用着的——停用的运行时同样拿不到。 - MCP 只看服务登记着、启用着没有:连不连得上、工具还在不在,要连一次才知道,发起时不去等它, - 上次没连上或者工具清单里没有的只在日志里提一句。 - """ - from app.db.models import CustomTool, DataSource, McpServer - from app.tools.datasource import QUERY_PREFIX, SCHEMA_PREFIX - from app.tools.registry import all_specs - - bindings = _bindings(spec) - if not bindings: - return [] - builtin = all_specs() - sources = {n for (n,) in await session.execute(select(DataSource.name).where(DataSource.enabled.is_(True)))} - custom = {n for (n,) in await session.execute(select(CustomTool.name).where(CustomTool.enabled.is_(True)))} - servers = {r.name: r for r in (await session.execute(select(McpServer))).scalars()} - missing: list[tuple[str, str, str]] = [] - for who, name in bindings: - if name.startswith("mcp:"): - server, _, tool = name[4:].partition("/") - row = servers.get(server) - if row is None or not row.enabled: - missing.append((who, name, "tools")) - elif row.status == "error": - logger.warning("%s 绑的 %s:MCP 服务 %s 上次没连上,运行时可能拿不到这个工具", who, name, server) - elif tool not in ("", "*") and row.tools_cache and tool not in row.tools_cache: - logger.warning("%s 绑的 %s:MCP 服务 %s 上次交回的工具清单里没有 %s", who, name, server, tool) - elif name in builtin: - continue - elif name.startswith((QUERY_PREFIX, SCHEMA_PREFIX)): - source = name[len(QUERY_PREFIX):] if name.startswith(QUERY_PREFIX) else name[len(SCHEMA_PREFIX):] - if source not in sources: - missing.append((who, name, "data")) - elif name not in custom: - missing.append((who, name, "tools")) - return missing - - -async def _refuse_missing_tools(session: AsyncSession, graph: dict[str, Any]) -> None: - """绑定的工具在本机不存在就不发起:以前要跑到那一步才失败,前面的步骤白跑、钱白花。""" - try: - spec = GraphSpec.model_validate(graph) - except Exception: # noqa: BLE001 - 图本身不成形交给 run_manager.start 报 - return - missing = await missing_tools(session, spec) - if not missing: - return - lines = ";".join(f"{who}绑定的 {name}" for who, name, _ in missing[:8]) - more = f"等 {len(missing)} 处" if len(missing) > 8 else "" - places = {where for _, _, where in missing} - if places == {"data"}: - fix = "请到「数据」页接入,或在节点中重新选择" - elif places == {"tools"}: - fix = "请到「工具」页接入,或在节点中重新选择" - else: - fix = "数据源工具请到「数据」页接入,其他工具请到「工具」页接入,或在节点中重新选择" - raise CodedHTTPException(422, f"绑定的工具不存在:{lines}{more}。{fix}", TOOL_MISSING) +def _refused(e: LaunchRefused) -> HTTPException: + """发起被拒的 HTTP 响应,和规则搬去 engine/launch.py 之前逐字相同:带机器码的回 {detail, code}, + 不带的回 {detail}。""" + if e.code: + return CodedHTTPException(e.status_code, e.message, e.code) + return HTTPException(e.status_code, e.message) #: workflow_id 取这个值时只要没挂在任何工作流上的运行:画布上没保存就跑的临时图, diff --git a/backend/app/engine/launch.py b/backend/app/engine/launch.py new file mode 100644 index 00000000..3186a690 --- /dev/null +++ b/backend/app/engine/launch.py @@ -0,0 +1,273 @@ +"""发起一次运行:HTTP 发起(api/runs.py 的 start_run)和定时任务到点发起共用的那套规则。 + +以前这些规则全写在 start_run 里、只认 HTTP 请求:定时任务要发起正式运行,要么另抄一份(两份规则迟早 +走样,正式运行「只能从当前发布版本发起」这条最要紧的规则就可能在一边被绕过),要么假装成一次 HTTP 请求。 +现在两边都调 launch_run。拒绝时抛 LaunchRefused:start_run 把它换回以前的 HTTP 响应(状态码、detail、 +code 逐字不变),调度器据此记一次未能发起。 + +发起前检查绑定的工具(missing_tools),以及它和 runner 固定数据版本共用的取绑定的办法(tool_bindings、 +node_setting)也放在这里:runner 以前要从接口层导入它们。 +""" +from __future__ import annotations + +import logging +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core import artifact_store +from app.core.events import EventType +from app.data.catalog_impact import catalog_drift +from app.db.models import Run, Workflow, WorkflowVersion +from app.engine.governance import governance_note, unresolved_caliber_upgrades +from app.engine.runner import run_manager +from app.engine.schema import GraphSpec, NodeType, type_label + +logger = logging.getLogger(__name__) + +#: 发起时发现绑定的工具在本机不存在(见 missing_tools) +TOOL_MISSING = "run_tool_missing" + + +class LaunchRefused(Exception): + """不发起这次运行。status_code 和 message 就是 HTTP 发起时回的状态码和 detail;code 是给程序认的机器码, + 没有的是 None(HTTP 响应里也就没有 code 字段)。""" + + def __init__(self, status_code: int, message: str, code: str | None = None) -> None: + super().__init__(message) + self.status_code = status_code + self.message = message + self.code = code + + +async def launch_run( + session: AsyncSession, + *, + workflow_id: str | None = None, + graph: dict[str, Any] | None = None, + input_payload: dict[str, Any] | None = None, + memory_scope: str | None = None, + collection: str | None = None, + run_class: str = "exploratory", + version: int | None = None, + actor: str | None, +) -> Run: + """发起一次运行,返回刚建好的运行记录(执行已经交给 run_manager)。不发起就抛 LaunchRefused。 + + 参数和 POST /api/runs 的请求体一一对应(input 叫 input_payload);actor 是记在运行上的发起人。 + """ + # 没挂在工作流上(画布没保存就发起、问数据当场搭的)。老数据里是「临时图」,前端两种都认 + name = "未保存的工作流" + run_version: int | None = None + version_hash: str | None = None + governed = False + formal = run_class == "formal" + + if formal: + # 正式运行的全部语义就这一条规则:必须引用一个不可变的已发布版本。 + # 传裸 graph、或工作流还没发布过,都进不了 formal。 + if not workflow_id: + raise LaunchRefused(400, "正式运行需要从已保存的工作流发起:请求中没有指定工作流") + if graph is not None: + raise LaunchRefused(400, "正式运行只能使用已发布的版本,画布上未发布的修改不会生效。请先保存并发布") + workflow = await session.get(Workflow, workflow_id) + if not workflow: + raise LaunchRefused(404, "工作流不存在,可能已被删除") + # 只看 published_version:status 反映的是当前画布(改过图就退回 draft), + # 而 formal 用的是已发布的那一版,两者本来就可以不一致 + if not workflow.published_version: + raise LaunchRefused(409, f"「{workflow.name}」还没有发布版本,请先在编排页发布") + run_version = version or workflow.published_version + # "不可变"和"过过闸"是两件事。快照确实不可变,但 PATCH 保存出来的 + # 草稿版本同样有快照——只认 version 存不存在的话,工作流只要发布过 + # 任意一个版本,之后所有草稿都能以 formal 身份启动,发布期的 + # validate_graph + lint_for_publish 全部绕过。 + if run_version != workflow.published_version: + raise LaunchRefused( + 409, + f"v{run_version} 不是当前发布版本(v{workflow.published_version})。" + "正式运行只能使用当前发布版本;如需运行这一版,请先发布它。", + ) + snapshot = ( + await session.execute( + select(WorkflowVersion).where( + WorkflowVersion.workflow_id == workflow.id, + WorkflowVersion.version == run_version, + ) + ) + ).scalar_one_or_none() + if not snapshot: + raise LaunchRefused(404, f"版本 v{run_version} 不存在") + graph = snapshot.graph + version_hash = snapshot.graph_hash or artifact_store.graph_hash(graph) + name = workflow.name + # 按这一版发布时记下的级别;记级别之前发布的老版本没有,照旧看工作流现在的 status + governed = (snapshot.level == "governed") if snapshot.level else workflow.status == "governed" + + try: + spec = GraphSpec.model_validate(graph) + except ValueError as e: + # 快照里的图不成形。以前这里没接,由 main.py 的 ValueError 兜底回 400、detail 是校验原话; + # 原样换成拒绝,HTTP 响应不变,调度器也拿得到一个明确的拒绝 + raise LaunchRefused(400, str(e)) from e + # 口径升版是被迫处置的事件:引用的方法卡有新版本而未声明策略,拒绝启动 + errors, upgrade_events = await unresolved_caliber_upgrades(session, spec) + if errors: + raise LaunchRefused(409, ";".join(errors)) + # 数据目录和口径卡不同:目录是给人和助手看的说明,不是运行的输入。发布之后变了只提醒、不拦—— + # 拦下来等于让一次文字修订卡住正式出具 + catalog_changes = await catalog_drift(session, snapshot.catalog_versions) + else: + upgrade_events = [] + catalog_changes = [] + if workflow_id: + workflow = await session.get(Workflow, workflow_id) + if not workflow: + raise LaunchRefused(404, "工作流不存在,可能已被删除") + graph = graph or workflow.graph + name = workflow.name + if not graph: + raise LaunchRefused(400, "没有要运行的内容:请求中既没有工作流 ID,也没有工作流内容") + await _refuse_missing_tools(session, graph) + + from app.api.settings import resolve_run_scope + + memory_scope, collection = await resolve_run_scope(session, memory_scope, collection) + try: + run = await run_manager.start( + graph=graph, + input_payload=input_payload if input_payload is not None else {}, + workflow_id=workflow_id, + workflow_name=name, + memory_scope=memory_scope, + collection=collection, + run_class="formal" if formal else "exploratory", + version=run_version, + version_hash=version_hash, + started_by=actor, + ) + except ValueError as e: + raise LaunchRefused(400, str(e)) from e + + if formal: + # 按不按受管出具在发起这一刻定下来、记进封存范围:status 反映的是当前画布,中途改图、 + # 审批停下再恢复时再去读它,报告节点和出口复核可能各读到不同的值 + await run_manager.note(run.id, "log", **governance_note( + governed=governed, workflow_name=name, version=run_version)) + if catalog_changes: + # 落在封存范围内:之后回看这次正式出具,能知道当时的目录和发布时不一样、差在哪几张表。 + # 每张表带 impact:direct 写死的 SQL 用到的,possible Agent 可能查询的(界面分开说) + await run_manager.note(run.id, EventType.CATALOG_DRIFT, version=run_version, count=len(catalog_changes), + tables=catalog_changes) + # 已声明策略的升版记入事件流,出具物上能看到"这期换口径了、怎么处置的" + for ev in upgrade_events: + payload_ev = dict(ev) + node = payload_ev.pop("node_id", None) + await run_manager.note(run.id, "caliber.upgrade", node_id=node, **payload_ev) + return run + + +# -------------------------------------------------------------------------- +# 发起前检查绑定的工具 +# -------------------------------------------------------------------------- + + +def node_setting(spec: GraphSpec, node: Any, key: str) -> Any: + """节点配置的实际取值,和运行时 NodeContext.cfg 同一个规则:节点上没写(None、空串)就取图级 defaults。""" + value = node.config.get(key) + return spec.defaults.get(key) if value in (None, "") else value + + +def tool_bindings(spec: GraphSpec) -> list[tuple[str, str]]: + """图里每一处绑定的工具:(谁绑的, 工具名)。调用工具节点的 tool、Agent 的 tools、协作成员的 tools。 + + 取值和运行时一致(node_setting):Agent 节点上没写 tools、协作节点上没写成员,运行时用的是 + 图级 defaults 里的,这里也算它们——不然发起前检查放过了缺的工具,发起时固定数据版本也漏了源。 + + 模板拼出来的名字({{ vars.tool }})也在内:发起前检查(_bindings)跳过它们,发起时固定数据 + 版本(engine/runner.py)要知道有没有这种名字——有的话说不准会用到哪个数据源。 + """ + out: list[tuple[str, str]] = [] + for node in spec.nodes: + who = f"「{node.title}」({type_label(node.type)})" + if node.type == NodeType.TOOL: + names = [node_setting(spec, node, "tool")] + out.extend((who, n) for n in names if isinstance(n, str) and n.strip()) + elif node.type == NodeType.AGENT: + out.extend((who, n) for n in node_setting(spec, node, "tools") or [] + if isinstance(n, str) and n.strip()) + elif node.type == NodeType.SUPERVISOR: + for member in node_setting(spec, node, "agents") or []: + if not isinstance(member, dict): + continue + label = f"「{node.title}」的成员「{member.get('name') or '?'}」" + out.extend((label, n) for n in member.get("tools") or [] if isinstance(n, str) and n.strip()) + return [(who, name.strip()) for who, name in out] + + +def _bindings(spec: GraphSpec) -> list[tuple[str, str]]: + """发起前要检查的绑定。模板拼出来的工具名({{ vars.tool }})发起时还不知道是谁,交给运行时。""" + return [(who, name) for who, name in tool_bindings(spec) if "{{" not in name] + + +async def missing_tools(session: AsyncSession, spec: GraphSpec) -> list[tuple[str, str, str]]: + """本机不存在的绑定:(谁绑的, 工具名, 去哪接入:data / tools)。 + + 解析和运行时 build_tools 一致:内置(静态注册)、mcp:<服务>/<工具>、db_query__ / db_schema__ + <数据源>、其余按自定义工具。数据源和自定义工具只认启用着的——停用的运行时同样拿不到。 + MCP 只看服务登记着、启用着没有:连不连得上、工具还在不在,要连一次才知道,发起时不去等它, + 上次没连上或者工具清单里没有的只在日志里提一句。 + """ + from app.db.models import CustomTool, DataSource, McpServer + from app.tools.datasource import QUERY_PREFIX, SCHEMA_PREFIX + from app.tools.registry import all_specs + + bindings = _bindings(spec) + if not bindings: + return [] + builtin = all_specs() + sources = {n for (n,) in await session.execute(select(DataSource.name).where(DataSource.enabled.is_(True)))} + custom = {n for (n,) in await session.execute(select(CustomTool.name).where(CustomTool.enabled.is_(True)))} + servers = {r.name: r for r in (await session.execute(select(McpServer))).scalars()} + missing: list[tuple[str, str, str]] = [] + for who, name in bindings: + if name.startswith("mcp:"): + server, _, tool = name[4:].partition("/") + row = servers.get(server) + if row is None or not row.enabled: + missing.append((who, name, "tools")) + elif row.status == "error": + logger.warning("%s 绑的 %s:MCP 服务 %s 上次没连上,运行时可能拿不到这个工具", who, name, server) + elif tool not in ("", "*") and row.tools_cache and tool not in row.tools_cache: + logger.warning("%s 绑的 %s:MCP 服务 %s 上次交回的工具清单里没有 %s", who, name, server, tool) + elif name in builtin: + continue + elif name.startswith((QUERY_PREFIX, SCHEMA_PREFIX)): + source = name[len(QUERY_PREFIX):] if name.startswith(QUERY_PREFIX) else name[len(SCHEMA_PREFIX):] + if source not in sources: + missing.append((who, name, "data")) + elif name not in custom: + missing.append((who, name, "tools")) + return missing + + +async def _refuse_missing_tools(session: AsyncSession, graph: dict[str, Any]) -> None: + """绑定的工具在本机不存在就不发起:以前要执行到那一步才失败,前面的步骤白做、钱白花。""" + try: + spec = GraphSpec.model_validate(graph) + except Exception: # noqa: BLE001 - 图本身不成形交给 run_manager.start 报 + return + missing = await missing_tools(session, spec) + if not missing: + return + lines = ";".join(f"{who}绑定的 {name}" for who, name, _ in missing[:8]) + more = f"等 {len(missing)} 处" if len(missing) > 8 else "" + places = {where for _, _, where in missing} + if places == {"data"}: + fix = "请到「数据」页接入,或在节点中重新选择" + elif places == {"tools"}: + fix = "请到「工具」页接入,或在节点中重新选择" + else: + fix = "数据源工具请到「数据」页接入,其他工具请到「工具」页接入,或在节点中重新选择" + raise LaunchRefused(422, f"绑定的工具不存在:{lines}{more}。{fix}", TOOL_MISSING) diff --git a/backend/app/engine/runner.py b/backend/app/engine/runner.py index 1be770d4..74151f86 100644 --- a/backend/app/engine/runner.py +++ b/backend/app/engine/runner.py @@ -1062,11 +1062,12 @@ async def _data_versions(session: Any, spec: GraphSpec) -> dict[str, Any]: async def _bound_sources(session: Any, spec: GraphSpec) -> tuple[set[str], bool]: """图里(含子工作流)绑定的数据源工具指向的源名,以及有没有带模板、发起时定不下来的工具名。 - 绑定的取法和发起前检查同一个(api/runs.py 的 tool_bindings),子工作流照运行时的取法 + 绑定的取法和发起前检查同一个(engine/launch.py 的 tool_bindings),子工作流照运行时的取法 读出来接着找:固定了版本的读那一版,没固定的读工作流现在的图。读不到、图不成形的跳过—— 那个节点运行时自己会报错。同一个工作流(同一版)只读一次,互相引用也不会转圈。 """ - from app.api.runs import node_setting, tool_bindings + # 在函数里导入:launch 在模块级导入了本模块(它要 run_manager) + from app.engine.launch import node_setting, tool_bindings from app.db.models import WorkflowVersion from app.tools.datasource import source_of_tool diff --git a/backend/tests/test_run_launch.py b/backend/tests/test_run_launch.py new file mode 100644 index 00000000..4fb698a1 --- /dev/null +++ b/backend/tests/test_run_launch.py @@ -0,0 +1,337 @@ +"""发起运行的规则搬到 engine/launch.py:HTTP 发起和定时任务到点发起共用同一个 launch_run。 + +- 每一种拒绝:POST /api/runs 的状态码、detail、code 先按搬家之前的原话写死(这一组先在旧代码上通过); + 直接调 launch_run 抛 LaunchRefused,和 HTTP 的返回逐条一致 +- 发起之后的治理事件(受管级别那条日志、口径升版处置)在服务里写:不经 HTTP 发起的正式运行一样有 +- 旧的导入路径(app.api.runs 的 missing_tools、node_setting、tool_bindings、TOOL_MISSING)照旧可用; + runner 不再从接口层导入 +""" +from __future__ import annotations + +import asyncio +import inspect +import uuid + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import select + +from app.db.base import SessionLocal +from app.db.models import Run, RunEvent, Workflow, WorkflowVersion +from app.engine.runner import run_manager +from app.engine.schema import GraphSpec +from app.main import app + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +def node(nid, ntype, label=None, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}} + + +def chain(*nodes): + return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +SIMPLE = chain(node("start", "input"), node("out", "output", fields=[{"name": "r", "value": "完成"}])) + + +def _tag() -> str: + return uuid.uuid4().hex[:8] + + +async def workflow(graph=SIMPLE, *, published=None, versions=(), snapshot=None, name=None) -> tuple[str, str]: + """建一个工作流。published 是当前发布版本号,versions 是要建快照的版本号,snapshot 是快照里的图(默认同 graph)。""" + name = name or f"发起测试·周报-{_tag()}" + async with SessionLocal() as session: + wf = Workflow(name=name, graph=graph, status="published" if published else "draft", + published_version=published, version=max(versions, default=1)) + session.add(wf) + await session.flush() + for v in versions: + session.add(WorkflowVersion(workflow_id=wf.id, version=v, + graph=snapshot if snapshot is not None else graph)) + await session.commit() + return wf.id, name + + +def tool_graph(*tools: tuple[str, str]) -> dict: + """每个 (节点名, 工具名) 一个调用工具节点。""" + middle = [node(f"t{i}", "tool", label, tool=tool, args={}) for i, (label, tool) in enumerate(tools)] + return chain(node("start", "input"), *middle, node("out", "output", fields=[{"name": "r", "value": "完成"}])) + + +# -------------------------------------------------------------------------- +# 每一种拒绝:(请求体, (状态码, detail, code))。detail 是 None 时只比两边一致,另由 check 查要点 +# -------------------------------------------------------------------------- + + +async def case_formal_without_workflow(): + return {"run_class": "formal"}, (400, "正式运行需要从已保存的工作流发起:请求中没有指定工作流", None) + + +async def case_formal_with_canvas_graph(): + wf, _ = await workflow(published=1, versions=(1,)) + return ({"run_class": "formal", "workflow_id": wf, "graph": SIMPLE}, + (400, "正式运行只能使用已发布的版本,画布上未发布的修改不会生效。请先保存并发布", None)) + + +async def case_formal_workflow_gone(): + return {"run_class": "formal", "workflow_id": f"gone{_tag()}"}, (404, "工作流不存在,可能已被删除", None) + + +async def case_formal_never_published(): + wf, name = await workflow() + return {"run_class": "formal", "workflow_id": wf}, (409, f"「{name}」还没有发布版本,请先在编排页发布", None) + + +async def case_formal_not_the_published_version(): + wf, _ = await workflow(published=2, versions=(1, 2)) + return ({"run_class": "formal", "workflow_id": wf, "version": 1}, + (409, "v1 不是当前发布版本(v2)。正式运行只能使用当前发布版本;如需运行这一版,请先发布它。", None)) + + +async def case_formal_snapshot_missing(): + wf, _ = await workflow(published=3, versions=(1, 2)) + return {"run_class": "formal", "workflow_id": wf}, (404, "版本 v3 不存在", None) + + +async def case_formal_snapshot_malformed(): + """快照里的图不成形:以前由 main.py 的 ValueError 兜底回 400,detail 是校验的原话。""" + broken = {"nodes": "坏掉的图"} + wf, _ = await workflow(published=1, versions=(1,), snapshot=broken) + with pytest.raises(ValueError) as caught: + GraphSpec.model_validate(broken) + return {"run_class": "formal", "workflow_id": wf}, (400, str(caught.value), None) + + +def library(version="v3"): + return chain(node("start", "input"), + node("caliber", "metrics", "周报口径卡", caliber="周报口径", caliber_version=version, + metrics=[{"id": "gmv", "name": "销售额", "unit": "元", "expression": "vars.kpi.gmv"}]), + node("out", "output", fields=[{"name": "r", "value": "{{ nodes.caliber.text }}"}])) + + +def weekly(lib_id: str, **card) -> dict: + return chain(node("start", "input"), + node("fetch", "transform", mode="expression", expression="{'gmv': 4321.5}", assign_to="kpi"), + node("card", "metrics", "本地口径卡", + caliber_from={"workflow_id": lib_id, "workflow_version": 1, "node_id": "caliber"}, **card), + node("out", "output", fields=[{"name": "r", "value": "{{ nodes.card.text }}"}])) + + +async def upstream_with_two_versions() -> str: + async with SessionLocal() as session: + lib = Workflow(name=f"口径库-{_tag()}", graph=library(), status="published", published_version=2, version=2) + session.add(lib) + await session.flush() + session.add(WorkflowVersion(workflow_id=lib.id, version=1, graph=library())) + session.add(WorkflowVersion(workflow_id=lib.id, version=2, graph=library(version="v4"))) + await session.commit() + return lib.id + + +async def case_formal_caliber_upgrade_undeclared(): + lib = await upstream_with_two_versions() + graph = weekly(lib) + wf, _ = await workflow(graph, published=1, versions=(1,)) + return {"run_class": "formal", "workflow_id": wf}, (409, None, None) + + +def check_formal_caliber_upgrade_undeclared(detail: str) -> None: + assert "「本地口径卡」" in detail and "v1" in detail and "v2" in detail and "「上游发布新版本时」" in detail + + +async def case_saved_workflow_gone(): + return {"workflow_id": f"gone{_tag()}"}, (404, "工作流不存在,可能已被删除", None) + + +async def case_nothing_to_run(): + return {}, (400, "没有要运行的内容:请求中既没有工作流 ID,也没有工作流内容", None) + + +async def case_saved_workflow_is_empty(): + wf, _ = await workflow({}) + return {"workflow_id": wf}, (400, "没有要运行的内容:请求中既没有工作流 ID,也没有工作流内容", None) + + +async def case_missing_datasource_tool(): + source = f"gone{_tag()}" + return ({"graph": tool_graph(("取数", f"db_query__{source}"))}, + (422, f"绑定的工具不存在:「取数」(调用工具)绑定的 db_query__{source}。请到「数据」页接入,或在节点中重新选择", + "run_tool_missing")) + + +async def case_missing_custom_tool(): + tool = f"ghost_{_tag()}" + return ({"graph": tool_graph(("查编码", tool))}, + (422, f"绑定的工具不存在:「查编码」(调用工具)绑定的 {tool}。请到「工具」页接入,或在节点中重新选择", + "run_tool_missing")) + + +async def case_missing_tools_of_both_kinds(): + source, tool = f"gone{_tag()}", f"ghost_{_tag()}" + return ({"graph": tool_graph(("取数", f"db_schema__{source}"), ("查编码", tool))}, + (422, f"绑定的工具不存在:「取数」(调用工具)绑定的 db_schema__{source};「查编码」(调用工具)绑定的 {tool}。" + "数据源工具请到「数据」页接入,其他工具请到「工具」页接入,或在节点中重新选择", "run_tool_missing")) + + +async def case_more_than_eight_missing_tools(): + tag = _tag() + tools = [(f"第{i}步", f"ghost_{tag}_{i}") for i in range(10)] + listed = ";".join(f"「{label}」(调用工具)绑定的 {tool}" for label, tool in tools[:8]) + return ({"graph": tool_graph(*tools)}, + (422, f"绑定的工具不存在:{listed}等 10 处。请到「工具」页接入,或在节点中重新选择", "run_tool_missing")) + + +async def case_formal_missing_tool(): + source = f"gone{_tag()}" + wf, _ = await workflow(tool_graph(("取数", f"db_query__{source}")), published=1, versions=(1,)) + return ({"run_class": "formal", "workflow_id": wf}, + (422, f"绑定的工具不存在:「取数」(调用工具)绑定的 db_query__{source}。请到「数据」页接入,或在节点中重新选择", + "run_tool_missing")) + + +async def case_graph_fails_validation(): + return ({"graph": {"nodes": [], "edges": []}}, + (400, "工作流未通过校验,请先在画布上修正标红的节点再运行:画布上还没有节点,请先从节点库拖入一个节点", None)) + + +#: 「不存在」的工作流、数据源、自定义工具都带随机后缀,库里不会恰好有同名的 +CASES = {name[len("case_"):]: fn for name, fn in dict(globals()).items() if name.startswith("case_")} + + +def _body(status: int, detail: str, code: str | None) -> dict: + return {"detail": detail, **({"code": code} if code else {})} + + +async def _run_count() -> int: + async with SessionLocal() as session: + return len((await session.execute(select(Run.id))).all()) + + +@pytest.mark.parametrize("case", list(CASES)) +async def test_http_refusals_are_unchanged(client, case): + """HTTP 这一侧逐字写死:搬家前后都得是这些状态码和原话。""" + body, (status, detail, code) = await CASES[case]() + before = await _run_count() + res = await client.post("/api/runs", json=body) + assert res.status_code == status, res.text + got = res.json() + if detail is None: + globals()[f"check_{case}"](got["detail"]) + assert set(got) == ({"detail", "code"} if code else {"detail"}), got + else: + assert got == _body(status, detail, code) + assert await _run_count() == before, "被拒的请求不该留下运行记录" + + +def _launch_kwargs(body: dict) -> dict: + """请求体换成 launch_run 的参数:input 对应 input_payload,其余同名。""" + out = {k: v for k, v in body.items() if k != "input"} + if "input" in body: + out["input_payload"] = body["input"] + return out + + +@pytest.mark.parametrize("case", list(CASES)) +async def test_launch_run_refuses_exactly_like_http(client, case): + from app.engine.launch import LaunchRefused, launch_run + + body, (status, detail, code) = await CASES[case]() + before = await _run_count() + async with SessionLocal() as session: + with pytest.raises(LaunchRefused) as caught: + await launch_run(session, actor="定时任务", **_launch_kwargs(body)) + refused = caught.value + assert (refused.status_code, refused.code) == (status, code) + if detail is not None: + assert refused.message == detail + res = await client.post("/api/runs", json=body) + assert res.status_code == refused.status_code + assert res.json() == _body(refused.status_code, refused.message, refused.code) + assert await _run_count() == before + + +# -------------------------------------------------------------------------- +# 发起成功:参数照转、治理事件在服务里写 +# -------------------------------------------------------------------------- + + +async def _settle(run_id: str) -> Run: + for _ in range(400): + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row.status in ("succeeded", "failed", "cancelled") and row.manifest_seq is not None: + return row + await asyncio.sleep(0.03) + raise AssertionError(f"运行 {run_id} 没有结束:{row.status}") + + +async def _events(run_id: str, etype: str) -> list[RunEvent]: + async with SessionLocal() as session: + return list((await session.execute( + select(RunEvent).where(RunEvent.run_id == run_id, RunEvent.type == etype).order_by(RunEvent.seq) + )).scalars()) + + +async def test_a_formal_launch_records_the_governance_notes(): + """不经 HTTP 发起的正式运行(定时任务)也记下按什么级别出具、口径升版怎么处置的。""" + from app.engine.launch import launch_run + + lib = await upstream_with_two_versions() + wf, name = await workflow(weekly(lib, upgrade_policy="dual"), published=1, versions=(1,)) + async with SessionLocal() as session: + run = await launch_run(session, workflow_id=wf, run_class="formal", input_payload={"week": "第 40 周"}, + actor="定时任务") + assert (run.run_class, run.version, run.workflow_name, run.started_by) == ("formal", 1, name, "定时任务") + assert run.version_hash and run.input == {"week": "第 40 周"} + row = await _settle(run.id) + assert row.status == "succeeded", row.error + notes = [e.data for e in await _events(run.id, "log") if e.data.get("code") == "run_governance"] + assert len(notes) == 1 and notes[0]["governed"] is False and "v1" in notes[0]["message"] + [upgrade] = await _events(run.id, "caliber.upgrade") + assert upgrade.node_id == "card" and upgrade.data["policy"] == "dual" and upgrade.data["latest"] == 2 + + +async def test_an_exploratory_launch_takes_the_saved_graph_and_the_run_defaults(): + from app.engine.launch import launch_run + + wf, name = await workflow() + async with SessionLocal() as session: + run = await launch_run(session, workflow_id=wf, actor=None) + assert (run.run_class, run.workflow_id, run.workflow_name, run.version) == ("exploratory", wf, name, None) + assert (run.memory_scope, run.collection) == ("default", "default") and run.input == {} + row = await _settle(run.id) + assert row.status == "succeeded", row.error + assert [e for e in await _events(run.id, "log") if e.data.get("code") == "run_governance"] == [] + + +# -------------------------------------------------------------------------- +# 旧的导入路径 +# -------------------------------------------------------------------------- + + +def test_the_old_import_paths_still_work(): + from app.api import runs as runs_api + from app.engine import launch + + for name in ("missing_tools", "node_setting", "tool_bindings", "TOOL_MISSING"): + assert getattr(runs_api, name) is getattr(launch, name), name + + +def test_the_runner_no_longer_imports_from_the_api_layer(): + from app.engine import runner + + assert "app.api.runs" not in inspect.getsource(runner) From 0eaaf713469e84876a3f838b789437b158fc6f0f Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 00:07:07 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=E8=BF=90=E8=A1=8C=E8=AE=B0=E4=B8=8B?= =?UTF-8?q?=E9=80=89=E9=A1=B9=E5=BF=AB=E7=85=A7=E5=92=8C=E6=9D=A5=E6=BA=90?= =?UTF-8?q?=EF=BC=9Aruns=20=E5=8A=A0=20options=EF=BC=88JSON=EF=BC=8C?= =?UTF-8?q?=E9=BB=98=E8=AE=A4=20{}=EF=BC=89=E5=92=8C=20schedule=5Fid?= =?UTF-8?q?=EF=BC=88=E5=8F=AF=E7=A9=BA=EF=BC=8C=E5=BB=BA=E7=B4=A2=E5=BC=95?= =?UTF-8?q?=EF=BC=89=E4=B8=A4=E5=88=97=E3=80=82=E5=90=8E=E9=9D=A2=E7=9A=84?= =?UTF-8?q?=E6=A0=B8=E6=9F=A5=E8=AE=BE=E7=BD=AE=E8=A6=81=E5=9C=A8=E5=8F=91?= =?UTF-8?q?=E8=B5=B7=E6=97=B6=E5=AE=9A=E4=B8=8B=E3=80=81=E6=95=B4=E6=AC=A1?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E6=B2=BF=E7=94=A8=EF=BC=8C=E5=AE=A1=E6=89=B9?= =?UTF-8?q?=E5=81=9C=E4=B8=8B=E6=9C=9F=E9=97=B4=E6=94=B9=E4=BA=86=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=EF=BC=8C=E6=81=A2=E5=A4=8D=E5=92=8C=E9=87=8D=E6=94=BE?= =?UTF-8?q?=E4=BB=8D=E6=8C=89=E5=8F=91=E8=B5=B7=E6=97=B6=E7=9A=84=EF=BC=88?= =?UTF-8?q?=E5=90=8C=20approval=5Fdefault=20=E7=9A=84=E9=81=93=E7=90=86?= =?UTF-8?q?=EF=BC=89=EF=BC=9B=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1=E8=A6=81?= =?UTF-8?q?=E8=83=BD=E6=89=BE=E5=9B=9E=E8=87=AA=E5=B7=B1=E5=8F=91=E8=B5=B7?= =?UTF-8?q?=E7=9A=84=E8=BF=90=E8=A1=8C=E3=80=82RunManager.start=20?= =?UTF-8?q?=E6=94=B6=20options=E3=80=81schedule=5Fid=20=E5=B9=B6=E8=90=BD?= =?UTF-8?q?=E5=BA=93=EF=BC=8C=E9=80=89=E9=A1=B9=E5=85=88=E6=B7=B1=E6=8B=B7?= =?UTF-8?q?=E4=B8=80=E4=BB=BD=EF=BC=8C=E8=B0=83=E7=94=A8=E6=96=B9=E4=B9=8B?= =?UTF-8?q?=E5=90=8E=E6=94=B9=E8=87=AA=E5=B7=B1=E7=9A=84=20dict=20?= =?UTF-8?q?=E4=B8=8D=E5=BD=B1=E5=93=8D=E5=BF=AB=E7=85=A7=EF=BC=9B=E9=80=89?= =?UTF-8?q?=E9=A1=B9=E9=9D=9E=E7=A9=BA=E6=97=B6=E5=86=99=E8=BF=9B=E6=AF=8F?= =?UTF-8?q?=E4=B8=80=E6=AE=B5=E7=9A=84=20run.started=EF=BC=88=E8=BF=9B?= =?UTF-8?q?=E5=B0=81=E5=AD=98=E8=8C=83=E5=9B=B4=EF=BC=89=EF=BC=8C=E4=B8=BA?= =?UTF-8?q?=E7=A9=BA=E6=97=B6=20run.started=20=E5=92=8C=E4=BB=A5=E5=89=8D?= =?UTF-8?q?=E9=80=90=E5=AD=97=E7=9B=B8=E5=90=8C=EF=BC=9B=E6=81=A2=E5=A4=8D?= =?UTF-8?q?=E3=80=81=E7=BB=A7=E7=BB=AD=E8=BF=90=E8=A1=8C=E7=85=A7=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E8=AE=B0=E5=BD=95=E6=B2=BF=E7=94=A8=EF=BC=8C=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E4=BB=8E=20RunContext.extra=20=E7=9A=84=20run=5Foptio?= =?UTF-8?q?ns=20=E8=AF=BB=E5=90=8C=E4=B8=80=E4=BB=BD=E6=8B=B7=E8=B4=9D?= =?UTF-8?q?=E3=80=82RunOut=20=E5=B8=A6=20schedule=5Fid=EF=BC=8CGET=20/api/?= =?UTF-8?q?runs=20=E5=8F=AF=E6=8C=89=20schedule=5Fid=20=E7=AD=9B=EF=BC=8Cl?= =?UTF-8?q?aunch=5Frun=20=E5=8E=9F=E6=A0=B7=E5=BE=80=E4=B8=8B=E4=BA=A4?= =?UTF-8?q?=E3=80=82=E6=97=A7=E5=BA=93=E8=B5=B0=20=5FCOLUMN=5FMIGRATIONS?= =?UTF-8?q?=20=E8=A1=A5=E5=88=97=EF=BC=8C=E5=AD=98=E9=87=8F=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E5=9B=9E=E5=A1=AB=20{}=20=E5=92=8C=20NULL=EF=BC=9Bcre?= =?UTF-8?q?ate=5Fall=20=E4=B8=8D=E7=A2=B0=E5=B7=B2=E6=9C=89=E7=9A=84?= =?UTF-8?q?=E8=A1=A8=EF=BC=8C=E6=96=B0=E5=88=97=E7=9A=84=E7=B4=A2=E5=BC=95?= =?UTF-8?q?=E5=8F=A6=E7=94=B1=20=5FINDEX=5FMIGRATIONS=20=E8=A1=A5=E5=BB=BA?= =?UTF-8?q?=EF=BC=8C=E7=B4=A2=E5=BC=95=E5=90=8D=E5=92=8C=E6=96=B0=E5=BA=93?= =?UTF-8?q?=20create=5Fall=20=E5=BB=BA=E5=87=BA=E6=9D=A5=E7=9A=84=E7=9B=B8?= =?UTF-8?q?=E5=90=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/runs.py | 5 + backend/app/db/base.py | 15 ++ backend/app/db/models.py | 5 + backend/app/engine/launch.py | 5 + backend/app/engine/runner.py | 25 ++- backend/tests/test_run_options.py | 268 ++++++++++++++++++++++++++++++ 6 files changed, 321 insertions(+), 2 deletions(-) create mode 100644 backend/tests/test_run_options.py diff --git a/backend/app/api/runs.py b/backend/app/api/runs.py index 058ad518..00fd3637 100644 --- a/backend/app/api/runs.py +++ b/backend/app/api/runs.py @@ -68,6 +68,8 @@ class RunOut(BaseModel): #: 发起时实际生效的记忆域和知识库 memory_scope: str | None = None collection: str | None = None + #: 发起它的定时任务。为空 = 手动发起 + schedule_id: str | None = None # 时间一律带时区(db.base.UTCDateTime 读出来就是 UTC aware) created_at: Any = None started_at: Any = None @@ -118,6 +120,7 @@ async def list_runs( status: str | None = Query(default=None, description="逗号分隔,如 failed,cancelled"), run_class: str | None = None, q: str | None = Query(default=None, description="按工作流名模糊匹配"), + schedule_id: str | None = Query(default=None, description="只要这个定时任务发起的运行"), limit: int = Query(default=50, ge=1, le=200), before: datetime | None = Query(default=None, description="翻页游标:只要 created_at 早于它的"), session: AsyncSession = Depends(get_session), @@ -138,6 +141,8 @@ async def list_runs( stmt = stmt.where(Run.run_class.in_(classes)) if q and q.strip(): stmt = stmt.where(Run.workflow_name.ilike(f"%{_like(q.strip())}%", escape="\\")) + if schedule_id: + stmt = stmt.where(Run.schedule_id == schedule_id) if before is not None: stmt = stmt.where(Run.created_at < before) return list((await session.execute(stmt)).scalars()) diff --git a/backend/app/db/base.py b/backend/app/db/base.py index 2f1e5a4a..68ee478b 100644 --- a/backend/app/db/base.py +++ b/backend/app/db/base.py @@ -200,6 +200,17 @@ _COLUMN_MIGRATIONS: list[tuple[str, str, str]] = [ ("import_stagings", "edits", "JSON"), ("import_stagings", "answers_dropped", "JSON"), ("import_stagings", "redraft_sha256", "VARCHAR(64)"), + # 运行选项(发起时的快照,整次运行沿用)和发起它的定时任务。存量运行回填 {} 和 NULL: + # 没有选项,也不是定时任务发起的 + ("runs", "options", "JSON DEFAULT '{}'"), + ("runs", "schedule_id", "VARCHAR(32)"), +] + +#: 迁移补上的列要建的索引:(索引名, 表, 列)。create_all 不碰已有的表,表上新列的索引也就不会建。 +#: 索引名照 NAMING_CONVENTION 取,和新库 create_all 建出来的同名,IF NOT EXISTS 才不会重复建 +_INDEX_MIGRATIONS: list[tuple[str, str, str]] = [ + # 定时任务的触发历史按它找这个任务发起的运行 + ("ix_runs_schedule_id", "runs", "schedule_id"), ] @@ -210,6 +221,10 @@ def _migrate(conn) -> None: # pragma: no cover - 同步回调 cols = {row[1] for row in conn.execute(text(f"PRAGMA table_info({table})"))} if cols and column not in cols: conn.execute(text(f"ALTER TABLE {table} ADD COLUMN {column} {ddl}")) + for name, table, column in _INDEX_MIGRATIONS: + cols = {row[1] for row in conn.execute(text(f"PRAGMA table_info({table})"))} + if column in cols: + conn.execute(text(f"CREATE INDEX IF NOT EXISTS {name} ON {table} ({column})")) async def init_db() -> None: diff --git a/backend/app/db/models.py b/backend/app/db/models.py index 403b9177..3d7cc860 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -491,6 +491,11 @@ class Run(Base, TimestampMixin): # 发起时钉住的上传表格版本:{source_id: {"snapshot": 快照 id, "name": 源名}}。续跑、恢复沿用, # 回收(table_versions.gc)不删任何运行引用的快照。None 是升级前发起的运行 data_versions: Mapped[dict[str, Any] | None] = mapped_column(default=None) + # 发起时定下、整次运行沿用的运行选项(比如核查设置的快照)。审批停下期间改了设置,恢复和重放 + # 仍按发起时的这份——同 approval_default 的道理。非空时写进 run.started,进封存范围。存量运行是 {} + options: Mapped[dict[str, Any]] = mapped_column(default=dict) + # 发起它的定时任务,手动发起的为空。不设外键:定时任务删掉之后,运行记录照旧是追溯凭证,来源 id 留着 + schedule_id: Mapped[str | None] = mapped_column(String(32), default=None, index=True) events: Mapped[list["RunEvent"]] = relationship( back_populates="run", cascade="all, delete-orphan" diff --git a/backend/app/engine/launch.py b/backend/app/engine/launch.py index 3186a690..71d7a9aa 100644 --- a/backend/app/engine/launch.py +++ b/backend/app/engine/launch.py @@ -52,10 +52,13 @@ async def launch_run( run_class: str = "exploratory", version: int | None = None, actor: str | None, + schedule_id: str | None = None, + options: dict[str, Any] | None = None, ) -> Run: """发起一次运行,返回刚建好的运行记录(执行已经交给 run_manager)。不发起就抛 LaunchRefused。 参数和 POST /api/runs 的请求体一一对应(input 叫 input_payload);actor 是记在运行上的发起人。 + schedule_id 是发起它的定时任务,options 是整次运行沿用的选项快照,都原样交给 run_manager.start。 """ # 没挂在工作流上(画布没保存就发起、问数据当场搭的)。老数据里是「临时图」,前端两种都认 name = "未保存的工作流" @@ -146,6 +149,8 @@ async def launch_run( version=run_version, version_hash=version_hash, started_by=actor, + options=options, + schedule_id=schedule_id, ) except ValueError as e: raise LaunchRefused(400, str(e)) from e diff --git a/backend/app/engine/runner.py b/backend/app/engine/runner.py index 74151f86..b5462af4 100644 --- a/backend/app/engine/runner.py +++ b/backend/app/engine/runner.py @@ -2,6 +2,7 @@ from __future__ import annotations import asyncio import contextlib +import copy import json import logging import time @@ -33,6 +34,10 @@ logger = logging.getLogger(__name__) # 轨迹里正文靠 node.finished 的 preview,思考靠调用结束时的单条 llm.thinking 汇总 _EPHEMERAL = {EventType.LLM_TOKEN, EventType.LLM_THINKING_DELTA} +#: RunContext.extra 里放这次运行的选项(Run.options)的键。节点从这里读发起时定下的快照、不读设置的 +#: 当前值:审批停下期间改了设置,恢复后的那一段也按发起时的,重放和正式运行才可复现 +RUN_OPTIONS_KEY = "run_options" + class RunManager: """负责一次运行的完整生命周期:启动、串流、中断、恢复、取消。""" @@ -177,7 +182,10 @@ class RunManager: version_hash: str | None = None, started_by: str | None = None, approval_default: str | None = None, + options: dict[str, Any] | None = None, + schedule_id: str | None = None, ) -> Run: + """建运行记录、交给执行任务。options 是这次运行的选项快照,schedule_id 是发起它的定时任务(可空)。""" spec = GraphSpec.model_validate(graph) report = validate_graph(spec) if not report.ok: @@ -187,6 +195,8 @@ class RunManager: ) if approval_default is None: approval_default = await _approval_default(run_class=run_class) + # 拷一份:调用方手上的 dict 之后再改,不该改到这次运行的快照 + options = copy.deepcopy(options) if options else {} async with SessionLocal() as session: from app.data.table_versions import pin_lock @@ -214,6 +224,8 @@ class RunManager: tool_trust=tool_trust, agent_limits=agent_limits, data_versions=data_versions, + options=options, + schedule_id=schedule_id, ) run.thread_id = run.id # 一个 run 一条 checkpoint 线程 session.add(run) @@ -232,6 +244,7 @@ class RunManager: tool_trust=tool_trust, agent_limits=agent_limits, data_versions=data_versions, + options=options, ) ) return run @@ -585,6 +598,7 @@ class RunManager: tool_trust: dict[str, str] | None = None, agent_limits: dict[str, Any] | None = None, data_versions: dict[str, Any] | None = None, + options: dict[str, Any] | None = None, resumed: list[str] | tuple[str, ...] = (), actors: dict[str, str | None] | None = None, ) -> None: @@ -631,7 +645,10 @@ class RunManager: # 运行记录里的那份,含运行中途第一次用到时补上的;None 是升级前的运行 "data_versions": data_versions, # 这一段是按哪一版重放协议跑的,恢复时据此认出升级前做下的工作 - "replay_protocol": PROTOCOL}, + "replay_protocol": PROTOCOL, + # 发起时定下的运行选项(核查设置之类),每一段都记、记的是同一份。没有选项的 + # 运行不带这个键,和加选项之前的 run.started 逐字相同 + **({"options": options} if options else {})}, ) run_ctx = RunContext( @@ -648,7 +665,9 @@ class RunManager: # 的 run_versions)。复制一份:运行中途补固定的源就地写进这个 dict,同一段执行里 # 后面的节点都看得到,但不该碰到别处(运行记录、事件)手上的那个对象 extra={"resumed": set(resumed), "actors": dict(actors or {}), - RUN_VERSIONS_KEY: dict(data_versions) if data_versions is not None else None}, + RUN_VERSIONS_KEY: dict(data_versions) if data_versions is not None else None, + # 同理也是一份拷贝:节点读它,不该改到运行记录那份 + RUN_OPTIONS_KEY: copy.deepcopy(options or {})}, ) app = compile_graph(spec, run_ctx).compile(checkpointer=self.checkpointer) # 循环按自己声明的轮数另记一份预算,全局上限只管没人把关的环 @@ -1025,6 +1044,8 @@ def _carried(run: Run) -> dict[str, Any]: "agent_limits": run.agent_limits, # 固定的数据版本照运行记录来,不重新读数据源的当前指针:中途重传过,续跑查的仍是发起时那一版 "data_versions": run.data_versions, + # 运行选项同理:照发起时的快照,不重新读设置 + "options": run.options or {}, } diff --git a/backend/tests/test_run_options.py b/backend/tests/test_run_options.py new file mode 100644 index 00000000..ef456a27 --- /dev/null +++ b/backend/tests/test_run_options.py @@ -0,0 +1,268 @@ +"""运行选项和来源:Run.options 是发起时定下、整次运行沿用的选项快照,Run.schedule_id 是发起它的定时任务。 + +- 旧库升级时补这两列和 schedule_id 的索引;存量运行的选项读出来是 {},来源为空 +- RunManager.start 收下两者并落库;选项非空时写进 run.started(进封存范围),为空时 run.started 和以前 + 逐字相同;恢复、继续运行沿用发起时的选项,节点从 RunContext.extra 读得到同一份 +- 接口带出 schedule_id,运行列表能按定时任务筛;launch_run 原样往下交 +""" +from __future__ import annotations + +import asyncio +import uuid + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import create_engine, select, text + +from app.db.base import SessionLocal, engine as app_engine +from app.db.models import Run, RunEvent +from app.engine import compiler +from app.engine.runner import run_manager +from app.engine.schema import NodeType +from app.main import app + +#: 加 options 之前 run.started 就有的键:选项为空时一个不多、一个不少 +LEGACY_STARTED_KEYS = {"nodes", "resumed", "memory_scope", "collection", "approval_default", "tool_trust", + "agent_limits", "data_versions", "replay_protocol"} +OPTIONS = {"verify": {"dual_check": True, "max_queries": 6}, "note": "第 40 周例行核查"} + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def chain(*nodes): + return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +SIMPLE = chain(node("start", "input"), node("out", "output", fields=[{"name": "r", "value": "完成"}])) +GATED = chain(node("start", "input"), + node("before", "transform", mode="template", template="前"), + node("gate", "human", mode="approve", title="放行吗"), + node("after", "transform", mode="template", template="后"), + node("out", "output", fields=[{"name": "r", "value": "{{ nodes.after }}"}])) + + +async def _settle(run_id: str, statuses=("succeeded", "failed", "cancelled")) -> Run: + for _ in range(400): + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row.status in statuses and (row.status == "interrupted" or row.manifest_seq is not None): + await run_manager.wait_idle(run_id) + return row + await asyncio.sleep(0.03) + raise AssertionError(f"运行 {run_id} 停在 {row.status}") + + +async def _started(run_id: str) -> list[dict]: + async with SessionLocal() as session: + return [e.data for e in (await session.execute( + select(RunEvent).where(RunEvent.run_id == run_id, RunEvent.type == "run.started").order_by(RunEvent.seq) + )).scalars()] + + +def _schedule() -> str: + return uuid.uuid4().hex + + +# -------------------------------------------------------------------------- +# 旧库升级 +# -------------------------------------------------------------------------- + + +def test_old_databases_get_both_columns_and_the_index(): + from app.db.base import _migrate + + db = create_engine("sqlite://") + with db.begin() as conn: + conn.execute(text("CREATE TABLE runs (id VARCHAR(32) PRIMARY KEY, workflow_name VARCHAR(200), " + "status VARCHAR(20), last_seq INTEGER)")) + conn.execute(text("INSERT INTO runs (id, workflow_name, status, last_seq) VALUES ('r1', '周报', 'succeeded', 9)")) + _migrate(conn) + _migrate(conn) # 再执行一次不出错:列和索引都已经在了 + cols = {row[1] for row in conn.execute(text("PRAGMA table_info(runs)"))} + assert {"options", "schedule_id"} <= cols + assert conn.execute(text("SELECT options, schedule_id FROM runs WHERE id = 'r1'")).one() == ("{}", None) + indexes = {row[1]: row for row in conn.execute(text("PRAGMA index_list(runs)"))} + assert "ix_runs_schedule_id" in indexes + assert [r[2] for r in conn.execute(text("PRAGMA index_info(ix_runs_schedule_id)"))] == ["schedule_id"] + + +def test_a_missing_table_is_left_alone(): + """还没有 runs 表的库(create_all 之前不会这样,这里只确认迁移不替它建索引、不报错)。""" + from app.db.base import _migrate + + db = create_engine("sqlite://") + with db.begin() as conn: + _migrate(conn) + assert conn.execute(text("SELECT name FROM sqlite_master WHERE name = 'ix_runs_schedule_id'")).first() is None + + +async def test_new_databases_index_schedule_id(): + async with app_engine.connect() as conn: + names = {row[1] for row in (await conn.execute(text("PRAGMA index_list(runs)"))).all()} + assert "ix_runs_schedule_id" in names + + +async def test_a_migrated_row_reads_back_empty_options(): + """迁移回填的是字符串 '{}':经 ORM 读出来是空 dict,不是字符串,也不是 None。""" + async with SessionLocal() as session: + run = Run(workflow_name="迁移回填", status="succeeded", graph={}, input={}) + session.add(run) + await session.commit() + run_id = run.id + async with app_engine.begin() as conn: + await conn.execute(text("UPDATE runs SET options = '{}' WHERE id = :id"), {"id": run_id}) + async with SessionLocal() as session: + row = await session.get(Run, run_id) + assert row.options == {} and row.schedule_id is None + + +# -------------------------------------------------------------------------- +# 发起:落库、run.started、沿用 +# -------------------------------------------------------------------------- + + +async def test_options_and_schedule_are_stored_and_sealed(): + sid = _schedule() + run = await run_manager.start(graph=SIMPLE, input_payload={}, options=OPTIONS, schedule_id=sid) + assert (run.options, run.schedule_id) == (OPTIONS, sid) + row = await _settle(run.id) + assert row.status == "succeeded", row.error + assert (row.options, row.schedule_id) == (OPTIONS, sid) + [started] = await _started(run.id) + assert started["options"] == OPTIONS + assert set(started) == LEGACY_STARTED_KEYS | {"options"} + + +@pytest.mark.parametrize("options", [None, {}]) +async def test_without_options_run_started_is_unchanged(options): + run = await run_manager.start(graph=SIMPLE, input_payload={}, options=options) + row = await _settle(run.id) + assert row.options == {} and row.schedule_id is None + [started] = await _started(run.id) + assert set(started) == LEGACY_STARTED_KEYS, "选项为空时 run.started 多出了键" + + +async def test_the_options_belong_to_the_run_not_to_the_callers_dict(): + mine = {"verify": {"dual_check": True}} + run = await run_manager.start(graph=SIMPLE, input_payload={}, options=mine) + mine["verify"]["dual_check"] = False + mine["late"] = 1 + await _settle(run.id) + async with SessionLocal() as session: + assert (await session.get(Run, run.id)).options == {"verify": {"dual_check": True}} + + +@pytest.fixture +def seen_options(monkeypatch): + """记下每次执行「数据整形」节点时它从运行上下文里读到的选项。""" + from app.engine.runner import RUN_OPTIONS_KEY + + seen: list = [] + original = compiler.RUNNERS[NodeType.TRANSFORM] + + async def spy(state, ctx): + seen.append(ctx.run.extra.get(RUN_OPTIONS_KEY)) + return await original(state, ctx) + + monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, spy) + return seen + + +async def test_resume_keeps_the_options_the_run_started_with(seen_options): + run = await run_manager.start(graph=GATED, input_payload={}, options=OPTIONS) + assert (await _settle(run.id, ("interrupted",))).status == "interrupted" + await run_manager.resume(run.id, {"approved": True}) + row = await _settle(run.id) + assert row.status == "succeeded", row.error + assert seen_options == [OPTIONS, OPTIONS], "恢复之后节点读到的选项变了" + first, second = await _started(run.id) + assert first["options"] == second["options"] == OPTIONS and second["resumed"] is True + + +async def test_continue_keeps_the_options_too(seen_options, monkeypatch): + original = compiler.RUNNERS[NodeType.TRANSFORM] + calls = {"n": 0} + + async def flaky(state, ctx): + calls["n"] += 1 + if ctx.node.id == "after" and calls["n"] <= 2: + raise RuntimeError("临时故障") + return await original(state, ctx) + + monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, flaky) + graph = chain(node("start", "input"), + node("before", "transform", mode="template", template="前"), + node("after", "transform", mode="template", template="后"), + node("out", "output", fields=[{"name": "r", "value": "{{ nodes.after }}"}])) + run = await run_manager.start(graph=graph, input_payload={}, options=OPTIONS) + assert (await _settle(run.id)).status == "failed" + await run_manager.continue_failed(run.id) + row = await _settle(run.id) + assert row.status == "succeeded", row.error + assert [s.get("options") for s in await _started(run.id)] == [OPTIONS, OPTIONS] + + +async def test_nodes_read_an_empty_dict_when_there_are_no_options(seen_options): + run = await run_manager.start(graph=GATED, input_payload={}) + await _settle(run.id, ("interrupted",)) + assert seen_options == [{}] + + +# -------------------------------------------------------------------------- +# 接口:带出来源、按定时任务筛、launch_run 照转 +# -------------------------------------------------------------------------- + + +async def test_the_api_shows_and_filters_by_schedule(client): + sid, other = _schedule(), _schedule() + mine = [await run_manager.start(graph=SIMPLE, input_payload={}, schedule_id=sid) for _ in range(2)] + theirs = await run_manager.start(graph=SIMPLE, input_payload={}, schedule_id=other) + manual = await run_manager.start(graph=SIMPLE, input_payload={}) + for run in (*mine, theirs, manual): + await _settle(run.id) + + got = (await client.get(f"/api/runs/{mine[0].id}")).json() + assert got["schedule_id"] == sid + assert (await client.get(f"/api/runs/{manual.id}")).json()["schedule_id"] is None + + listed = (await client.get("/api/runs", params={"schedule_id": sid})).json() + assert sorted(r["id"] for r in listed) == sorted(r.id for r in mine) + assert all(r["schedule_id"] == sid for r in listed) + # 不带这个参数时照旧全列 + everything = {r["id"] for r in (await client.get("/api/runs", params={"limit": 200})).json()} + assert {theirs.id, manual.id} <= everything + + +async def test_launch_run_passes_both_through(): + from app.engine.launch import launch_run + + sid = _schedule() + async with SessionLocal() as session: + run = await launch_run(session, graph=SIMPLE, actor="定时任务", schedule_id=sid, options=OPTIONS) + row = await _settle(run.id) + assert (row.schedule_id, row.options, row.started_by) == (sid, OPTIONS, "定时任务") + [started] = await _started(run.id) + assert started["options"] == OPTIONS + + +async def test_an_http_launch_has_no_schedule_and_no_options(client): + res = await client.post("/api/runs", json={"graph": SIMPLE}) + assert res.status_code == 201, res.text + assert res.json()["schedule_id"] is None + row = await _settle(res.json()["id"]) + assert row.options == {} From 3dbd69b561e274a1ceb559ea8124e36dbb06418d Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 00:14:40 +0800 Subject: [PATCH 3/7] =?UTF-8?q?=E5=BE=80=E5=81=9C=E4=B8=8B=E7=9A=84?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E9=87=8C=E8=BF=BD=E5=8A=A0=E4=BA=8B=E4=BB=B6?= =?UTF-8?q?=E7=BB=9F=E4=B8=80=E8=B5=B0=20engine/run=5Fappend.py=20?= =?UTF-8?q?=E7=9A=84=20append=5Fevents(run=5Fid,=20items,=20*,=20require,?= =?UTF-8?q?=20seal=3DNone)=EF=BC=9A=E5=AE=A1=E6=89=B9=E8=B4=A8=E8=AF=A2?= =?UTF-8?q?=E8=A6=81=E5=9C=A8=E8=BF=90=E8=A1=8C=E5=81=9C=E5=9C=A8=E5=AE=A1?= =?UTF-8?q?=E6=89=B9=E6=97=B6=E6=8A=8A=E9=97=AE=E7=AD=94=E5=86=99=E8=BF=9B?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E3=80=81=E9=9A=8F=E8=BF=90=E8=A1=8C=E7=BB=93?= =?UTF-8?q?=E6=9D=9F=E4=B8=80=E8=B5=B7=E5=B0=81=E5=AD=98=EF=BC=8C=E6=8E=A2?= =?UTF-8?q?=E7=B4=A2=E8=BF=90=E8=A1=8C=E5=B0=81=E5=AD=98=E4=B9=8B=E5=90=8E?= =?UTF-8?q?=E8=BF=98=E8=A6=81=E8=83=BD=E8=BF=BD=E5=8A=A0=E6=89=B9=E6=B3=A8?= =?UTF-8?q?=E3=80=82=E4=BA=8B=E4=BB=B6=E6=B5=81=E6=98=AF=E5=B0=81=E5=AD=98?= =?UTF-8?q?=E7=9A=84=E4=BE=9D=E6=8D=AE=EF=BC=8C=E6=B8=85=E5=8D=95=E6=8C=89?= =?UTF-8?q?=20seq=20=E6=8E=92=E5=BA=8F=E3=80=81run=5Fevents=20=E7=9A=84=20?= =?UTF-8?q?(run=5Fid,=20seq)=20=E5=8F=88=E4=B8=8D=E5=94=AF=E4=B8=80?= =?UTF-8?q?=EF=BC=8C=E6=92=9E=E5=8F=B7=E4=BC=9A=E8=AE=A9=E5=B0=81=E5=AD=98?= =?UTF-8?q?=E6=A0=B8=E5=AF=B9=E6=97=B6=E5=AF=B9=E6=97=B6=E4=B8=8D=E5=AF=B9?= =?UTF-8?q?=EF=BC=8C=E6=89=80=E4=BB=A5=E7=85=A7=20=5Fappend=5Fjudged=20?= =?UTF-8?q?=E7=9A=84=E5=81=9A=E6=B3=95=EF=BC=9A=E5=90=8C=E4=B8=80=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E5=9C=A8=E8=BF=9B=E7=A8=8B=E5=86=85=E6=8E=92=E9=98=9F?= =?UTF-8?q?=EF=BC=8C=E8=AF=BB=20last=5Fseq=20=E5=90=8E=E4=BB=8E=20bus=20?= =?UTF-8?q?=E6=8E=A5=E5=8F=B7=EF=BC=88=E5=8F=A6=E5=8F=96=E5=BA=93=E9=87=8C?= =?UTF-8?q?=E5=AE=9E=E9=99=85=E7=9A=84=E6=9C=80=E5=A4=A7=E5=BA=8F=E5=8F=B7?= =?UTF-8?q?=E5=85=9C=E5=BA=95=EF=BC=8Clast=5Fseq=20=E8=A2=AB=E5=88=AB?= =?UTF-8?q?=E5=A4=84=E5=86=99=E5=B0=8F=E4=BA=86=E4=B9=9F=E4=B8=8D=E6=92=9E?= =?UTF-8?q?=E5=8F=B7=EF=BC=89=EF=BC=8C=E5=86=99=E4=BA=8B=E4=BB=B6=E5=92=8C?= =?UTF-8?q?=E6=9D=A1=E4=BB=B6=E6=9B=B4=E6=96=B0=20last=5Fseq=20=E5=9C=A8?= =?UTF-8?q?=E5=90=8C=E4=B8=80=E4=BA=8B=E5=8A=A1=E9=87=8C=E2=80=94=E2=80=94?= =?UTF-8?q?interrupted=20=E8=A6=81=E6=B1=82=E8=BF=90=E8=A1=8C=E4=BB=8D?= =?UTF-8?q?=E5=81=9C=E5=9C=A8=E5=AE=A1=E6=89=B9=EF=BC=8Csealed=20=E8=A6=81?= =?UTF-8?q?=E6=B1=82=E4=BB=8D=E6=98=AF=E7=BB=88=E6=80=81=E4=B8=94=E5=B0=81?= =?UTF-8?q?=E5=AD=98=E6=B2=A1=E5=8F=98=EF=BC=8C=E4=BB=BB=E4=B8=80=E6=9D=A1?= =?UTF-8?q?=E8=90=BD=E7=A9=BA=E5=B0=B1=E5=9B=9E=E6=BB=9A=E3=80=81=E8=BF=94?= =?UTF-8?q?=E5=9B=9E=20None=EF=BC=8C=E8=B0=83=E7=94=A8=E6=96=B9=E6=8D=AE?= =?UTF-8?q?=E6=AD=A4=E8=AF=B4=E6=98=8E=E6=9C=AA=E5=86=99=E5=85=A5=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E8=AE=B0=E5=BD=95=EF=BC=9B=E6=8F=90=E4=BA=A4=E4=B9=8B?= =?UTF-8?q?=E5=90=8E=E6=89=8D=E5=B9=BF=E6=92=AD=E3=80=82sealed=20=E7=9A=84?= =?UTF-8?q?=E4=BA=8B=E4=BB=B6=E5=B8=A6=20post=5Fseal:=20true=EF=BC=9B?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E8=AE=B0=E5=BD=95=E4=B8=8A=E7=9A=84=E5=B0=81?= =?UTF-8?q?=E5=AD=98=E8=8B=A5=E6=97=A9=E4=BA=8E=E6=9C=80=E5=90=8E=E4=B8=80?= =?UTF-8?q?=E6=9D=A1=E7=BB=93=E6=9D=9F=E4=BA=8B=E4=BB=B6=EF=BC=88=E7=BB=A7?= =?UTF-8?q?=E7=BB=AD=E8=BF=90=E8=A1=8C=E5=90=8E=E5=8F=88=E7=BB=93=E6=9D=9F?= =?UTF-8?q?=E3=80=81=E8=BF=98=E6=B2=A1=E9=87=8D=E6=96=B0=E5=B0=81=E5=AD=98?= =?UTF-8?q?=E7=9A=84=E9=82=A3=E4=B8=80=E4=B8=8B=EF=BC=89=EF=BC=8C=E4=B8=8D?= =?UTF-8?q?=E7=AE=97=E7=8E=B0=E5=9C=A8=E7=9A=84=E5=B0=81=E5=AD=98=EF=BC=8C?= =?UTF-8?q?=E4=B8=8D=E8=BF=BD=E5=8A=A0=EF=BC=8C=E5=85=8D=E5=BE=97=E4=BA=8B?= =?UTF-8?q?=E4=BB=B6=E8=A2=AB=E6=8E=A5=E4=B8=8B=E6=9D=A5=E7=9A=84=E5=B0=81?= =?UTF-8?q?=E5=AD=98=E6=94=B6=E8=BF=9B=E5=8E=BB=EF=BC=9B=E8=B0=83=E7=94=A8?= =?UTF-8?q?=E6=96=B9=E5=8F=AF=E4=BC=A0=20seal=20=E5=9B=BA=E5=AE=9A?= =?UTF-8?q?=E5=BC=80=E5=A7=8B=E6=97=B6=E7=9C=8B=E5=88=B0=E7=9A=84=E5=B0=81?= =?UTF-8?q?=E5=AD=98=E3=80=82=E4=BA=8B=E4=BB=B6=E7=B1=BB=E5=9E=8B=E5=8A=A0?= =?UTF-8?q?=20approval.inquiry=E3=80=82=E6=B5=8B=E8=AF=95=E7=94=A8?= =?UTF-8?q?=E4=BA=BA=E5=B7=A5=E5=AE=A1=E6=89=B9=E5=81=9C=E4=BD=8F=E7=9A=84?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=EF=BC=9A=E8=BF=BD=E5=8A=A0=E4=B8=A4=E6=9D=A1?= =?UTF-8?q?=E5=90=8E=E6=89=B9=E5=87=86=E3=80=81=E5=B0=81=E5=AD=98=EF=BC=8C?= =?UTF-8?q?verify=20=E9=80=9A=E8=BF=87=E4=B8=94=E6=94=B9=E5=8A=A8=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0=E7=9A=84=E4=BA=8B=E4=BB=B6=20verify=20=E5=B0=B1?= =?UTF-8?q?=E5=AF=B9=E4=B8=8D=E4=B8=8A=EF=BC=9B=E6=81=A2=E5=A4=8D=E3=80=81?= =?UTF-8?q?=E7=BB=A7=E7=BB=AD=E8=BF=90=E8=A1=8C=E3=80=81=E9=87=8D=E6=96=B0?= =?UTF-8?q?=E5=B0=81=E5=AD=98=E3=80=81=E5=88=AB=E5=A4=84=E5=86=99=E5=85=A5?= =?UTF-8?q?=E6=8A=A2=E5=9C=A8=E4=B8=AD=E9=97=B4=E6=97=B6=E9=83=BD=E4=B8=8D?= =?UTF-8?q?=E5=86=99=E3=80=81last=5Fseq=20=E4=B8=8D=E5=8A=A8=EF=BC=9B?= =?UTF-8?q?=E8=BF=9E=E7=BB=AD=E5=92=8C=E5=B9=B6=E5=8F=91=E8=BF=BD=E5=8A=A0?= =?UTF-8?q?=E5=BA=8F=E5=8F=B7=E4=B8=A5=E6=A0=BC=E9=80=92=E5=A2=9E=E3=80=81?= =?UTF-8?q?=E5=85=A8=E6=B5=81=E6=97=A0=E9=87=8D=E5=A4=8D=EF=BC=9B=E6=81=A2?= =?UTF-8?q?=E5=A4=8D=E5=90=8E=E5=89=8D=E9=9D=A2=E7=9A=84=E8=8A=82=E7=82=B9?= =?UTF-8?q?=E4=B8=8D=E9=87=8D=E5=A4=8D=E6=89=A7=E8=A1=8C=EF=BC=8CAgent=20?= =?UTF-8?q?=E9=87=8C=E6=89=B9=E8=BF=87=E7=9A=84=E5=B7=A5=E5=85=B7=E4=B8=8D?= =?UTF-8?q?=E9=87=8D=E5=A4=8D=E6=89=A7=E8=A1=8C=EF=BC=88=E5=90=AB=E5=8D=87?= =?UTF-8?q?=E7=BA=A7=E5=89=8D=E5=86=99=E4=B8=8B=E7=9A=84=E6=96=AD=E7=82=B9?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/core/events.py | 4 + backend/app/engine/run_append.py | 147 ++++++++++ backend/tests/test_run_append.py | 469 +++++++++++++++++++++++++++++++ 3 files changed, 620 insertions(+) create mode 100644 backend/app/engine/run_append.py create mode 100644 backend/tests/test_run_append.py diff --git a/backend/app/core/events.py b/backend/app/core/events.py index 95c5211a..b23b4936 100644 --- a/backend/app/core/events.py +++ b/backend/app/core/events.py @@ -65,6 +65,10 @@ class EventType(StrEnum): # 人工介入 HUMAN_REQUESTED = "human.requested" HUMAN_RESOLVED = "human.resolved" + # 审批质询:运行停在审批时,审批人就待审的报告问的一个问题和得到的回答(engine/run_append.py 追加, + # node_id 为空)。这次问答的用量写在事件数据的 usage 里,计进运行用量(runner._usage_of_events)。 + # 停在审批时追加的随运行结束一起封存;封存之后追加的带 post_seal: true,是批注不是证据 + APPROVAL_INQUIRY = "approval.inquiry" # 出具与口径治理 ISSUANCE = "issuance" # 出具契约的判定结果(档位 / 缺数据 / 未回指数字) diff --git a/backend/app/engine/run_append.py b/backend/app/engine/run_append.py new file mode 100644 index 00000000..f4f20be5 --- /dev/null +++ b/backend/app/engine/run_append.py @@ -0,0 +1,147 @@ +"""往一次停下的运行里追加事件:运行停在审批时的审批质询、封存之后的批注。 + +事件流是封存的依据:清单按 seq 排序算哈希,而 run_events 的 (run_id, seq) 没有唯一约束——两条事件撞了号, +排序就说不准,封存核对时对时不对。所以追加和 runner 发事件守同一套序号:接在库里最后一条后面,从 bus 取号 +(同一进程里和执行任务共用一个计数器),写事件和条件更新 last_seq 在同一个事务里,谁先提交算谁的。做法同 +api/evidence.py 的 _append_judged(按需裁判的判定)。 + +require 说明往哪种运行里追加: +- "interrupted":运行停在审批、还没封存。事件落在以后的封存范围里,随运行结束一起封存。 +- "sealed":已经封存的终态运行。事件带 post_seal: true,落在 manifest_seq 之后,不进封存范围,verify 照旧一致。 + +没抢到(读完运行之后它被恢复、被放弃、继续运行后重新封存了,或者别处刚往里写过事件)就一条都不写、返回 None, +调用方据此说明「未写入运行记录」。 +""" +from __future__ import annotations + +import asyncio +import contextlib +from collections.abc import AsyncIterator, Sequence +from typing import Any, Literal + +from sqlalchemy import func, select, update + +from app.core.bus import bus +from app.core.events import EventType, RunEventModel +from app.db.base import SessionLocal +from app.db.models import Run, RunEvent +from app.engine.runner import _stage, run_manager + +#: 运行的终态。封存过的运行只会停在这几种上 +_TERMINAL = ("succeeded", "failed", "cancelled") +#: 宣布运行结束的事件。封存盖住了最后一条这种事件,才是这次结束之后封的(见 _current_seal) +_ENDINGS = (EventType.RUN_FINISHED, EventType.RUN_FAILED, EventType.RUN_CANCELLED) + +#: 一条要追加的事件:(事件类型, node_id, 数据) +Item = tuple[EventType | str, str | None, dict[str, Any] | None] + + +class _RunLocks: + """每次运行一把锁,没人用了就扔:同一次运行的追加在本进程里排队,读 last_seq 到条件更新之间不会被 + 另一次追加插进来——插进来的话后提交的那次条件更新落空,明明没有冲突也成了「未写入」。""" + + def __init__(self) -> None: + self._locks: dict[str, asyncio.Lock] = {} + self._users: dict[str, int] = {} + + @contextlib.asynccontextmanager + async def hold(self, key: str) -> AsyncIterator[None]: + lock = self._locks.setdefault(key, asyncio.Lock()) + self._users[key] = self._users.get(key, 0) + 1 + try: + async with lock: + yield + finally: + self._users[key] -= 1 + if not self._users[key]: + del self._users[key], self._locks[key] + + +_APPENDING = _RunLocks() + + +async def append_events( + run_id: str, + items: Sequence[Item], + *, + require: Literal["interrupted", "sealed"], + seal: tuple[int | None, str | None] | None = None, +) -> list[RunEventModel] | None: + """把 items 依次接在运行记录的最后一条后面,返回写入的事件(seq 严格递增);没写入返回 None。 + + seal 只用于 "sealed":调用方开始做事时看到的 (manifest_seq, manifest_hash)。做的事要花几秒(问模型), + 这期间继续运行、重新封存了的话,结果依据的已经不是现在的封存,不追加。不传就按这次读到的封存。 + items 为空时什么都不做,返回空列表。 + """ + if require not in ("interrupted", "sealed"): + raise ValueError(f"require 只能是 interrupted 或 sealed,收到的是 {require!r}") + # 事件类型先验过:写错了在取号之前就报出来 + entries = [(EventType(str(kind)), node_id, dict(data or {})) for kind, node_id, data in items] + if not entries: + return [] + sealed = require == "sealed" + for _, _, data in entries: + if sealed: + data["post_seal"] = True + else: + # 停在审批时追加的事件要进封存范围,不能自称在封存之后 + data.pop("post_seal", None) + + async with _APPENDING.hold(run_id): + async with SessionLocal() as session: + run = await session.get(Run, run_id) + if run is None: + return None + if sealed: + if run.status not in _TERMINAL or not run.manifest_hash \ + or (seal is not None and (run.manifest_seq, run.manifest_hash) != tuple(seal)) \ + or not await _current_seal(session, run): + return None + guard = (Run.status.in_(_TERMINAL), _same(Run.manifest_seq, run.manifest_seq), + _same(Run.manifest_hash, run.manifest_hash)) + else: + if run.status != "interrupted": + return None + guard = (Run.status == "interrupted",) + last = run.last_seq + # 进程重启过的话内存里的序号从 0 起:接在库里最后一条后面。last_seq 按说就是最后一条,另取一次 + # 实际的最大序号兜底——别处的追加把 last_seq 写小了,也不至于在这里撞号 + stored = await session.scalar(select(func.max(RunEvent.seq)).where(RunEvent.run_id == run_id)) + bus.set_seq(run_id, max(last or 0, stored or 0)) + events = [run_manager._event(run_id, kind, node_id=node_id, data=data) + for kind, node_id, data in entries] + # 核对和写入是同一行上的条件更新,和恢复、放弃、继续运行(runner._claim / _abandon)抢的是同一行 + claimed = await session.execute( + update(Run).where(Run.id == run_id, _same(Run.last_seq, last), *guard) + .values(last_seq=events[-1].seq)) + if claimed.rowcount != 1: + # 读完运行之后它变了。烧掉几个序号不要紧:不落库的事件(流式增量)本来就占序号, + # 序号只要求递增、不撞号 + await session.rollback() + return None + for event in events: + await _stage(session, event, run) + await session.commit() + # 先落库再广播,理由同 RunManager._emit + for event in events: + await bus.publish(event) + return events + + +def _same(column: Any, value: Any) -> Any: + return column.is_(None) if value is None else column == value + + +async def _current_seal(session: Any, run: Run) -> bool: + """运行记录上的封存是不是这次结束之后封的:manifest_seq 盖住了最后一条宣布结束的事件。 + + 继续运行之后再次结束时,终态和终态事件先提交、封存后算。这中间运行记录上还是上一次结束时的封存, + 当成现在的封存去追加,事件会被接下来的封存收进去,post_seal 就成了假话。manifest_seq 为空的是加这一列 + 之前封存的老运行,当时的封存范围按 verify_manifest 的老口径算,这里不拦。 + """ + if run.manifest_seq is None: + return True + ended = await session.scalar( + select(func.max(RunEvent.seq)).where(RunEvent.run_id == run.id, + RunEvent.type.in_([str(t) for t in _ENDINGS]))) + return ended is None or run.manifest_seq >= ended diff --git a/backend/tests/test_run_append.py b/backend/tests/test_run_append.py new file mode 100644 index 00000000..2a7707b3 --- /dev/null +++ b/backend/tests/test_run_append.py @@ -0,0 +1,469 @@ +"""往停下的运行里追加事件(engine/run_append.py 的 append_events)。 + +- 停在审批时追加:事件接在最后一条后面、落进以后的封存范围。批准、运行结束封存之后 verify 照旧一致, + 改动追加的事件 verify 就对不上——它们确实在封存范围里 +- 追加途中运行被恢复(条件更新落空):什么都不写,返回 None,last_seq 不动 +- 已封存的运行追加:带 post_seal、落在 manifest_seq 之后,verify 照旧一致;封存不是这次结束之后封的、 + 封存和调用方开始时看到的不同,都不追加 +- 连续追加、并发追加,序号严格递增,整条事件流没有重复的序号 +- 恢复后的重放不受影响:前面的节点不重复执行,Agent 里批过的工具不重复执行(含升级前写下的断点) +""" +from __future__ import annotations + +import asyncio +import json +import sqlite3 +from collections import Counter + +import pytest +from langchain_core.messages import AIMessage, ToolMessage +from pydantic import BaseModel +from sqlalchemy import select + +from app.core.bus import bus +from app.core.config import settings +from app.core.events import EventType +from app.db.base import SessionLocal +from app.db.models import Run, RunEvent +from app.engine import compiler +from app.engine.context import NodeError +from app.engine.runner import RunManager, run_manager, verify_manifest +from app.engine.schema import NodeType +from app.tools.registry import register + +#: Agent 场景里工具真正执行的次数。注册表是进程级的,工具只注册一次,计数按测试清零 +CALLS: list[str] = [] + + +class _TallyArgs(BaseModel): + label: str + + +@register("inquiry_tally", "测试用:每执行一次记一笔", "test", _TallyArgs, dangerous=True, needs_context=False) +async def _tally(label: str) -> str: + CALLS.append(label) + return json.dumps({"label": label, "n": len(CALLS)}, ensure_ascii=False) + + +@pytest.fixture(autouse=True) +async def engine_up(): + CALLS.clear() + await run_manager.setup() + yield + await run_manager.shutdown() + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def chain(*nodes): + return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +GATED = chain(node("start", "input"), + node("work", "transform", mode="template", template="本周销售额 4321.5 元"), + node("gate", "human", mode="approve", title="这份周报可以出具吗"), + node("out", "output", fields=[{"name": "r", "value": "{{ nodes.work }}"}])) +SIMPLE = chain(node("start", "input"), node("out", "output", fields=[{"name": "r", "value": "完成"}])) + + +def inquiry(question: str) -> tuple: + return (EventType.APPROVAL_INQUIRY, None, {"question": question, "answer": f"关于「{question}」:按订单表核对一致"}) + + +@pytest.fixture +def executed(monkeypatch) -> Counter: + """每个「数据整形」节点真正执行了几次。""" + seen: Counter = Counter() + original = compiler.RUNNERS[NodeType.TRANSFORM] + + async def counting(state, ctx): + seen[ctx.node.id] += 1 + return await original(state, ctx) + + monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, counting) + return seen + + +async def _row(run_id: str) -> Run: + async with SessionLocal() as session: + return await session.get(Run, run_id) + + +async def _until(run_id: str, statuses: tuple[str, ...], sealed: bool = False) -> Run: + for _ in range(500): + row = await _row(run_id) + if row.status in statuses and (not sealed or row.manifest_seq is not None): + await run_manager.wait_idle(run_id) + return await _row(run_id) + await asyncio.sleep(0.03) + raise AssertionError(f"运行 {run_id} 停在 {row.status}") + + +async def _paused(graph=GATED) -> Run: + run = await run_manager.start(graph=graph, input_payload={}) + return await _until(run.id, ("interrupted",)) + + +async def _finished(graph=SIMPLE) -> Run: + run = await run_manager.start(graph=graph, input_payload={}) + return await _until(run.id, ("succeeded", "failed"), sealed=True) + + +async def _events(run_id: str, etype: str | None = None) -> list[RunEvent]: + async with SessionLocal() as session: + query = select(RunEvent).where(RunEvent.run_id == run_id) + if etype: + query = query.where(RunEvent.type == etype) + return list((await session.execute(query.order_by(RunEvent.seq))).scalars()) + + +def _strictly_increasing(seqs: list[int]) -> bool: + return all(a < b for a, b in zip(seqs, seqs[1:])) + + +async def _no_repeated_seq(run_id: str) -> None: + seqs = [e.seq for e in await _events(run_id)] + assert len(seqs) == len(set(seqs)), f"事件流里有重复的序号:{seqs}" + + +# -------------------------------------------------------------------------- +# 停在审批时追加 +# -------------------------------------------------------------------------- + + +async def test_appended_while_waiting_are_sealed_with_the_run(executed): + from app.engine.run_append import append_events + + paused = await _paused() + subscription = bus.attach(paused.id) + try: + written = await append_events(paused.id, [inquiry("销售额为什么涨了"), inquiry("退款算进去了吗")], + require="interrupted") + assert written is not None and len(written) == 2 + assert written[0].seq > paused.last_seq and _strictly_increasing([e.seq for e in written]) + # 提交之后才广播:在线的客户端收到的就是落了库的这两条 + stream = subscription.stream() + live = [await asyncio.wait_for(stream.__anext__(), 2) for _ in range(2)] + assert [e.seq for e in live] == [e.seq for e in written] + finally: + subscription.close() + row = await _row(paused.id) + assert row.status == "interrupted" and row.last_seq == written[-1].seq + stored = await _events(paused.id, "approval.inquiry") + assert [(e.seq, e.node_id, e.data["question"]) for e in stored] == [ + (written[0].seq, None, "销售额为什么涨了"), (written[1].seq, None, "退款算进去了吗")] + assert all("post_seal" not in e.data for e in stored), "进封存范围的事件不该标成封存之后" + + await run_manager.resume(paused.id, {"approved": True}) + final = await _until(paused.id, ("succeeded", "failed"), sealed=True) + assert final.status == "succeeded", final.error + assert final.manifest_seq > written[-1].seq + check = await verify_manifest(paused.id) + assert check["ok"] is True and check["sealed_at"] == final.manifest_seq + # 恢复之后前面的节点没有再执行一遍 + assert executed["work"] == 1 + assert len([e for e in await _events(paused.id, "node.started") if e.node_id == "work"]) == 1 + await _no_repeated_seq(paused.id) + + # 追加的事件确实在封存范围里:改它一个字,封存核对就对不上 + async with SessionLocal() as session: + event = await session.get(RunEvent, stored[0].id) + event.data = {**event.data, "answer": "改过的回答"} + await session.commit() + assert (await verify_manifest(paused.id))["ok"] is False + + +async def test_a_resume_that_wins_the_row_leaves_nothing_behind(monkeypatch): + """读完运行、写入之前,恢复先把运行占成了 running:条件更新落空,一条都不写、last_seq 不动。""" + from app.engine.run_append import append_events + + paused = await _paused() + set_seq = bus.set_seq + + def resumed_in_between(run_id, value): + if run_id == paused.id: + db = sqlite3.connect(settings.db_path, timeout=5) + try: + db.execute("UPDATE runs SET status = 'running' WHERE id = ?", (paused.id,)) + db.commit() + finally: + db.close() + set_seq(run_id, value) + + monkeypatch.setattr(bus, "set_seq", resumed_in_between) + assert await append_events(paused.id, [inquiry("毛利率怎么算的")], require="interrupted") is None + row = await _row(paused.id) + assert row.status == "running" and row.last_seq == paused.last_seq + assert await _events(paused.id, "approval.inquiry") == [] + + +async def test_events_written_elsewhere_in_between_win(monkeypatch): + """读完运行、写入之前,别处(另一个进程,或者没走这把锁的追加)已经接着写了一条:状态没变,last_seq 变了。 + 条件更新落空、不写——否则两边各拿着同一个序号,事件流里撞号。""" + from app.engine.run_append import append_events + + paused = await _paused() + taken = paused.last_seq + 1 + set_seq = bus.set_seq + + def written_in_between(run_id, value): + if run_id == paused.id: + db = sqlite3.connect(settings.db_path, timeout=5) + try: + db.execute("INSERT INTO run_events (id, run_id, seq, type, node_id, ts, data) " + "VALUES ('elsewhere-1', ?, ?, 'log', NULL, 0, '{}')", (paused.id, taken)) + db.execute("UPDATE runs SET last_seq = ? WHERE id = ?", (taken, paused.id)) + db.commit() + finally: + db.close() + set_seq(run_id, value) + + monkeypatch.setattr(bus, "set_seq", written_in_between) + bus._seq.pop(paused.id, None) # 另一个进程的序号本进程不知道 + assert await append_events(paused.id, [inquiry("同时写进来的")], require="interrupted") is None + assert (await _row(paused.id)).last_seq == taken + assert await _events(paused.id, "approval.inquiry") == [] + await _no_repeated_seq(paused.id) + + +async def test_a_last_seq_left_behind_does_not_cause_a_clash(): + """last_seq 比库里实际的最后一条小(别处的追加交错着把它写小了):照实际的最后一条接号,顺手把 last_seq 写对。""" + from app.engine.run_append import append_events + + paused = await _paused() + async with SessionLocal() as session: + run = await session.get(Run, paused.id) + run.last_seq = paused.last_seq - 3 + await session.commit() + bus._seq.pop(paused.id, None) + written = await append_events(paused.id, [inquiry("接在哪里")], require="interrupted") + assert written[0].seq == paused.last_seq + 1 + assert (await _row(paused.id)).last_seq == written[0].seq + await _no_repeated_seq(paused.id) + + +async def test_a_run_that_already_moved_on_gets_nothing(executed): + from app.engine.run_append import append_events + + paused = await _paused() + await run_manager.resume(paused.id, {"approved": True}) + final = await _until(paused.id, ("succeeded",), sealed=True) + assert await append_events(paused.id, [inquiry("晚到的问题")], require="interrupted") is None + assert (await _row(paused.id)).last_seq == final.last_seq + assert await _events(paused.id, "approval.inquiry") == [] + + +# -------------------------------------------------------------------------- +# 封存之后追加 +# -------------------------------------------------------------------------- + + +async def test_post_seal_appends_stay_outside_the_seal(): + from app.engine.run_append import append_events + + final = await _finished() + assert final.status == "succeeded", final.error + before = await verify_manifest(final.id) + written = await append_events(final.id, [inquiry("这个数是含税的吗"), inquiry("和上周口径一样吗")], + require="sealed") + assert written is not None and all(e.data["post_seal"] is True for e in written) + assert written[0].seq > final.manifest_seq and _strictly_increasing([e.seq for e in written]) + row = await _row(final.id) + assert (row.status, row.manifest_seq, row.manifest_hash) == ("succeeded", final.manifest_seq, final.manifest_hash) + assert row.last_seq == written[-1].seq + after = await verify_manifest(final.id) + assert after["ok"] is True and (after["sealed_at"], after["events"]) == (before["sealed_at"], before["events"]) + assert [e.data["post_seal"] for e in await _events(final.id, "approval.inquiry")] == [True, True] + await _no_repeated_seq(final.id) + + +async def test_a_seal_pinned_by_the_caller_must_still_be_the_seal(): + from app.engine.run_append import append_events + + final = await _finished() + seal = (final.manifest_seq, final.manifest_hash) + assert await append_events(final.id, [inquiry("甲")], require="sealed", seal=(seal[0] - 1, seal[1])) is None + assert await append_events(final.id, [inquiry("乙")], require="sealed", seal=(seal[0], "0" * 64)) is None + assert await _events(final.id, "approval.inquiry") == [] + written = await append_events(final.id, [inquiry("丙")], require="sealed", seal=seal) + assert written is not None and written[0].data["question"] == "丙" + + +@pytest.mark.parametrize("change", [ + "UPDATE runs SET status = 'running' WHERE id = ?", # 继续运行占住了它 + "UPDATE runs SET manifest_hash = 'resealed' WHERE id = ?", # 继续运行之后重新封存了 + "UPDATE runs SET manifest_seq = manifest_seq + 1 WHERE id = ?", +], ids=["continued", "resealed-hash", "resealed-seq"]) +async def test_a_sealed_run_that_changes_in_between_gets_nothing(monkeypatch, change): + """封存之后追加也是核对和写入同一行上的条件更新:读完运行、写入之前它被继续运行占住、或者重新封存了,就不写。""" + from app.engine.run_append import append_events + + final = await _finished() + set_seq = bus.set_seq + + def changed_in_between(run_id, value): + if run_id == final.id: + db = sqlite3.connect(settings.db_path, timeout=5) + try: + db.execute(change, (final.id,)) + db.commit() + finally: + db.close() + set_seq(run_id, value) + + monkeypatch.setattr(bus, "set_seq", changed_in_between) + assert await append_events(final.id, [inquiry("这期为什么少了")], require="sealed") is None + assert (await _row(final.id)).last_seq == final.last_seq + assert await _events(final.id, "approval.inquiry") == [] + + +async def test_a_seal_left_from_an_earlier_ending_is_not_the_seal(monkeypatch): + """继续运行后又失败:终态和 run.failed 先提交、封存后算。这中间运行记录上还是上一次结束时的封存, + 这时追加的事件会被接下来的封存收进去,post_seal 就成了假话——不追加。""" + from app.engine.run_append import append_events + + original = compiler.RUNNERS[NodeType.TRANSFORM] + + async def boom(state, ctx): + if ctx.node.id == "boom": + raise NodeError(ctx.node.id, "数据源暂时不可用") + return await original(state, ctx) + + monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, boom) + first = await _finished(chain(node("start", "input"), node("boom", "transform", mode="template", template="x"), + node("out", "output", fields=[{"name": "r", "value": "x"}]))) + assert first.status == "failed" + + entered, release = asyncio.Event(), asyncio.Event() + real_seal = RunManager._seal + + async def held_seal(self, run_id): + if run_id == first.id and not entered.is_set(): + entered.set() + await release.wait() + return await real_seal(self, run_id) + + monkeypatch.setattr(RunManager, "_seal", held_seal) + await run_manager.continue_failed(first.id) + await asyncio.wait_for(entered.wait(), 10) + try: + row = await _row(first.id) + failures = [e.seq for e in await _events(first.id, "run.failed")] + assert row.status == "failed" and row.manifest_seq == first.manifest_seq < failures[-1] + assert await append_events(first.id, [inquiry("为什么又失败了")], require="sealed") is None + finally: + release.set() + await run_manager.wait_idle(first.id) + resealed = await _row(first.id) + assert resealed.manifest_seq >= failures[-1] + written = await append_events(first.id, [inquiry("为什么又失败了")], require="sealed") + assert written is not None and written[0].seq > resealed.manifest_seq + assert (await verify_manifest(first.id))["ok"] is True + + +async def test_the_wrong_kind_of_run_gets_nothing(): + from app.engine.run_append import append_events + + paused, final = await _paused(), await _finished() + assert await append_events(paused.id, [inquiry("甲")], require="sealed") is None, "停在审批的运行还没封存" + assert await append_events(final.id, [inquiry("乙")], require="interrupted") is None, "已结束的运行不在等审批" + assert await append_events("no-such-run", [inquiry("丙")], require="interrupted") is None + assert await append_events(paused.id, [], require="interrupted") == [] + with pytest.raises(ValueError): + await append_events(paused.id, [inquiry("丁")], require="running") + with pytest.raises(ValueError): + await append_events(paused.id, [("approval.unknown", None, {})], require="interrupted") + for run in (paused, final): + assert (await _row(run.id)).last_seq == run.last_seq + assert await _events(run.id, "approval.inquiry") == [] + + +# -------------------------------------------------------------------------- +# 序号 +# -------------------------------------------------------------------------- + + +async def test_seqs_strictly_increase_and_never_repeat(): + from app.engine.run_append import append_events + + paused = await _paused() + seqs: list[int] = [] + for batch in ([inquiry("一")], [inquiry("二"), inquiry("三")], [inquiry("四")]): + written = await append_events(paused.id, batch, require="interrupted") + seqs += [e.seq for e in written] + assert seqs[0] > paused.last_seq and _strictly_increasing(seqs) + + # 同时来几个:同一运行在本进程里排队,读 last_seq 到写入之间不会被另一次插进来,所以一个都不落空 + results = await asyncio.gather(*(append_events(paused.id, [inquiry(f"并发{i}")], require="interrupted") + for i in range(6))) + assert all(r is not None for r in results), "并发的追加有落空的" + together = sorted(r[0].seq for r in results) + assert together[0] > seqs[-1] and _strictly_increasing(together) + assert (await _row(paused.id)).last_seq == together[-1] + await _no_repeated_seq(paused.id) + + await run_manager.resume(paused.id, {"approved": True}) + final = await _until(paused.id, ("succeeded",), sealed=True) + assert (await verify_manifest(final.id))["ok"] is True + await _no_repeated_seq(final.id) + + +async def test_a_restarted_process_continues_after_the_stored_events(): + """进程重启过,内存里的序号从 0 起:追加接在库里最后一条后面。""" + from app.engine.run_append import append_events + + paused = await _paused() + bus._seq.pop(paused.id, None) + written = await append_events(paused.id, [inquiry("重启之后的问题")], require="interrupted") + assert written[0].seq == paused.last_seq + 1 + await _no_repeated_seq(paused.id) + + +# -------------------------------------------------------------------------- +# 恢复后的重放:Agent 里批过的工具不重复执行 +# -------------------------------------------------------------------------- + + +def _script_two_calls(monkeypatch) -> None: + from app.providers import mock_model + + def _decide(self, messages): + done = sum(isinstance(m, ToolMessage) for m in messages) + if self.tools and done < 2: + return AIMessage(content="", tool_calls=[ + {"name": "inquiry_tally", "args": {"label": f"第{done + 1}次"}, "id": f"call_{done + 1}"}]) + return AIMessage(content="两笔都记好了。") + + monkeypatch.setattr(mock_model.MockChatModel, "_decide", _decide) + + +@pytest.mark.parametrize("older_pause", [False, True], ids=["current", "older-protocol"]) +async def test_inquiries_do_not_disturb_the_replay(monkeypatch, older_pause): + """Agent 停在工具审批上时追加问答,批准后工具各执行一次。升级前写下的断点要按事件流判断能不能 + 安全恢复(runner._stale_replay 只看 tool.start / human.resolved),追加的问答不影响判断。""" + from app.engine import replay, runner + from app.engine.run_append import append_events + + _script_two_calls(monkeypatch) + if older_pause: + monkeypatch.setattr(runner, "PROTOCOL", runner.PROTOCOL - 1) + graph = chain(node("start", "input"), + node("bot", "agent", prompt="记两笔", tools=["inquiry_tally"], approval="always", max_steps=5), + node("out", "output", fields=[{"name": "r", "value": "{{ nodes.bot.text }}"}])) + run = await run_manager.start(graph=graph, input_payload={}) + for round_ in range(2): + paused = await _until(run.id, ("interrupted", "failed", "succeeded")) + assert paused.status == "interrupted", (paused.status, paused.error) + assert await append_events(run.id, [inquiry(f"第{round_ + 1}笔记的是什么")], require="interrupted") + if older_pause: + monkeypatch.setattr(runner, "PROTOCOL", replay.PROTOCOL) + await run_manager.resume(run.id, {"approved": True}) + await asyncio.sleep(0.05) + final = await _until(run.id, ("succeeded", "failed"), sealed=True) + assert final.status == "succeeded", final.error + assert CALLS == ["第1次", "第2次"], "批过的工具又执行了一次" + assert len(await _events(run.id, "approval.inquiry")) == 2 + assert (await verify_manifest(run.id))["ok"] is True + await _no_repeated_seq(run.id) From 6b272de6f9254f3639929646dcb7f184c6669c81 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 00:18:22 +0800 Subject: [PATCH 4/7] =?UTF-8?q?=E5=AE=A1=E6=89=B9=E8=B4=A8=E8=AF=A2?= =?UTF-8?q?=E7=9A=84=E7=94=A8=E9=87=8F=E8=AE=A1=E8=BF=9B=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E7=94=A8=E9=87=8F=EF=BC=9A=E8=B4=A8=E8=AF=A2=E4=B8=8D=E5=9C=A8?= =?UTF-8?q?=E5=9B=BE=E9=87=8C=E6=89=A7=E8=A1=8C=E3=80=81=E4=B8=8D=E5=8F=91?= =?UTF-8?q?=20llm.end=EF=BC=8C=E8=8A=B1=E7=9A=84=20token=20=E5=92=8C?= =?UTF-8?q?=E9=87=91=E9=A2=9D=E5=86=99=E5=9C=A8=20approval.inquiry=20?= =?UTF-8?q?=E4=BA=8B=E4=BB=B6=E6=95=B0=E6=8D=AE=E7=9A=84=20usage=20?= =?UTF-8?q?=E9=87=8C=EF=BC=88input=5Ftokens=E3=80=81output=5Ftokens?= =?UTF-8?q?=E3=80=81cost=5Fusd=E3=80=81calls=EF=BC=8C=E5=8F=AF=E5=B8=A6=20?= =?UTF-8?q?unpriced=5Fcalls=EF=BC=89=EF=BC=8C=E4=BB=A5=E5=89=8D=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E7=94=A8=E9=87=8F=E5=8F=AA=E8=AE=A4=E5=9B=BE=E7=9A=84?= =?UTF-8?q?=E7=8A=B6=E6=80=81=E5=92=8C=20llm.end=EF=BC=8C=E8=B4=A8?= =?UTF-8?q?=E8=AF=A2=E8=8A=B1=E7=9A=84=E9=92=B1=E5=9C=A8=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E9=A1=B5=E4=B8=8A=E7=9C=8B=E4=B8=8D=E8=A7=81=E3=80=82=5Fusage?= =?UTF-8?q?=5Fof=5Fevents=20=E6=8A=8A=E8=B4=A8=E8=AF=A2=E7=9A=84=E7=94=A8?= =?UTF-8?q?=E9=87=8F=E5=8D=95=E7=8B=AC=E5=8A=A0=E6=80=BB=E6=94=BE=E5=9C=A8?= =?UTF-8?q?=20inquiry=20=E4=B8=8B=EF=BC=9Allm.end=20=E8=BF=99=E6=9C=AC?= =?UTF-8?q?=E8=B4=A6=E8=A6=81=E5=92=8C=E7=8A=B6=E6=80=81=E9=82=A3=E6=9C=AC?= =?UTF-8?q?=E6=AF=94=E5=A4=A7=E5=B0=8F=EF=BC=8C=E8=B4=A8=E8=AF=A2=E8=8A=B1?= =?UTF-8?q?=E7=9A=84=E4=B8=A4=E6=9C=AC=E9=87=8C=E9=83=BD=E6=B2=A1=E6=9C=89?= =?UTF-8?q?=EF=BC=8C=E6=B7=B7=E8=BF=9B=E5=8E=BB=E6=AF=94=E4=BC=9A=E8=A2=AB?= =?UTF-8?q?=E7=8A=B6=E6=80=81=E9=82=A3=E6=9C=AC=E7=9B=96=E6=8E=89=E3=80=82?= =?UTF-8?q?=5Ffinalize=20=E7=85=A7=E6=97=A7=E7=AE=97=E5=87=BA=E5=9B=BE?= =?UTF-8?q?=E7=9A=84=E7=94=A8=E9=87=8F=EF=BC=88=E6=88=90=E5=8A=9F=E5=8F=96?= =?UTF-8?q?=E7=8A=B6=E6=80=81=EF=BC=8C=E5=85=B6=E4=BD=99=E4=B8=A4=E6=9C=AC?= =?UTF-8?q?=E5=8F=96=E5=A4=A7=EF=BC=89=E5=90=8E=E5=86=8D=E5=8A=A0=E4=B8=8A?= =?UTF-8?q?=E8=B4=A8=E8=AF=A2=E7=9A=84=EF=BC=8C=E6=98=8E=E7=BB=86=E5=8F=A6?= =?UTF-8?q?=E8=AE=B0=E5=9C=A8=E8=BF=90=E8=A1=8C=E7=94=A8=E9=87=8F=E7=9A=84?= =?UTF-8?q?=20inquiry=20=E4=B8=8B=EF=BC=9B=E5=81=9C=E5=9C=A8=E5=AE=A1?= =?UTF-8?q?=E6=89=B9=E6=97=B6=E8=A2=AB=E6=94=BE=E5=BC=83=E7=9A=84=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=EF=BC=8C=E6=94=BE=E5=BC=83=E6=97=B6=E8=A1=A5=E4=B8=8A?= =?UTF-8?q?=E8=BF=99=E6=AC=A1=E5=81=9C=E4=B8=8B=E6=9C=9F=E9=97=B4=E7=9A=84?= =?UTF-8?q?=E8=B4=A8=E8=AF=A2=E3=80=82=E5=81=9C=E8=BF=87=E4=B8=A4=E6=AC=A1?= =?UTF-8?q?=E3=80=81=E4=B8=A4=E6=AC=A1=E9=83=BD=E9=97=AE=E8=BF=87=E7=9A=84?= =?UTF-8?q?=EF=BC=8C=E6=8C=89=E4=B8=8A=E6=AC=A1=E8=AE=B0=E4=B8=8B=E7=9A=84?= =?UTF-8?q?=20inquiry=20=E5=85=88=E5=87=8F=E5=90=8E=E5=8A=A0=EF=BC=8C?= =?UTF-8?q?=E6=AF=8F=E6=AC=A1=E8=B4=A8=E8=AF=A2=E5=8F=AA=E8=AE=A1=E4=B8=80?= =?UTF-8?q?=E9=81=8D=E3=80=82=E6=B2=A1=E6=9C=89=E8=B4=A8=E8=AF=A2=E7=9A=84?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E7=94=A8=E9=87=8F=E5=92=8C=E4=BB=A5=E5=89=8D?= =?UTF-8?q?=E9=80=90=E5=AD=97=E7=9B=B8=E5=90=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/engine/runner.py | 59 ++++++- backend/tests/test_inquiry_usage.py | 248 ++++++++++++++++++++++++++++ 2 files changed, 303 insertions(+), 4 deletions(-) create mode 100644 backend/tests/test_inquiry_usage.py diff --git a/backend/app/engine/runner.py b/backend/app/engine/runner.py index b5462af4..f4b64364 100644 --- a/backend/app/engine/runner.py +++ b/backend/app/engine/runner.py @@ -551,7 +551,12 @@ class RunManager: closed = await _close_approvals(session, run_id, actor, now) run = await session.get(Run, run_id) timing = await _timing(session, run, 0, now) - run.usage = {**(run.usage or {}), "duration_ms": timing["active_ms"], **timing} + usage = dict(run.usage or {}) + # 停下时记的用量里还没有这次停在审批期间问的质询,这里补上(上次停下时计过的不再计) + inquiry = (await _usage_of_events(session, run_id)).get("inquiry") + if inquiry or usage.get("inquiry"): + usage = _with_inquiry(usage, inquiry) + run.usage = {**usage, "duration_ms": timing["active_ms"], **timing} run.error = "用户取消:等待审批时放弃了这次运行" if closed else "用户取消:挂起期间放弃了这次运行" bus.set_seq(run_id, run.last_seq or 0) # 和状态同一个事务提交,理由同 _finalize @@ -857,6 +862,7 @@ class RunManager: if run: timing = await _timing(session, run, elapsed_ms, now) events_usage = await _usage_of_events(session, run_id) + inquiry = events_usage.pop("inquiry", None) if status != "succeeded": # 状态里的用量只算跑完了的节点。失败、取消、停在审批上的那个节点已经 # 花出去的调用(校验修复、agent 的前几步)只在 llm.end 里——以前这部分 @@ -865,6 +871,9 @@ class RunManager: elif events_usage.get("unpriced_calls"): # 金额估不出来的调用只有 llm.end 数得出(状态里的用量只有金额,0 分不出是免费还是没价) usage = {**usage, "unpriced_calls": events_usage["unpriced_calls"]} + if inquiry: + # 审批质询不在图里执行,状态和 llm.end 两本账里都没有它:比完再加上 + usage = _with_inquiry(usage, inquiry) # duration_ms 是各段执行时长之和(= active_ms)。以前用这一段的耗时 # 覆盖它:87 秒、中间等过一次审批的运行,列表上写着 22ms usage.update(duration_ms=timing["active_ms"], **timing) @@ -1270,18 +1279,35 @@ async def _close_approvals(session: Any, run_id: str, actor: str | None, now: da _USAGE_KEYS = ("input_tokens", "output_tokens", "cost_usd", "calls") +#: 审批质询的用量里计进运行用量的键:unpriced_calls 是其中金额估不出来的调用次数(模型不在价格目录里) +_INQUIRY_KEYS = (*_USAGE_KEYS, "unpriced_calls") async def _usage_of_events(session: Any, run_id: str) -> dict[str, Any]: - """按 llm.end 事件逐次加总的用量:每次模型调用都有一条,不管节点最后成没成。""" + """按事件加总的用量。 + + llm.end 逐次加总:每次模型调用都有一条,不管节点最后成没成。审批质询不在图里执行、不发 llm.end, + 它的用量写在 approval.inquiry 事件数据的 usage 里,单独加总放在 inquiry 下(没有质询就没有这个键): + llm.end 这本账要和状态里的用量比大小(_spent),质询花的两本里都没有,混进去比会被状态那本盖掉。 + """ rows = await session.execute( - select(RunEvent.data).where(RunEvent.run_id == run_id, RunEvent.type == EventType.LLM_END) + select(RunEvent.type, RunEvent.data).where( + RunEvent.run_id == run_id, RunEvent.type.in_([EventType.LLM_END, EventType.APPROVAL_INQUIRY])) ) from app.providers.catalog import unpriced_call total: dict[str, Any] = dict.fromkeys(_USAGE_KEYS, 0) unpriced = 0 - for (data,) in rows: + inquiry: dict[str, Any] = {} + for kind, data in rows: + if kind == EventType.APPROVAL_INQUIRY: + spent = (data or {}).get("usage") + if isinstance(spent, dict): + for key in _INQUIRY_KEYS: + value = spent.get(key) + if isinstance(value, (int, float)) and not isinstance(value, bool): + inquiry[key] = inquiry.get(key, 0) + value + continue for key in _USAGE_KEYS: value = (data or {}).get(key, 1 if key == "calls" else 0) if isinstance(value, (int, float)) and not isinstance(value, bool): @@ -1291,9 +1317,34 @@ async def _usage_of_events(session: Any, run_id: str) -> dict[str, Any]: if unpriced: # 有几次调用的金额估不出来(模型不在价格目录里):cost_usd 只是估得出的那部分,界面据此不写 $0 total["unpriced_calls"] = unpriced + if inquiry: + if "cost_usd" in inquiry: + inquiry["cost_usd"] = round(inquiry["cost_usd"], 6) + total["inquiry"] = inquiry return total +def _with_inquiry(usage: dict[str, Any], inquiry: dict[str, Any] | None) -> dict[str, Any]: + """把审批质询的用量计进运行用量,明细另放在 inquiry 下(界面要说清其中多少是质询花的)。 + + usage 里已经计过一回的(上次停下时记下的 inquiry)先减掉再加:停过两次、两次都问过的运行, + 每次质询只计一遍。 + """ + out = dict(usage) + counted = out.pop("inquiry", None) or {} + for key in _INQUIRY_KEYS: + delta = (inquiry or {}).get(key, 0) - counted.get(key, 0) + if delta: + out[key] = (out.get(key) or 0) + delta + if isinstance(out.get("cost_usd"), float): + out["cost_usd"] = round(out["cost_usd"], 6) + if "input_tokens" in out or "output_tokens" in out: + out["total_tokens"] = int(out.get("input_tokens") or 0) + int(out.get("output_tokens") or 0) + if inquiry: + out["inquiry"] = dict(inquiry) + return out + + def _spent(state_usage: dict[str, Any], events_usage: dict[str, Any]) -> dict[str, Any]: """两本账取大的那本。新引擎每次调用都发 llm.end,事件那本不会更少; 加这条事件之前的老运行(agent 节点不发 llm.end)状态那本更全。""" diff --git a/backend/tests/test_inquiry_usage.py b/backend/tests/test_inquiry_usage.py new file mode 100644 index 00000000..c29dfd43 --- /dev/null +++ b/backend/tests/test_inquiry_usage.py @@ -0,0 +1,248 @@ +"""审批质询的用量计进运行用量。 + +质询不在图里执行、不发 llm.end,它花的 token 和金额写在 approval.inquiry 事件数据的 usage 里 +(input_tokens、output_tokens、cost_usd、calls,可带 unpriced_calls)。以前运行用量只认图的状态和 llm.end, +质询花的钱在运行页上看不见。 + +- 运行结束(成功、失败)时:图的用量照旧(成功取状态、其余两本账取大),再加上质询的;明细另放在 inquiry 下 +- 停在审批时被放弃:停下时记的用量里还没有这次停下期间的质询,放弃时补上 +- 停过两次、两次都问过:每次质询只计一遍 +- 没有质询的运行,用量和以前逐字相同 +""" +from __future__ import annotations + +import asyncio + +import pytest +from sqlalchemy import select + +from app.core.events import EventType +from app.db.base import SessionLocal +from app.db.models import Run, RunEvent +from app.engine import compiler +from app.engine.context import NodeError +from app.engine.runner import run_manager +from app.engine.schema import NodeType + +A = {"input_tokens": 1200, "output_tokens": 300, "cost_usd": 0.0123, "calls": 2} +B = {"input_tokens": 800, "output_tokens": 100, "cost_usd": 0.004, "calls": 1, "unpriced_calls": 1} + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def chain(*nodes): + return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +def gated(*middle, gates: int = 1) -> dict: + nodes = [node("start", "input"), *middle] + nodes += [node(f"gate{i}", "human", mode="approve", title=f"第 {i + 1} 道审批") for i in range(gates)] + nodes.append(node("out", "output", fields=[{"name": "r", "value": "完成"}])) + return chain(*nodes) + + +def ask(question: str, usage: dict | None) -> tuple: + data = {"question": question, "answer": "按订单表核对一致"} + if usage is not None: + data["usage"] = usage + return (EventType.APPROVAL_INQUIRY, None, data) + + +async def _row(run_id: str) -> Run: + async with SessionLocal() as session: + return await session.get(Run, run_id) + + +async def _until(run_id: str, statuses: tuple[str, ...]) -> Run: + for _ in range(500): + row = await _row(run_id) + if row.status in statuses and (row.status == "interrupted" or row.manifest_seq is not None): + await run_manager.wait_idle(run_id) + return await _row(run_id) + await asyncio.sleep(0.03) + raise AssertionError(f"运行 {run_id} 停在 {row.status}") + + +async def _ask_while_paused(run_id: str, *items: tuple) -> None: + from app.engine.run_append import append_events + + assert await append_events(run_id, list(items), require="interrupted"), "质询没写进运行" + + +def _tokens(usage: dict) -> tuple: + return (usage.get("input_tokens"), usage.get("output_tokens"), usage.get("total_tokens"), + usage.get("calls"), usage.get("unpriced_calls")) + + +# -------------------------------------------------------------------------- +# 按事件加总 +# -------------------------------------------------------------------------- + + +async def test_usage_of_events_keeps_inquiries_apart(): + """llm.end 照旧逐次加总;质询的另记在 inquiry 下(_finalize 比完两本账再加,见下面几条)。""" + from app.engine.runner import _usage_of_events + + async with SessionLocal() as session: + run = Run(workflow_name="用量加总", status="succeeded", graph={}, input={}) + session.add(run) + await session.flush() + rows = [ + ("llm.end", {"model": "claude-sonnet-4-5", "input_tokens": 100, "output_tokens": 20, "cost_usd": 0.001, + "calls": 1}), + ("llm.end", {"model": "claude-sonnet-4-5", "input_tokens": 50, "output_tokens": 5, "cost_usd": 0.0005}), + ("approval.inquiry", {"question": "甲", "usage": A}), + ("approval.inquiry", {"question": "乙", "usage": B}), + ("approval.inquiry", {"question": "没问模型就答了"}), + ("approval.inquiry", {"question": "写坏的用量", "usage": "很多"}), + # 别的事件里的金额不算:按需裁判的判定另有自己的账 + ("evidence.judged", {"cost_usd": 9.9, "calls": 9}), + ] + for seq, (kind, data) in enumerate(rows, start=1): + session.add(RunEvent(run_id=run.id, seq=seq, type=kind, node_id=None, ts=float(seq), data=data)) + await session.commit() + got = await _usage_of_events(session, run.id) + assert {k: got[k] for k in ("input_tokens", "output_tokens", "calls")} == { + "input_tokens": 150, "output_tokens": 25, "calls": 2} + assert got["cost_usd"] == pytest.approx(0.0015) + assert got["inquiry"] == {"input_tokens": 2000, "output_tokens": 400, "cost_usd": pytest.approx(0.0163), + "calls": 3, "unpriced_calls": 1} + + +async def test_without_inquiries_there_is_no_inquiry_key(): + from app.engine.runner import _usage_of_events + + async with SessionLocal() as session: + run = Run(workflow_name="没有质询", status="succeeded", graph={}, input={}) + session.add(run) + await session.flush() + session.add(RunEvent(run_id=run.id, seq=1, type="llm.end", node_id="ask", ts=1.0, + data={"model": "claude-sonnet-4-5", "input_tokens": 10, "output_tokens": 2, + "cost_usd": 0.0001, "calls": 1})) + await session.commit() + got = await _usage_of_events(session, run.id) + assert set(got) == {"input_tokens", "output_tokens", "cost_usd", "calls"} + + +# -------------------------------------------------------------------------- +# 运行用量 +# -------------------------------------------------------------------------- + + +async def test_a_succeeded_run_counts_the_inquiry(): + run = await run_manager.start(graph=gated(), input_payload={}) + await _until(run.id, ("interrupted",)) + await _ask_while_paused(run.id, ask("销售额为什么涨了", A)) + await run_manager.resume(run.id, {"approved": True}) + row = await _until(run.id, ("succeeded",)) + # 图里没有模型调用,用量全是这次质询的 + assert _tokens(row.usage) == (1200, 300, 1500, 2, None) + assert row.usage["cost_usd"] == pytest.approx(0.0123) + assert row.usage["inquiry"] == A + assert row.usage["wall_ms"] >= row.usage["active_ms"] + + +async def test_the_inquiry_is_added_on_top_of_the_model_calls(): + """图里有模型调用:成功的运行取状态里的用量,再加上质询的。""" + graph = gated(node("ask", "llm", prompt="上周到店多少人")) + run = await run_manager.start(graph=graph, input_payload={}) + await _until(run.id, ("interrupted",)) + await _ask_while_paused(run.id, ask("到店人数怎么算的", B)) + await run_manager.resume(run.id, {"approved": True}) + row = await _until(run.id, ("succeeded",)) + async with SessionLocal() as session: + ends = [e.data for e in (await session.execute(select(RunEvent).where( + RunEvent.run_id == run.id, RunEvent.type == "llm.end"))).scalars()] + assert ends, "图里的模型调用没有记 llm.end" + model_in = sum(e["input_tokens"] for e in ends) + model_out = sum(e["output_tokens"] for e in ends) + unpriced = sum(1 for e in ends if e.get("unpriced")) + assert row.usage["input_tokens"] == model_in + B["input_tokens"] + assert row.usage["output_tokens"] == model_out + B["output_tokens"] + assert row.usage["total_tokens"] == model_in + model_out + B["input_tokens"] + B["output_tokens"] + assert row.usage["calls"] == len(ends) + B["calls"] + assert row.usage["unpriced_calls"] == unpriced + B["unpriced_calls"] + assert row.usage["cost_usd"] == pytest.approx(sum(e["cost_usd"] for e in ends) + B["cost_usd"]) + + +async def test_a_failed_run_counts_the_inquiry_too(monkeypatch): + original = compiler.RUNNERS[NodeType.TRANSFORM] + + async def boom(state, ctx): + if ctx.node.id == "boom": + raise NodeError(ctx.node.id, "数据源暂时不可用") + return await original(state, ctx) + + monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, boom) + graph = chain(node("start", "input"), node("gate", "human", mode="approve", title="放行吗"), + node("boom", "transform", mode="template", template="x"), + node("out", "output", fields=[{"name": "r", "value": "x"}])) + run = await run_manager.start(graph=graph, input_payload={}) + await _until(run.id, ("interrupted",)) + await _ask_while_paused(run.id, ask("口径对吗", A)) + await run_manager.resume(run.id, {"approved": True}) + row = await _until(run.id, ("failed",)) + assert _tokens(row.usage) == (1200, 300, 1500, 2, None) + assert row.usage["inquiry"] == A + + +async def test_each_inquiry_counts_once_across_two_pauses(): + run = await run_manager.start(graph=gated(gates=2), input_payload={}) + await _until(run.id, ("interrupted",)) + await _ask_while_paused(run.id, ask("第一问", A)) + await run_manager.resume(run.id, {"approved": True}) + await asyncio.sleep(0.05) + paused = await _until(run.id, ("interrupted",)) + # 第二次停下时记的用量已经有第一问 + assert paused.usage["input_tokens"] == A["input_tokens"] and paused.usage["inquiry"] == A + await _ask_while_paused(run.id, ask("第二问", B)) + await run_manager.resume(run.id, {"approved": True}) + row = await _until(run.id, ("succeeded",)) + assert _tokens(row.usage) == (2000, 400, 2400, 3, 1) + assert row.usage["cost_usd"] == pytest.approx(0.0163) + assert row.usage["inquiry"] == {"input_tokens": 2000, "output_tokens": 400, "cost_usd": pytest.approx(0.0163), + "calls": 3, "unpriced_calls": 1} + + +@pytest.mark.parametrize("asked_before", [False, True], ids=["first-pause", "second-pause"]) +async def test_abandoning_a_paused_run_counts_what_was_asked(asked_before): + """停在审批时放弃:停下时记的用量里还没有这次停下期间的质询,放弃时补上;上一次停下时已经计过的不再计。""" + run = await run_manager.start(graph=gated(gates=2 if asked_before else 1), input_payload={}) + await _until(run.id, ("interrupted",)) + if asked_before: + await _ask_while_paused(run.id, ask("第一问", A)) + await run_manager.resume(run.id, {"approved": True}) + await asyncio.sleep(0.05) + await _until(run.id, ("interrupted",)) + await _ask_while_paused(run.id, ask("最后一问", B)) + assert await run_manager.cancel(run.id) == "cancelled" + row = await _until(run.id, ("cancelled",)) + if asked_before: + assert _tokens(row.usage) == (2000, 400, 2400, 3, 1) + assert row.usage["cost_usd"] == pytest.approx(0.0163) + else: + assert _tokens(row.usage) == (800, 100, 900, 1, 1) and row.usage["inquiry"] == B + assert row.usage["wall_ms"] >= row.usage["active_ms"] + + +async def test_runs_without_inquiries_keep_their_usage(): + run = await run_manager.start(graph=gated(), input_payload={}) + await _until(run.id, ("interrupted",)) + await run_manager.resume(run.id, {"approved": True}) + row = await _until(run.id, ("succeeded",)) + assert set(row.usage) == {"duration_ms", "wall_ms", "active_ms", "wait_ms"} + + other = await run_manager.start(graph=gated(), input_payload={}) + await _until(other.id, ("interrupted",)) + assert await run_manager.cancel(other.id) == "cancelled" + row = await _until(other.id, ("cancelled",)) + assert set(row.usage) == {"duration_ms", "wall_ms", "active_ms", "wait_ms"} From f8b285763040d69495781fc79262494a0084d55a Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 01:17:18 +0800 Subject: [PATCH 5/7] =?UTF-8?q?=E8=BF=9B=E7=A8=8B=E9=87=8D=E5=90=AF?= =?UTF-8?q?=E5=90=8E=E6=8E=A5=E5=8F=B7=E4=B8=8D=E5=86=8D=E6=92=9E=E4=B8=8A?= =?UTF-8?q?=E5=B7=B2=E6=9C=89=E7=9A=84=E5=BA=8F=E5=8F=B7=EF=BC=9A=E4=BB=A5?= =?UTF-8?q?=E5=89=8D=E6=81=A2=E5=A4=8D=E3=80=81=E7=BB=A7=E7=BB=AD=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E3=80=81=E6=94=BE=E5=BC=83=E3=80=81=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E6=97=B6=E8=A1=A5=E6=97=A5=E5=BF=97=E9=83=BD=E5=8F=AA=E6=8C=89?= =?UTF-8?q?=20run.last=5Fseq=20=E6=8E=A5=E5=8F=B7=EF=BC=8C=E8=80=8C=20last?= =?UTF-8?q?=5Fseq=20=E4=BC=9A=E8=A2=AB=E5=86=99=E5=B0=8F=E3=80=82=E6=8C=89?= =?UTF-8?q?=E9=9C=80=E8=A3=81=E5=88=A4=E8=AE=B0=E5=88=A4=E5=AE=9A=EF=BC=88?= =?UTF-8?q?api/evidence.py=20=E7=9A=84=20=5Fappend=5Fjudged=EF=BC=89?= =?UTF-8?q?=E7=9A=84=E6=9D=A1=E4=BB=B6=E6=9B=B4=E6=96=B0=E4=B8=8D=E5=B8=A6?= =?UTF-8?q?=20last=5Fseq=20=E6=9D=A1=E4=BB=B6=EF=BC=9A=E5=AE=83=E5=8F=96?= =?UTF-8?q?=E4=BA=86=E5=8F=B7=E8=BF=98=E6=B2=A1=E5=86=99=EF=BC=8C=E5=B0=81?= =?UTF-8?q?=E5=AD=98=E5=90=8E=E8=BF=BD=E5=8A=A0=EF=BC=88run=5Fappend?= =?UTF-8?q?=EF=BC=89=E6=8A=A2=E5=85=88=E6=8F=90=E4=BA=A4=E4=BA=86=EF=BC=8C?= =?UTF-8?q?=E5=AE=83=E5=86=8D=E6=8F=90=E4=BA=A4=E6=97=B6=E6=8A=8A=20last?= =?UTF-8?q?=5Fseq=20=E5=86=99=E5=9B=9E=E8=87=AA=E5=B7=B1=E9=82=A3=E4=B8=AA?= =?UTF-8?q?=E6=9B=B4=E5=B0=8F=E7=9A=84=E5=8F=B7=E3=80=82=E5=90=8C=E4=B8=80?= =?UTF-8?q?=E8=BF=9B=E7=A8=8B=E9=87=8C=E5=BA=8F=E5=8F=B7=E8=AE=A1=E6=95=B0?= =?UTF-8?q?=E5=8F=AA=E5=A2=9E=E4=B8=8D=E5=87=8F=EF=BC=8C=E4=B8=8D=E7=A2=8D?= =?UTF-8?q?=E4=BA=8B=EF=BC=9B=E9=87=8D=E5=90=AF=E5=90=8E=E8=AE=A1=E6=95=B0?= =?UTF-8?q?=E4=BB=8E=200=20=E8=B5=B7=EF=BC=8C=E7=BB=A7=E7=BB=AD=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E5=86=99=E5=87=BA=E7=9A=84=20run.resumed=20=E5=B0=B1?= =?UTF-8?q?=E5=92=8C=E5=B0=81=E5=AD=98=E5=90=8E=E8=BF=BD=E5=8A=A0=E7=9A=84?= =?UTF-8?q?=E4=BA=8B=E4=BB=B6=E5=90=8C=E5=8F=B7=EF=BC=8C=E9=87=8D=E6=96=B0?= =?UTF-8?q?=E5=B0=81=E5=AD=98=E6=97=B6=E4=B8=A4=E6=9D=A1=E5=90=8C=E5=8F=B7?= =?UTF-8?q?=E7=9A=84=E4=BA=8B=E4=BB=B6=E4=B8=80=E8=B5=B7=E8=BF=9B=E4=BA=86?= =?UTF-8?q?=E6=B8=85=E5=8D=95=EF=BC=8C=E8=80=8C=E6=B8=85=E5=8D=95=E6=8C=89?= =?UTF-8?q?=E5=BA=8F=E5=8F=B7=E6=8E=92=E5=BA=8F=E7=AE=97=E5=93=88=E5=B8=8C?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - runner 加 _seq_floor:接号起点取 last_seq 和库里实际最大序号中大的那个。恢复、继续运行、放弃都改用它,run_append 原来手写的同一段兜底也换成它 - 启动时(setup)先把落后的 last_seq 按库里实际的最后一条改正,再处理上次留下的运行中记录。_append_judged 规格不许改,它重启后也只按 last_seq 接号,不改正的话它第一次追加就撞号;有 (run_id, seq) 索引,五千次运行、一百五十万条事件实测约 7 毫秒。updated_at 不动,这是补记录,不是运行有了变化 - _stage 不带运行对象时的更新只往大里改:另开会话记的事件和执行任务同时在写,号大的先提交、号小的后提交,以前会把 last_seq 写小 测试:last_seq 写小、清掉本进程计数之后继续运行、恢复、放弃都接在实际的最后一条后面,全流无重复序号、verify 通过;_stage 先提交大号再提交小号,last_seq 不变;按真实的交错(裁判停在条件更新之前,封存后追加先提交)把 last_seq 写小,重启(shutdown、清计数、setup)后 last_seq 改正、别的运行原样不动,再裁判、继续运行都不撞号。去掉启动时的改正,只有裁判那条变红;去掉 _seq_floor,继续运行、恢复、放弃三条变红;_stage 改回无条件写,对应那条变红 --- backend/app/engine/run_append.py | 9 +- backend/app/engine/runner.py | 35 ++++++- backend/tests/test_run_append.py | 158 +++++++++++++++++++++++++++++++ 3 files changed, 192 insertions(+), 10 deletions(-) diff --git a/backend/app/engine/run_append.py b/backend/app/engine/run_append.py index f4f20be5..f7a515ad 100644 --- a/backend/app/engine/run_append.py +++ b/backend/app/engine/run_append.py @@ -25,7 +25,7 @@ from app.core.bus import bus from app.core.events import EventType, RunEventModel from app.db.base import SessionLocal from app.db.models import Run, RunEvent -from app.engine.runner import _stage, run_manager +from app.engine.runner import _seq_floor, _stage, run_manager #: 运行的终态。封存过的运行只会停在这几种上 _TERMINAL = ("succeeded", "failed", "cancelled") @@ -104,10 +104,9 @@ async def append_events( return None guard = (Run.status == "interrupted",) last = run.last_seq - # 进程重启过的话内存里的序号从 0 起:接在库里最后一条后面。last_seq 按说就是最后一条,另取一次 - # 实际的最大序号兜底——别处的追加把 last_seq 写小了,也不至于在这里撞号 - stored = await session.scalar(select(func.max(RunEvent.seq)).where(RunEvent.run_id == run_id)) - bus.set_seq(run_id, max(last or 0, stored or 0)) + # 进程重启过的话内存里的序号从 0 起:接在库里最后一条后面。last_seq 按说就是最后一条,_seq_floor + # 另取一次实际的最大序号兜底——别处的追加把 last_seq 写小了,也不至于在这里撞号 + bus.set_seq(run_id, await _seq_floor(session, run_id, last)) events = [run_manager._event(run_id, kind, node_id=node_id, data=data) for kind, node_id, data in entries] # 核对和写入是同一行上的条件更新,和恢复、放弃、继续运行(runner._claim / _abandon)抢的是同一行 diff --git a/backend/app/engine/runner.py b/backend/app/engine/runner.py index f4b64364..15b1c0a6 100644 --- a/backend/app/engine/runner.py +++ b/backend/app/engine/runner.py @@ -13,7 +13,7 @@ from typing import Any from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver from langgraph.errors import GraphRecursionError from langgraph.types import Command -from sqlalchemy import select, update +from sqlalchemy import case, func, select, update from app.core.bus import bus from app.core.config import settings @@ -66,6 +66,15 @@ class RunManager: # 一个 checkpoint 都没写过,"恢复"它只会从 START 用默认值重跑一遍, # 还丢掉 run.input——那不是恢复,是伪造一份结果。 async with SessionLocal() as session: + # 先把落后于库里实际最后一条的 last_seq 改正(为什么会落后见 _seq_floor)。内存里的序号计数刚清零, + # 接下来各处都按 last_seq 接号:本模块的几处另外兜了底,按需裁判记判定(api/evidence.py 的 + # _append_judged)没有,不改正的话它重启后第一次追加就撞上已有的序号。有 (run_id, seq) 索引, + # 每次运行一次索引查找。updated_at 不动:这是补记录,不是运行有了变化 + latest = select(func.max(RunEvent.seq)).where(RunEvent.run_id == Run.id).scalar_subquery() + await session.execute( + update(Run).where(func.coalesce(Run.last_seq, 0) < latest) + .values(last_seq=latest, updated_at=Run.updated_at) + .execution_options(synchronize_session=False)) stranded = list((await session.execute( select(Run.id, Run.last_seq).where(Run.status == "running") )).all()) @@ -392,7 +401,7 @@ class RunManager: .values(status="resolved") ) run = await session.get(Run, run_id) - bus.set_seq(run_id, run.last_seq) + bus.set_seq(run_id, await _seq_floor(session, run_id, run.last_seq)) await session.commit() await session.refresh(run) @@ -492,7 +501,7 @@ class RunManager: **({"graph": spec.model_dump(mode="json")} if graph is not None else {}), **({"started_by": actor} if actor else {})) run = await session.get(Run, run_id) - bus.set_seq(run_id, run.last_seq) + bus.set_seq(run_id, await _seq_floor(session, run_id, run.last_seq)) await session.commit() await session.refresh(run) @@ -558,7 +567,7 @@ class RunManager: usage = _with_inquiry(usage, inquiry) run.usage = {**usage, "duration_ms": timing["active_ms"], **timing} run.error = "用户取消:等待审批时放弃了这次运行" if closed else "用户取消:挂起期间放弃了这次运行" - bus.set_seq(run_id, run.last_seq or 0) + bus.set_seq(run_id, await _seq_floor(session, run_id, run.last_seq)) # 和状态同一个事务提交,理由同 _finalize event = self._event(run_id, EventType.RUN_CANCELLED, data={ "timing": timing, "actor": actor, @@ -1260,11 +1269,27 @@ async def _stage(session: Any, event: RunEventModel, run: Run | None = None) -> if run is not None: run.last_seq = event.seq else: + # 只往大里改:另开会话记的事件(note)和执行任务同时在写时,取号和提交的先后可以不一样,号小的后提交, + # 无条件写会把 last_seq 写小(后果见 _seq_floor) await session.execute( - update(Run).where(Run.id == event.run_id).values(last_seq=event.seq) + update(Run).where(Run.id == event.run_id) + .values(last_seq=case((Run.last_seq > event.seq, Run.last_seq), else_=event.seq)) + .execution_options(synchronize_session=False) ) +async def _seq_floor(session: Any, run_id: str, last_seq: int | None) -> int: + """接着一次运行往下编号的起点:last_seq 和库里实际最大的序号取大的。 + + last_seq 按说就是最后一条,但有的写入会把它写小:按需裁判记判定(api/evidence.py 的 _append_judged) + 的条件更新不带 last_seq 条件,它取了号还没写、封存后追加(engine/run_append.py)抢先提交了,它再提交时 + 把 last_seq 写回自己那个更小的号。同一进程里序号计数只增不减,不碍事;进程重启后计数从 0 起,只按 + last_seq 接号就会撞上已有的序号。清单按序号排序算哈希,撞了号封存核对就时对时不对。 + """ + stored = await session.scalar(select(func.max(RunEvent.seq)).where(RunEvent.run_id == run_id)) + return max(last_seq or 0, stored or 0) + + async def _close_approvals(session: Any, run_id: str, actor: str | None, now: datetime) -> int: """运行已经结束,它还没处理完的审批一并关掉。返回关掉了几条。""" rows = list((await session.execute( diff --git a/backend/tests/test_run_append.py b/backend/tests/test_run_append.py index 2a7707b3..1052fef4 100644 --- a/backend/tests/test_run_append.py +++ b/backend/tests/test_run_append.py @@ -6,6 +6,8 @@ - 已封存的运行追加:带 post_seal、落在 manifest_seq 之后,verify 照旧一致;封存不是这次结束之后封的、 封存和调用方开始时看到的不同,都不追加 - 连续追加、并发追加,序号严格递增,整条事件流没有重复的序号 +- last_seq 落后于库里实际的最后一条(按需裁判的追加和封存后追加交错提交时会这样):进程重启后恢复、 + 继续运行、放弃、按需裁判都接在实际的最后一条后面,不撞号;runner 记事件不把 last_seq 写小 - 恢复后的重放不受影响:前面的节点不重复执行,Agent 里批过的工具不重复执行(含升级前写下的断点) """ from __future__ import annotations @@ -66,6 +68,9 @@ GATED = chain(node("start", "input"), node("gate", "human", mode="approve", title="这份周报可以出具吗"), node("out", "output", fields=[{"name": "r", "value": "{{ nodes.work }}"}])) SIMPLE = chain(node("start", "input"), node("out", "output", fields=[{"name": "r", "value": "完成"}])) +#: 「boom」节点按 flaky 的开关失败:先让运行失败、封存,再关掉开关继续运行 +FLAKY = chain(node("start", "input"), node("boom", "transform", mode="template", template="门店 17 家"), + node("out", "output", fields=[{"name": "r", "value": "{{ nodes.boom }}"}])) def inquiry(question: str) -> tuple: @@ -86,6 +91,21 @@ def executed(monkeypatch) -> Counter: return seen +@pytest.fixture +def flaky(monkeypatch) -> dict: + """FLAKY 里的「boom」在 fail 为真时失败。""" + state = {"fail": True} + original = compiler.RUNNERS[NodeType.TRANSFORM] + + async def boom(st, ctx): + if ctx.node.id == "boom" and state["fail"]: + raise NodeError(ctx.node.id, "数据源暂时不可用") + return await original(st, ctx) + + monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, boom) + return state + + async def _row(run_id: str) -> Run: async with SessionLocal() as session: return await session.get(Run, run_id) @@ -128,6 +148,16 @@ async def _no_repeated_seq(run_id: str) -> None: assert len(seqs) == len(set(seqs)), f"事件流里有重复的序号:{seqs}" +def _sql(statement: str, *params) -> None: + """另开一条连接改库:模拟别处(恢复、继续运行、另一个写入者)抢在中间提交。""" + db = sqlite3.connect(settings.db_path, timeout=5) + try: + db.execute(statement, params) + db.commit() + finally: + db.close() + + # -------------------------------------------------------------------------- # 停在审批时追加 # -------------------------------------------------------------------------- @@ -421,6 +451,134 @@ async def test_a_restarted_process_continues_after_the_stored_events(): await _no_repeated_seq(paused.id) +@pytest.mark.parametrize("action", ["continue", "resume", "abandon"]) +async def test_the_runner_numbers_after_the_stored_events_when_last_seq_lags(flaky, action): + """last_seq 比库里实际的最后一条小(怎么来的见下一条),本进程又没有这次运行的序号计数(重启过):继续运行、 + 恢复、放弃都接在实际的最后一条后面。只按 last_seq 接,写出的 run.resumed / run.cancelled 撞上已有的序号, + 重新封存时两条同号的事件一起进了清单。""" + if action == "continue": + run = await _finished(FLAKY) + assert run.status == "failed" + else: + run = await _paused() + stored = (await _events(run.id))[-1].seq + _sql("UPDATE runs SET last_seq = last_seq - 3 WHERE id = ?", run.id) + bus._seq.pop(run.id, None) + + if action == "continue": + flaky["fail"] = False + await run_manager.continue_failed(run.id) + first_new = "run.resumed" + elif action == "resume": + await run_manager.resume(run.id, {"approved": True}) + first_new = "run.resumed" + else: + assert await run_manager.cancel(run.id) == "cancelled" + first_new = "run.cancelled" + final = await _until(run.id, ("succeeded", "cancelled"), sealed=True) + assert final.status == ("cancelled" if action == "abandon" else "succeeded"), final.error + assert [e.seq for e in await _events(run.id, first_new)][-1] == stored + 1 + await _no_repeated_seq(run.id) + assert (await verify_manifest(run.id))["ok"] is True + + +async def test_runner_event_writes_never_lower_last_seq(): + """runner 记事件时 last_seq 只往大里改。取号和提交的先后可以不一样:别处另开会话记的事件(run_manager.note) + 和执行任务同时在写,号大的先提交、号小的后提交,以前后提交的那条把 last_seq 写回小的。""" + from app.engine.runner import _stage + + paused = await _paused() + early = run_manager._event(paused.id, EventType.LOG, data={"level": "info", "message": "先取号"}) + late = run_manager._event(paused.id, EventType.LOG, data={"level": "info", "message": "后取号"}) + for event in (late, early): + async with SessionLocal() as session: + await _stage(session, event) + await session.commit() + assert (await _row(paused.id)).last_seq == late.seq + + +def _judged_args(run: Run) -> tuple: + """直接调按需裁判记判定的那一步(api/evidence.py 的 _append_judged)要的参数:问过一次模型、什么都没判成。""" + from app.api import evidence + + sealed = evidence._Sealed(run_id=run.id, graph={}, seal={"sealed": True, "ok": True}, events=[], + manifest_hash=run.manifest_hash, run_manifest_seq=run.manifest_seq) + report = evidence._Report(node_id="out", doc_artifact="doc-1", ok=None, repairs=None, stats=None, doc=None, + hash_ok=None, fields=[]) + outcome = {key: {} for key in ("verdicts", "keys", "unjudged", "skipped", "budget")} + outcome.update(model="judge-model", priced=True, same_model=False, writer_model=None, cost_usd=0.0021, + calls=1, duration_ms=1200, limits_hit=[], gaps=[], notes=[]) + return sealed, report, ["u1"], outcome + + +async def test_a_judge_interleaved_with_a_post_seal_append_survives_a_restart(flaky, monkeypatch): + """按需裁判记判定(_append_judged)的条件更新不带 last_seq 条件:它取了号、还没写,封存后追加抢先提交了, + 它再提交时把 last_seq 写回自己那个更小的号。同一进程里序号计数只增不减,不会撞号;进程重启后计数从 0 起, + 各处又按 last_seq 接号——裁判自己也是。所以启动时按库里实际的最后一条改正 last_seq:之后再裁判、继续运行, + 都接在实际的最后一条后面。""" + from app.api import evidence + from app.engine.run_append import append_events + + first = await _finished(FLAKY) + assert first.status == "failed" + held, opened = asyncio.Event(), evidence.SessionLocal + + class HeldBeforeUpdate: + """裁判的会话:执行条件更新之前停住,等封存后追加先提交。""" + + def __init__(self): + self._cm = opened() + + async def __aenter__(self): + session = await self._cm.__aenter__() + execute = session.execute + + async def held_execute(statement, *args, **kwargs): + if getattr(statement, "is_update", False): + await held.wait() + return await execute(statement, *args, **kwargs) + + session.execute = held_execute + return session + + async def __aexit__(self, *exc): + return await self._cm.__aexit__(*exc) + + monkeypatch.setattr(evidence, "SessionLocal", HeldBeforeUpdate) + judging = asyncio.create_task(evidence._append_judged(*_judged_args(first))) + for _ in range(200): # 裁判取到号了 + if bus._seq.get(first.id, 0) > first.last_seq: + break + await asyncio.sleep(0.01) + written = await append_events(first.id, [inquiry("这次为什么失败")], require="sealed") + held.set() + assert await judging == {"seq": first.last_seq + 1} + assert written is not None and written[0].seq == first.last_seq + 2 + assert (await _row(first.id)).last_seq == first.last_seq + 1, "裁判后提交,last_seq 被写回了小的号" + monkeypatch.setattr(evidence, "SessionLocal", opened) + + # 进程重启:内存里的序号计数没了,启动时 setup 照常执行一遍。只改落后的那条,别的运行原样不动 + lagging = await _row(first.id) + others = [await _finished(), await _paused()] + await run_manager.shutdown() + bus._seq.pop(first.id, None) + await run_manager.setup() + repaired = await _row(first.id) + assert (repaired.last_seq, repaired.updated_at) == (written[-1].seq, lagging.updated_at) + for other in others: + now = await _row(other.id) + assert (now.last_seq, now.updated_at) == (other.last_seq, other.updated_at) + assert await evidence._append_judged(*_judged_args(first)) == {"seq": written[-1].seq + 1} + await _no_repeated_seq(first.id) + + flaky["fail"] = False + await run_manager.continue_failed(first.id) + final = await _until(first.id, ("succeeded",), sealed=True) + assert final.manifest_seq > written[-1].seq + 1 + await _no_repeated_seq(first.id) + assert (await verify_manifest(first.id))["ok"] is True + + # -------------------------------------------------------------------------- # 恢复后的重放:Agent 里批过的工具不重复执行 # -------------------------------------------------------------------------- From fc57ed0e3234f1a1ae0cab066f1bfd8e26b43e8d Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 01:18:03 +0800 Subject: [PATCH 6/7] =?UTF-8?q?=E8=BF=BD=E5=8A=A0=E4=BA=8B=E4=BB=B6?= =?UTF-8?q?=E6=94=B6=E7=B4=A7=E5=87=A0=E5=A4=84=EF=BC=8C=E5=B0=81=E5=AD=98?= =?UTF-8?q?=E5=90=8E=E8=BF=BD=E5=8A=A0=E7=9A=84=E8=B4=A8=E8=AF=A2=E4=B8=8D?= =?UTF-8?q?=E8=AE=A1=E8=BF=9B=E8=BF=90=E8=A1=8C=E7=94=A8=E9=87=8F=E3=80=82?= =?UTF-8?q?=E5=A4=8D=E6=A0=B8=E6=8F=90=E5=87=BA=EF=BC=9A=E5=81=9C=E5=9C=A8?= =?UTF-8?q?=E5=AE=A1=E6=89=B9=E6=97=B6=E8=BF=BD=E5=8A=A0=E5=8F=AA=E7=9C=8B?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E8=BF=98=E5=81=9C=E7=9D=80=EF=BC=8C=E5=85=B3?= =?UTF-8?q?=E4=BA=8E=E7=AC=AC=E4=B8=80=E5=BC=A0=E5=AE=A1=E6=89=B9=E5=8D=A1?= =?UTF-8?q?=E7=9A=84=E8=B4=A8=E8=AF=A2=E9=97=AE=E5=AE=8C=E6=97=B6=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E5=8F=AF=E8=83=BD=E5=B7=B2=E7=BB=8F=E6=89=B9=E8=BF=87?= =?UTF-8?q?=E5=AE=83=E3=80=81=E5=81=9C=E5=88=B0=E4=BA=86=E7=AC=AC=E4=BA=8C?= =?UTF-8?q?=E5=BC=A0=E4=B8=8A=EF=BC=8C=E9=97=AE=E7=AD=94=E7=85=A7=E6=A0=B7?= =?UTF-8?q?=E5=86=99=E8=BF=9B=E5=8E=BB=EF=BC=9B=E6=B2=A1=E6=9C=89=E7=BB=88?= =?UTF-8?q?=E6=80=81=E4=BA=8B=E4=BB=B6=E7=9A=84=E8=80=81=E5=B0=81=E5=AD=98?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E8=BF=BD=E5=8A=A0=E4=B9=8B=E5=90=8E=20verify?= =?UTF-8?q?=20=E7=94=B1=E4=B8=80=E8=87=B4=E5=8F=98=E6=88=90=E4=B8=8D?= =?UTF-8?q?=E4=B8=80=E8=87=B4=EF=BC=9B=E8=BF=94=E5=9B=9E=E7=BB=99=E8=B0=83?= =?UTF-8?q?=E7=94=A8=E6=96=B9=E7=9A=84=E4=BA=8B=E4=BB=B6=20id=20=E5=92=8C?= =?UTF-8?q?=E5=BA=93=E9=87=8C=E7=9A=84=E4=B8=8D=E4=B8=80=E6=A0=B7=EF=BC=9B?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E8=AF=81=E6=98=8E=E4=B8=8D=E4=BA=86=E5=85=88?= =?UTF-8?q?=E8=90=BD=E5=BA=93=E5=86=8D=E5=B9=BF=E6=92=AD=EF=BC=9B=E5=B0=81?= =?UTF-8?q?=E5=AD=98=E5=90=8E=E8=BF=BD=E5=8A=A0=E7=9A=84=E8=B4=A8=E8=AF=A2?= =?UTF-8?q?=E5=B9=B3=E6=97=B6=E4=B8=8D=E8=AE=A1=E7=94=A8=E9=87=8F=EF=BC=8C?= =?UTF-8?q?=E7=BB=A7=E7=BB=AD=E8=BF=90=E8=A1=8C=E4=B9=8B=E5=90=8E=E5=8F=88?= =?UTF-8?q?=E8=A2=AB=E8=AE=A1=E8=BF=9B=E5=8E=BB=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - append_events 加 approval_id(只用于 interrupted):问的是哪张审批卡,那张卡仍待审批才写。核对放在和其余条件同一次条件更新里(EXISTS 待审批的那条),批过了、不是这次运行的卡都不写、返回 None。条件更新写明不同步会话里的运行:条件里带了 EXISTS 时 SQLAlchemy 会悄悄换一种同步办法、给语句加上 RETURNING - manifest_seq 为空的老封存运行,封到哪一条是 verify_manifest 按事件类型推出来的:有终态事件的封到最后一条终态事件,追加在后面不碍事,照旧追加;没有终态事件的封到最后一行,追加一条范围就挪到它身上,我们自己写的批注被当成篡改,现在不追加 - _stage 落库时用事件自己的 id:广播出去的、交回给调用方的和库里的是同一个 id - 运行用量不计数据带 post_seal 的质询:运行用量是这次运行自己花的,结束时就定下了、写进了终态事件;封存后的批注另有自己的账,同按需裁判的判定。继续运行之后它落进新的封存范围,照样不计 测试:两道审批的运行批过第一道、停在第二道,钉住第一道的质询不写、钉住第二道的写,读完运行之后那张卡被批了也不写,返回的 id 和序号与落库的一致;广播时查库,每条都已落库,落空的一条都不广播(把广播挪到条件更新之前这条就变红,以前的测试照样通过);老封存三种(无终态、run.finished、run.failed)追加前后 verify 都一致,无终态的不写;失败封存的运行追加 post_seal 质询后继续运行,运行用量里没有它。去掉审批卡的核对、改回老封存一律追加、改回计 post_seal、去掉落库 id,对应的测试各自变红 --- backend/app/engine/run_append.py | 40 +++++--- backend/app/engine/runner.py | 13 ++- backend/tests/test_inquiry_usage.py | 31 +++++++ backend/tests/test_run_append.py | 139 +++++++++++++++++++++++++++- 4 files changed, 205 insertions(+), 18 deletions(-) diff --git a/backend/app/engine/run_append.py b/backend/app/engine/run_append.py index f7a515ad..70de2784 100644 --- a/backend/app/engine/run_append.py +++ b/backend/app/engine/run_append.py @@ -9,8 +9,8 @@ require 说明往哪种运行里追加: - "interrupted":运行停在审批、还没封存。事件落在以后的封存范围里,随运行结束一起封存。 - "sealed":已经封存的终态运行。事件带 post_seal: true,落在 manifest_seq 之后,不进封存范围,verify 照旧一致。 -没抢到(读完运行之后它被恢复、被放弃、继续运行后重新封存了,或者别处刚往里写过事件)就一条都不写、返回 None, -调用方据此说明「未写入运行记录」。 +没抢到(读完运行之后它被恢复、被放弃、继续运行后重新封存了,问的那张审批卡已经批过了,或者别处刚往里写过 +事件)就一条都不写、返回 None,调用方据此说明「未写入运行记录」。 """ from __future__ import annotations @@ -19,12 +19,12 @@ import contextlib from collections.abc import AsyncIterator, Sequence from typing import Any, Literal -from sqlalchemy import func, select, update +from sqlalchemy import exists, func, select, update from app.core.bus import bus from app.core.events import EventType, RunEventModel from app.db.base import SessionLocal -from app.db.models import Run, RunEvent +from app.db.models import Approval, Run, RunEvent from app.engine.runner import _seq_floor, _stage, run_manager #: 运行的终态。封存过的运行只会停在这几种上 @@ -66,15 +66,21 @@ async def append_events( *, require: Literal["interrupted", "sealed"], seal: tuple[int | None, str | None] | None = None, + approval_id: str | None = None, ) -> list[RunEventModel] | None: - """把 items 依次接在运行记录的最后一条后面,返回写入的事件(seq 严格递增);没写入返回 None。 + """把 items 依次接在运行记录的最后一条后面,返回写入的事件(seq 严格递增,id 和落库的相同);没写入返回 None。 - seal 只用于 "sealed":调用方开始做事时看到的 (manifest_seq, manifest_hash)。做的事要花几秒(问模型), - 这期间继续运行、重新封存了的话,结果依据的已经不是现在的封存,不追加。不传就按这次读到的封存。 + 调用方开始做事时看到的状态可以钉住,做的事要花几秒(问模型),这期间变了就不追加: + - seal 只用于 "sealed":当时的 (manifest_seq, manifest_hash)。继续运行、重新封存了的话,结果依据的已经 + 不是现在的封存。不传就按这次读到的封存。 + - approval_id 只用于 "interrupted":问的是哪张审批卡。那张卡仍待审批才写——批过了,运行可能已经停到了 + 下一张卡上,仍是「停在审批」,关于上一张的问答却会写进这一次停下里。 items 为空时什么都不做,返回空列表。 """ if require not in ("interrupted", "sealed"): raise ValueError(f"require 只能是 interrupted 或 sealed,收到的是 {require!r}") + if approval_id is not None and require != "interrupted": + raise ValueError("approval_id 只用于 interrupted:封存之后没有待审批的卡") # 事件类型先验过:写错了在取号之前就报出来 entries = [(EventType(str(kind)), node_id, dict(data or {})) for kind, node_id, data in items] if not entries: @@ -103,16 +109,21 @@ async def append_events( if run.status != "interrupted": return None guard = (Run.status == "interrupted",) + if approval_id is not None: + guard += (exists().where(Approval.id == approval_id, Approval.run_id == run_id, + Approval.status == "pending"),) last = run.last_seq # 进程重启过的话内存里的序号从 0 起:接在库里最后一条后面。last_seq 按说就是最后一条,_seq_floor # 另取一次实际的最大序号兜底——别处的追加把 last_seq 写小了,也不至于在这里撞号 bus.set_seq(run_id, await _seq_floor(session, run_id, last)) events = [run_manager._event(run_id, kind, node_id=node_id, data=data) for kind, node_id, data in entries] - # 核对和写入是同一行上的条件更新,和恢复、放弃、继续运行(runner._claim / _abandon)抢的是同一行 + # 核对和写入是同一行上的条件更新,和恢复、放弃、继续运行(runner._claim / _abandon)抢的是同一行。 + # 不同步会话里的 run(下面 _stage 会把它的 last_seq 改成同一个值):不写明的话,条件里带了 EXISTS 时 + # SQLAlchemy 会悄悄换一种同步办法、给这条语句加上 RETURNING claimed = await session.execute( update(Run).where(Run.id == run_id, _same(Run.last_seq, last), *guard) - .values(last_seq=events[-1].seq)) + .values(last_seq=events[-1].seq).execution_options(synchronize_session=False)) if claimed.rowcount != 1: # 读完运行之后它变了。烧掉几个序号不要紧:不落库的事件(流式增量)本来就占序号, # 序号只要求递增、不撞号 @@ -135,12 +146,15 @@ async def _current_seal(session: Any, run: Run) -> bool: """运行记录上的封存是不是这次结束之后封的:manifest_seq 盖住了最后一条宣布结束的事件。 继续运行之后再次结束时,终态和终态事件先提交、封存后算。这中间运行记录上还是上一次结束时的封存, - 当成现在的封存去追加,事件会被接下来的封存收进去,post_seal 就成了假话。manifest_seq 为空的是加这一列 - 之前封存的老运行,当时的封存范围按 verify_manifest 的老口径算,这里不拦。 + 当成现在的封存去追加,事件会被接下来的封存收进去,post_seal 就成了假话。 + + manifest_seq 为空的是加这一列之前封存的老运行,封到哪一条是 verify_manifest 按老口径推出来的:有终态事件的 + 封到最后一条终态事件,追加在后面不碍事;没有的封到最后一行,追加一条,范围就挪到它身上,我们自己写的 + 批注被当成篡改。所以老运行要有终态事件才追加。 """ - if run.manifest_seq is None: - return True ended = await session.scalar( select(func.max(RunEvent.seq)).where(RunEvent.run_id == run.id, RunEvent.type.in_([str(t) for t in _ENDINGS]))) + if run.manifest_seq is None: + return ended is not None return ended is None or run.manifest_seq >= ended diff --git a/backend/app/engine/runner.py b/backend/app/engine/runner.py index 15b1c0a6..a1dd51f8 100644 --- a/backend/app/engine/runner.py +++ b/backend/app/engine/runner.py @@ -1263,8 +1263,11 @@ async def _claim( async def _stage(session: Any, event: RunEventModel, run: Run | None = None) -> None: - """把一条事件写进这个 session,随它一起提交。run 已经在 session 里时直接改它的 last_seq。""" - session.add(RunEvent(run_id=event.run_id, seq=event.seq, type=str(event.type), + """把一条事件写进这个 session,随它一起提交。run 已经在 session 里时直接改它的 last_seq。 + + 落库的 id 用事件自己的:广播出去的、交回给调用方的和库里的是同一个 id,回头按 id 找得到。 + """ + session.add(RunEvent(id=event.id, run_id=event.run_id, seq=event.seq, type=str(event.type), node_id=event.node_id, ts=event.ts, data=event.data)) if run is not None: run.last_seq = event.seq @@ -1314,6 +1317,10 @@ async def _usage_of_events(session: Any, run_id: str) -> dict[str, Any]: llm.end 逐次加总:每次模型调用都有一条,不管节点最后成没成。审批质询不在图里执行、不发 llm.end, 它的用量写在 approval.inquiry 事件数据的 usage 里,单独加总放在 inquiry 下(没有质询就没有这个键): llm.end 这本账要和状态里的用量比大小(_spent),质询花的两本里都没有,混进去比会被状态那本盖掉。 + + 封存之后追加的质询(数据带 post_seal)不算:运行用量是这次运行自己花的,结束时就定下了、写进了终态事件; + 事后的批注另有自己的账,同按需裁判的判定。继续运行之后它落进新的封存范围,照样不算——不然同一条质询 + 计不计,全看之后有没有继续运行。 """ rows = await session.execute( select(RunEvent.type, RunEvent.data).where( @@ -1327,7 +1334,7 @@ async def _usage_of_events(session: Any, run_id: str) -> dict[str, Any]: for kind, data in rows: if kind == EventType.APPROVAL_INQUIRY: spent = (data or {}).get("usage") - if isinstance(spent, dict): + if isinstance(spent, dict) and not (data or {}).get("post_seal"): for key in _INQUIRY_KEYS: value = spent.get(key) if isinstance(value, (int, float)) and not isinstance(value, bool): diff --git a/backend/tests/test_inquiry_usage.py b/backend/tests/test_inquiry_usage.py index c29dfd43..d3f28f8e 100644 --- a/backend/tests/test_inquiry_usage.py +++ b/backend/tests/test_inquiry_usage.py @@ -7,6 +7,7 @@ - 运行结束(成功、失败)时:图的用量照旧(成功取状态、其余两本账取大),再加上质询的;明细另放在 inquiry 下 - 停在审批时被放弃:停下时记的用量里还没有这次停下期间的质询,放弃时补上 - 停过两次、两次都问过:每次质询只计一遍 +- 封存之后追加的质询(post_seal)不计进运行用量,继续运行、重新封存之后也不计 - 没有质询的运行,用量和以前逐字相同 """ from __future__ import annotations @@ -213,6 +214,36 @@ async def test_each_inquiry_counts_once_across_two_pauses(): "calls": 3, "unpriced_calls": 1} +async def test_post_seal_inquiries_stay_out_of_the_run_usage(monkeypatch): + """运行用量是这次运行自己花的,结束时就定下了、写进了封存范围里的终态事件。封存之后追加的质询是事后的批注, + 和按需裁判的判定一样另有自己的账,不计进来。以前平时不计,继续运行、重新算用量时又被计了进去:同一条质询 + 计不计,全看之后有没有继续运行。""" + from app.engine.run_append import append_events + + original = compiler.RUNNERS[NodeType.TRANSFORM] + state = {"fail": True} + + async def flaky(st, ctx): + if ctx.node.id == "boom" and state["fail"]: + raise NodeError(ctx.node.id, "数据源暂时不可用") + return await original(st, ctx) + + monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, flaky) + graph = chain(node("start", "input"), node("boom", "transform", mode="template", template="x"), + node("out", "output", fields=[{"name": "r", "value": "x"}])) + run = await run_manager.start(graph=graph, input_payload={}) + failed = await _until(run.id, ("failed",)) + written = await append_events(run.id, [ask("这次为什么失败", A)], require="sealed") + assert written and written[0].data["post_seal"] is True + assert set(failed.usage) == {"duration_ms", "wall_ms", "active_ms", "wait_ms"} + + state["fail"] = False + await run_manager.continue_failed(run.id) + row = await _until(run.id, ("succeeded",)) + assert row.manifest_seq > written[0].seq, "继续运行之后重新封存,这条质询落进了新的封存范围" + assert set(row.usage) == {"duration_ms", "wall_ms", "active_ms", "wait_ms"} + + @pytest.mark.parametrize("asked_before", [False, True], ids=["first-pause", "second-pause"]) async def test_abandoning_a_paused_run_counts_what_was_asked(asked_before): """停在审批时放弃:停下时记的用量里还没有这次停下期间的质询,放弃时补上;上一次停下时已经计过的不再计。""" diff --git a/backend/tests/test_run_append.py b/backend/tests/test_run_append.py index 1052fef4..e73d5f8d 100644 --- a/backend/tests/test_run_append.py +++ b/backend/tests/test_run_append.py @@ -3,8 +3,10 @@ - 停在审批时追加:事件接在最后一条后面、落进以后的封存范围。批准、运行结束封存之后 verify 照旧一致, 改动追加的事件 verify 就对不上——它们确实在封存范围里 - 追加途中运行被恢复(条件更新落空):什么都不写,返回 None,last_seq 不动 +- 质询指明问的是哪张审批卡(approval_id)时,那张卡仍待审批才写;返回的事件和落库的 id、序号一样 +- 先落库再广播,没写入的一条都不广播 - 已封存的运行追加:带 post_seal、落在 manifest_seq 之后,verify 照旧一致;封存不是这次结束之后封的、 - 封存和调用方开始时看到的不同,都不追加 + 封存和调用方开始时看到的不同、没有终态事件的老封存,都不追加 - 连续追加、并发追加,序号严格递增,整条事件流没有重复的序号 - last_seq 落后于库里实际的最后一条(按需裁判的追加和封存后追加交错提交时会这样):进程重启后恢复、 继续运行、放弃、按需裁判都接在实际的最后一条后面,不撞号;runner 记事件不把 last_seq 写小 @@ -26,7 +28,7 @@ from app.core.bus import bus from app.core.config import settings from app.core.events import EventType from app.db.base import SessionLocal -from app.db.models import Run, RunEvent +from app.db.models import Approval, Run, RunEvent from app.engine import compiler from app.engine.context import NodeError from app.engine.runner import RunManager, run_manager, verify_manifest @@ -68,6 +70,10 @@ GATED = chain(node("start", "input"), node("gate", "human", mode="approve", title="这份周报可以出具吗"), node("out", "output", fields=[{"name": "r", "value": "{{ nodes.work }}"}])) SIMPLE = chain(node("start", "input"), node("out", "output", fields=[{"name": "r", "value": "完成"}])) +TWO_GATES = chain(node("start", "input"), + node("g1", "human", mode="approve", title="第一道:口径对吗"), + node("g2", "human", mode="approve", title="第二道:可以出具吗"), + node("out", "output", fields=[{"name": "r", "value": "完成"}])) #: 「boom」节点按 flaky 的开关失败:先让运行失败、封存,再关掉开关继续运行 FLAKY = chain(node("start", "input"), node("boom", "transform", mode="template", template="门店 17 家"), node("out", "output", fields=[{"name": "r", "value": "{{ nodes.boom }}"}])) @@ -148,6 +154,12 @@ async def _no_repeated_seq(run_id: str) -> None: assert len(seqs) == len(set(seqs)), f"事件流里有重复的序号:{seqs}" +async def _pending(run_id: str) -> list[Approval]: + async with SessionLocal() as session: + return list((await session.execute( + select(Approval).where(Approval.run_id == run_id, Approval.status == "pending"))).scalars()) + + def _sql(statement: str, *params) -> None: """另开一条连接改库:模拟别处(恢复、继续运行、另一个写入者)抢在中间提交。""" db = sqlite3.connect(settings.db_path, timeout=5) @@ -285,6 +297,89 @@ async def test_a_run_that_already_moved_on_gets_nothing(executed): assert await _events(paused.id, "approval.inquiry") == [] +async def test_an_inquiry_about_an_approval_needs_that_approval_still_pending(monkeypatch): + """质询问的是某一张审批卡:开始问模型时那张卡还在等,问完时运行可能已经批过它、停到了下一张上。只看 + 「运行停在审批」的话,关于第一张的问答会写进第二次停下里。传 approval_id 时那张卡仍待审批才写, + 和其余条件在同一次条件更新里核对。""" + from app.engine.run_append import append_events + + paused = await _paused(TWO_GATES) + [first] = await _pending(paused.id) + await run_manager.resume(paused.id, {"approved": True}, approval_id=first.id) + await asyncio.sleep(0.05) + later = await _until(paused.id, ("interrupted",)) + [second] = await _pending(paused.id) + assert (first.node_id, second.node_id) == ("g1", "g2") + + assert await append_events(paused.id, [inquiry("第一道的口径")], require="interrupted", + approval_id=first.id) is None + assert await append_events(paused.id, [inquiry("不存在的卡")], require="interrupted", + approval_id="no-such-approval") is None + assert (await _row(paused.id)).last_seq == later.last_seq + assert await _events(paused.id, "approval.inquiry") == [] + + written = await append_events(paused.id, [inquiry("出具前再确认一次")], require="interrupted", + approval_id=second.id) + assert written is not None + # 返回的就是落了库的那几条:调用方按 id 或序号回头找,两边对得上 + stored = await _events(paused.id, "approval.inquiry") + assert [(e.id, e.seq) for e in stored] == [(e.id, e.seq) for e in written] + + # 读完运行、写入之前那张卡被人批了(并行的审批还没凑齐,运行仍停着):不写 + set_seq = bus.set_seq + + def answered_in_between(run_id, value): + if run_id == paused.id: + _sql("UPDATE approvals SET status = 'answered' WHERE id = ?", second.id) + set_seq(run_id, value) + + monkeypatch.setattr(bus, "set_seq", answered_in_between) + assert await append_events(paused.id, [inquiry("批完才问到的")], require="interrupted", + approval_id=second.id) is None + assert (await _row(paused.id)).last_seq == written[-1].seq + assert len(await _events(paused.id, "approval.inquiry")) == 1 + with pytest.raises(ValueError): + await append_events(paused.id, [inquiry("封存后的卡")], require="sealed", approval_id=second.id) + + +async def test_published_only_once_stored_and_never_when_not_written(monkeypatch): + """先落库再广播:在线的客户端收到的每一条库里都已经有了(订阅者靠「先读历史、再接实时」拼完整,见 + RunManager._emit)。没写进运行记录的一条都不广播,否则界面上会多出一条运行记录里没有的问答。""" + from app.engine.run_append import append_events + + published: list[tuple[int, int]] = [] + publish = bus.publish + + async def stored_when_published(event): + if event.type == EventType.APPROVAL_INQUIRY: + db = sqlite3.connect(settings.db_path, timeout=5) + try: + (count,) = db.execute("SELECT count(*) FROM run_events WHERE run_id = ? AND seq = ?", + (event.run_id, event.seq)).fetchone() + finally: + db.close() + published.append((event.seq, count)) + await publish(event) + + monkeypatch.setattr(bus, "publish", stored_when_published) + paused = await _paused() + written = await append_events(paused.id, [inquiry("客单价怎么算的"), inquiry("含不含退款")], + require="interrupted") + assert published == [(e.seq, 1) for e in written] + + published.clear() + set_seq = bus.set_seq + + def resumed_in_between(run_id, value): + if run_id == paused.id: + _sql("UPDATE runs SET status = 'running' WHERE id = ?", paused.id) + set_seq(run_id, value) + + monkeypatch.setattr(bus, "set_seq", resumed_in_between) + assert await append_events(paused.id, [inquiry("恢复之后才到的")], require="interrupted") is None + assert published == [] + + # -------------------------------------------------------------------------- # 封存之后追加 # -------------------------------------------------------------------------- @@ -393,6 +488,46 @@ async def test_a_seal_left_from_an_earlier_ending_is_not_the_seal(monkeypatch): assert (await verify_manifest(first.id))["ok"] is True +async def _legacy_sealed(ending: str | None) -> Run: + """加 manifest_seq 这一列之前封存的老运行:只记了清单哈希,封到哪一条要由 verify_manifest 按事件类型推。""" + from app.core.artifact_store import manifest_hash + + rows = [(1, "run.started", None, {"nodes": 2}), (2, "node.started", "out", {}), + (3, "node.finished", "out", {"output": {"r": "完成"}})] + if ending: + rows.append((4, ending, None, {})) + # 老口径:run.finished 在算哈希之后才落库,不在清单里;run.failed / run.cancelled 在 + covered = rows[:-1] if ending == "run.finished" else rows + async with SessionLocal() as session: + run = Run(workflow_name="老运行·月报", status="failed" if ending == "run.failed" else "succeeded", + graph={}, input={}, last_seq=rows[-1][0], manifest_hash=manifest_hash(covered), manifest_seq=None) + session.add(run) + await session.flush() + for seq, kind, node_id, data in rows: + session.add(RunEvent(run_id=run.id, seq=seq, type=kind, node_id=node_id, ts=float(seq), data=data)) + await session.commit() + return run + + +@pytest.mark.parametrize("ending", [None, "run.finished", "run.failed"], ids=["no-ending", "finished", "failed"]) +async def test_a_legacy_seal_takes_appends_only_when_its_range_does_not_move(ending): + """老封存的范围是推出来的:有终态事件的,封到最后一条终态事件为止,追加在后面不进范围;没有终态事件的, + 封到最后一行——追加一条,范围就跟着挪到它身上,我们自己写的批注被当成篡改。这种不追加。""" + from app.engine.run_append import append_events + + legacy = await _legacy_sealed(ending) + before = await verify_manifest(legacy.id) + assert before["ok"] is True and before["legacy"] is True + written = await append_events(legacy.id, [inquiry("这期为什么少了一家店")], require="sealed") + if ending is None: + assert written is None + assert (await _row(legacy.id)).last_seq == legacy.last_seq + else: + assert written is not None and written[0].data["post_seal"] is True + after = await verify_manifest(legacy.id) + assert after["ok"] is True and (after["sealed_at"], after["events"]) == (before["sealed_at"], before["events"]) + + async def test_the_wrong_kind_of_run_gets_nothing(): from app.engine.run_append import append_events From c8363206bd98879335db53c04a83ff794fde8f51 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 01:18:43 +0800 Subject: [PATCH 7/7] =?UTF-8?q?=E5=8F=91=E8=B5=B7=E6=97=B6=E5=B0=B1?= =?UTF-8?q?=E5=AE=9A=E4=B8=8B=E7=9A=84=E4=BA=8B=E4=BB=B6=E5=92=8C=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E8=AE=B0=E5=BD=95=E5=90=8C=E4=B8=80=E4=B8=AA=E4=BA=8B?= =?UTF-8?q?=E5=8A=A1=E5=86=99=E8=BF=9B=E4=BA=8B=E4=BB=B6=E6=B5=81=EF=BC=9A?= =?UTF-8?q?=E5=8F=97=E7=AE=A1=E7=BA=A7=E5=88=AB=E9=82=A3=E6=9D=A1=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E3=80=81=E7=9B=AE=E5=BD=95=E5=8F=98=E5=8C=96=E3=80=81?= =?UTF-8?q?=E5=8F=A3=E5=BE=84=E5=8D=87=E7=89=88=E5=A4=84=E7=BD=AE=E4=BB=A5?= =?UTF-8?q?=E5=89=8D=E5=9C=A8=20run=5Fmanager.start=20=E8=BF=94=E5=9B=9E?= =?UTF-8?q?=E4=B9=8B=E5=90=8E=E5=8F=A6=E5=A4=96=E5=86=99=EF=BC=8C=E8=BF=99?= =?UTF-8?q?=E6=97=B6=E8=BF=90=E8=A1=8C=E5=B7=B2=E7=BB=8F=E5=BB=BA=E5=A5=BD?= =?UTF-8?q?=E3=80=81=E6=89=A7=E8=A1=8C=E4=BB=BB=E5=8A=A1=E5=B7=B2=E7=BB=8F?= =?UTF-8?q?=E8=B5=B7=E4=BA=86=E3=80=82=E5=85=B6=E4=B8=AD=E4=BB=BB=E4=BD=95?= =?UTF-8?q?=E4=B8=80=E6=AC=A1=E5=86=99=E5=A4=B1=E8=B4=A5=EF=BC=88=E5=BA=93?= =?UTF-8?q?=E8=A2=AB=E9=94=81=E8=B6=85=E8=BF=87=E7=AD=89=E5=BE=85=E6=97=B6?= =?UTF-8?q?=E9=95=BF=E4=B9=8B=E7=B1=BB=EF=BC=89=EF=BC=8Claunch=5Frun=20?= =?UTF-8?q?=E6=8A=9B=E5=87=BA=E6=99=AE=E9=80=9A=E5=BC=82=E5=B8=B8=EF=BC=8C?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E5=8D=B4=E5=9C=A8=E6=89=A7=E8=A1=8C=EF=BC=9B?= =?UTF-8?q?=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1=E6=8C=89=E3=80=8CLaunchRef?= =?UTF-8?q?used=20=E4=BB=A5=E5=A4=96=E7=9A=84=E5=BC=82=E5=B8=B8=E8=AE=B0?= =?UTF-8?q?=E6=9C=AA=E8=83=BD=E5=8F=91=E8=B5=B7=E3=80=8D=E4=BC=9A=E6=8A=8A?= =?UTF-8?q?=E4=B8=80=E6=AC=A1=E6=AD=A3=E5=9C=A8=E6=89=A7=E8=A1=8C=E7=9A=84?= =?UTF-8?q?=E6=AD=A3=E5=BC=8F=E8=BF=90=E8=A1=8C=E8=AE=B0=E6=88=90=E6=9C=AA?= =?UTF-8?q?=E8=83=BD=E5=8F=91=E8=B5=B7=EF=BC=8C=E5=B0=81=E5=AD=98=E8=8C=83?= =?UTF-8?q?=E5=9B=B4=E9=87=8C=E4=B9=9F=E5=B0=91=E4=BA=86=E5=8F=97=E7=AE=A1?= =?UTF-8?q?=E7=BA=A7=E5=88=AB=E9=82=A3=E6=9D=A1=E6=97=A5=E5=BF=97=EF=BC=88?= =?UTF-8?q?governed=5Fformal=20=E6=89=BE=E4=B8=8D=E5=88=B0=E5=AE=83?= =?UTF-8?q?=E5=B0=B1=E9=80=80=E5=9B=9E=E7=9C=8B=E5=B7=A5=E4=BD=9C=E6=B5=81?= =?UTF-8?q?=E7=8E=B0=E5=9C=A8=E7=9A=84=20status=EF=BC=8C=E4=B8=AD=E9=80=94?= =?UTF-8?q?=E6=94=B9=E8=BF=87=E7=94=BB=E5=B8=83=E7=9A=84=E4=BC=9A=E6=8C=89?= =?UTF-8?q?=E5=B7=B2=E5=8F=91=E5=B8=83=E7=BA=A7=E5=88=AB=E5=87=BA=E5=85=B7?= =?UTF-8?q?=EF=BC=89=E3=80=82=E5=8F=A6=E5=A4=96=E8=BF=99=E5=87=A0=E6=9D=A1?= =?UTF-8?q?=E5=92=8C=E6=89=A7=E8=A1=8C=E6=8A=A2=E5=85=88=E5=90=8E=EF=BC=8C?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E7=BB=93=E6=9D=9F=E5=BE=97=E5=BF=AB=E6=97=B6?= =?UTF-8?q?=E5=8F=AF=E8=83=BD=E8=90=BD=E5=88=B0=E5=B0=81=E5=AD=98=E4=B9=8B?= =?UTF-8?q?=E5=90=8E=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - run_manager.start 加 notes:(类型, node_id, 数据) 的列表,先 flush 拿到运行的 id,和运行记录同一个事务提交,提交之后广播,最后才建执行任务。写不进去就不建运行,launch_run 抛出的任何异常都说明运行没有开始执行 - launch_run 在发起前把这几条事件备好交给 start,不再事后用 run_manager.note 写。顺序和以前实际落库的一样(日志、目录变化、口径升版处置,然后 run.started),现在是确定的 - 没有 notes 的发起(探索运行、问数据)和以前完全相同 没有照复核建议的「写失败就记日志、照常返回运行」:那样正式运行会在缺了受管级别记录的情况下照常出具,可能悄悄降成已发布级别。 测试:用触发器让口径升版处置那条写不进去,launch_run 抛出 IntegrityError、库里没有多出运行(旧代码留下一条正在执行的运行,这条变红);服务发起的正式运行里,受管级别日志和口径升版处置排在 run.started 前面,都在封存范围里 --- backend/app/engine/launch.py | 41 ++++++++++++++++++-------------- backend/app/engine/runner.py | 22 ++++++++++++++--- backend/tests/test_run_launch.py | 38 ++++++++++++++++++++++++++--- 3 files changed, 77 insertions(+), 24 deletions(-) diff --git a/backend/app/engine/launch.py b/backend/app/engine/launch.py index 71d7a9aa..087be199 100644 --- a/backend/app/engine/launch.py +++ b/backend/app/engine/launch.py @@ -3,7 +3,8 @@ 以前这些规则全写在 start_run 里、只认 HTTP 请求:定时任务要发起正式运行,要么另抄一份(两份规则迟早 走样,正式运行「只能从当前发布版本发起」这条最要紧的规则就可能在一边被绕过),要么假装成一次 HTTP 请求。 现在两边都调 launch_run。拒绝时抛 LaunchRefused:start_run 把它换回以前的 HTTP 响应(状态码、detail、 -code 逐字不变),调度器据此记一次未能发起。 +code 逐字不变),调度器据此记一次未能发起。别的异常(库出错之类)也说明运行没有开始执行:发起时定下的事件 +(受管级别、目录变化、口径升版处置)和运行记录同一个事务提交,执行任务是最后一步才建的。 发起前检查绑定的工具(missing_tools),以及它和 runner 固定数据版本共用的取绑定的办法(tool_bindings、 node_setting)也放在这里:runner 以前要从接口层导入它们。 @@ -55,7 +56,8 @@ async def launch_run( schedule_id: str | None = None, options: dict[str, Any] | None = None, ) -> Run: - """发起一次运行,返回刚建好的运行记录(执行已经交给 run_manager)。不发起就抛 LaunchRefused。 + """发起一次运行,返回刚建好的运行记录(执行已经交给 run_manager)。不发起就抛 LaunchRefused;抛别的异常时 + 运行也没有开始执行。 参数和 POST /api/runs 的请求体一一对应(input 叫 input_payload);actor 是记在运行上的发起人。 schedule_id 是发起它的定时任务,options 是整次运行沿用的选项快照,都原样交给 run_manager.start。 @@ -137,6 +139,24 @@ async def launch_run( from app.api.settings import resolve_run_scope memory_scope, collection = await resolve_run_scope(session, memory_scope, collection) + # 发起时就定下的事件交给 run_manager.start,和运行记录同一个事务写进事件流(在封存范围内、排在 run.started + # 前面)。以前运行建好、开始执行之后才另外写:写失败时运行已经在执行,这里却抛出异常,定时任务会把一次 + # 正在执行的正式运行记成未能发起 + notes: list[tuple[EventType | str, str | None, dict[str, Any]]] = [] + if formal: + # 按不按受管出具在发起这一刻定下来:status 反映的是当前画布,中途改图、审批停下再恢复时再去读它, + # 报告节点和出口复核可能各读到不同的值 + notes.append(("log", None, governance_note(governed=governed, workflow_name=name, version=run_version))) + if catalog_changes: + # 之后回看这次正式出具,能知道当时的目录和发布时不一样、差在哪几张表。每张表带 impact:direct 写死的 + # SQL 用到的,possible Agent 可能查询的(界面分开说) + notes.append((EventType.CATALOG_DRIFT, None, + {"version": run_version, "count": len(catalog_changes), "tables": catalog_changes})) + # 已声明策略的升版记入事件流,出具物上能看到"这期换口径了、怎么处置的" + for ev in upgrade_events: + payload_ev = dict(ev) + node = payload_ev.pop("node_id", None) + notes.append(("caliber.upgrade", node, payload_ev)) try: run = await run_manager.start( graph=graph, @@ -151,25 +171,10 @@ async def launch_run( started_by=actor, options=options, schedule_id=schedule_id, + notes=notes, ) except ValueError as e: raise LaunchRefused(400, str(e)) from e - - if formal: - # 按不按受管出具在发起这一刻定下来、记进封存范围:status 反映的是当前画布,中途改图、 - # 审批停下再恢复时再去读它,报告节点和出口复核可能各读到不同的值 - await run_manager.note(run.id, "log", **governance_note( - governed=governed, workflow_name=name, version=run_version)) - if catalog_changes: - # 落在封存范围内:之后回看这次正式出具,能知道当时的目录和发布时不一样、差在哪几张表。 - # 每张表带 impact:direct 写死的 SQL 用到的,possible Agent 可能查询的(界面分开说) - await run_manager.note(run.id, EventType.CATALOG_DRIFT, version=run_version, count=len(catalog_changes), - tables=catalog_changes) - # 已声明策略的升版记入事件流,出具物上能看到"这期换口径了、怎么处置的" - for ev in upgrade_events: - payload_ev = dict(ev) - node = payload_ev.pop("node_id", None) - await run_manager.note(run.id, "caliber.upgrade", node_id=node, **payload_ev) return run diff --git a/backend/app/engine/runner.py b/backend/app/engine/runner.py index a1dd51f8..4677078b 100644 --- a/backend/app/engine/runner.py +++ b/backend/app/engine/runner.py @@ -6,7 +6,7 @@ import copy import json import logging import time -from collections.abc import Callable +from collections.abc import Callable, Sequence from datetime import datetime, timezone from typing import Any @@ -172,7 +172,7 @@ class RunManager: self, run_id: str, event_type: EventType | str, *, node_id: str | None = None, **data: Any, ) -> None: - """给 API 层用的事件入口(比如 formal 启动时记录口径升版处置)。""" + """给运行之外的代码往运行里记一条事件。发起时就定下的事件不走这里,交给 start 的 notes。""" await self._emit(run_id, event_type, node_id=node_id, data=data) # ---------------- 启动 ---------------- @@ -193,8 +193,14 @@ class RunManager: approval_default: str | None = None, options: dict[str, Any] | None = None, schedule_id: str | None = None, + notes: Sequence[tuple[EventType | str, str | None, dict[str, Any]]] = (), ) -> Run: - """建运行记录、交给执行任务。options 是这次运行的选项快照,schedule_id 是发起它的定时任务(可空)。""" + """建运行记录、交给执行任务。options 是这次运行的选项快照,schedule_id 是发起它的定时任务(可空)。 + + notes 是发起时就定下的事件 (类型, node_id, 数据),比如正式运行按什么级别出具、口径升版怎么处置。它们和 + 运行记录同一个事务提交,排在 run.started 前面:写不进去就不建运行,不会出现运行已经在执行、调用方却 + 只拿到一个异常(定时任务会把它记成未能发起);也不会和执行抢先后,运行结束得快时落到封存之后。 + """ spec = GraphSpec.model_validate(graph) report = validate_graph(spec) if not report.ok: @@ -238,8 +244,18 @@ class RunManager: ) run.thread_id = run.id # 一个 run 一条 checkpoint 线程 session.add(run) + staged: list[RunEventModel] = [] + if notes: + await session.flush() # 事件要带上运行的 id + staged = [self._event(run.id, kind, node_id=node_id, data=data) + for kind, node_id, data in notes] + for event in staged: + await _stage(session, event, run) await session.commit() await session.refresh(run) + # 先落库再广播,理由同 _emit;执行任务还没建,run.started 必定排在它们后面 + for event in staged: + await bus.publish(event) self._tasks[run.id] = asyncio.create_task( self._drive( diff --git a/backend/tests/test_run_launch.py b/backend/tests/test_run_launch.py index 4fb698a1..edd38cf0 100644 --- a/backend/tests/test_run_launch.py +++ b/backend/tests/test_run_launch.py @@ -2,7 +2,8 @@ - 每一种拒绝:POST /api/runs 的状态码、detail、code 先按搬家之前的原话写死(这一组先在旧代码上通过); 直接调 launch_run 抛 LaunchRefused,和 HTTP 的返回逐条一致 -- 发起之后的治理事件(受管级别那条日志、口径升版处置)在服务里写:不经 HTTP 发起的正式运行一样有 +- 发起时定下的治理事件(受管级别那条日志、口径升版处置)在服务里写:不经 HTTP 发起的正式运行一样有; + 它们和运行记录同一个事务提交,排在 run.started 前面、在封存范围里,写不进去就不建运行 - 旧的导入路径(app.api.runs 的 missing_tools、node_setting、tool_bindings、TOOL_MISSING)照旧可用; runner 不再从接口层导入 """ @@ -299,10 +300,41 @@ async def test_a_formal_launch_records_the_governance_notes(): assert run.version_hash and run.input == {"week": "第 40 周"} row = await _settle(run.id) assert row.status == "succeeded", row.error - notes = [e.data for e in await _events(run.id, "log") if e.data.get("code") == "run_governance"] - assert len(notes) == 1 and notes[0]["governed"] is False and "v1" in notes[0]["message"] + notes = [e for e in await _events(run.id, "log") if e.data.get("code") == "run_governance"] + assert len(notes) == 1 and notes[0].data["governed"] is False and "v1" in notes[0].data["message"] [upgrade] = await _events(run.id, "caliber.upgrade") assert upgrade.node_id == "card" and upgrade.data["policy"] == "dual" and upgrade.data["latest"] == 2 + # 发起时就有的事件排在 run.started 前面,都在封存范围里 + [started] = await _events(run.id, "run.started") + assert notes[0].seq < upgrade.seq < started.seq <= row.manifest_seq + + +async def test_launch_events_are_written_with_the_run_or_not_at_all(): + """发起时定下的事件和运行记录同一个事务提交:写不进去就不建运行,launch_run 抛出异常就是没有发起。以前运行 + 建好、开始执行之后才另外写,写失败时运行已经在执行,调用方(定时任务)只拿到一个异常,会把一次正在执行的 + 正式运行记成未能发起,封存范围里也少了受管级别那条日志。""" + from sqlalchemy import text + from sqlalchemy.exc import IntegrityError + + from app.db.base import engine + from app.engine.launch import launch_run + + lib = await upstream_with_two_versions() + wf, _ = await workflow(weekly(lib, upgrade_policy="dual"), published=1, versions=(1,)) + before = await _run_count() + # 口径升版处置那条写不进去。走异步引擎建删触发器:同步连接等写锁时会卡住事件循环,而持锁的正是这个循环里的写入 + async with engine.begin() as conn: + await conn.execute(text( + "CREATE TRIGGER refuse_upgrade_note BEFORE INSERT ON run_events WHEN NEW.type = 'caliber.upgrade' " + "BEGIN SELECT RAISE(ABORT, 'refused by test'); END")) + try: + async with SessionLocal() as session: + with pytest.raises(IntegrityError): + await launch_run(session, workflow_id=wf, run_class="formal", actor="定时任务") + assert await _run_count() == before, "没写进治理事件,却留下了一条运行" + finally: + async with engine.begin() as conn: + await conn.execute(text("DROP TRIGGER IF EXISTS refuse_upgrade_note")) async def test_an_exploratory_launch_takes_the_saved_graph_and_the_run_defaults():