合入期 0 运行时底座:发起运行的规则抽成 launch_run,运行记下选项快照和定时来源,往停下或已封存的运行里追加事件统一走 append_events,质询用量计进运行用量

This commit is contained in:
ThearchyHelios
2026-10-05 02:52:16 +08:00
11 changed files with 2311 additions and 237 deletions
+28 -220
View File
@@ -2,7 +2,6 @@ from __future__ import annotations
import asyncio
import contextlib
import logging
from datetime import datetime
from typing import Any
from urllib.parse import unquote
@@ -12,24 +11,17 @@ from pydantic import BaseModel, Field
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select as _select
from app.api.coded import CodedHTTPException
from app.core.bus import bus
from app.core import artifact_store
from app.core.events import EventType
from app.data.catalog_impact import catalog_drift
from app.db.base import SessionLocal, get_session
from app.db.models import Approval, Artifact, Run, RunEvent, Workflow, WorkflowVersion
from app.engine.governance import governance_note, unresolved_caliber_upgrades
from app.db.models import Approval, Artifact, Run, RunEvent
# 发起规则和发起前检查绑定的工具搬到了 engine/launch.py。这几个名字以前从这里导入,照旧可用
from app.engine.launch import ( # noqa: F401
TOOL_MISSING, LaunchRefused, launch_run, missing_tools, node_setting, tool_bindings,
)
from app.engine.runner import run_manager
from app.engine.schema import GraphSpec, NodeType, type_label
router = APIRouter(prefix="/api/runs", tags=["runs"])
logger = logging.getLogger(__name__)
#: 发起时发现绑定的工具在本机不存在(见 missing_tools)
TOOL_MISSING = "run_tool_missing"
def actor_of(header: str | None) -> str | None:
@@ -76,6 +68,8 @@ class RunOut(BaseModel):
#: 发起时实际生效的记忆域和知识库
memory_scope: str | None = None
collection: str | None = None
#: 发起它的定时任务。为空 = 手动发起
schedule_id: str | None = None
# 时间一律带时区(db.base.UTCDateTime 读出来就是 UTC aware)
created_at: Any = None
started_at: Any = None
@@ -90,218 +84,29 @@ async def start_run(
session: AsyncSession = Depends(get_session),
x_actor: str | None = Header(default=None),
) -> Run:
actor = actor_of(x_actor)
# 没挂在工作流上(画布没保存就跑、问数据当场搭的)。老数据里是「临时图」,前端两种都认
name = "未保存的工作流"
version: int | None = None
version_hash: str | None = None
governed = False
if payload.run_class == "formal":
# 正式运行的全部语义就这一条规则:必须引用一个不可变的已发布版本。
# 传裸 graph、或工作流还没发布过,都进不了 formal。
if not payload.workflow_id:
raise HTTPException(400, "正式运行需要从已保存的工作流发起:请求中没有指定工作流")
if payload.graph is not None:
raise HTTPException(400, "正式运行只能使用已发布的版本,画布上未发布的修改不会生效。请先保存并发布")
workflow = await session.get(Workflow, payload.workflow_id)
if not workflow:
raise HTTPException(404, "工作流不存在,可能已被删除")
# 只看 published_version:status 反映的是当前画布(改过图就退回 draft),
# 而 formal 跑的是已发布的那一版,两者本来就可以不一致
if not workflow.published_version:
raise HTTPException(409, f"「{workflow.name}」还没有发布版本,请先在编排页发布")
version = payload.version or workflow.published_version
# "不可变"和"过过闸"是两件事。快照确实不可变,但 PATCH 保存出来的
# 草稿版本同样有快照——只认 version 存不存在的话,工作流只要发布过
# 任意一个版本,之后所有草稿都能以 formal 身份启动,发布期的
# validate_graph + lint_for_publish 全部绕过。
if version != workflow.published_version:
raise HTTPException(
409,
f"v{version} 不是当前发布版本(v{workflow.published_version})。"
"正式运行只能使用当前发布版本;如需运行这一版,请先发布它。",
)
snapshot = (
await session.execute(
_select(WorkflowVersion).where(
WorkflowVersion.workflow_id == workflow.id,
WorkflowVersion.version == version,
)
)
).scalar_one_or_none()
if not snapshot:
raise HTTPException(404, f"版本 v{version} 不存在")
graph = snapshot.graph
version_hash = snapshot.graph_hash or artifact_store.graph_hash(graph)
name = workflow.name
# 按这一版发布时记下的级别;记级别之前发布的老版本没有,照旧看工作流现在的 status
governed = (snapshot.level == "governed") if snapshot.level else workflow.status == "governed"
# 口径升版是被迫处置的事件:钉住的方法卡有新版本而未声明策略,拒绝启动
spec = GraphSpec.model_validate(graph)
errors, upgrade_events = await unresolved_caliber_upgrades(session, spec)
if errors:
raise HTTPException(409, ";".join(errors))
# 数据目录和口径卡不同:目录是给人和助手看的说明,不是运行的输入。发布之后变了只提醒、不拦——
# 拦下来等于让一次文字修订卡住正式出具
catalog_changes = await catalog_drift(session, snapshot.catalog_versions)
else:
graph = payload.graph
upgrade_events = []
catalog_changes = []
if payload.workflow_id:
workflow = await session.get(Workflow, payload.workflow_id)
if not workflow:
raise HTTPException(404, "工作流不存在,可能已被删除")
graph = graph or workflow.graph
name = workflow.name
if not graph:
raise HTTPException(400, "没有要运行的内容:请求中既没有工作流 ID,也没有工作流内容")
await _refuse_missing_tools(session, graph)
from app.api.settings import resolve_run_scope
memory_scope, collection = await resolve_run_scope(
session, payload.memory_scope, payload.collection
)
"""发起规则全在 engine/launch.py(定时任务到点发起也用它),这里只把拒绝换回 HTTP 响应。"""
try:
run = await run_manager.start(
graph=graph,
input_payload=payload.input,
return await launch_run(
session,
workflow_id=payload.workflow_id,
workflow_name=name,
memory_scope=memory_scope,
collection=collection,
run_class="formal" if payload.run_class == "formal" else "exploratory",
version=version,
version_hash=version_hash,
started_by=actor,
graph=payload.graph,
input_payload=payload.input,
memory_scope=payload.memory_scope,
collection=payload.collection,
run_class=payload.run_class,
version=payload.version,
actor=actor_of(x_actor),
)
except ValueError as e:
raise HTTPException(400, str(e)) from e
if payload.run_class == "formal":
# 按不按受管出具在发起这一刻定下来、记进封存范围:status 反映的是当前画布,中途改图、
# 审批停下再恢复时再去读它,报告节点和出口复核可能各读到不同的值
await run_manager.note(run.id, "log", **governance_note(
governed=governed, workflow_name=name, version=version))
if catalog_changes:
# 落在封存范围内:之后回看这次正式出具,能知道当时的目录和发布时不一样、差在哪几张表。
# 每张表带 impact:direct 写死的 SQL 用到的,possible Agent 可能查询的(界面分开说)
await run_manager.note(run.id, EventType.CATALOG_DRIFT, version=version, count=len(catalog_changes),
tables=catalog_changes)
# 已声明策略的升版记入事件流,出具物上能看到"这期换口径了、怎么处置的"
for ev in upgrade_events:
payload_ev = dict(ev)
node = payload_ev.pop("node_id", None)
await run_manager.note(run.id, "caliber.upgrade", node_id=node, **payload_ev)
return run
except LaunchRefused as e:
raise _refused(e) from e
# --------------------------------------------------------------------------
# 发起前检查绑定的工具
# --------------------------------------------------------------------------
def node_setting(spec: GraphSpec, node: Any, key: str) -> Any:
"""节点配置的实际取值,和运行时 NodeContext.cfg 同一个规则:节点上没写(None、空串)就取图级 defaults。"""
value = node.config.get(key)
return spec.defaults.get(key) if value in (None, "") else value
def tool_bindings(spec: GraphSpec) -> list[tuple[str, str]]:
"""图里每一处绑定的工具:(谁绑的, 工具名)。调用工具节点的 tool、Agent 的 tools、协作成员的 tools。
取值和运行时一致(node_setting):Agent 节点上没写 tools、协作节点上没写成员,运行时用的是
图级 defaults 里的,这里也算它们——不然发起前检查放过了缺的工具,发起时固定数据版本也漏了源。
模板拼出来的名字({{ vars.tool }})也在内:发起前检查(_bindings)跳过它们,发起时固定数据
版本(engine/runner.py)要知道有没有这种名字——有的话说不准会用到哪个数据源。
"""
out: list[tuple[str, str]] = []
for node in spec.nodes:
who = f"「{node.title}」({type_label(node.type)})"
if node.type == NodeType.TOOL:
names = [node_setting(spec, node, "tool")]
out.extend((who, n) for n in names if isinstance(n, str) and n.strip())
elif node.type == NodeType.AGENT:
out.extend((who, n) for n in node_setting(spec, node, "tools") or []
if isinstance(n, str) and n.strip())
elif node.type == NodeType.SUPERVISOR:
for member in node_setting(spec, node, "agents") or []:
if not isinstance(member, dict):
continue
label = f"「{node.title}」的成员「{member.get('name') or '?'}」"
out.extend((label, n) for n in member.get("tools") or [] if isinstance(n, str) and n.strip())
return [(who, name.strip()) for who, name in out]
def _bindings(spec: GraphSpec) -> list[tuple[str, str]]:
"""发起前要检查的绑定。模板拼出来的工具名({{ vars.tool }})发起时还不知道是谁,交给运行时。"""
return [(who, name) for who, name in tool_bindings(spec) if "{{" not in name]
async def missing_tools(session: AsyncSession, spec: GraphSpec) -> list[tuple[str, str, str]]:
"""本机不存在的绑定:(谁绑的, 工具名, 去哪接入:data / tools)。
解析和运行时 build_tools 一致:内置(静态注册)、mcp:<服务>/<工具>、db_query__ / db_schema__
<数据源>、其余按自定义工具。数据源和自定义工具只认启用着的——停用的运行时同样拿不到。
MCP 只看服务登记着、启用着没有:连不连得上、工具还在不在,要连一次才知道,发起时不去等它,
上次没连上或者工具清单里没有的只在日志里提一句。
"""
from app.db.models import CustomTool, DataSource, McpServer
from app.tools.datasource import QUERY_PREFIX, SCHEMA_PREFIX
from app.tools.registry import all_specs
bindings = _bindings(spec)
if not bindings:
return []
builtin = all_specs()
sources = {n for (n,) in await session.execute(select(DataSource.name).where(DataSource.enabled.is_(True)))}
custom = {n for (n,) in await session.execute(select(CustomTool.name).where(CustomTool.enabled.is_(True)))}
servers = {r.name: r for r in (await session.execute(select(McpServer))).scalars()}
missing: list[tuple[str, str, str]] = []
for who, name in bindings:
if name.startswith("mcp:"):
server, _, tool = name[4:].partition("/")
row = servers.get(server)
if row is None or not row.enabled:
missing.append((who, name, "tools"))
elif row.status == "error":
logger.warning("%s 绑的 %s:MCP 服务 %s 上次没连上,运行时可能拿不到这个工具", who, name, server)
elif tool not in ("", "*") and row.tools_cache and tool not in row.tools_cache:
logger.warning("%s 绑的 %s:MCP 服务 %s 上次交回的工具清单里没有 %s", who, name, server, tool)
elif name in builtin:
continue
elif name.startswith((QUERY_PREFIX, SCHEMA_PREFIX)):
source = name[len(QUERY_PREFIX):] if name.startswith(QUERY_PREFIX) else name[len(SCHEMA_PREFIX):]
if source not in sources:
missing.append((who, name, "data"))
elif name not in custom:
missing.append((who, name, "tools"))
return missing
async def _refuse_missing_tools(session: AsyncSession, graph: dict[str, Any]) -> None:
"""绑定的工具在本机不存在就不发起:以前要跑到那一步才失败,前面的步骤白跑、钱白花。"""
try:
spec = GraphSpec.model_validate(graph)
except Exception: # noqa: BLE001 - 图本身不成形交给 run_manager.start 报
return
missing = await missing_tools(session, spec)
if not missing:
return
lines = ";".join(f"{who}绑定的 {name}" for who, name, _ in missing[:8])
more = f"等 {len(missing)} 处" if len(missing) > 8 else ""
places = {where for _, _, where in missing}
if places == {"data"}:
fix = "请到「数据」页接入,或在节点中重新选择"
elif places == {"tools"}:
fix = "请到「工具」页接入,或在节点中重新选择"
else:
fix = "数据源工具请到「数据」页接入,其他工具请到「工具」页接入,或在节点中重新选择"
raise CodedHTTPException(422, f"绑定的工具不存在:{lines}{more}。{fix}", TOOL_MISSING)
def _refused(e: LaunchRefused) -> HTTPException:
"""发起被拒的 HTTP 响应,和规则搬去 engine/launch.py 之前逐字相同:带机器码的回 {detail, code},
不带的回 {detail}。"""
if e.code:
return CodedHTTPException(e.status_code, e.message, e.code)
return HTTPException(e.status_code, e.message)
#: workflow_id 取这个值时只要没挂在任何工作流上的运行:画布上没保存就跑的临时图,
@@ -315,6 +120,7 @@ async def list_runs(
status: str | None = Query(default=None, description="逗号分隔,如 failed,cancelled"),
run_class: str | None = None,
q: str | None = Query(default=None, description="按工作流名模糊匹配"),
schedule_id: str | None = Query(default=None, description="只要这个定时任务发起的运行"),
limit: int = Query(default=50, ge=1, le=200),
before: datetime | None = Query(default=None, description="翻页游标:只要 created_at 早于它的"),
session: AsyncSession = Depends(get_session),
@@ -335,6 +141,8 @@ async def list_runs(
stmt = stmt.where(Run.run_class.in_(classes))
if q and q.strip():
stmt = stmt.where(Run.workflow_name.ilike(f"%{_like(q.strip())}%", escape="\\"))
if schedule_id:
stmt = stmt.where(Run.schedule_id == schedule_id)
if before is not None:
stmt = stmt.where(Run.created_at < before)
return list((await session.execute(stmt)).scalars())
+4
View File
@@ -65,6 +65,10 @@ class EventType(StrEnum):
# 人工介入
HUMAN_REQUESTED = "human.requested"
HUMAN_RESOLVED = "human.resolved"
# 审批质询:运行停在审批时,审批人就待审的报告问的一个问题和得到的回答(engine/run_append.py 追加,
# node_id 为空)。这次问答的用量写在事件数据的 usage 里,计进运行用量(runner._usage_of_events)。
# 停在审批时追加的随运行结束一起封存;封存之后追加的带 post_seal: true,是批注不是证据
APPROVAL_INQUIRY = "approval.inquiry"
# 出具与口径治理
ISSUANCE = "issuance" # 出具契约的判定结果(档位 / 缺数据 / 未回指数字)
+15
View File
@@ -200,6 +200,17 @@ _COLUMN_MIGRATIONS: list[tuple[str, str, str]] = [
("import_stagings", "edits", "JSON"),
("import_stagings", "answers_dropped", "JSON"),
("import_stagings", "redraft_sha256", "VARCHAR(64)"),
# 运行选项(发起时的快照,整次运行沿用)和发起它的定时任务。存量运行回填 {} 和 NULL:
# 没有选项,也不是定时任务发起的
("runs", "options", "JSON DEFAULT '{}'"),
("runs", "schedule_id", "VARCHAR(32)"),
]
#: 迁移补上的列要建的索引:(索引名, 表, 列)。create_all 不碰已有的表,表上新列的索引也就不会建。
#: 索引名照 NAMING_CONVENTION 取,和新库 create_all 建出来的同名,IF NOT EXISTS 才不会重复建
_INDEX_MIGRATIONS: list[tuple[str, str, str]] = [
# 定时任务的触发历史按它找这个任务发起的运行
("ix_runs_schedule_id", "runs", "schedule_id"),
]
@@ -210,6 +221,10 @@ def _migrate(conn) -> None: # pragma: no cover - 同步回调
cols = {row[1] for row in conn.execute(text(f"PRAGMA table_info({table})"))}
if cols and column not in cols:
conn.execute(text(f"ALTER TABLE {table} ADD COLUMN {column} {ddl}"))
for name, table, column in _INDEX_MIGRATIONS:
cols = {row[1] for row in conn.execute(text(f"PRAGMA table_info({table})"))}
if column in cols:
conn.execute(text(f"CREATE INDEX IF NOT EXISTS {name} ON {table} ({column})"))
async def init_db() -> None:
+5
View File
@@ -491,6 +491,11 @@ class Run(Base, TimestampMixin):
# 发起时钉住的上传表格版本:{source_id: {"snapshot": 快照 id, "name": 源名}}。续跑、恢复沿用,
# 回收(table_versions.gc)不删任何运行引用的快照。None 是升级前发起的运行
data_versions: Mapped[dict[str, Any] | None] = mapped_column(default=None)
# 发起时定下、整次运行沿用的运行选项(比如核查设置的快照)。审批停下期间改了设置,恢复和重放
# 仍按发起时的这份——同 approval_default 的道理。非空时写进 run.started,进封存范围。存量运行是 {}
options: Mapped[dict[str, Any]] = mapped_column(default=dict)
# 发起它的定时任务,手动发起的为空。不设外键:定时任务删掉之后,运行记录照旧是追溯凭证,来源 id 留着
schedule_id: Mapped[str | None] = mapped_column(String(32), default=None, index=True)
events: Mapped[list["RunEvent"]] = relationship(
back_populates="run", cascade="all, delete-orphan"
+283
View File
@@ -0,0 +1,283 @@
"""发起一次运行:HTTP 发起(api/runs.py 的 start_run)和定时任务到点发起共用的那套规则。
以前这些规则全写在 start_run 里、只认 HTTP 请求:定时任务要发起正式运行,要么另抄一份(两份规则迟早
走样,正式运行「只能从当前发布版本发起」这条最要紧的规则就可能在一边被绕过),要么假装成一次 HTTP 请求。
现在两边都调 launch_run。拒绝时抛 LaunchRefused:start_run 把它换回以前的 HTTP 响应(状态码、detail、
code 逐字不变),调度器据此记一次未能发起。别的异常(库出错之类)也说明运行没有开始执行:发起时定下的事件
(受管级别、目录变化、口径升版处置)和运行记录同一个事务提交,执行任务是最后一步才建的。
发起前检查绑定的工具(missing_tools),以及它和 runner 固定数据版本共用的取绑定的办法(tool_bindings、
node_setting)也放在这里:runner 以前要从接口层导入它们。
"""
from __future__ import annotations
import logging
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core import artifact_store
from app.core.events import EventType
from app.data.catalog_impact import catalog_drift
from app.db.models import Run, Workflow, WorkflowVersion
from app.engine.governance import governance_note, unresolved_caliber_upgrades
from app.engine.runner import run_manager
from app.engine.schema import GraphSpec, NodeType, type_label
logger = logging.getLogger(__name__)
#: 发起时发现绑定的工具在本机不存在(见 missing_tools)
TOOL_MISSING = "run_tool_missing"
class LaunchRefused(Exception):
"""不发起这次运行。status_code 和 message 就是 HTTP 发起时回的状态码和 detail;code 是给程序认的机器码,
没有的是 None(HTTP 响应里也就没有 code 字段)。"""
def __init__(self, status_code: int, message: str, code: str | None = None) -> None:
super().__init__(message)
self.status_code = status_code
self.message = message
self.code = code
async def launch_run(
session: AsyncSession,
*,
workflow_id: str | None = None,
graph: dict[str, Any] | None = None,
input_payload: dict[str, Any] | None = None,
memory_scope: str | None = None,
collection: str | None = None,
run_class: str = "exploratory",
version: int | None = None,
actor: str | None,
schedule_id: str | None = None,
options: dict[str, Any] | None = None,
) -> Run:
"""发起一次运行,返回刚建好的运行记录(执行已经交给 run_manager)。不发起就抛 LaunchRefused;抛别的异常时
运行也没有开始执行。
参数和 POST /api/runs 的请求体一一对应(input 叫 input_payload);actor 是记在运行上的发起人。
schedule_id 是发起它的定时任务,options 是整次运行沿用的选项快照,都原样交给 run_manager.start。
"""
# 没挂在工作流上(画布没保存就发起、问数据当场搭的)。老数据里是「临时图」,前端两种都认
name = "未保存的工作流"
run_version: int | None = None
version_hash: str | None = None
governed = False
formal = run_class == "formal"
if formal:
# 正式运行的全部语义就这一条规则:必须引用一个不可变的已发布版本。
# 传裸 graph、或工作流还没发布过,都进不了 formal。
if not workflow_id:
raise LaunchRefused(400, "正式运行需要从已保存的工作流发起:请求中没有指定工作流")
if graph is not None:
raise LaunchRefused(400, "正式运行只能使用已发布的版本,画布上未发布的修改不会生效。请先保存并发布")
workflow = await session.get(Workflow, workflow_id)
if not workflow:
raise LaunchRefused(404, "工作流不存在,可能已被删除")
# 只看 published_version:status 反映的是当前画布(改过图就退回 draft),
# 而 formal 用的是已发布的那一版,两者本来就可以不一致
if not workflow.published_version:
raise LaunchRefused(409, f"「{workflow.name}」还没有发布版本,请先在编排页发布")
run_version = version or workflow.published_version
# "不可变"和"过过闸"是两件事。快照确实不可变,但 PATCH 保存出来的
# 草稿版本同样有快照——只认 version 存不存在的话,工作流只要发布过
# 任意一个版本,之后所有草稿都能以 formal 身份启动,发布期的
# validate_graph + lint_for_publish 全部绕过。
if run_version != workflow.published_version:
raise LaunchRefused(
409,
f"v{run_version} 不是当前发布版本(v{workflow.published_version})。"
"正式运行只能使用当前发布版本;如需运行这一版,请先发布它。",
)
snapshot = (
await session.execute(
select(WorkflowVersion).where(
WorkflowVersion.workflow_id == workflow.id,
WorkflowVersion.version == run_version,
)
)
).scalar_one_or_none()
if not snapshot:
raise LaunchRefused(404, f"版本 v{run_version} 不存在")
graph = snapshot.graph
version_hash = snapshot.graph_hash or artifact_store.graph_hash(graph)
name = workflow.name
# 按这一版发布时记下的级别;记级别之前发布的老版本没有,照旧看工作流现在的 status
governed = (snapshot.level == "governed") if snapshot.level else workflow.status == "governed"
try:
spec = GraphSpec.model_validate(graph)
except ValueError as e:
# 快照里的图不成形。以前这里没接,由 main.py 的 ValueError 兜底回 400、detail 是校验原话;
# 原样换成拒绝,HTTP 响应不变,调度器也拿得到一个明确的拒绝
raise LaunchRefused(400, str(e)) from e
# 口径升版是被迫处置的事件:引用的方法卡有新版本而未声明策略,拒绝启动
errors, upgrade_events = await unresolved_caliber_upgrades(session, spec)
if errors:
raise LaunchRefused(409, ";".join(errors))
# 数据目录和口径卡不同:目录是给人和助手看的说明,不是运行的输入。发布之后变了只提醒、不拦——
# 拦下来等于让一次文字修订卡住正式出具
catalog_changes = await catalog_drift(session, snapshot.catalog_versions)
else:
upgrade_events = []
catalog_changes = []
if workflow_id:
workflow = await session.get(Workflow, workflow_id)
if not workflow:
raise LaunchRefused(404, "工作流不存在,可能已被删除")
graph = graph or workflow.graph
name = workflow.name
if not graph:
raise LaunchRefused(400, "没有要运行的内容:请求中既没有工作流 ID,也没有工作流内容")
await _refuse_missing_tools(session, graph)
from app.api.settings import resolve_run_scope
memory_scope, collection = await resolve_run_scope(session, memory_scope, collection)
# 发起时就定下的事件交给 run_manager.start,和运行记录同一个事务写进事件流(在封存范围内、排在 run.started
# 前面)。以前运行建好、开始执行之后才另外写:写失败时运行已经在执行,这里却抛出异常,定时任务会把一次
# 正在执行的正式运行记成未能发起
notes: list[tuple[EventType | str, str | None, dict[str, Any]]] = []
if formal:
# 按不按受管出具在发起这一刻定下来:status 反映的是当前画布,中途改图、审批停下再恢复时再去读它,
# 报告节点和出口复核可能各读到不同的值
notes.append(("log", None, governance_note(governed=governed, workflow_name=name, version=run_version)))
if catalog_changes:
# 之后回看这次正式出具,能知道当时的目录和发布时不一样、差在哪几张表。每张表带 impact:direct 写死的
# SQL 用到的,possible Agent 可能查询的(界面分开说)
notes.append((EventType.CATALOG_DRIFT, None,
{"version": run_version, "count": len(catalog_changes), "tables": catalog_changes}))
# 已声明策略的升版记入事件流,出具物上能看到"这期换口径了、怎么处置的"
for ev in upgrade_events:
payload_ev = dict(ev)
node = payload_ev.pop("node_id", None)
notes.append(("caliber.upgrade", node, payload_ev))
try:
run = await run_manager.start(
graph=graph,
input_payload=input_payload if input_payload is not None else {},
workflow_id=workflow_id,
workflow_name=name,
memory_scope=memory_scope,
collection=collection,
run_class="formal" if formal else "exploratory",
version=run_version,
version_hash=version_hash,
started_by=actor,
options=options,
schedule_id=schedule_id,
notes=notes,
)
except ValueError as e:
raise LaunchRefused(400, str(e)) from e
return run
# --------------------------------------------------------------------------
# 发起前检查绑定的工具
# --------------------------------------------------------------------------
def node_setting(spec: GraphSpec, node: Any, key: str) -> Any:
"""节点配置的实际取值,和运行时 NodeContext.cfg 同一个规则:节点上没写(None、空串)就取图级 defaults。"""
value = node.config.get(key)
return spec.defaults.get(key) if value in (None, "") else value
def tool_bindings(spec: GraphSpec) -> list[tuple[str, str]]:
"""图里每一处绑定的工具:(谁绑的, 工具名)。调用工具节点的 tool、Agent 的 tools、协作成员的 tools。
取值和运行时一致(node_setting):Agent 节点上没写 tools、协作节点上没写成员,运行时用的是
图级 defaults 里的,这里也算它们——不然发起前检查放过了缺的工具,发起时固定数据版本也漏了源。
模板拼出来的名字({{ vars.tool }})也在内:发起前检查(_bindings)跳过它们,发起时固定数据
版本(engine/runner.py)要知道有没有这种名字——有的话说不准会用到哪个数据源。
"""
out: list[tuple[str, str]] = []
for node in spec.nodes:
who = f"「{node.title}」({type_label(node.type)})"
if node.type == NodeType.TOOL:
names = [node_setting(spec, node, "tool")]
out.extend((who, n) for n in names if isinstance(n, str) and n.strip())
elif node.type == NodeType.AGENT:
out.extend((who, n) for n in node_setting(spec, node, "tools") or []
if isinstance(n, str) and n.strip())
elif node.type == NodeType.SUPERVISOR:
for member in node_setting(spec, node, "agents") or []:
if not isinstance(member, dict):
continue
label = f"「{node.title}」的成员「{member.get('name') or '?'}」"
out.extend((label, n) for n in member.get("tools") or [] if isinstance(n, str) and n.strip())
return [(who, name.strip()) for who, name in out]
def _bindings(spec: GraphSpec) -> list[tuple[str, str]]:
"""发起前要检查的绑定。模板拼出来的工具名({{ vars.tool }})发起时还不知道是谁,交给运行时。"""
return [(who, name) for who, name in tool_bindings(spec) if "{{" not in name]
async def missing_tools(session: AsyncSession, spec: GraphSpec) -> list[tuple[str, str, str]]:
"""本机不存在的绑定:(谁绑的, 工具名, 去哪接入:data / tools)。
解析和运行时 build_tools 一致:内置(静态注册)、mcp:<服务>/<工具>、db_query__ / db_schema__
<数据源>、其余按自定义工具。数据源和自定义工具只认启用着的——停用的运行时同样拿不到。
MCP 只看服务登记着、启用着没有:连不连得上、工具还在不在,要连一次才知道,发起时不去等它,
上次没连上或者工具清单里没有的只在日志里提一句。
"""
from app.db.models import CustomTool, DataSource, McpServer
from app.tools.datasource import QUERY_PREFIX, SCHEMA_PREFIX
from app.tools.registry import all_specs
bindings = _bindings(spec)
if not bindings:
return []
builtin = all_specs()
sources = {n for (n,) in await session.execute(select(DataSource.name).where(DataSource.enabled.is_(True)))}
custom = {n for (n,) in await session.execute(select(CustomTool.name).where(CustomTool.enabled.is_(True)))}
servers = {r.name: r for r in (await session.execute(select(McpServer))).scalars()}
missing: list[tuple[str, str, str]] = []
for who, name in bindings:
if name.startswith("mcp:"):
server, _, tool = name[4:].partition("/")
row = servers.get(server)
if row is None or not row.enabled:
missing.append((who, name, "tools"))
elif row.status == "error":
logger.warning("%s 绑的 %s:MCP 服务 %s 上次没连上,运行时可能拿不到这个工具", who, name, server)
elif tool not in ("", "*") and row.tools_cache and tool not in row.tools_cache:
logger.warning("%s 绑的 %s:MCP 服务 %s 上次交回的工具清单里没有 %s", who, name, server, tool)
elif name in builtin:
continue
elif name.startswith((QUERY_PREFIX, SCHEMA_PREFIX)):
source = name[len(QUERY_PREFIX):] if name.startswith(QUERY_PREFIX) else name[len(SCHEMA_PREFIX):]
if source not in sources:
missing.append((who, name, "data"))
elif name not in custom:
missing.append((who, name, "tools"))
return missing
async def _refuse_missing_tools(session: AsyncSession, graph: dict[str, Any]) -> None:
"""绑定的工具在本机不存在就不发起:以前要执行到那一步才失败,前面的步骤白做、钱白花。"""
try:
spec = GraphSpec.model_validate(graph)
except Exception: # noqa: BLE001 - 图本身不成形交给 run_manager.start 报
return
missing = await missing_tools(session, spec)
if not missing:
return
lines = ";".join(f"{who}绑定的 {name}" for who, name, _ in missing[:8])
more = f"等 {len(missing)} 处" if len(missing) > 8 else ""
places = {where for _, _, where in missing}
if places == {"data"}:
fix = "请到「数据」页接入,或在节点中重新选择"
elif places == {"tools"}:
fix = "请到「工具」页接入,或在节点中重新选择"
else:
fix = "数据源工具请到「数据」页接入,其他工具请到「工具」页接入,或在节点中重新选择"
raise LaunchRefused(422, f"绑定的工具不存在:{lines}{more}。{fix}", TOOL_MISSING)
+160
View File
@@ -0,0 +1,160 @@
"""往一次停下的运行里追加事件:运行停在审批时的审批质询、封存之后的批注。
事件流是封存的依据:清单按 seq 排序算哈希,而 run_events 的 (run_id, seq) 没有唯一约束——两条事件撞了号,
排序就说不准,封存核对时对时不对。所以追加和 runner 发事件守同一套序号:接在库里最后一条后面,从 bus 取号
(同一进程里和执行任务共用一个计数器),写事件和条件更新 last_seq 在同一个事务里,谁先提交算谁的。做法同
api/evidence.py 的 _append_judged(按需裁判的判定)。
require 说明往哪种运行里追加:
- "interrupted":运行停在审批、还没封存。事件落在以后的封存范围里,随运行结束一起封存。
- "sealed":已经封存的终态运行。事件带 post_seal: true,落在 manifest_seq 之后,不进封存范围,verify 照旧一致。
没抢到(读完运行之后它被恢复、被放弃、继续运行后重新封存了,问的那张审批卡已经批过了,或者别处刚往里写过
事件)就一条都不写、返回 None,调用方据此说明「未写入运行记录」。
"""
from __future__ import annotations
import asyncio
import contextlib
from collections.abc import AsyncIterator, Sequence
from typing import Any, Literal
from sqlalchemy import exists, func, select, update
from app.core.bus import bus
from app.core.events import EventType, RunEventModel
from app.db.base import SessionLocal
from app.db.models import Approval, Run, RunEvent
from app.engine.runner import _seq_floor, _stage, run_manager
#: 运行的终态。封存过的运行只会停在这几种上
_TERMINAL = ("succeeded", "failed", "cancelled")
#: 宣布运行结束的事件。封存盖住了最后一条这种事件,才是这次结束之后封的(见 _current_seal)
_ENDINGS = (EventType.RUN_FINISHED, EventType.RUN_FAILED, EventType.RUN_CANCELLED)
#: 一条要追加的事件:(事件类型, node_id, 数据)
Item = tuple[EventType | str, str | None, dict[str, Any] | None]
class _RunLocks:
"""每次运行一把锁,没人用了就扔:同一次运行的追加在本进程里排队,读 last_seq 到条件更新之间不会被
另一次追加插进来——插进来的话后提交的那次条件更新落空,明明没有冲突也成了「未写入」。"""
def __init__(self) -> None:
self._locks: dict[str, asyncio.Lock] = {}
self._users: dict[str, int] = {}
@contextlib.asynccontextmanager
async def hold(self, key: str) -> AsyncIterator[None]:
lock = self._locks.setdefault(key, asyncio.Lock())
self._users[key] = self._users.get(key, 0) + 1
try:
async with lock:
yield
finally:
self._users[key] -= 1
if not self._users[key]:
del self._users[key], self._locks[key]
_APPENDING = _RunLocks()
async def append_events(
run_id: str,
items: Sequence[Item],
*,
require: Literal["interrupted", "sealed"],
seal: tuple[int | None, str | None] | None = None,
approval_id: str | None = None,
) -> list[RunEventModel] | None:
"""把 items 依次接在运行记录的最后一条后面,返回写入的事件(seq 严格递增,id 和落库的相同);没写入返回 None。
调用方开始做事时看到的状态可以钉住,做的事要花几秒(问模型),这期间变了就不追加:
- seal 只用于 "sealed":当时的 (manifest_seq, manifest_hash)。继续运行、重新封存了的话,结果依据的已经
不是现在的封存。不传就按这次读到的封存。
- approval_id 只用于 "interrupted":问的是哪张审批卡。那张卡仍待审批才写——批过了,运行可能已经停到了
下一张卡上,仍是「停在审批」,关于上一张的问答却会写进这一次停下里。
items 为空时什么都不做,返回空列表。
"""
if require not in ("interrupted", "sealed"):
raise ValueError(f"require 只能是 interrupted 或 sealed,收到的是 {require!r}")
if approval_id is not None and require != "interrupted":
raise ValueError("approval_id 只用于 interrupted:封存之后没有待审批的卡")
# 事件类型先验过:写错了在取号之前就报出来
entries = [(EventType(str(kind)), node_id, dict(data or {})) for kind, node_id, data in items]
if not entries:
return []
sealed = require == "sealed"
for _, _, data in entries:
if sealed:
data["post_seal"] = True
else:
# 停在审批时追加的事件要进封存范围,不能自称在封存之后
data.pop("post_seal", None)
async with _APPENDING.hold(run_id):
async with SessionLocal() as session:
run = await session.get(Run, run_id)
if run is None:
return None
if sealed:
if run.status not in _TERMINAL or not run.manifest_hash \
or (seal is not None and (run.manifest_seq, run.manifest_hash) != tuple(seal)) \
or not await _current_seal(session, run):
return None
guard = (Run.status.in_(_TERMINAL), _same(Run.manifest_seq, run.manifest_seq),
_same(Run.manifest_hash, run.manifest_hash))
else:
if run.status != "interrupted":
return None
guard = (Run.status == "interrupted",)
if approval_id is not None:
guard += (exists().where(Approval.id == approval_id, Approval.run_id == run_id,
Approval.status == "pending"),)
last = run.last_seq
# 进程重启过的话内存里的序号从 0 起:接在库里最后一条后面。last_seq 按说就是最后一条,_seq_floor
# 另取一次实际的最大序号兜底——别处的追加把 last_seq 写小了,也不至于在这里撞号
bus.set_seq(run_id, await _seq_floor(session, run_id, last))
events = [run_manager._event(run_id, kind, node_id=node_id, data=data)
for kind, node_id, data in entries]
# 核对和写入是同一行上的条件更新,和恢复、放弃、继续运行(runner._claim / _abandon)抢的是同一行。
# 不同步会话里的 run(下面 _stage 会把它的 last_seq 改成同一个值):不写明的话,条件里带了 EXISTS 时
# SQLAlchemy 会悄悄换一种同步办法、给这条语句加上 RETURNING
claimed = await session.execute(
update(Run).where(Run.id == run_id, _same(Run.last_seq, last), *guard)
.values(last_seq=events[-1].seq).execution_options(synchronize_session=False))
if claimed.rowcount != 1:
# 读完运行之后它变了。烧掉几个序号不要紧:不落库的事件(流式增量)本来就占序号,
# 序号只要求递增、不撞号
await session.rollback()
return None
for event in events:
await _stage(session, event, run)
await session.commit()
# 先落库再广播,理由同 RunManager._emit
for event in events:
await bus.publish(event)
return events
def _same(column: Any, value: Any) -> Any:
return column.is_(None) if value is None else column == value
async def _current_seal(session: Any, run: Run) -> bool:
"""运行记录上的封存是不是这次结束之后封的:manifest_seq 盖住了最后一条宣布结束的事件。
继续运行之后再次结束时,终态和终态事件先提交、封存后算。这中间运行记录上还是上一次结束时的封存,
当成现在的封存去追加,事件会被接下来的封存收进去,post_seal 就成了假话。
manifest_seq 为空的是加这一列之前封存的老运行,封到哪一条是 verify_manifest 按老口径推出来的:有终态事件的
封到最后一条终态事件,追加在后面不碍事;没有的封到最后一行,追加一条,范围就挪到它身上,我们自己写的
批注被当成篡改。所以老运行要有终态事件才追加。
"""
ended = await session.scalar(
select(func.max(RunEvent.seq)).where(RunEvent.run_id == run.id,
RunEvent.type.in_([str(t) for t in _ENDINGS])))
if run.manifest_seq is None:
return ended is not None
return ended is None or run.manifest_seq >= ended
+138 -17
View File
@@ -2,17 +2,18 @@ from __future__ import annotations
import asyncio
import contextlib
import copy
import json
import logging
import time
from collections.abc import Callable
from collections.abc import Callable, Sequence
from datetime import datetime, timezone
from typing import Any
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
from langgraph.errors import GraphRecursionError
from langgraph.types import Command
from sqlalchemy import select, update
from sqlalchemy import case, func, select, update
from app.core.bus import bus
from app.core.config import settings
@@ -33,6 +34,10 @@ logger = logging.getLogger(__name__)
# 轨迹里正文靠 node.finished 的 preview,思考靠调用结束时的单条 llm.thinking 汇总
_EPHEMERAL = {EventType.LLM_TOKEN, EventType.LLM_THINKING_DELTA}
#: RunContext.extra 里放这次运行的选项(Run.options)的键。节点从这里读发起时定下的快照、不读设置的
#: 当前值:审批停下期间改了设置,恢复后的那一段也按发起时的,重放和正式运行才可复现
RUN_OPTIONS_KEY = "run_options"
class RunManager:
"""负责一次运行的完整生命周期:启动、串流、中断、恢复、取消。"""
@@ -61,6 +66,15 @@ class RunManager:
# 一个 checkpoint 都没写过,"恢复"它只会从 START 用默认值重跑一遍,
# 还丢掉 run.input——那不是恢复,是伪造一份结果。
async with SessionLocal() as session:
# 先把落后于库里实际最后一条的 last_seq 改正(为什么会落后见 _seq_floor)。内存里的序号计数刚清零,
# 接下来各处都按 last_seq 接号:本模块的几处另外兜了底,按需裁判记判定(api/evidence.py 的
# _append_judged)没有,不改正的话它重启后第一次追加就撞上已有的序号。有 (run_id, seq) 索引,
# 每次运行一次索引查找。updated_at 不动:这是补记录,不是运行有了变化
latest = select(func.max(RunEvent.seq)).where(RunEvent.run_id == Run.id).scalar_subquery()
await session.execute(
update(Run).where(func.coalesce(Run.last_seq, 0) < latest)
.values(last_seq=latest, updated_at=Run.updated_at)
.execution_options(synchronize_session=False))
stranded = list((await session.execute(
select(Run.id, Run.last_seq).where(Run.status == "running")
)).all())
@@ -158,7 +172,7 @@ class RunManager:
self, run_id: str, event_type: EventType | str, *, node_id: str | None = None,
**data: Any,
) -> None:
"""给 API 层用的事件入口(比如 formal 启动时记录口径升版处置)。"""
"""给运行之外的代码往运行里记一条事件。发起时就定下的事件不走这里,交给 start 的 notes。"""
await self._emit(run_id, event_type, node_id=node_id, data=data)
# ---------------- 启动 ----------------
@@ -177,7 +191,16 @@ class RunManager:
version_hash: str | None = None,
started_by: str | None = None,
approval_default: str | None = None,
options: dict[str, Any] | None = None,
schedule_id: str | None = None,
notes: Sequence[tuple[EventType | str, str | None, dict[str, Any]]] = (),
) -> Run:
"""建运行记录、交给执行任务。options 是这次运行的选项快照,schedule_id 是发起它的定时任务(可空)。
notes 是发起时就定下的事件 (类型, node_id, 数据),比如正式运行按什么级别出具、口径升版怎么处置。它们和
运行记录同一个事务提交,排在 run.started 前面:写不进去就不建运行,不会出现运行已经在执行、调用方却
只拿到一个异常(定时任务会把它记成未能发起);也不会和执行抢先后,运行结束得快时落到封存之后。
"""
spec = GraphSpec.model_validate(graph)
report = validate_graph(spec)
if not report.ok:
@@ -187,6 +210,8 @@ class RunManager:
)
if approval_default is None:
approval_default = await _approval_default(run_class=run_class)
# 拷一份:调用方手上的 dict 之后再改,不该改到这次运行的快照
options = copy.deepcopy(options) if options else {}
async with SessionLocal() as session:
from app.data.table_versions import pin_lock
@@ -214,11 +239,23 @@ class RunManager:
tool_trust=tool_trust,
agent_limits=agent_limits,
data_versions=data_versions,
options=options,
schedule_id=schedule_id,
)
run.thread_id = run.id # 一个 run 一条 checkpoint 线程
session.add(run)
staged: list[RunEventModel] = []
if notes:
await session.flush() # 事件要带上运行的 id
staged = [self._event(run.id, kind, node_id=node_id, data=data)
for kind, node_id, data in notes]
for event in staged:
await _stage(session, event, run)
await session.commit()
await session.refresh(run)
# 先落库再广播,理由同 _emit;执行任务还没建,run.started 必定排在它们后面
for event in staged:
await bus.publish(event)
self._tasks[run.id] = asyncio.create_task(
self._drive(
@@ -232,6 +269,7 @@ class RunManager:
tool_trust=tool_trust,
agent_limits=agent_limits,
data_versions=data_versions,
options=options,
)
)
return run
@@ -379,7 +417,7 @@ class RunManager:
.values(status="resolved")
)
run = await session.get(Run, run_id)
bus.set_seq(run_id, run.last_seq)
bus.set_seq(run_id, await _seq_floor(session, run_id, run.last_seq))
await session.commit()
await session.refresh(run)
@@ -479,7 +517,7 @@ class RunManager:
**({"graph": spec.model_dump(mode="json")} if graph is not None else {}),
**({"started_by": actor} if actor else {}))
run = await session.get(Run, run_id)
bus.set_seq(run_id, run.last_seq)
bus.set_seq(run_id, await _seq_floor(session, run_id, run.last_seq))
await session.commit()
await session.refresh(run)
@@ -538,9 +576,14 @@ class RunManager:
closed = await _close_approvals(session, run_id, actor, now)
run = await session.get(Run, run_id)
timing = await _timing(session, run, 0, now)
run.usage = {**(run.usage or {}), "duration_ms": timing["active_ms"], **timing}
usage = dict(run.usage or {})
# 停下时记的用量里还没有这次停在审批期间问的质询,这里补上(上次停下时计过的不再计)
inquiry = (await _usage_of_events(session, run_id)).get("inquiry")
if inquiry or usage.get("inquiry"):
usage = _with_inquiry(usage, inquiry)
run.usage = {**usage, "duration_ms": timing["active_ms"], **timing}
run.error = "用户取消:等待审批时放弃了这次运行" if closed else "用户取消:挂起期间放弃了这次运行"
bus.set_seq(run_id, run.last_seq or 0)
bus.set_seq(run_id, await _seq_floor(session, run_id, run.last_seq))
# 和状态同一个事务提交,理由同 _finalize
event = self._event(run_id, EventType.RUN_CANCELLED, data={
"timing": timing, "actor": actor,
@@ -585,6 +628,7 @@ class RunManager:
tool_trust: dict[str, str] | None = None,
agent_limits: dict[str, Any] | None = None,
data_versions: dict[str, Any] | None = None,
options: dict[str, Any] | None = None,
resumed: list[str] | tuple[str, ...] = (),
actors: dict[str, str | None] | None = None,
) -> None:
@@ -631,7 +675,10 @@ class RunManager:
# 运行记录里的那份,含运行中途第一次用到时补上的;None 是升级前的运行
"data_versions": data_versions,
# 这一段是按哪一版重放协议跑的,恢复时据此认出升级前做下的工作
"replay_protocol": PROTOCOL},
"replay_protocol": PROTOCOL,
# 发起时定下的运行选项(核查设置之类),每一段都记、记的是同一份。没有选项的
# 运行不带这个键,和加选项之前的 run.started 逐字相同
**({"options": options} if options else {})},
)
run_ctx = RunContext(
@@ -648,7 +695,9 @@ class RunManager:
# 的 run_versions)。复制一份:运行中途补固定的源就地写进这个 dict,同一段执行里
# 后面的节点都看得到,但不该碰到别处(运行记录、事件)手上的那个对象
extra={"resumed": set(resumed), "actors": dict(actors or {}),
RUN_VERSIONS_KEY: dict(data_versions) if data_versions is not None else None},
RUN_VERSIONS_KEY: dict(data_versions) if data_versions is not None else None,
# 同理也是一份拷贝:节点读它,不该改到运行记录那份
RUN_OPTIONS_KEY: copy.deepcopy(options or {})},
)
app = compile_graph(spec, run_ctx).compile(checkpointer=self.checkpointer)
# 循环按自己声明的轮数另记一份预算,全局上限只管没人把关的环
@@ -838,6 +887,7 @@ class RunManager:
if run:
timing = await _timing(session, run, elapsed_ms, now)
events_usage = await _usage_of_events(session, run_id)
inquiry = events_usage.pop("inquiry", None)
if status != "succeeded":
# 状态里的用量只算跑完了的节点。失败、取消、停在审批上的那个节点已经
# 花出去的调用(校验修复、agent 的前几步)只在 llm.end 里——以前这部分
@@ -846,6 +896,9 @@ class RunManager:
elif events_usage.get("unpriced_calls"):
# 金额估不出来的调用只有 llm.end 数得出(状态里的用量只有金额,0 分不出是免费还是没价)
usage = {**usage, "unpriced_calls": events_usage["unpriced_calls"]}
if inquiry:
# 审批质询不在图里执行,状态和 llm.end 两本账里都没有它:比完再加上
usage = _with_inquiry(usage, inquiry)
# duration_ms 是各段执行时长之和(= active_ms)。以前用这一段的耗时
# 覆盖它:87 秒、中间等过一次审批的运行,列表上写着 22ms
usage.update(duration_ms=timing["active_ms"], **timing)
@@ -1025,6 +1078,8 @@ def _carried(run: Run) -> dict[str, Any]:
"agent_limits": run.agent_limits,
# 固定的数据版本照运行记录来,不重新读数据源的当前指针:中途重传过,续跑查的仍是发起时那一版
"data_versions": run.data_versions,
# 运行选项同理:照发起时的快照,不重新读设置
"options": run.options or {},
}
@@ -1062,11 +1117,12 @@ async def _data_versions(session: Any, spec: GraphSpec) -> dict[str, Any]:
async def _bound_sources(session: Any, spec: GraphSpec) -> tuple[set[str], bool]:
"""图里(含子工作流)绑定的数据源工具指向的源名,以及有没有带模板、发起时定不下来的工具名。
绑定的取法和发起前检查同一个(api/runs.py 的 tool_bindings),子工作流照运行时的取法
绑定的取法和发起前检查同一个(engine/launch.py 的 tool_bindings),子工作流照运行时的取法
读出来接着找:固定了版本的读那一版,没固定的读工作流现在的图。读不到、图不成形的跳过——
那个节点运行时自己会报错。同一个工作流(同一版)只读一次,互相引用也不会转圈。
"""
from app.api.runs import node_setting, tool_bindings
# 在函数里导入:launch 在模块级导入了本模块(它要 run_manager)
from app.engine.launch import node_setting, tool_bindings
from app.db.models import WorkflowVersion
from app.tools.datasource import source_of_tool
@@ -1223,17 +1279,36 @@ async def _claim(
async def _stage(session: Any, event: RunEventModel, run: Run | None = None) -> None:
"""把一条事件写进这个 session,随它一起提交。run 已经在 session 里时直接改它的 last_seq。"""
session.add(RunEvent(run_id=event.run_id, seq=event.seq, type=str(event.type),
"""把一条事件写进这个 session,随它一起提交。run 已经在 session 里时直接改它的 last_seq。
落库的 id 用事件自己的:广播出去的、交回给调用方的和库里的是同一个 id,回头按 id 找得到。
"""
session.add(RunEvent(id=event.id, run_id=event.run_id, seq=event.seq, type=str(event.type),
node_id=event.node_id, ts=event.ts, data=event.data))
if run is not None:
run.last_seq = event.seq
else:
# 只往大里改:另开会话记的事件(note)和执行任务同时在写时,取号和提交的先后可以不一样,号小的后提交,
# 无条件写会把 last_seq 写小(后果见 _seq_floor)
await session.execute(
update(Run).where(Run.id == event.run_id).values(last_seq=event.seq)
update(Run).where(Run.id == event.run_id)
.values(last_seq=case((Run.last_seq > event.seq, Run.last_seq), else_=event.seq))
.execution_options(synchronize_session=False)
)
async def _seq_floor(session: Any, run_id: str, last_seq: int | None) -> int:
"""接着一次运行往下编号的起点:last_seq 和库里实际最大的序号取大的。
last_seq 按说就是最后一条,但有的写入会把它写小:按需裁判记判定(api/evidence.py 的 _append_judged)
的条件更新不带 last_seq 条件,它取了号还没写、封存后追加(engine/run_append.py)抢先提交了,它再提交时
把 last_seq 写回自己那个更小的号。同一进程里序号计数只增不减,不碍事;进程重启后计数从 0 起,只按
last_seq 接号就会撞上已有的序号。清单按序号排序算哈希,撞了号封存核对就时对时不对。
"""
stored = await session.scalar(select(func.max(RunEvent.seq)).where(RunEvent.run_id == run_id))
return max(last_seq or 0, stored or 0)
async def _close_approvals(session: Any, run_id: str, actor: str | None, now: datetime) -> int:
"""运行已经结束,它还没处理完的审批一并关掉。返回关掉了几条。"""
rows = list((await session.execute(
@@ -1248,18 +1323,39 @@ async def _close_approvals(session: Any, run_id: str, actor: str | None, now: da
_USAGE_KEYS = ("input_tokens", "output_tokens", "cost_usd", "calls")
#: 审批质询的用量里计进运行用量的键:unpriced_calls 是其中金额估不出来的调用次数(模型不在价格目录里)
_INQUIRY_KEYS = (*_USAGE_KEYS, "unpriced_calls")
async def _usage_of_events(session: Any, run_id: str) -> dict[str, Any]:
"""按 llm.end 事件逐次加总的用量:每次模型调用都有一条,不管节点最后成没成。"""
"""按事件加总的用量。
llm.end 逐次加总:每次模型调用都有一条,不管节点最后成没成。审批质询不在图里执行、不发 llm.end,
它的用量写在 approval.inquiry 事件数据的 usage 里,单独加总放在 inquiry 下(没有质询就没有这个键):
llm.end 这本账要和状态里的用量比大小(_spent),质询花的两本里都没有,混进去比会被状态那本盖掉。
封存之后追加的质询(数据带 post_seal)不算:运行用量是这次运行自己花的,结束时就定下了、写进了终态事件;
事后的批注另有自己的账,同按需裁判的判定。继续运行之后它落进新的封存范围,照样不算——不然同一条质询
计不计,全看之后有没有继续运行。
"""
rows = await session.execute(
select(RunEvent.data).where(RunEvent.run_id == run_id, RunEvent.type == EventType.LLM_END)
select(RunEvent.type, RunEvent.data).where(
RunEvent.run_id == run_id, RunEvent.type.in_([EventType.LLM_END, EventType.APPROVAL_INQUIRY]))
)
from app.providers.catalog import unpriced_call
total: dict[str, Any] = dict.fromkeys(_USAGE_KEYS, 0)
unpriced = 0
for (data,) in rows:
inquiry: dict[str, Any] = {}
for kind, data in rows:
if kind == EventType.APPROVAL_INQUIRY:
spent = (data or {}).get("usage")
if isinstance(spent, dict) and not (data or {}).get("post_seal"):
for key in _INQUIRY_KEYS:
value = spent.get(key)
if isinstance(value, (int, float)) and not isinstance(value, bool):
inquiry[key] = inquiry.get(key, 0) + value
continue
for key in _USAGE_KEYS:
value = (data or {}).get(key, 1 if key == "calls" else 0)
if isinstance(value, (int, float)) and not isinstance(value, bool):
@@ -1269,9 +1365,34 @@ async def _usage_of_events(session: Any, run_id: str) -> dict[str, Any]:
if unpriced:
# 有几次调用的金额估不出来(模型不在价格目录里):cost_usd 只是估得出的那部分,界面据此不写 $0
total["unpriced_calls"] = unpriced
if inquiry:
if "cost_usd" in inquiry:
inquiry["cost_usd"] = round(inquiry["cost_usd"], 6)
total["inquiry"] = inquiry
return total
def _with_inquiry(usage: dict[str, Any], inquiry: dict[str, Any] | None) -> dict[str, Any]:
"""把审批质询的用量计进运行用量,明细另放在 inquiry 下(界面要说清其中多少是质询花的)。
usage 里已经计过一回的(上次停下时记下的 inquiry)先减掉再加:停过两次、两次都问过的运行,
每次质询只计一遍。
"""
out = dict(usage)
counted = out.pop("inquiry", None) or {}
for key in _INQUIRY_KEYS:
delta = (inquiry or {}).get(key, 0) - counted.get(key, 0)
if delta:
out[key] = (out.get(key) or 0) + delta
if isinstance(out.get("cost_usd"), float):
out["cost_usd"] = round(out["cost_usd"], 6)
if "input_tokens" in out or "output_tokens" in out:
out["total_tokens"] = int(out.get("input_tokens") or 0) + int(out.get("output_tokens") or 0)
if inquiry:
out["inquiry"] = dict(inquiry)
return out
def _spent(state_usage: dict[str, Any], events_usage: dict[str, Any]) -> dict[str, Any]:
"""两本账取大的那本。新引擎每次调用都发 llm.end,事件那本不会更少;
加这条事件之前的老运行(agent 节点不发 llm.end)状态那本更全。"""
+279
View File
@@ -0,0 +1,279 @@
"""审批质询的用量计进运行用量。
质询不在图里执行、不发 llm.end,它花的 token 和金额写在 approval.inquiry 事件数据的 usage 里
(input_tokens、output_tokens、cost_usd、calls,可带 unpriced_calls)。以前运行用量只认图的状态和 llm.end,
质询花的钱在运行页上看不见。
- 运行结束(成功、失败)时:图的用量照旧(成功取状态、其余两本账取大),再加上质询的;明细另放在 inquiry 下
- 停在审批时被放弃:停下时记的用量里还没有这次停下期间的质询,放弃时补上
- 停过两次、两次都问过:每次质询只计一遍
- 封存之后追加的质询(post_seal)不计进运行用量,继续运行、重新封存之后也不计
- 没有质询的运行,用量和以前逐字相同
"""
from __future__ import annotations
import asyncio
import pytest
from sqlalchemy import select
from app.core.events import EventType
from app.db.base import SessionLocal
from app.db.models import Run, RunEvent
from app.engine import compiler
from app.engine.context import NodeError
from app.engine.runner import run_manager
from app.engine.schema import NodeType
A = {"input_tokens": 1200, "output_tokens": 300, "cost_usd": 0.0123, "calls": 2}
B = {"input_tokens": 800, "output_tokens": 100, "cost_usd": 0.004, "calls": 1, "unpriced_calls": 1}
@pytest.fixture(autouse=True)
async def engine_up():
await run_manager.setup()
yield
await run_manager.shutdown()
def node(nid, ntype, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
def chain(*nodes):
return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
def gated(*middle, gates: int = 1) -> dict:
nodes = [node("start", "input"), *middle]
nodes += [node(f"gate{i}", "human", mode="approve", title=f"第 {i + 1} 道审批") for i in range(gates)]
nodes.append(node("out", "output", fields=[{"name": "r", "value": "完成"}]))
return chain(*nodes)
def ask(question: str, usage: dict | None) -> tuple:
data = {"question": question, "answer": "按订单表核对一致"}
if usage is not None:
data["usage"] = usage
return (EventType.APPROVAL_INQUIRY, None, data)
async def _row(run_id: str) -> Run:
async with SessionLocal() as session:
return await session.get(Run, run_id)
async def _until(run_id: str, statuses: tuple[str, ...]) -> Run:
for _ in range(500):
row = await _row(run_id)
if row.status in statuses and (row.status == "interrupted" or row.manifest_seq is not None):
await run_manager.wait_idle(run_id)
return await _row(run_id)
await asyncio.sleep(0.03)
raise AssertionError(f"运行 {run_id} 停在 {row.status}")
async def _ask_while_paused(run_id: str, *items: tuple) -> None:
from app.engine.run_append import append_events
assert await append_events(run_id, list(items), require="interrupted"), "质询没写进运行"
def _tokens(usage: dict) -> tuple:
return (usage.get("input_tokens"), usage.get("output_tokens"), usage.get("total_tokens"),
usage.get("calls"), usage.get("unpriced_calls"))
# --------------------------------------------------------------------------
# 按事件加总
# --------------------------------------------------------------------------
async def test_usage_of_events_keeps_inquiries_apart():
"""llm.end 照旧逐次加总;质询的另记在 inquiry 下(_finalize 比完两本账再加,见下面几条)。"""
from app.engine.runner import _usage_of_events
async with SessionLocal() as session:
run = Run(workflow_name="用量加总", status="succeeded", graph={}, input={})
session.add(run)
await session.flush()
rows = [
("llm.end", {"model": "claude-sonnet-4-5", "input_tokens": 100, "output_tokens": 20, "cost_usd": 0.001,
"calls": 1}),
("llm.end", {"model": "claude-sonnet-4-5", "input_tokens": 50, "output_tokens": 5, "cost_usd": 0.0005}),
("approval.inquiry", {"question": "甲", "usage": A}),
("approval.inquiry", {"question": "乙", "usage": B}),
("approval.inquiry", {"question": "没问模型就答了"}),
("approval.inquiry", {"question": "写坏的用量", "usage": "很多"}),
# 别的事件里的金额不算:按需裁判的判定另有自己的账
("evidence.judged", {"cost_usd": 9.9, "calls": 9}),
]
for seq, (kind, data) in enumerate(rows, start=1):
session.add(RunEvent(run_id=run.id, seq=seq, type=kind, node_id=None, ts=float(seq), data=data))
await session.commit()
got = await _usage_of_events(session, run.id)
assert {k: got[k] for k in ("input_tokens", "output_tokens", "calls")} == {
"input_tokens": 150, "output_tokens": 25, "calls": 2}
assert got["cost_usd"] == pytest.approx(0.0015)
assert got["inquiry"] == {"input_tokens": 2000, "output_tokens": 400, "cost_usd": pytest.approx(0.0163),
"calls": 3, "unpriced_calls": 1}
async def test_without_inquiries_there_is_no_inquiry_key():
from app.engine.runner import _usage_of_events
async with SessionLocal() as session:
run = Run(workflow_name="没有质询", status="succeeded", graph={}, input={})
session.add(run)
await session.flush()
session.add(RunEvent(run_id=run.id, seq=1, type="llm.end", node_id="ask", ts=1.0,
data={"model": "claude-sonnet-4-5", "input_tokens": 10, "output_tokens": 2,
"cost_usd": 0.0001, "calls": 1}))
await session.commit()
got = await _usage_of_events(session, run.id)
assert set(got) == {"input_tokens", "output_tokens", "cost_usd", "calls"}
# --------------------------------------------------------------------------
# 运行用量
# --------------------------------------------------------------------------
async def test_a_succeeded_run_counts_the_inquiry():
run = await run_manager.start(graph=gated(), input_payload={})
await _until(run.id, ("interrupted",))
await _ask_while_paused(run.id, ask("销售额为什么涨了", A))
await run_manager.resume(run.id, {"approved": True})
row = await _until(run.id, ("succeeded",))
# 图里没有模型调用,用量全是这次质询的
assert _tokens(row.usage) == (1200, 300, 1500, 2, None)
assert row.usage["cost_usd"] == pytest.approx(0.0123)
assert row.usage["inquiry"] == A
assert row.usage["wall_ms"] >= row.usage["active_ms"]
async def test_the_inquiry_is_added_on_top_of_the_model_calls():
"""图里有模型调用:成功的运行取状态里的用量,再加上质询的。"""
graph = gated(node("ask", "llm", prompt="上周到店多少人"))
run = await run_manager.start(graph=graph, input_payload={})
await _until(run.id, ("interrupted",))
await _ask_while_paused(run.id, ask("到店人数怎么算的", B))
await run_manager.resume(run.id, {"approved": True})
row = await _until(run.id, ("succeeded",))
async with SessionLocal() as session:
ends = [e.data for e in (await session.execute(select(RunEvent).where(
RunEvent.run_id == run.id, RunEvent.type == "llm.end"))).scalars()]
assert ends, "图里的模型调用没有记 llm.end"
model_in = sum(e["input_tokens"] for e in ends)
model_out = sum(e["output_tokens"] for e in ends)
unpriced = sum(1 for e in ends if e.get("unpriced"))
assert row.usage["input_tokens"] == model_in + B["input_tokens"]
assert row.usage["output_tokens"] == model_out + B["output_tokens"]
assert row.usage["total_tokens"] == model_in + model_out + B["input_tokens"] + B["output_tokens"]
assert row.usage["calls"] == len(ends) + B["calls"]
assert row.usage["unpriced_calls"] == unpriced + B["unpriced_calls"]
assert row.usage["cost_usd"] == pytest.approx(sum(e["cost_usd"] for e in ends) + B["cost_usd"])
async def test_a_failed_run_counts_the_inquiry_too(monkeypatch):
original = compiler.RUNNERS[NodeType.TRANSFORM]
async def boom(state, ctx):
if ctx.node.id == "boom":
raise NodeError(ctx.node.id, "数据源暂时不可用")
return await original(state, ctx)
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, boom)
graph = chain(node("start", "input"), node("gate", "human", mode="approve", title="放行吗"),
node("boom", "transform", mode="template", template="x"),
node("out", "output", fields=[{"name": "r", "value": "x"}]))
run = await run_manager.start(graph=graph, input_payload={})
await _until(run.id, ("interrupted",))
await _ask_while_paused(run.id, ask("口径对吗", A))
await run_manager.resume(run.id, {"approved": True})
row = await _until(run.id, ("failed",))
assert _tokens(row.usage) == (1200, 300, 1500, 2, None)
assert row.usage["inquiry"] == A
async def test_each_inquiry_counts_once_across_two_pauses():
run = await run_manager.start(graph=gated(gates=2), input_payload={})
await _until(run.id, ("interrupted",))
await _ask_while_paused(run.id, ask("第一问", A))
await run_manager.resume(run.id, {"approved": True})
await asyncio.sleep(0.05)
paused = await _until(run.id, ("interrupted",))
# 第二次停下时记的用量已经有第一问
assert paused.usage["input_tokens"] == A["input_tokens"] and paused.usage["inquiry"] == A
await _ask_while_paused(run.id, ask("第二问", B))
await run_manager.resume(run.id, {"approved": True})
row = await _until(run.id, ("succeeded",))
assert _tokens(row.usage) == (2000, 400, 2400, 3, 1)
assert row.usage["cost_usd"] == pytest.approx(0.0163)
assert row.usage["inquiry"] == {"input_tokens": 2000, "output_tokens": 400, "cost_usd": pytest.approx(0.0163),
"calls": 3, "unpriced_calls": 1}
async def test_post_seal_inquiries_stay_out_of_the_run_usage(monkeypatch):
"""运行用量是这次运行自己花的,结束时就定下了、写进了封存范围里的终态事件。封存之后追加的质询是事后的批注,
和按需裁判的判定一样另有自己的账,不计进来。以前平时不计,继续运行、重新算用量时又被计了进去:同一条质询
计不计,全看之后有没有继续运行。"""
from app.engine.run_append import append_events
original = compiler.RUNNERS[NodeType.TRANSFORM]
state = {"fail": True}
async def flaky(st, ctx):
if ctx.node.id == "boom" and state["fail"]:
raise NodeError(ctx.node.id, "数据源暂时不可用")
return await original(st, ctx)
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, flaky)
graph = chain(node("start", "input"), node("boom", "transform", mode="template", template="x"),
node("out", "output", fields=[{"name": "r", "value": "x"}]))
run = await run_manager.start(graph=graph, input_payload={})
failed = await _until(run.id, ("failed",))
written = await append_events(run.id, [ask("这次为什么失败", A)], require="sealed")
assert written and written[0].data["post_seal"] is True
assert set(failed.usage) == {"duration_ms", "wall_ms", "active_ms", "wait_ms"}
state["fail"] = False
await run_manager.continue_failed(run.id)
row = await _until(run.id, ("succeeded",))
assert row.manifest_seq > written[0].seq, "继续运行之后重新封存,这条质询落进了新的封存范围"
assert set(row.usage) == {"duration_ms", "wall_ms", "active_ms", "wait_ms"}
@pytest.mark.parametrize("asked_before", [False, True], ids=["first-pause", "second-pause"])
async def test_abandoning_a_paused_run_counts_what_was_asked(asked_before):
"""停在审批时放弃:停下时记的用量里还没有这次停下期间的质询,放弃时补上;上一次停下时已经计过的不再计。"""
run = await run_manager.start(graph=gated(gates=2 if asked_before else 1), input_payload={})
await _until(run.id, ("interrupted",))
if asked_before:
await _ask_while_paused(run.id, ask("第一问", A))
await run_manager.resume(run.id, {"approved": True})
await asyncio.sleep(0.05)
await _until(run.id, ("interrupted",))
await _ask_while_paused(run.id, ask("最后一问", B))
assert await run_manager.cancel(run.id) == "cancelled"
row = await _until(run.id, ("cancelled",))
if asked_before:
assert _tokens(row.usage) == (2000, 400, 2400, 3, 1)
assert row.usage["cost_usd"] == pytest.approx(0.0163)
else:
assert _tokens(row.usage) == (800, 100, 900, 1, 1) and row.usage["inquiry"] == B
assert row.usage["wall_ms"] >= row.usage["active_ms"]
async def test_runs_without_inquiries_keep_their_usage():
run = await run_manager.start(graph=gated(), input_payload={})
await _until(run.id, ("interrupted",))
await run_manager.resume(run.id, {"approved": True})
row = await _until(run.id, ("succeeded",))
assert set(row.usage) == {"duration_ms", "wall_ms", "active_ms", "wait_ms"}
other = await run_manager.start(graph=gated(), input_payload={})
await _until(other.id, ("interrupted",))
assert await run_manager.cancel(other.id) == "cancelled"
row = await _until(other.id, ("cancelled",))
assert set(row.usage) == {"duration_ms", "wall_ms", "active_ms", "wait_ms"}
+762
View File
@@ -0,0 +1,762 @@
"""往停下的运行里追加事件(engine/run_append.py 的 append_events)。
- 停在审批时追加:事件接在最后一条后面、落进以后的封存范围。批准、运行结束封存之后 verify 照旧一致,
改动追加的事件 verify 就对不上——它们确实在封存范围里
- 追加途中运行被恢复(条件更新落空):什么都不写,返回 None,last_seq 不动
- 质询指明问的是哪张审批卡(approval_id)时,那张卡仍待审批才写;返回的事件和落库的 id、序号一样
- 先落库再广播,没写入的一条都不广播
- 已封存的运行追加:带 post_seal、落在 manifest_seq 之后,verify 照旧一致;封存不是这次结束之后封的、
封存和调用方开始时看到的不同、没有终态事件的老封存,都不追加
- 连续追加、并发追加,序号严格递增,整条事件流没有重复的序号
- last_seq 落后于库里实际的最后一条(按需裁判的追加和封存后追加交错提交时会这样):进程重启后恢复、
继续运行、放弃、按需裁判都接在实际的最后一条后面,不撞号;runner 记事件不把 last_seq 写小
- 恢复后的重放不受影响:前面的节点不重复执行,Agent 里批过的工具不重复执行(含升级前写下的断点)
"""
from __future__ import annotations
import asyncio
import json
import sqlite3
from collections import Counter
import pytest
from langchain_core.messages import AIMessage, ToolMessage
from pydantic import BaseModel
from sqlalchemy import select
from app.core.bus import bus
from app.core.config import settings
from app.core.events import EventType
from app.db.base import SessionLocal
from app.db.models import Approval, Run, RunEvent
from app.engine import compiler
from app.engine.context import NodeError
from app.engine.runner import RunManager, run_manager, verify_manifest
from app.engine.schema import NodeType
from app.tools.registry import register
#: Agent 场景里工具真正执行的次数。注册表是进程级的,工具只注册一次,计数按测试清零
CALLS: list[str] = []
class _TallyArgs(BaseModel):
label: str
@register("inquiry_tally", "测试用:每执行一次记一笔", "test", _TallyArgs, dangerous=True, needs_context=False)
async def _tally(label: str) -> str:
CALLS.append(label)
return json.dumps({"label": label, "n": len(CALLS)}, ensure_ascii=False)
@pytest.fixture(autouse=True)
async def engine_up():
CALLS.clear()
await run_manager.setup()
yield
await run_manager.shutdown()
def node(nid, ntype, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
def chain(*nodes):
return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
GATED = chain(node("start", "input"),
node("work", "transform", mode="template", template="本周销售额 4321.5 元"),
node("gate", "human", mode="approve", title="这份周报可以出具吗"),
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.work }}"}]))
SIMPLE = chain(node("start", "input"), node("out", "output", fields=[{"name": "r", "value": "完成"}]))
TWO_GATES = chain(node("start", "input"),
node("g1", "human", mode="approve", title="第一道:口径对吗"),
node("g2", "human", mode="approve", title="第二道:可以出具吗"),
node("out", "output", fields=[{"name": "r", "value": "完成"}]))
#: 「boom」节点按 flaky 的开关失败:先让运行失败、封存,再关掉开关继续运行
FLAKY = chain(node("start", "input"), node("boom", "transform", mode="template", template="门店 17 家"),
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.boom }}"}]))
def inquiry(question: str) -> tuple:
return (EventType.APPROVAL_INQUIRY, None, {"question": question, "answer": f"关于「{question}」:按订单表核对一致"})
@pytest.fixture
def executed(monkeypatch) -> Counter:
"""每个「数据整形」节点真正执行了几次。"""
seen: Counter = Counter()
original = compiler.RUNNERS[NodeType.TRANSFORM]
async def counting(state, ctx):
seen[ctx.node.id] += 1
return await original(state, ctx)
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, counting)
return seen
@pytest.fixture
def flaky(monkeypatch) -> dict:
"""FLAKY 里的「boom」在 fail 为真时失败。"""
state = {"fail": True}
original = compiler.RUNNERS[NodeType.TRANSFORM]
async def boom(st, ctx):
if ctx.node.id == "boom" and state["fail"]:
raise NodeError(ctx.node.id, "数据源暂时不可用")
return await original(st, ctx)
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, boom)
return state
async def _row(run_id: str) -> Run:
async with SessionLocal() as session:
return await session.get(Run, run_id)
async def _until(run_id: str, statuses: tuple[str, ...], sealed: bool = False) -> Run:
for _ in range(500):
row = await _row(run_id)
if row.status in statuses and (not sealed or row.manifest_seq is not None):
await run_manager.wait_idle(run_id)
return await _row(run_id)
await asyncio.sleep(0.03)
raise AssertionError(f"运行 {run_id} 停在 {row.status}")
async def _paused(graph=GATED) -> Run:
run = await run_manager.start(graph=graph, input_payload={})
return await _until(run.id, ("interrupted",))
async def _finished(graph=SIMPLE) -> Run:
run = await run_manager.start(graph=graph, input_payload={})
return await _until(run.id, ("succeeded", "failed"), sealed=True)
async def _events(run_id: str, etype: str | None = None) -> list[RunEvent]:
async with SessionLocal() as session:
query = select(RunEvent).where(RunEvent.run_id == run_id)
if etype:
query = query.where(RunEvent.type == etype)
return list((await session.execute(query.order_by(RunEvent.seq))).scalars())
def _strictly_increasing(seqs: list[int]) -> bool:
return all(a < b for a, b in zip(seqs, seqs[1:]))
async def _no_repeated_seq(run_id: str) -> None:
seqs = [e.seq for e in await _events(run_id)]
assert len(seqs) == len(set(seqs)), f"事件流里有重复的序号:{seqs}"
async def _pending(run_id: str) -> list[Approval]:
async with SessionLocal() as session:
return list((await session.execute(
select(Approval).where(Approval.run_id == run_id, Approval.status == "pending"))).scalars())
def _sql(statement: str, *params) -> None:
"""另开一条连接改库:模拟别处(恢复、继续运行、另一个写入者)抢在中间提交。"""
db = sqlite3.connect(settings.db_path, timeout=5)
try:
db.execute(statement, params)
db.commit()
finally:
db.close()
# --------------------------------------------------------------------------
# 停在审批时追加
# --------------------------------------------------------------------------
async def test_appended_while_waiting_are_sealed_with_the_run(executed):
from app.engine.run_append import append_events
paused = await _paused()
subscription = bus.attach(paused.id)
try:
written = await append_events(paused.id, [inquiry("销售额为什么涨了"), inquiry("退款算进去了吗")],
require="interrupted")
assert written is not None and len(written) == 2
assert written[0].seq > paused.last_seq and _strictly_increasing([e.seq for e in written])
# 提交之后才广播:在线的客户端收到的就是落了库的这两条
stream = subscription.stream()
live = [await asyncio.wait_for(stream.__anext__(), 2) for _ in range(2)]
assert [e.seq for e in live] == [e.seq for e in written]
finally:
subscription.close()
row = await _row(paused.id)
assert row.status == "interrupted" and row.last_seq == written[-1].seq
stored = await _events(paused.id, "approval.inquiry")
assert [(e.seq, e.node_id, e.data["question"]) for e in stored] == [
(written[0].seq, None, "销售额为什么涨了"), (written[1].seq, None, "退款算进去了吗")]
assert all("post_seal" not in e.data for e in stored), "进封存范围的事件不该标成封存之后"
await run_manager.resume(paused.id, {"approved": True})
final = await _until(paused.id, ("succeeded", "failed"), sealed=True)
assert final.status == "succeeded", final.error
assert final.manifest_seq > written[-1].seq
check = await verify_manifest(paused.id)
assert check["ok"] is True and check["sealed_at"] == final.manifest_seq
# 恢复之后前面的节点没有再执行一遍
assert executed["work"] == 1
assert len([e for e in await _events(paused.id, "node.started") if e.node_id == "work"]) == 1
await _no_repeated_seq(paused.id)
# 追加的事件确实在封存范围里:改它一个字,封存核对就对不上
async with SessionLocal() as session:
event = await session.get(RunEvent, stored[0].id)
event.data = {**event.data, "answer": "改过的回答"}
await session.commit()
assert (await verify_manifest(paused.id))["ok"] is False
async def test_a_resume_that_wins_the_row_leaves_nothing_behind(monkeypatch):
"""读完运行、写入之前,恢复先把运行占成了 running:条件更新落空,一条都不写、last_seq 不动。"""
from app.engine.run_append import append_events
paused = await _paused()
set_seq = bus.set_seq
def resumed_in_between(run_id, value):
if run_id == paused.id:
db = sqlite3.connect(settings.db_path, timeout=5)
try:
db.execute("UPDATE runs SET status = 'running' WHERE id = ?", (paused.id,))
db.commit()
finally:
db.close()
set_seq(run_id, value)
monkeypatch.setattr(bus, "set_seq", resumed_in_between)
assert await append_events(paused.id, [inquiry("毛利率怎么算的")], require="interrupted") is None
row = await _row(paused.id)
assert row.status == "running" and row.last_seq == paused.last_seq
assert await _events(paused.id, "approval.inquiry") == []
async def test_events_written_elsewhere_in_between_win(monkeypatch):
"""读完运行、写入之前,别处(另一个进程,或者没走这把锁的追加)已经接着写了一条:状态没变,last_seq 变了。
条件更新落空、不写——否则两边各拿着同一个序号,事件流里撞号。"""
from app.engine.run_append import append_events
paused = await _paused()
taken = paused.last_seq + 1
set_seq = bus.set_seq
def written_in_between(run_id, value):
if run_id == paused.id:
db = sqlite3.connect(settings.db_path, timeout=5)
try:
db.execute("INSERT INTO run_events (id, run_id, seq, type, node_id, ts, data) "
"VALUES ('elsewhere-1', ?, ?, 'log', NULL, 0, '{}')", (paused.id, taken))
db.execute("UPDATE runs SET last_seq = ? WHERE id = ?", (taken, paused.id))
db.commit()
finally:
db.close()
set_seq(run_id, value)
monkeypatch.setattr(bus, "set_seq", written_in_between)
bus._seq.pop(paused.id, None) # 另一个进程的序号本进程不知道
assert await append_events(paused.id, [inquiry("同时写进来的")], require="interrupted") is None
assert (await _row(paused.id)).last_seq == taken
assert await _events(paused.id, "approval.inquiry") == []
await _no_repeated_seq(paused.id)
async def test_a_last_seq_left_behind_does_not_cause_a_clash():
"""last_seq 比库里实际的最后一条小(别处的追加交错着把它写小了):照实际的最后一条接号,顺手把 last_seq 写对。"""
from app.engine.run_append import append_events
paused = await _paused()
async with SessionLocal() as session:
run = await session.get(Run, paused.id)
run.last_seq = paused.last_seq - 3
await session.commit()
bus._seq.pop(paused.id, None)
written = await append_events(paused.id, [inquiry("接在哪里")], require="interrupted")
assert written[0].seq == paused.last_seq + 1
assert (await _row(paused.id)).last_seq == written[0].seq
await _no_repeated_seq(paused.id)
async def test_a_run_that_already_moved_on_gets_nothing(executed):
from app.engine.run_append import append_events
paused = await _paused()
await run_manager.resume(paused.id, {"approved": True})
final = await _until(paused.id, ("succeeded",), sealed=True)
assert await append_events(paused.id, [inquiry("晚到的问题")], require="interrupted") is None
assert (await _row(paused.id)).last_seq == final.last_seq
assert await _events(paused.id, "approval.inquiry") == []
async def test_an_inquiry_about_an_approval_needs_that_approval_still_pending(monkeypatch):
"""质询问的是某一张审批卡:开始问模型时那张卡还在等,问完时运行可能已经批过它、停到了下一张上。只看
「运行停在审批」的话,关于第一张的问答会写进第二次停下里。传 approval_id 时那张卡仍待审批才写,
和其余条件在同一次条件更新里核对。"""
from app.engine.run_append import append_events
paused = await _paused(TWO_GATES)
[first] = await _pending(paused.id)
await run_manager.resume(paused.id, {"approved": True}, approval_id=first.id)
await asyncio.sleep(0.05)
later = await _until(paused.id, ("interrupted",))
[second] = await _pending(paused.id)
assert (first.node_id, second.node_id) == ("g1", "g2")
assert await append_events(paused.id, [inquiry("第一道的口径")], require="interrupted",
approval_id=first.id) is None
assert await append_events(paused.id, [inquiry("不存在的卡")], require="interrupted",
approval_id="no-such-approval") is None
assert (await _row(paused.id)).last_seq == later.last_seq
assert await _events(paused.id, "approval.inquiry") == []
written = await append_events(paused.id, [inquiry("出具前再确认一次")], require="interrupted",
approval_id=second.id)
assert written is not None
# 返回的就是落了库的那几条:调用方按 id 或序号回头找,两边对得上
stored = await _events(paused.id, "approval.inquiry")
assert [(e.id, e.seq) for e in stored] == [(e.id, e.seq) for e in written]
# 读完运行、写入之前那张卡被人批了(并行的审批还没凑齐,运行仍停着):不写
set_seq = bus.set_seq
def answered_in_between(run_id, value):
if run_id == paused.id:
_sql("UPDATE approvals SET status = 'answered' WHERE id = ?", second.id)
set_seq(run_id, value)
monkeypatch.setattr(bus, "set_seq", answered_in_between)
assert await append_events(paused.id, [inquiry("批完才问到的")], require="interrupted",
approval_id=second.id) is None
assert (await _row(paused.id)).last_seq == written[-1].seq
assert len(await _events(paused.id, "approval.inquiry")) == 1
with pytest.raises(ValueError):
await append_events(paused.id, [inquiry("封存后的卡")], require="sealed", approval_id=second.id)
async def test_published_only_once_stored_and_never_when_not_written(monkeypatch):
"""先落库再广播:在线的客户端收到的每一条库里都已经有了(订阅者靠「先读历史、再接实时」拼完整,见
RunManager._emit)。没写进运行记录的一条都不广播,否则界面上会多出一条运行记录里没有的问答。"""
from app.engine.run_append import append_events
published: list[tuple[int, int]] = []
publish = bus.publish
async def stored_when_published(event):
if event.type == EventType.APPROVAL_INQUIRY:
db = sqlite3.connect(settings.db_path, timeout=5)
try:
(count,) = db.execute("SELECT count(*) FROM run_events WHERE run_id = ? AND seq = ?",
(event.run_id, event.seq)).fetchone()
finally:
db.close()
published.append((event.seq, count))
await publish(event)
monkeypatch.setattr(bus, "publish", stored_when_published)
paused = await _paused()
written = await append_events(paused.id, [inquiry("客单价怎么算的"), inquiry("含不含退款")],
require="interrupted")
assert published == [(e.seq, 1) for e in written]
published.clear()
set_seq = bus.set_seq
def resumed_in_between(run_id, value):
if run_id == paused.id:
_sql("UPDATE runs SET status = 'running' WHERE id = ?", paused.id)
set_seq(run_id, value)
monkeypatch.setattr(bus, "set_seq", resumed_in_between)
assert await append_events(paused.id, [inquiry("恢复之后才到的")], require="interrupted") is None
assert published == []
# --------------------------------------------------------------------------
# 封存之后追加
# --------------------------------------------------------------------------
async def test_post_seal_appends_stay_outside_the_seal():
from app.engine.run_append import append_events
final = await _finished()
assert final.status == "succeeded", final.error
before = await verify_manifest(final.id)
written = await append_events(final.id, [inquiry("这个数是含税的吗"), inquiry("和上周口径一样吗")],
require="sealed")
assert written is not None and all(e.data["post_seal"] is True for e in written)
assert written[0].seq > final.manifest_seq and _strictly_increasing([e.seq for e in written])
row = await _row(final.id)
assert (row.status, row.manifest_seq, row.manifest_hash) == ("succeeded", final.manifest_seq, final.manifest_hash)
assert row.last_seq == written[-1].seq
after = await verify_manifest(final.id)
assert after["ok"] is True and (after["sealed_at"], after["events"]) == (before["sealed_at"], before["events"])
assert [e.data["post_seal"] for e in await _events(final.id, "approval.inquiry")] == [True, True]
await _no_repeated_seq(final.id)
async def test_a_seal_pinned_by_the_caller_must_still_be_the_seal():
from app.engine.run_append import append_events
final = await _finished()
seal = (final.manifest_seq, final.manifest_hash)
assert await append_events(final.id, [inquiry("甲")], require="sealed", seal=(seal[0] - 1, seal[1])) is None
assert await append_events(final.id, [inquiry("乙")], require="sealed", seal=(seal[0], "0" * 64)) is None
assert await _events(final.id, "approval.inquiry") == []
written = await append_events(final.id, [inquiry("丙")], require="sealed", seal=seal)
assert written is not None and written[0].data["question"] == "丙"
@pytest.mark.parametrize("change", [
"UPDATE runs SET status = 'running' WHERE id = ?", # 继续运行占住了它
"UPDATE runs SET manifest_hash = 'resealed' WHERE id = ?", # 继续运行之后重新封存了
"UPDATE runs SET manifest_seq = manifest_seq + 1 WHERE id = ?",
], ids=["continued", "resealed-hash", "resealed-seq"])
async def test_a_sealed_run_that_changes_in_between_gets_nothing(monkeypatch, change):
"""封存之后追加也是核对和写入同一行上的条件更新:读完运行、写入之前它被继续运行占住、或者重新封存了,就不写。"""
from app.engine.run_append import append_events
final = await _finished()
set_seq = bus.set_seq
def changed_in_between(run_id, value):
if run_id == final.id:
db = sqlite3.connect(settings.db_path, timeout=5)
try:
db.execute(change, (final.id,))
db.commit()
finally:
db.close()
set_seq(run_id, value)
monkeypatch.setattr(bus, "set_seq", changed_in_between)
assert await append_events(final.id, [inquiry("这期为什么少了")], require="sealed") is None
assert (await _row(final.id)).last_seq == final.last_seq
assert await _events(final.id, "approval.inquiry") == []
async def test_a_seal_left_from_an_earlier_ending_is_not_the_seal(monkeypatch):
"""继续运行后又失败:终态和 run.failed 先提交、封存后算。这中间运行记录上还是上一次结束时的封存,
这时追加的事件会被接下来的封存收进去,post_seal 就成了假话——不追加。"""
from app.engine.run_append import append_events
original = compiler.RUNNERS[NodeType.TRANSFORM]
async def boom(state, ctx):
if ctx.node.id == "boom":
raise NodeError(ctx.node.id, "数据源暂时不可用")
return await original(state, ctx)
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, boom)
first = await _finished(chain(node("start", "input"), node("boom", "transform", mode="template", template="x"),
node("out", "output", fields=[{"name": "r", "value": "x"}])))
assert first.status == "failed"
entered, release = asyncio.Event(), asyncio.Event()
real_seal = RunManager._seal
async def held_seal(self, run_id):
if run_id == first.id and not entered.is_set():
entered.set()
await release.wait()
return await real_seal(self, run_id)
monkeypatch.setattr(RunManager, "_seal", held_seal)
await run_manager.continue_failed(first.id)
await asyncio.wait_for(entered.wait(), 10)
try:
row = await _row(first.id)
failures = [e.seq for e in await _events(first.id, "run.failed")]
assert row.status == "failed" and row.manifest_seq == first.manifest_seq < failures[-1]
assert await append_events(first.id, [inquiry("为什么又失败了")], require="sealed") is None
finally:
release.set()
await run_manager.wait_idle(first.id)
resealed = await _row(first.id)
assert resealed.manifest_seq >= failures[-1]
written = await append_events(first.id, [inquiry("为什么又失败了")], require="sealed")
assert written is not None and written[0].seq > resealed.manifest_seq
assert (await verify_manifest(first.id))["ok"] is True
async def _legacy_sealed(ending: str | None) -> Run:
"""加 manifest_seq 这一列之前封存的老运行:只记了清单哈希,封到哪一条要由 verify_manifest 按事件类型推。"""
from app.core.artifact_store import manifest_hash
rows = [(1, "run.started", None, {"nodes": 2}), (2, "node.started", "out", {}),
(3, "node.finished", "out", {"output": {"r": "完成"}})]
if ending:
rows.append((4, ending, None, {}))
# 老口径:run.finished 在算哈希之后才落库,不在清单里;run.failed / run.cancelled 在
covered = rows[:-1] if ending == "run.finished" else rows
async with SessionLocal() as session:
run = Run(workflow_name="老运行·月报", status="failed" if ending == "run.failed" else "succeeded",
graph={}, input={}, last_seq=rows[-1][0], manifest_hash=manifest_hash(covered), manifest_seq=None)
session.add(run)
await session.flush()
for seq, kind, node_id, data in rows:
session.add(RunEvent(run_id=run.id, seq=seq, type=kind, node_id=node_id, ts=float(seq), data=data))
await session.commit()
return run
@pytest.mark.parametrize("ending", [None, "run.finished", "run.failed"], ids=["no-ending", "finished", "failed"])
async def test_a_legacy_seal_takes_appends_only_when_its_range_does_not_move(ending):
"""老封存的范围是推出来的:有终态事件的,封到最后一条终态事件为止,追加在后面不进范围;没有终态事件的,
封到最后一行——追加一条,范围就跟着挪到它身上,我们自己写的批注被当成篡改。这种不追加。"""
from app.engine.run_append import append_events
legacy = await _legacy_sealed(ending)
before = await verify_manifest(legacy.id)
assert before["ok"] is True and before["legacy"] is True
written = await append_events(legacy.id, [inquiry("这期为什么少了一家店")], require="sealed")
if ending is None:
assert written is None
assert (await _row(legacy.id)).last_seq == legacy.last_seq
else:
assert written is not None and written[0].data["post_seal"] is True
after = await verify_manifest(legacy.id)
assert after["ok"] is True and (after["sealed_at"], after["events"]) == (before["sealed_at"], before["events"])
async def test_the_wrong_kind_of_run_gets_nothing():
from app.engine.run_append import append_events
paused, final = await _paused(), await _finished()
assert await append_events(paused.id, [inquiry("甲")], require="sealed") is None, "停在审批的运行还没封存"
assert await append_events(final.id, [inquiry("乙")], require="interrupted") is None, "已结束的运行不在等审批"
assert await append_events("no-such-run", [inquiry("丙")], require="interrupted") is None
assert await append_events(paused.id, [], require="interrupted") == []
with pytest.raises(ValueError):
await append_events(paused.id, [inquiry("丁")], require="running")
with pytest.raises(ValueError):
await append_events(paused.id, [("approval.unknown", None, {})], require="interrupted")
for run in (paused, final):
assert (await _row(run.id)).last_seq == run.last_seq
assert await _events(run.id, "approval.inquiry") == []
# --------------------------------------------------------------------------
# 序号
# --------------------------------------------------------------------------
async def test_seqs_strictly_increase_and_never_repeat():
from app.engine.run_append import append_events
paused = await _paused()
seqs: list[int] = []
for batch in ([inquiry("一")], [inquiry("二"), inquiry("三")], [inquiry("四")]):
written = await append_events(paused.id, batch, require="interrupted")
seqs += [e.seq for e in written]
assert seqs[0] > paused.last_seq and _strictly_increasing(seqs)
# 同时来几个:同一运行在本进程里排队,读 last_seq 到写入之间不会被另一次插进来,所以一个都不落空
results = await asyncio.gather(*(append_events(paused.id, [inquiry(f"并发{i}")], require="interrupted")
for i in range(6)))
assert all(r is not None for r in results), "并发的追加有落空的"
together = sorted(r[0].seq for r in results)
assert together[0] > seqs[-1] and _strictly_increasing(together)
assert (await _row(paused.id)).last_seq == together[-1]
await _no_repeated_seq(paused.id)
await run_manager.resume(paused.id, {"approved": True})
final = await _until(paused.id, ("succeeded",), sealed=True)
assert (await verify_manifest(final.id))["ok"] is True
await _no_repeated_seq(final.id)
async def test_a_restarted_process_continues_after_the_stored_events():
"""进程重启过,内存里的序号从 0 起:追加接在库里最后一条后面。"""
from app.engine.run_append import append_events
paused = await _paused()
bus._seq.pop(paused.id, None)
written = await append_events(paused.id, [inquiry("重启之后的问题")], require="interrupted")
assert written[0].seq == paused.last_seq + 1
await _no_repeated_seq(paused.id)
@pytest.mark.parametrize("action", ["continue", "resume", "abandon"])
async def test_the_runner_numbers_after_the_stored_events_when_last_seq_lags(flaky, action):
"""last_seq 比库里实际的最后一条小(怎么来的见下一条),本进程又没有这次运行的序号计数(重启过):继续运行、
恢复、放弃都接在实际的最后一条后面。只按 last_seq 接,写出的 run.resumed / run.cancelled 撞上已有的序号,
重新封存时两条同号的事件一起进了清单。"""
if action == "continue":
run = await _finished(FLAKY)
assert run.status == "failed"
else:
run = await _paused()
stored = (await _events(run.id))[-1].seq
_sql("UPDATE runs SET last_seq = last_seq - 3 WHERE id = ?", run.id)
bus._seq.pop(run.id, None)
if action == "continue":
flaky["fail"] = False
await run_manager.continue_failed(run.id)
first_new = "run.resumed"
elif action == "resume":
await run_manager.resume(run.id, {"approved": True})
first_new = "run.resumed"
else:
assert await run_manager.cancel(run.id) == "cancelled"
first_new = "run.cancelled"
final = await _until(run.id, ("succeeded", "cancelled"), sealed=True)
assert final.status == ("cancelled" if action == "abandon" else "succeeded"), final.error
assert [e.seq for e in await _events(run.id, first_new)][-1] == stored + 1
await _no_repeated_seq(run.id)
assert (await verify_manifest(run.id))["ok"] is True
async def test_runner_event_writes_never_lower_last_seq():
"""runner 记事件时 last_seq 只往大里改。取号和提交的先后可以不一样:别处另开会话记的事件(run_manager.note)
和执行任务同时在写,号大的先提交、号小的后提交,以前后提交的那条把 last_seq 写回小的。"""
from app.engine.runner import _stage
paused = await _paused()
early = run_manager._event(paused.id, EventType.LOG, data={"level": "info", "message": "先取号"})
late = run_manager._event(paused.id, EventType.LOG, data={"level": "info", "message": "后取号"})
for event in (late, early):
async with SessionLocal() as session:
await _stage(session, event)
await session.commit()
assert (await _row(paused.id)).last_seq == late.seq
def _judged_args(run: Run) -> tuple:
"""直接调按需裁判记判定的那一步(api/evidence.py 的 _append_judged)要的参数:问过一次模型、什么都没判成。"""
from app.api import evidence
sealed = evidence._Sealed(run_id=run.id, graph={}, seal={"sealed": True, "ok": True}, events=[],
manifest_hash=run.manifest_hash, run_manifest_seq=run.manifest_seq)
report = evidence._Report(node_id="out", doc_artifact="doc-1", ok=None, repairs=None, stats=None, doc=None,
hash_ok=None, fields=[])
outcome = {key: {} for key in ("verdicts", "keys", "unjudged", "skipped", "budget")}
outcome.update(model="judge-model", priced=True, same_model=False, writer_model=None, cost_usd=0.0021,
calls=1, duration_ms=1200, limits_hit=[], gaps=[], notes=[])
return sealed, report, ["u1"], outcome
async def test_a_judge_interleaved_with_a_post_seal_append_survives_a_restart(flaky, monkeypatch):
"""按需裁判记判定(_append_judged)的条件更新不带 last_seq 条件:它取了号、还没写,封存后追加抢先提交了,
它再提交时把 last_seq 写回自己那个更小的号。同一进程里序号计数只增不减,不会撞号;进程重启后计数从 0 起,
各处又按 last_seq 接号——裁判自己也是。所以启动时按库里实际的最后一条改正 last_seq:之后再裁判、继续运行,
都接在实际的最后一条后面。"""
from app.api import evidence
from app.engine.run_append import append_events
first = await _finished(FLAKY)
assert first.status == "failed"
held, opened = asyncio.Event(), evidence.SessionLocal
class HeldBeforeUpdate:
"""裁判的会话:执行条件更新之前停住,等封存后追加先提交。"""
def __init__(self):
self._cm = opened()
async def __aenter__(self):
session = await self._cm.__aenter__()
execute = session.execute
async def held_execute(statement, *args, **kwargs):
if getattr(statement, "is_update", False):
await held.wait()
return await execute(statement, *args, **kwargs)
session.execute = held_execute
return session
async def __aexit__(self, *exc):
return await self._cm.__aexit__(*exc)
monkeypatch.setattr(evidence, "SessionLocal", HeldBeforeUpdate)
judging = asyncio.create_task(evidence._append_judged(*_judged_args(first)))
for _ in range(200): # 裁判取到号了
if bus._seq.get(first.id, 0) > first.last_seq:
break
await asyncio.sleep(0.01)
written = await append_events(first.id, [inquiry("这次为什么失败")], require="sealed")
held.set()
assert await judging == {"seq": first.last_seq + 1}
assert written is not None and written[0].seq == first.last_seq + 2
assert (await _row(first.id)).last_seq == first.last_seq + 1, "裁判后提交,last_seq 被写回了小的号"
monkeypatch.setattr(evidence, "SessionLocal", opened)
# 进程重启:内存里的序号计数没了,启动时 setup 照常执行一遍。只改落后的那条,别的运行原样不动
lagging = await _row(first.id)
others = [await _finished(), await _paused()]
await run_manager.shutdown()
bus._seq.pop(first.id, None)
await run_manager.setup()
repaired = await _row(first.id)
assert (repaired.last_seq, repaired.updated_at) == (written[-1].seq, lagging.updated_at)
for other in others:
now = await _row(other.id)
assert (now.last_seq, now.updated_at) == (other.last_seq, other.updated_at)
assert await evidence._append_judged(*_judged_args(first)) == {"seq": written[-1].seq + 1}
await _no_repeated_seq(first.id)
flaky["fail"] = False
await run_manager.continue_failed(first.id)
final = await _until(first.id, ("succeeded",), sealed=True)
assert final.manifest_seq > written[-1].seq + 1
await _no_repeated_seq(first.id)
assert (await verify_manifest(first.id))["ok"] is True
# --------------------------------------------------------------------------
# 恢复后的重放:Agent 里批过的工具不重复执行
# --------------------------------------------------------------------------
def _script_two_calls(monkeypatch) -> None:
from app.providers import mock_model
def _decide(self, messages):
done = sum(isinstance(m, ToolMessage) for m in messages)
if self.tools and done < 2:
return AIMessage(content="", tool_calls=[
{"name": "inquiry_tally", "args": {"label": f"第{done + 1}次"}, "id": f"call_{done + 1}"}])
return AIMessage(content="两笔都记好了。")
monkeypatch.setattr(mock_model.MockChatModel, "_decide", _decide)
@pytest.mark.parametrize("older_pause", [False, True], ids=["current", "older-protocol"])
async def test_inquiries_do_not_disturb_the_replay(monkeypatch, older_pause):
"""Agent 停在工具审批上时追加问答,批准后工具各执行一次。升级前写下的断点要按事件流判断能不能
安全恢复(runner._stale_replay 只看 tool.start / human.resolved),追加的问答不影响判断。"""
from app.engine import replay, runner
from app.engine.run_append import append_events
_script_two_calls(monkeypatch)
if older_pause:
monkeypatch.setattr(runner, "PROTOCOL", runner.PROTOCOL - 1)
graph = chain(node("start", "input"),
node("bot", "agent", prompt="记两笔", tools=["inquiry_tally"], approval="always", max_steps=5),
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.bot.text }}"}]))
run = await run_manager.start(graph=graph, input_payload={})
for round_ in range(2):
paused = await _until(run.id, ("interrupted", "failed", "succeeded"))
assert paused.status == "interrupted", (paused.status, paused.error)
assert await append_events(run.id, [inquiry(f"第{round_ + 1}笔记的是什么")], require="interrupted")
if older_pause:
monkeypatch.setattr(runner, "PROTOCOL", replay.PROTOCOL)
await run_manager.resume(run.id, {"approved": True})
await asyncio.sleep(0.05)
final = await _until(run.id, ("succeeded", "failed"), sealed=True)
assert final.status == "succeeded", final.error
assert CALLS == ["第1次", "第2次"], "批过的工具又执行了一次"
assert len(await _events(run.id, "approval.inquiry")) == 2
assert (await verify_manifest(run.id))["ok"] is True
await _no_repeated_seq(run.id)
+369
View File
@@ -0,0 +1,369 @@
"""发起运行的规则搬到 engine/launch.py:HTTP 发起和定时任务到点发起共用同一个 launch_run。
- 每一种拒绝:POST /api/runs 的状态码、detail、code 先按搬家之前的原话写死(这一组先在旧代码上通过);
直接调 launch_run 抛 LaunchRefused,和 HTTP 的返回逐条一致
- 发起时定下的治理事件(受管级别那条日志、口径升版处置)在服务里写:不经 HTTP 发起的正式运行一样有;
它们和运行记录同一个事务提交,排在 run.started 前面、在封存范围里,写不进去就不建运行
- 旧的导入路径(app.api.runs 的 missing_tools、node_setting、tool_bindings、TOOL_MISSING)照旧可用;
runner 不再从接口层导入
"""
from __future__ import annotations
import asyncio
import inspect
import uuid
import pytest
from httpx import ASGITransport, AsyncClient
from sqlalchemy import select
from app.db.base import SessionLocal
from app.db.models import Run, RunEvent, Workflow, WorkflowVersion
from app.engine.runner import run_manager
from app.engine.schema import GraphSpec
from app.main import app
@pytest.fixture(autouse=True)
async def engine_up():
await run_manager.setup()
yield
await run_manager.shutdown()
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
def node(nid, ntype, label=None, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}}
def chain(*nodes):
return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
SIMPLE = chain(node("start", "input"), node("out", "output", fields=[{"name": "r", "value": "完成"}]))
def _tag() -> str:
return uuid.uuid4().hex[:8]
async def workflow(graph=SIMPLE, *, published=None, versions=(), snapshot=None, name=None) -> tuple[str, str]:
"""建一个工作流。published 是当前发布版本号,versions 是要建快照的版本号,snapshot 是快照里的图(默认同 graph)。"""
name = name or f"发起测试·周报-{_tag()}"
async with SessionLocal() as session:
wf = Workflow(name=name, graph=graph, status="published" if published else "draft",
published_version=published, version=max(versions, default=1))
session.add(wf)
await session.flush()
for v in versions:
session.add(WorkflowVersion(workflow_id=wf.id, version=v,
graph=snapshot if snapshot is not None else graph))
await session.commit()
return wf.id, name
def tool_graph(*tools: tuple[str, str]) -> dict:
"""每个 (节点名, 工具名) 一个调用工具节点。"""
middle = [node(f"t{i}", "tool", label, tool=tool, args={}) for i, (label, tool) in enumerate(tools)]
return chain(node("start", "input"), *middle, node("out", "output", fields=[{"name": "r", "value": "完成"}]))
# --------------------------------------------------------------------------
# 每一种拒绝:(请求体, (状态码, detail, code))。detail 是 None 时只比两边一致,另由 check 查要点
# --------------------------------------------------------------------------
async def case_formal_without_workflow():
return {"run_class": "formal"}, (400, "正式运行需要从已保存的工作流发起:请求中没有指定工作流", None)
async def case_formal_with_canvas_graph():
wf, _ = await workflow(published=1, versions=(1,))
return ({"run_class": "formal", "workflow_id": wf, "graph": SIMPLE},
(400, "正式运行只能使用已发布的版本,画布上未发布的修改不会生效。请先保存并发布", None))
async def case_formal_workflow_gone():
return {"run_class": "formal", "workflow_id": f"gone{_tag()}"}, (404, "工作流不存在,可能已被删除", None)
async def case_formal_never_published():
wf, name = await workflow()
return {"run_class": "formal", "workflow_id": wf}, (409, f"「{name}」还没有发布版本,请先在编排页发布", None)
async def case_formal_not_the_published_version():
wf, _ = await workflow(published=2, versions=(1, 2))
return ({"run_class": "formal", "workflow_id": wf, "version": 1},
(409, "v1 不是当前发布版本(v2)。正式运行只能使用当前发布版本;如需运行这一版,请先发布它。", None))
async def case_formal_snapshot_missing():
wf, _ = await workflow(published=3, versions=(1, 2))
return {"run_class": "formal", "workflow_id": wf}, (404, "版本 v3 不存在", None)
async def case_formal_snapshot_malformed():
"""快照里的图不成形:以前由 main.py 的 ValueError 兜底回 400,detail 是校验的原话。"""
broken = {"nodes": "坏掉的图"}
wf, _ = await workflow(published=1, versions=(1,), snapshot=broken)
with pytest.raises(ValueError) as caught:
GraphSpec.model_validate(broken)
return {"run_class": "formal", "workflow_id": wf}, (400, str(caught.value), None)
def library(version="v3"):
return chain(node("start", "input"),
node("caliber", "metrics", "周报口径卡", caliber="周报口径", caliber_version=version,
metrics=[{"id": "gmv", "name": "销售额", "unit": "元", "expression": "vars.kpi.gmv"}]),
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.caliber.text }}"}]))
def weekly(lib_id: str, **card) -> dict:
return chain(node("start", "input"),
node("fetch", "transform", mode="expression", expression="{'gmv': 4321.5}", assign_to="kpi"),
node("card", "metrics", "本地口径卡",
caliber_from={"workflow_id": lib_id, "workflow_version": 1, "node_id": "caliber"}, **card),
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.card.text }}"}]))
async def upstream_with_two_versions() -> str:
async with SessionLocal() as session:
lib = Workflow(name=f"口径库-{_tag()}", graph=library(), status="published", published_version=2, version=2)
session.add(lib)
await session.flush()
session.add(WorkflowVersion(workflow_id=lib.id, version=1, graph=library()))
session.add(WorkflowVersion(workflow_id=lib.id, version=2, graph=library(version="v4")))
await session.commit()
return lib.id
async def case_formal_caliber_upgrade_undeclared():
lib = await upstream_with_two_versions()
graph = weekly(lib)
wf, _ = await workflow(graph, published=1, versions=(1,))
return {"run_class": "formal", "workflow_id": wf}, (409, None, None)
def check_formal_caliber_upgrade_undeclared(detail: str) -> None:
assert "「本地口径卡」" in detail and "v1" in detail and "v2" in detail and "「上游发布新版本时」" in detail
async def case_saved_workflow_gone():
return {"workflow_id": f"gone{_tag()}"}, (404, "工作流不存在,可能已被删除", None)
async def case_nothing_to_run():
return {}, (400, "没有要运行的内容:请求中既没有工作流 ID,也没有工作流内容", None)
async def case_saved_workflow_is_empty():
wf, _ = await workflow({})
return {"workflow_id": wf}, (400, "没有要运行的内容:请求中既没有工作流 ID,也没有工作流内容", None)
async def case_missing_datasource_tool():
source = f"gone{_tag()}"
return ({"graph": tool_graph(("取数", f"db_query__{source}"))},
(422, f"绑定的工具不存在:「取数」(调用工具)绑定的 db_query__{source}。请到「数据」页接入,或在节点中重新选择",
"run_tool_missing"))
async def case_missing_custom_tool():
tool = f"ghost_{_tag()}"
return ({"graph": tool_graph(("查编码", tool))},
(422, f"绑定的工具不存在:「查编码」(调用工具)绑定的 {tool}。请到「工具」页接入,或在节点中重新选择",
"run_tool_missing"))
async def case_missing_tools_of_both_kinds():
source, tool = f"gone{_tag()}", f"ghost_{_tag()}"
return ({"graph": tool_graph(("取数", f"db_schema__{source}"), ("查编码", tool))},
(422, f"绑定的工具不存在:「取数」(调用工具)绑定的 db_schema__{source};「查编码」(调用工具)绑定的 {tool}。"
"数据源工具请到「数据」页接入,其他工具请到「工具」页接入,或在节点中重新选择", "run_tool_missing"))
async def case_more_than_eight_missing_tools():
tag = _tag()
tools = [(f"第{i}步", f"ghost_{tag}_{i}") for i in range(10)]
listed = ";".join(f"「{label}」(调用工具)绑定的 {tool}" for label, tool in tools[:8])
return ({"graph": tool_graph(*tools)},
(422, f"绑定的工具不存在:{listed}等 10 处。请到「工具」页接入,或在节点中重新选择", "run_tool_missing"))
async def case_formal_missing_tool():
source = f"gone{_tag()}"
wf, _ = await workflow(tool_graph(("取数", f"db_query__{source}")), published=1, versions=(1,))
return ({"run_class": "formal", "workflow_id": wf},
(422, f"绑定的工具不存在:「取数」(调用工具)绑定的 db_query__{source}。请到「数据」页接入,或在节点中重新选择",
"run_tool_missing"))
async def case_graph_fails_validation():
return ({"graph": {"nodes": [], "edges": []}},
(400, "工作流未通过校验,请先在画布上修正标红的节点再运行:画布上还没有节点,请先从节点库拖入一个节点", None))
#: 「不存在」的工作流、数据源、自定义工具都带随机后缀,库里不会恰好有同名的
CASES = {name[len("case_"):]: fn for name, fn in dict(globals()).items() if name.startswith("case_")}
def _body(status: int, detail: str, code: str | None) -> dict:
return {"detail": detail, **({"code": code} if code else {})}
async def _run_count() -> int:
async with SessionLocal() as session:
return len((await session.execute(select(Run.id))).all())
@pytest.mark.parametrize("case", list(CASES))
async def test_http_refusals_are_unchanged(client, case):
"""HTTP 这一侧逐字写死:搬家前后都得是这些状态码和原话。"""
body, (status, detail, code) = await CASES[case]()
before = await _run_count()
res = await client.post("/api/runs", json=body)
assert res.status_code == status, res.text
got = res.json()
if detail is None:
globals()[f"check_{case}"](got["detail"])
assert set(got) == ({"detail", "code"} if code else {"detail"}), got
else:
assert got == _body(status, detail, code)
assert await _run_count() == before, "被拒的请求不该留下运行记录"
def _launch_kwargs(body: dict) -> dict:
"""请求体换成 launch_run 的参数:input 对应 input_payload,其余同名。"""
out = {k: v for k, v in body.items() if k != "input"}
if "input" in body:
out["input_payload"] = body["input"]
return out
@pytest.mark.parametrize("case", list(CASES))
async def test_launch_run_refuses_exactly_like_http(client, case):
from app.engine.launch import LaunchRefused, launch_run
body, (status, detail, code) = await CASES[case]()
before = await _run_count()
async with SessionLocal() as session:
with pytest.raises(LaunchRefused) as caught:
await launch_run(session, actor="定时任务", **_launch_kwargs(body))
refused = caught.value
assert (refused.status_code, refused.code) == (status, code)
if detail is not None:
assert refused.message == detail
res = await client.post("/api/runs", json=body)
assert res.status_code == refused.status_code
assert res.json() == _body(refused.status_code, refused.message, refused.code)
assert await _run_count() == before
# --------------------------------------------------------------------------
# 发起成功:参数照转、治理事件在服务里写
# --------------------------------------------------------------------------
async def _settle(run_id: str) -> Run:
for _ in range(400):
async with SessionLocal() as session:
row = await session.get(Run, run_id)
if row.status in ("succeeded", "failed", "cancelled") and row.manifest_seq is not None:
return row
await asyncio.sleep(0.03)
raise AssertionError(f"运行 {run_id} 没有结束:{row.status}")
async def _events(run_id: str, etype: str) -> list[RunEvent]:
async with SessionLocal() as session:
return list((await session.execute(
select(RunEvent).where(RunEvent.run_id == run_id, RunEvent.type == etype).order_by(RunEvent.seq)
)).scalars())
async def test_a_formal_launch_records_the_governance_notes():
"""不经 HTTP 发起的正式运行(定时任务)也记下按什么级别出具、口径升版怎么处置的。"""
from app.engine.launch import launch_run
lib = await upstream_with_two_versions()
wf, name = await workflow(weekly(lib, upgrade_policy="dual"), published=1, versions=(1,))
async with SessionLocal() as session:
run = await launch_run(session, workflow_id=wf, run_class="formal", input_payload={"week": "第 40 周"},
actor="定时任务")
assert (run.run_class, run.version, run.workflow_name, run.started_by) == ("formal", 1, name, "定时任务")
assert run.version_hash and run.input == {"week": "第 40 周"}
row = await _settle(run.id)
assert row.status == "succeeded", row.error
notes = [e for e in await _events(run.id, "log") if e.data.get("code") == "run_governance"]
assert len(notes) == 1 and notes[0].data["governed"] is False and "v1" in notes[0].data["message"]
[upgrade] = await _events(run.id, "caliber.upgrade")
assert upgrade.node_id == "card" and upgrade.data["policy"] == "dual" and upgrade.data["latest"] == 2
# 发起时就有的事件排在 run.started 前面,都在封存范围里
[started] = await _events(run.id, "run.started")
assert notes[0].seq < upgrade.seq < started.seq <= row.manifest_seq
async def test_launch_events_are_written_with_the_run_or_not_at_all():
"""发起时定下的事件和运行记录同一个事务提交:写不进去就不建运行,launch_run 抛出异常就是没有发起。以前运行
建好、开始执行之后才另外写,写失败时运行已经在执行,调用方(定时任务)只拿到一个异常,会把一次正在执行的
正式运行记成未能发起,封存范围里也少了受管级别那条日志。"""
from sqlalchemy import text
from sqlalchemy.exc import IntegrityError
from app.db.base import engine
from app.engine.launch import launch_run
lib = await upstream_with_two_versions()
wf, _ = await workflow(weekly(lib, upgrade_policy="dual"), published=1, versions=(1,))
before = await _run_count()
# 口径升版处置那条写不进去。走异步引擎建删触发器:同步连接等写锁时会卡住事件循环,而持锁的正是这个循环里的写入
async with engine.begin() as conn:
await conn.execute(text(
"CREATE TRIGGER refuse_upgrade_note BEFORE INSERT ON run_events WHEN NEW.type = 'caliber.upgrade' "
"BEGIN SELECT RAISE(ABORT, 'refused by test'); END"))
try:
async with SessionLocal() as session:
with pytest.raises(IntegrityError):
await launch_run(session, workflow_id=wf, run_class="formal", actor="定时任务")
assert await _run_count() == before, "没写进治理事件,却留下了一条运行"
finally:
async with engine.begin() as conn:
await conn.execute(text("DROP TRIGGER IF EXISTS refuse_upgrade_note"))
async def test_an_exploratory_launch_takes_the_saved_graph_and_the_run_defaults():
from app.engine.launch import launch_run
wf, name = await workflow()
async with SessionLocal() as session:
run = await launch_run(session, workflow_id=wf, actor=None)
assert (run.run_class, run.workflow_id, run.workflow_name, run.version) == ("exploratory", wf, name, None)
assert (run.memory_scope, run.collection) == ("default", "default") and run.input == {}
row = await _settle(run.id)
assert row.status == "succeeded", row.error
assert [e for e in await _events(run.id, "log") if e.data.get("code") == "run_governance"] == []
# --------------------------------------------------------------------------
# 旧的导入路径
# --------------------------------------------------------------------------
def test_the_old_import_paths_still_work():
from app.api import runs as runs_api
from app.engine import launch
for name in ("missing_tools", "node_setting", "tool_bindings", "TOOL_MISSING"):
assert getattr(runs_api, name) is getattr(launch, name), name
def test_the_runner_no_longer_imports_from_the_api_layer():
from app.engine import runner
assert "app.api.runs" not in inspect.getsource(runner)
+268
View File
@@ -0,0 +1,268 @@
"""运行选项和来源:Run.options 是发起时定下、整次运行沿用的选项快照,Run.schedule_id 是发起它的定时任务。
- 旧库升级时补这两列和 schedule_id 的索引;存量运行的选项读出来是 {},来源为空
- RunManager.start 收下两者并落库;选项非空时写进 run.started(进封存范围),为空时 run.started 和以前
逐字相同;恢复、继续运行沿用发起时的选项,节点从 RunContext.extra 读得到同一份
- 接口带出 schedule_id,运行列表能按定时任务筛;launch_run 原样往下交
"""
from __future__ import annotations
import asyncio
import uuid
import pytest
from httpx import ASGITransport, AsyncClient
from sqlalchemy import create_engine, select, text
from app.db.base import SessionLocal, engine as app_engine
from app.db.models import Run, RunEvent
from app.engine import compiler
from app.engine.runner import run_manager
from app.engine.schema import NodeType
from app.main import app
#: 加 options 之前 run.started 就有的键:选项为空时一个不多、一个不少
LEGACY_STARTED_KEYS = {"nodes", "resumed", "memory_scope", "collection", "approval_default", "tool_trust",
"agent_limits", "data_versions", "replay_protocol"}
OPTIONS = {"verify": {"dual_check": True, "max_queries": 6}, "note": "第 40 周例行核查"}
@pytest.fixture(autouse=True)
async def engine_up():
await run_manager.setup()
yield
await run_manager.shutdown()
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
def node(nid, ntype, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
def chain(*nodes):
return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
SIMPLE = chain(node("start", "input"), node("out", "output", fields=[{"name": "r", "value": "完成"}]))
GATED = chain(node("start", "input"),
node("before", "transform", mode="template", template="前"),
node("gate", "human", mode="approve", title="放行吗"),
node("after", "transform", mode="template", template="后"),
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.after }}"}]))
async def _settle(run_id: str, statuses=("succeeded", "failed", "cancelled")) -> Run:
for _ in range(400):
async with SessionLocal() as session:
row = await session.get(Run, run_id)
if row.status in statuses and (row.status == "interrupted" or row.manifest_seq is not None):
await run_manager.wait_idle(run_id)
return row
await asyncio.sleep(0.03)
raise AssertionError(f"运行 {run_id} 停在 {row.status}")
async def _started(run_id: str) -> list[dict]:
async with SessionLocal() as session:
return [e.data for e in (await session.execute(
select(RunEvent).where(RunEvent.run_id == run_id, RunEvent.type == "run.started").order_by(RunEvent.seq)
)).scalars()]
def _schedule() -> str:
return uuid.uuid4().hex
# --------------------------------------------------------------------------
# 旧库升级
# --------------------------------------------------------------------------
def test_old_databases_get_both_columns_and_the_index():
from app.db.base import _migrate
db = create_engine("sqlite://")
with db.begin() as conn:
conn.execute(text("CREATE TABLE runs (id VARCHAR(32) PRIMARY KEY, workflow_name VARCHAR(200), "
"status VARCHAR(20), last_seq INTEGER)"))
conn.execute(text("INSERT INTO runs (id, workflow_name, status, last_seq) VALUES ('r1', '周报', 'succeeded', 9)"))
_migrate(conn)
_migrate(conn) # 再执行一次不出错:列和索引都已经在了
cols = {row[1] for row in conn.execute(text("PRAGMA table_info(runs)"))}
assert {"options", "schedule_id"} <= cols
assert conn.execute(text("SELECT options, schedule_id FROM runs WHERE id = 'r1'")).one() == ("{}", None)
indexes = {row[1]: row for row in conn.execute(text("PRAGMA index_list(runs)"))}
assert "ix_runs_schedule_id" in indexes
assert [r[2] for r in conn.execute(text("PRAGMA index_info(ix_runs_schedule_id)"))] == ["schedule_id"]
def test_a_missing_table_is_left_alone():
"""还没有 runs 表的库(create_all 之前不会这样,这里只确认迁移不替它建索引、不报错)。"""
from app.db.base import _migrate
db = create_engine("sqlite://")
with db.begin() as conn:
_migrate(conn)
assert conn.execute(text("SELECT name FROM sqlite_master WHERE name = 'ix_runs_schedule_id'")).first() is None
async def test_new_databases_index_schedule_id():
async with app_engine.connect() as conn:
names = {row[1] for row in (await conn.execute(text("PRAGMA index_list(runs)"))).all()}
assert "ix_runs_schedule_id" in names
async def test_a_migrated_row_reads_back_empty_options():
"""迁移回填的是字符串 '{}':经 ORM 读出来是空 dict,不是字符串,也不是 None。"""
async with SessionLocal() as session:
run = Run(workflow_name="迁移回填", status="succeeded", graph={}, input={})
session.add(run)
await session.commit()
run_id = run.id
async with app_engine.begin() as conn:
await conn.execute(text("UPDATE runs SET options = '{}' WHERE id = :id"), {"id": run_id})
async with SessionLocal() as session:
row = await session.get(Run, run_id)
assert row.options == {} and row.schedule_id is None
# --------------------------------------------------------------------------
# 发起:落库、run.started、沿用
# --------------------------------------------------------------------------
async def test_options_and_schedule_are_stored_and_sealed():
sid = _schedule()
run = await run_manager.start(graph=SIMPLE, input_payload={}, options=OPTIONS, schedule_id=sid)
assert (run.options, run.schedule_id) == (OPTIONS, sid)
row = await _settle(run.id)
assert row.status == "succeeded", row.error
assert (row.options, row.schedule_id) == (OPTIONS, sid)
[started] = await _started(run.id)
assert started["options"] == OPTIONS
assert set(started) == LEGACY_STARTED_KEYS | {"options"}
@pytest.mark.parametrize("options", [None, {}])
async def test_without_options_run_started_is_unchanged(options):
run = await run_manager.start(graph=SIMPLE, input_payload={}, options=options)
row = await _settle(run.id)
assert row.options == {} and row.schedule_id is None
[started] = await _started(run.id)
assert set(started) == LEGACY_STARTED_KEYS, "选项为空时 run.started 多出了键"
async def test_the_options_belong_to_the_run_not_to_the_callers_dict():
mine = {"verify": {"dual_check": True}}
run = await run_manager.start(graph=SIMPLE, input_payload={}, options=mine)
mine["verify"]["dual_check"] = False
mine["late"] = 1
await _settle(run.id)
async with SessionLocal() as session:
assert (await session.get(Run, run.id)).options == {"verify": {"dual_check": True}}
@pytest.fixture
def seen_options(monkeypatch):
"""记下每次执行「数据整形」节点时它从运行上下文里读到的选项。"""
from app.engine.runner import RUN_OPTIONS_KEY
seen: list = []
original = compiler.RUNNERS[NodeType.TRANSFORM]
async def spy(state, ctx):
seen.append(ctx.run.extra.get(RUN_OPTIONS_KEY))
return await original(state, ctx)
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, spy)
return seen
async def test_resume_keeps_the_options_the_run_started_with(seen_options):
run = await run_manager.start(graph=GATED, input_payload={}, options=OPTIONS)
assert (await _settle(run.id, ("interrupted",))).status == "interrupted"
await run_manager.resume(run.id, {"approved": True})
row = await _settle(run.id)
assert row.status == "succeeded", row.error
assert seen_options == [OPTIONS, OPTIONS], "恢复之后节点读到的选项变了"
first, second = await _started(run.id)
assert first["options"] == second["options"] == OPTIONS and second["resumed"] is True
async def test_continue_keeps_the_options_too(seen_options, monkeypatch):
original = compiler.RUNNERS[NodeType.TRANSFORM]
calls = {"n": 0}
async def flaky(state, ctx):
calls["n"] += 1
if ctx.node.id == "after" and calls["n"] <= 2:
raise RuntimeError("临时故障")
return await original(state, ctx)
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, flaky)
graph = chain(node("start", "input"),
node("before", "transform", mode="template", template="前"),
node("after", "transform", mode="template", template="后"),
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.after }}"}]))
run = await run_manager.start(graph=graph, input_payload={}, options=OPTIONS)
assert (await _settle(run.id)).status == "failed"
await run_manager.continue_failed(run.id)
row = await _settle(run.id)
assert row.status == "succeeded", row.error
assert [s.get("options") for s in await _started(run.id)] == [OPTIONS, OPTIONS]
async def test_nodes_read_an_empty_dict_when_there_are_no_options(seen_options):
run = await run_manager.start(graph=GATED, input_payload={})
await _settle(run.id, ("interrupted",))
assert seen_options == [{}]
# --------------------------------------------------------------------------
# 接口:带出来源、按定时任务筛、launch_run 照转
# --------------------------------------------------------------------------
async def test_the_api_shows_and_filters_by_schedule(client):
sid, other = _schedule(), _schedule()
mine = [await run_manager.start(graph=SIMPLE, input_payload={}, schedule_id=sid) for _ in range(2)]
theirs = await run_manager.start(graph=SIMPLE, input_payload={}, schedule_id=other)
manual = await run_manager.start(graph=SIMPLE, input_payload={})
for run in (*mine, theirs, manual):
await _settle(run.id)
got = (await client.get(f"/api/runs/{mine[0].id}")).json()
assert got["schedule_id"] == sid
assert (await client.get(f"/api/runs/{manual.id}")).json()["schedule_id"] is None
listed = (await client.get("/api/runs", params={"schedule_id": sid})).json()
assert sorted(r["id"] for r in listed) == sorted(r.id for r in mine)
assert all(r["schedule_id"] == sid for r in listed)
# 不带这个参数时照旧全列
everything = {r["id"] for r in (await client.get("/api/runs", params={"limit": 200})).json()}
assert {theirs.id, manual.id} <= everything
async def test_launch_run_passes_both_through():
from app.engine.launch import launch_run
sid = _schedule()
async with SessionLocal() as session:
run = await launch_run(session, graph=SIMPLE, actor="定时任务", schedule_id=sid, options=OPTIONS)
row = await _settle(run.id)
assert (row.schedule_id, row.options, row.started_by) == (sid, OPTIONS, "定时任务")
[started] = await _started(run.id)
assert started["options"] == OPTIONS
async def test_an_http_launch_has_no_schedule_and_no_options(client):
res = await client.post("/api/runs", json={"graph": SIMPLE})
assert res.status_code == 201, res.text
assert res.json()["schedule_id"] is None
row = await _settle(res.json()["id"])
assert row.options == {}